kushi-agents 6.2.0 → 6.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "kushi-agents",
3
- "version": "6.2.0",
3
+ "version": "6.3.0",
4
4
  "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -1,198 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  // plugin/runners/pull-email.mjs
3
- // Deterministic email pull: one mailbox folder × one ISO week per call.
4
- // Writes to Evidence/<alias>/email/<safe-folder>/<week>/.
5
- //
6
- // Usage:
7
- // node plugin/runners/pull-email.mjs --project <P> --alias <A> --entity <folder-name>
8
- // [--mailbox <upn>] [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]
3
+ // Deterministic email pull via WorkIQ (HARD RULE per workiq-only.instructions.md).
9
4
 
10
- import path from 'node:path';
11
- import { promises as fs } from 'node:fs';
12
- import YAML from 'yaml';
13
- import { assertProject, loadConfig } from './lib/config.mjs';
14
- import { sourceDir } from './lib/layout.mjs';
15
- import { writeAtomic, safeSegment } from './lib/evidence.mjs';
16
- import { fetchAllPages, fetchWithRetry, encodeODataOp } from './lib/http.mjs';
17
- import { getToken, SCOPES } from './lib/identity.mjs';
18
- import { updateCell } from './lib/ledger.mjs';
19
- import { appendRunLog } from './lib/runlog.mjs';
20
- import { enqueue, clear } from './lib/deferred.mjs';
21
- import { currentIsoMonday, ymd, parseYmd } from './lib/weeks.mjs';
22
- import { emitLearningCandidate } from './lib/learnings.mjs';
5
+ import { runCli } from './lib/csc-pull.mjs';
23
6
 
24
- const SOURCE = 'email';
25
-
26
- function parseArgs(argv) {
27
- const args = { dryRun: false, force: false };
28
- for (let i = 0; i < argv.length; i++) {
29
- const a = argv[i];
30
- if (a === '--project') args.project = argv[++i];
31
- else if (a === '--alias') args.alias = argv[++i];
32
- else if (a === '--entity') args.entity = argv[++i];
33
- else if (a === '--mailbox') args.mailbox = argv[++i];
34
- else if (a === '--week') args.week = argv[++i];
35
- else if (a === '--dry-run') args.dryRun = true;
36
- else if (a === '--force') args.force = true;
37
- else if (a === '--fixture') args.fixture = argv[++i];
38
- else if (a === '--help' || a === '-h') args.help = true;
39
- }
40
- return args;
41
- }
42
-
43
- function help() {
44
- return `Usage: node pull-email.mjs --project <P> --alias <A> --entity <folder-name>
45
- [--mailbox <upn>] [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]`;
46
- }
47
-
48
- function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
49
- function configError(msg) { const e = new Error(msg); e.exitCode = 2; return e; }
50
- function authError(msg) { const e = new Error(msg); e.exitCode = 3; return e; }
51
-
52
- async function buildClient({ mailbox, fixture }) {
53
- if (fixture) {
54
- const data = JSON.parse(await fs.readFile(fixture, 'utf8'));
55
- return makeFixtureClient(data);
56
- }
57
- const token = await getToken(SCOPES.graph).catch(e => { throw authError(`token fetch failed: ${e.message}`); });
58
- const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' };
59
- const base = mailbox
60
- ? `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(mailbox)}`
61
- : `https://graph.microsoft.com/v1.0/me`;
62
- return {
63
- async findFolder(displayName) {
64
- const q = `${encodeODataOp('$filter')}=displayName eq '${displayName.replace(/'/g, "''")}'`;
65
- const url = `${base}/mailFolders?${q}&${encodeODataOp('$top')}=2`;
66
- const res = await fetchWithRetry(url, { headers });
67
- const body = await res.json();
68
- return (body.value && body.value[0]) || null;
69
- },
70
- async listMessages(folderId, fromIso, toIso) {
71
- const filter = `receivedDateTime ge ${fromIso} and receivedDateTime lt ${toIso}`;
72
- const url = `${base}/mailFolders/${encodeURIComponent(folderId)}/messages?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}&${encodeODataOp('$orderby')}=receivedDateTime desc&${encodeODataOp('$top')}=50`;
73
- const { items } = await fetchAllPages(url, { headers });
74
- return items;
75
- },
76
- };
77
- }
78
-
79
- function makeFixtureClient(data) {
80
- const foldersByName = new Map();
81
- for (const f of (data.folders || [])) foldersByName.set(f.displayName, f);
82
- return {
83
- async findFolder(name) { return foldersByName.get(name) || null; },
84
- async listMessages(folderId, fromIso, toIso) {
85
- if (data.throwOnListMessages) {
86
- const t = data.throwOnListMessages;
87
- const e = new Error(t.message || 'fixture-throw');
88
- if (t.status) e.status = t.status;
89
- throw e;
90
- }
91
- const all = (data.messagesByFolder && data.messagesByFolder[folderId]) || [];
92
- return all.filter(m => m.receivedDateTime >= fromIso && m.receivedDateTime < toIso);
93
- },
94
- };
95
- }
96
-
97
- function weekBounds(weekStartYmd) {
98
- const start = parseYmd(weekStartYmd);
99
- const end = new Date(start);
100
- end.setDate(end.getDate() + 7);
101
- return { fromIso: start.toISOString(), toIso: end.toISOString() };
102
- }
103
-
104
- async function main() {
105
- const args = parseArgs(process.argv.slice(2));
106
- if (args.help) { console.log(help()); return 0; }
107
- if (!args.project || !args.alias || !args.entity) {
108
- console.error(help());
109
- emit({ source: SOURCE, status: 'failed', errors: [{ signature: 'bad-args', message: 'required: --project --alias --entity' }] });
110
- return 2;
111
- }
112
- const projectRoot = await assertProject(args.project).catch(e => { throw configError(e.message); });
113
- await loadConfig(projectRoot, args.alias);
114
- const weekStart = args.week || ymd(currentIsoMonday());
115
- const { fromIso, toIso } = weekBounds(weekStart);
116
-
117
- const client = await buildClient({ mailbox: args.mailbox, fixture: args.fixture });
118
- const startedAt = new Date().toISOString();
119
-
120
- const folder = await client.findFolder(args.entity).catch(e => { throw e; });
121
- if (!folder) {
122
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
123
- last_status: 'failed', last_error: `folder not found: ${args.entity}`,
124
- });
125
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'failed', errors: [{ signature: 'folder-not-found', message: args.entity }] });
126
- return 0;
127
- }
128
-
129
- let messages;
130
- try {
131
- messages = await client.listMessages(folder.id, fromIso, toIso);
132
- } catch (e) {
133
- const retryable = !e.status || [429, 502, 503, 504].includes(e.status);
134
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: retryable ? 'deferred' : 'failed', last_error: e.message });
135
- if (retryable && !args.dryRun) await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'fetch-failed', reason: e.message });
136
- if (!retryable && !args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'fetch-failed', message: e.message, status: e.status }, context: { runner: 'pull-email' } });
137
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: retryable ? 'deferred' : 'failed', errors: [{ message: e.message, status: e.status }] });
138
- return retryable ? 1 : 0;
139
- }
140
-
141
- if (messages.length === 0) {
142
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'no-activity', items_pulled: 0, folder_id: folder.id, folder_name: folder.displayName });
143
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'no-activity', items_pulled: 0, files_written: [] });
144
- return 0;
145
- }
146
-
147
- const outDir = path.join(sourceDir(projectRoot, args.alias, SOURCE), safeSegment(args.entity), weekStart);
148
- const filesWritten = [];
149
- if (!args.dryRun) {
150
- const r1 = await writeAtomic(path.join(outDir, 'messages.yml'), YAML.stringify(messages));
151
- const r2 = await writeAtomic(path.join(outDir, 'index.md'), renderIndexMd({ folder, messages, weekStart, pulledAt: startedAt }), { skipIfUnchanged: false });
152
- for (const r of [r1, r2]) if (r.written !== false) filesWritten.push(path.relative(projectRoot, r.path));
153
- }
154
-
155
- await clear(projectRoot, args.alias, SOURCE, args.entity).catch(() => {});
156
-
157
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
158
- last_status: 'captured',
159
- items_pulled: messages.length,
160
- folder_id: folder.id,
161
- folder_name: folder.displayName,
162
- });
163
-
164
- if (!args.dryRun) {
165
- await appendRunLog(projectRoot, { runner: 'pull-email', alias: args.alias, entity: args.entity, week: weekStart, status: 'captured', items_pulled: messages.length });
166
- }
167
-
168
- emit({
169
- source: SOURCE, entity: args.entity, week: weekStart, status: 'captured',
170
- items_pulled: messages.length, folder_id: folder.id,
171
- files_written: filesWritten, ledger_key: `email::${args.entity}::${weekStart}`,
172
- });
173
- return 0;
174
- }
175
-
176
- function renderIndexMd({ folder, messages, weekStart, pulledAt }) {
177
- const lines = [
178
- `# Email — ${folder.displayName} — week ${weekStart}`,
179
- '',
180
- `- folder_id: ${folder.id}`,
181
- `- week_start: ${weekStart}`,
182
- `- messages: ${messages.length}`,
183
- `- pulled_at: ${pulledAt}`,
184
- '',
185
- '## Messages',
186
- ];
187
- for (const m of messages) {
188
- const from = (m.from && m.from.emailAddress && m.from.emailAddress.address) || '?';
189
- lines.push(`- [${m.receivedDateTime}] **${m.subject || '(no subject)'}** — ${from}`);
190
- }
191
- lines.push('');
192
- return lines.join('\n');
193
- }
194
-
195
- main().then(code => { process.exitCode = code; }).catch(e => {
196
- emit({ source: SOURCE, status: 'failed', errors: [{ message: e.message, code: e.exitCode || 'unknown' }] });
197
- process.exitCode = e.exitCode || 1;
198
- });
7
+ runCli('email').then(code => { process.exitCode = code; });
@@ -1,211 +1,8 @@
1
1
  #!/usr/bin/env node
2
2
  // plugin/runners/pull-meetings.mjs
3
- // Deterministic meeting pull: one occurrence per call.
4
- // Verbatim transcript required at meetings/<dir>/verbatim/transcript.txt.
5
- // BODY_UNAVAILABLE (no transcript yet) defers 4h (per RETRY_MIN_AGE_MIN.meetings).
6
- //
7
- // Usage:
8
- // node plugin/runners/pull-meetings.mjs --project <P> --alias <A> --entity <joinUrl-or-meetingId>
9
- // [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]
3
+ // Deterministic meetings pull via WorkIQ. CSC weekly file emitted via runCli.
4
+ // Verbatim transcript capture is handled inside pullSource (meetings-only branch).
10
5
 
11
- import path from 'node:path';
12
- import { promises as fs } from 'node:fs';
13
- import YAML from 'yaml';
14
- import { assertProject, loadConfig } from './lib/config.mjs';
15
- import { sourceDir } from './lib/layout.mjs';
16
- import { writeAtomic } from './lib/evidence.mjs';
17
- import { fetchWithRetry, encodeODataOp } from './lib/http.mjs';
18
- import { getToken, SCOPES } from './lib/identity.mjs';
19
- import { updateCell } from './lib/ledger.mjs';
20
- import { appendRunLog } from './lib/runlog.mjs';
21
- import { enqueue, clear } from './lib/deferred.mjs';
22
- import { shortHash } from './lib/dedup.mjs';
23
- import { currentIsoMonday, ymd } from './lib/weeks.mjs';
24
- import { emitLearningCandidate } from './lib/learnings.mjs';
6
+ import { runCli } from './lib/csc-pull.mjs';
25
7
 
26
- const SOURCE = 'meetings';
27
-
28
- function parseArgs(argv) {
29
- const args = { dryRun: false, force: false };
30
- for (let i = 0; i < argv.length; i++) {
31
- const a = argv[i];
32
- if (a === '--project') args.project = argv[++i];
33
- else if (a === '--alias') args.alias = argv[++i];
34
- else if (a === '--entity') args.entity = argv[++i];
35
- else if (a === '--week') args.week = argv[++i];
36
- else if (a === '--dry-run') args.dryRun = true;
37
- else if (a === '--force') args.force = true;
38
- else if (a === '--fixture') args.fixture = argv[++i];
39
- else if (a === '--help' || a === '-h') args.help = true;
40
- }
41
- return args;
42
- }
43
-
44
- function help() {
45
- return `Usage: node pull-meetings.mjs --project <P> --alias <A> --entity <joinUrl-or-meetingId>
46
- [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]`;
47
- }
48
-
49
- function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
50
- function configError(msg) { const e = new Error(msg); e.exitCode = 2; return e; }
51
- function authError(msg) { const e = new Error(msg); e.exitCode = 3; return e; }
52
-
53
- async function buildClient({ fixture }) {
54
- if (fixture) {
55
- const data = JSON.parse(await fs.readFile(fixture, 'utf8'));
56
- return makeFixtureClient(data);
57
- }
58
- const token = await getToken(SCOPES.graph).catch(e => { throw authError(`token fetch failed: ${e.message}`); });
59
- const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' };
60
- return {
61
- async resolveMeeting(entity) {
62
- // Treat joinUrl-shaped entities specially; otherwise treat as onlineMeeting id.
63
- if (/^https?:\/\//i.test(entity)) {
64
- const filter = `JoinWebUrl eq '${entity.replace(/'/g, "''")}'`;
65
- const url = `https://graph.microsoft.com/v1.0/me/onlineMeetings?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}`;
66
- const res = await fetchWithRetry(url, { headers });
67
- const body = await res.json();
68
- return (body.value && body.value[0]) || null;
69
- }
70
- const res = await fetchWithRetry(`https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(entity)}`, { headers });
71
- return await res.json();
72
- },
73
- async getTranscript(meetingId) {
74
- // List transcripts, fetch most recent .vtt content.
75
- const listUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(meetingId)}/transcripts`;
76
- let res;
77
- try { res = await fetchWithRetry(listUrl, { headers }); } catch (e) { if (e.status === 404) return null; throw e; }
78
- const body = await res.json();
79
- const list = body.value || [];
80
- if (list.length === 0) return null;
81
- list.sort((a, b) => String(b.createdDateTime || '').localeCompare(String(a.createdDateTime || '')));
82
- const tx = list[0];
83
- const contentUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(meetingId)}/transcripts/${encodeURIComponent(tx.id)}/content?$format=text/vtt`;
84
- const res2 = await fetchWithRetry(contentUrl, { headers: { ...headers, Accept: 'text/vtt' } });
85
- return await res2.text();
86
- },
87
- };
88
- }
89
-
90
- function makeFixtureClient(data) {
91
- return {
92
- async resolveMeeting(_entity) { return data.meeting || null; },
93
- async getTranscript(_id) {
94
- if (data.transcript === null) return null;
95
- return data.transcript || null;
96
- },
97
- };
98
- }
99
-
100
- async function main() {
101
- const args = parseArgs(process.argv.slice(2));
102
- if (args.help) { console.log(help()); return 0; }
103
- if (!args.project || !args.alias || !args.entity) {
104
- console.error(help());
105
- emit({ source: SOURCE, status: 'failed', errors: [{ signature: 'bad-args', message: 'required: --project --alias --entity' }] });
106
- return 2;
107
- }
108
- const projectRoot = await assertProject(args.project).catch(e => { throw configError(e.message); });
109
- await loadConfig(projectRoot, args.alias);
110
- const weekStart = args.week || ymd(currentIsoMonday());
111
-
112
- const client = await buildClient({ fixture: args.fixture });
113
- const startedAt = new Date().toISOString();
114
-
115
- let meeting;
116
- try { meeting = await client.resolveMeeting(args.entity); }
117
- catch (e) {
118
- const retryable = !e.status || [429, 502, 503, 504].includes(e.status);
119
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: retryable ? 'deferred' : 'failed', last_error: e.message });
120
- if (retryable && !args.dryRun) await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'fetch-failed', reason: e.message });
121
- if (!retryable && !args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'fetch-failed', message: e.message, status: e.status }, context: { runner: 'pull-meetings' } });
122
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: retryable ? 'deferred' : 'failed', errors: [{ message: e.message, status: e.status }] });
123
- return retryable ? 1 : 0;
124
- }
125
-
126
- if (!meeting) {
127
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'no-activity', items_pulled: 0 });
128
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'no-activity', items_pulled: 0, files_written: [] });
129
- return 0;
130
- }
131
-
132
- const meetingId = meeting.id || shortHash(args.entity);
133
- const dirHash = shortHash(args.entity);
134
- const outDir = path.join(sourceDir(projectRoot, args.alias, SOURCE), dirHash);
135
-
136
- let transcript;
137
- try { transcript = await client.getTranscript(meetingId); }
138
- catch (e) { transcript = null; }
139
-
140
- const filesWritten = [];
141
- if (!args.dryRun) {
142
- const r1 = await writeAtomic(path.join(outDir, 'meeting.yml'), YAML.stringify(meeting));
143
- if (r1.written !== false) filesWritten.push(path.relative(projectRoot, r1.path));
144
- }
145
-
146
- if (transcript == null) {
147
- // BODY_UNAVAILABLE — defer 4h
148
- if (!args.dryRun) {
149
- await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'body-unavailable', reason: 'transcript not yet available' });
150
- }
151
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
152
- last_status: 'body-unavailable',
153
- meeting_id: meetingId,
154
- transcript_available: false,
155
- });
156
- emit({
157
- source: SOURCE, entity: args.entity, week: weekStart, status: 'body-unavailable',
158
- meeting_id: meetingId, files_written: filesWritten,
159
- errors: [{ signature: 'body-unavailable', action_taken: 'deferred-retry-marker-written', retry_after_min: 240 }],
160
- });
161
- return 0;
162
- }
163
-
164
- if (!args.dryRun) {
165
- const r2 = await writeAtomic(path.join(outDir, 'verbatim', 'transcript.txt'), transcript);
166
- const r3 = await writeAtomic(path.join(outDir, 'index.md'), renderIndexMd({ meeting, transcriptChars: transcript.length, pulledAt: startedAt }), { skipIfUnchanged: false });
167
- for (const r of [r2, r3]) if (r.written !== false) filesWritten.push(path.relative(projectRoot, r.path));
168
- }
169
-
170
- await clear(projectRoot, args.alias, SOURCE, args.entity).catch(() => {});
171
-
172
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
173
- last_status: 'captured',
174
- items_pulled: 1,
175
- meeting_id: meetingId,
176
- transcript_available: true,
177
- transcript_chars: transcript.length,
178
- });
179
-
180
- if (!args.dryRun) {
181
- await appendRunLog(projectRoot, { runner: 'pull-meetings', alias: args.alias, entity: args.entity, week: weekStart, status: 'captured', transcript_chars: transcript.length });
182
- }
183
-
184
- emit({
185
- source: SOURCE, entity: args.entity, week: weekStart, status: 'captured',
186
- items_pulled: 1, meeting_id: meetingId, transcript_chars: transcript.length,
187
- files_written: filesWritten, ledger_key: `meetings::${args.entity}::${weekStart}`,
188
- });
189
- return 0;
190
- }
191
-
192
- function renderIndexMd({ meeting, transcriptChars, pulledAt }) {
193
- const lines = [
194
- `# Meeting — ${meeting.subject || '(no subject)'}`,
195
- '',
196
- `- meeting_id: ${meeting.id || '?'}`,
197
- `- start: ${(meeting.startDateTime) || '?'}`,
198
- `- end: ${(meeting.endDateTime) || '?'}`,
199
- `- transcript_chars: ${transcriptChars}`,
200
- `- pulled_at: ${pulledAt}`,
201
- '',
202
- 'Verbatim transcript at `verbatim/transcript.txt`.',
203
- '',
204
- ];
205
- return lines.join('\n');
206
- }
207
-
208
- main().then(code => { process.exitCode = code; }).catch(e => {
209
- emit({ source: SOURCE, status: 'failed', errors: [{ message: e.message, code: e.exitCode || 'unknown' }] });
210
- process.exitCode = e.exitCode || 1;
211
- });
8
+ runCli('meetings').then(code => { process.exitCode = code; });
@@ -1,243 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  // plugin/runners/pull-onenote.mjs
3
- // Deterministic OneNote pull: two-phase (enum pages, then per-page content).
4
- // BODY_UNAVAILABLE does NOT defer — re-resolves the section via search, then retries.
5
- // Writes to Evidence/<alias>/onenote/<section-file-id>/<week>/.
6
- //
7
- // Usage:
8
- // node plugin/runners/pull-onenote.mjs --project <P> --alias <A> --entity <section-file-id>
9
- // [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]
3
+ // Deterministic onenote pull via WorkIQ (HARD RULE per workiq-only.instructions.md).
10
4
 
11
- import path from 'node:path';
12
- import { promises as fs } from 'node:fs';
13
- import YAML from 'yaml';
14
- import { assertProject, loadConfig } from './lib/config.mjs';
15
- import { sourceDir } from './lib/layout.mjs';
16
- import { writeAtomic } from './lib/evidence.mjs';
17
- import { fetchAllPages, fetchWithRetry, encodeODataOp } from './lib/http.mjs';
18
- import { getToken, SCOPES } from './lib/identity.mjs';
19
- import { updateCell } from './lib/ledger.mjs';
20
- import { appendRunLog } from './lib/runlog.mjs';
21
- import { clear } from './lib/deferred.mjs';
22
- import { currentIsoMonday, ymd, parseYmd } from './lib/weeks.mjs';
23
- import { emitLearningCandidate } from './lib/learnings.mjs';
24
- import { readLedger, cellKey } from './lib/ledger.mjs';
5
+ import { runCli } from './lib/csc-pull.mjs';
25
6
 
26
- const SOURCE = 'onenote';
27
-
28
- function parseArgs(argv) {
29
- const args = { dryRun: false, force: false };
30
- for (let i = 0; i < argv.length; i++) {
31
- const a = argv[i];
32
- if (a === '--project') args.project = argv[++i];
33
- else if (a === '--alias') args.alias = argv[++i];
34
- else if (a === '--entity') args.entity = argv[++i];
35
- else if (a === '--week') args.week = argv[++i];
36
- else if (a === '--dry-run') args.dryRun = true;
37
- else if (a === '--force') args.force = true;
38
- else if (a === '--fixture') args.fixture = argv[++i];
39
- else if (a === '--help' || a === '-h') args.help = true;
40
- }
41
- return args;
42
- }
43
-
44
- function help() {
45
- return `Usage: node pull-onenote.mjs --project <P> --alias <A> --entity <section-file-id>
46
- [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]`;
47
- }
48
-
49
- function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
50
- function configError(msg) { const e = new Error(msg); e.exitCode = 2; return e; }
51
- function authError(msg) { const e = new Error(msg); e.exitCode = 3; return e; }
52
-
53
- async function buildClient({ fixture }) {
54
- if (fixture) {
55
- const data = JSON.parse(await fs.readFile(fixture, 'utf8'));
56
- return makeFixtureClient(data);
57
- }
58
- const token = await getToken(SCOPES.graph).catch(e => { throw authError(`token fetch failed: ${e.message}`); });
59
- const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' };
60
- return {
61
- async getSection(sectionId) {
62
- try {
63
- const res = await fetchWithRetry(`https://graph.microsoft.com/v1.0/me/onenote/sections/${encodeURIComponent(sectionId)}`, { headers });
64
- return await res.json();
65
- } catch (e) { if (e.status === 404) return null; throw e; }
66
- },
67
- async resolveSection(sectionFileIdOrName) {
68
- // Re-resolve via search across sections (used on body-unavailable)
69
- const url = `https://graph.microsoft.com/v1.0/me/onenote/sections?${encodeODataOp('$filter')}=id eq '${sectionFileIdOrName.replace(/'/g, "''")}'`;
70
- try {
71
- const res = await fetchWithRetry(url, { headers });
72
- const body = await res.json();
73
- return (body.value && body.value[0]) || null;
74
- } catch { return null; }
75
- },
76
- async listPages(sectionId, fromIso, toIso) {
77
- const filter = `lastModifiedDateTime ge ${fromIso} and lastModifiedDateTime lt ${toIso}`;
78
- const url = `https://graph.microsoft.com/v1.0/me/onenote/sections/${encodeURIComponent(sectionId)}/pages?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}&${encodeODataOp('$orderby')}=lastModifiedDateTime desc&${encodeODataOp('$top')}=50`;
79
- const { items } = await fetchAllPages(url, { headers });
80
- return items;
81
- },
82
- async getPageContent(pageId) {
83
- const url = `https://graph.microsoft.com/v1.0/me/onenote/pages/${encodeURIComponent(pageId)}/content`;
84
- try {
85
- const res = await fetchWithRetry(url, { headers: { ...headers, Accept: 'text/html' } });
86
- return await res.text();
87
- } catch (e) {
88
- if (e.status === 404 || e.status === 410) return null; // body-unavailable signal
89
- throw e;
90
- }
91
- },
92
- };
93
- }
94
-
95
- function makeFixtureClient(data) {
96
- return {
97
- async getSection(id) { return (data.section && data.section.id === id) ? data.section : null; },
98
- async resolveSection(id) { return (data.resolvedSection && data.resolvedSection.id === id) ? data.resolvedSection : null; },
99
- async listPages(_sid, fromIso, toIso) {
100
- return (data.pages || []).filter(p => p.lastModifiedDateTime >= fromIso && p.lastModifiedDateTime < toIso);
101
- },
102
- async getPageContent(pageId) {
103
- const p = (data.pages || []).find(p => p.id === pageId);
104
- return p ? p.content : null;
105
- },
106
- };
107
- }
108
-
109
- function weekBounds(weekStartYmd) {
110
- const start = parseYmd(weekStartYmd);
111
- const end = new Date(start);
112
- end.setDate(end.getDate() + 7);
113
- return { fromIso: start.toISOString(), toIso: end.toISOString() };
114
- }
115
-
116
- async function main() {
117
- const args = parseArgs(process.argv.slice(2));
118
- if (args.help) { console.log(help()); return 0; }
119
- if (!args.project || !args.alias || !args.entity) {
120
- console.error(help());
121
- emit({ source: SOURCE, status: 'failed', errors: [{ signature: 'bad-args', message: 'required: --project --alias --entity' }] });
122
- return 2;
123
- }
124
- const projectRoot = await assertProject(args.project).catch(e => { throw configError(e.message); });
125
- await loadConfig(projectRoot, args.alias);
126
- const weekStart = args.week || ymd(currentIsoMonday());
127
- const { fromIso, toIso } = weekBounds(weekStart);
128
-
129
- const client = await buildClient({ fixture: args.fixture });
130
- const startedAt = new Date().toISOString();
131
-
132
- // Phase 1: section enum
133
- let section;
134
- try { section = await client.getSection(args.entity); }
135
- catch (e) {
136
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'failed', last_error: e.message });
137
- if (!args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'section-fetch-failed', message: e.message, status: e.status }, context: { runner: 'pull-onenote' } });
138
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'failed', errors: [{ message: e.message }] });
139
- return 0;
140
- }
141
- if (!section) {
142
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'failed', last_error: 'section not found' });
143
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'failed', errors: [{ signature: 'section-not-found' }] });
144
- return 0;
145
- }
146
-
147
- const pages = await client.listPages(section.id, fromIso, toIso).catch(() => []);
148
- if (pages.length === 0) {
149
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'no-activity', items_pulled: 0, section_file_id: section.id });
150
- emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'no-activity', items_pulled: 0, files_written: [] });
151
- return 0;
152
- }
153
-
154
- // Phase 2: per-page content fetch (re-resolves section on body-unavailable; does NOT enqueue)
155
- const captures = [];
156
- const bodyUnavailable = [];
157
- for (const p of pages) {
158
- let content = await client.getPageContent(p.id).catch(() => null);
159
- if (content == null) {
160
- const resolved = await client.resolveSection(args.entity);
161
- if (resolved && resolved.id) {
162
- content = await client.getPageContent(p.id).catch(() => null);
163
- }
164
- }
165
- if (content == null) bodyUnavailable.push(p.id);
166
- else captures.push({ page: p, content });
167
- }
168
-
169
- const outDir = path.join(sourceDir(projectRoot, args.alias, SOURCE), section.id, weekStart);
170
- const filesWritten = [];
171
- if (!args.dryRun) {
172
- const r1 = await writeAtomic(path.join(outDir, 'pages.yml'), YAML.stringify(pages));
173
- if (r1.written !== false) filesWritten.push(path.relative(projectRoot, r1.path));
174
- for (const c of captures) {
175
- const r = await writeAtomic(path.join(outDir, 'content', `${c.page.id}.html`), c.content);
176
- if (r.written !== false) filesWritten.push(path.relative(projectRoot, r.path));
177
- }
178
- const r2 = await writeAtomic(path.join(outDir, 'index.md'), renderIndexMd({ section, pages, captures, bodyUnavailable, pulledAt: startedAt }), { skipIfUnchanged: false });
179
- if (r2.written !== false) filesWritten.push(path.relative(projectRoot, r2.path));
180
- }
181
-
182
- const status = bodyUnavailable.length === 0 ? 'captured' : (captures.length === 0 ? 'body-unavailable' : 'partial');
183
-
184
- if (status === 'body-unavailable' && !args.dryRun) {
185
- const prior = (await readLedger(projectRoot, args.alias).catch(() => ({ cells: {} })))
186
- .cells?.[cellKey(SOURCE, args.entity, weekStart)];
187
- const priorOccurrences = Number(prior?.body_unavailable_runs || 0);
188
- const occurrences = priorOccurrences + 1;
189
- if (occurrences >= 2) {
190
- await emitLearningCandidate({
191
- projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart,
192
- error: { signature: 'body-unavailable', message: `OneNote section ${section.id}: ${bodyUnavailable.length}/${pages.length} pages had no body across ${occurrences} runs`, occurrences },
193
- context: { runner: 'pull-onenote' },
194
- });
195
- }
196
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { body_unavailable_runs: occurrences });
197
- }
198
-
199
- await clear(projectRoot, args.alias, SOURCE, args.entity).catch(() => {});
200
- await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
201
- last_status: status,
202
- items_pulled: captures.length,
203
- pages_enumerated: pages.length,
204
- body_unavailable: bodyUnavailable.length || undefined,
205
- section_file_id: section.id,
206
- });
207
-
208
- if (!args.dryRun) {
209
- await appendRunLog(projectRoot, { runner: 'pull-onenote', alias: args.alias, entity: args.entity, week: weekStart, status, pages_enumerated: pages.length, pages_captured: captures.length });
210
- }
211
-
212
- emit({
213
- source: SOURCE, entity: args.entity, week: weekStart, status,
214
- items_pulled: captures.length, pages_enumerated: pages.length, body_unavailable: bodyUnavailable,
215
- files_written: filesWritten, ledger_key: `onenote::${args.entity}::${weekStart}`,
216
- });
217
- return 0;
218
- }
219
-
220
- function renderIndexMd({ section, pages, captures, bodyUnavailable, pulledAt }) {
221
- const lines = [
222
- `# OneNote — ${section.displayName || section.id}`,
223
- '',
224
- `- section_id: ${section.id}`,
225
- `- pages_enumerated: ${pages.length}`,
226
- `- pages_captured: ${captures.length}`,
227
- `- body_unavailable: ${bodyUnavailable.length}`,
228
- `- pulled_at: ${pulledAt}`,
229
- '',
230
- '## Pages',
231
- ];
232
- for (const p of pages) {
233
- const captured = captures.find(c => c.page.id === p.id) ? '✓' : '✗';
234
- lines.push(`- [${captured}] ${p.lastModifiedDateTime} — **${p.title || p.id}**`);
235
- }
236
- lines.push('');
237
- return lines.join('\n');
238
- }
239
-
240
- main().then(code => { process.exitCode = code; }).catch(e => {
241
- emit({ source: SOURCE, status: 'failed', errors: [{ message: e.message, code: e.exitCode || 'unknown' }] });
242
- process.exitCode = e.exitCode || 1;
243
- });
7
+ runCli('onenote').then(code => { process.exitCode = code; });