kushi-agents 6.2.1 → 6.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kushi-agents",
-  "version": "6.2.1",
+  "version": "6.3.0",
   "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
   "type": "module",
   "bin": {

package/plugin/runners/pull-email.mjs

@@ -1,198 +1,7 @@
 #!/usr/bin/env node
 // plugin/runners/pull-email.mjs
-// Deterministic email pull: one mailbox folder × one ISO week per call.
-// Writes to Evidence/<alias>/email/<safe-folder>/<week>/.
-//
-// Usage:
-//   node plugin/runners/pull-email.mjs --project <P> --alias <A> --entity <folder-name>
-//     [--mailbox <upn>] [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]
+// Deterministic email pull via WorkIQ (HARD RULE per workiq-only.instructions.md).
 
-import path from 'node:path';
-import { promises as fs } from 'node:fs';
-import YAML from 'yaml';
-import { assertProject, loadConfig } from './lib/config.mjs';
-import { sourceDir } from './lib/layout.mjs';
-import { writeAtomic, safeSegment } from './lib/evidence.mjs';
-import { fetchAllPages, fetchWithRetry, encodeODataOp } from './lib/http.mjs';
-import { getToken, SCOPES } from './lib/identity.mjs';
-import { updateCell } from './lib/ledger.mjs';
-import { appendRunLog } from './lib/runlog.mjs';
-import { enqueue, clear } from './lib/deferred.mjs';
-import { currentIsoMonday, ymd, parseYmd } from './lib/weeks.mjs';
-import { emitLearningCandidate } from './lib/learnings.mjs';
+import { runCli } from './lib/csc-pull.mjs';
 
-const SOURCE = 'email';
-
-function parseArgs(argv) {
-  const args = { dryRun: false, force: false };
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (a === '--project') args.project = argv[++i];
-    else if (a === '--alias') args.alias = argv[++i];
-    else if (a === '--entity') args.entity = argv[++i];
-    else if (a === '--mailbox') args.mailbox = argv[++i];
-    else if (a === '--week') args.week = argv[++i];
-    else if (a === '--dry-run') args.dryRun = true;
-    else if (a === '--force') args.force = true;
-    else if (a === '--fixture') args.fixture = argv[++i];
-    else if (a === '--help' || a === '-h') args.help = true;
-  }
-  return args;
-}
-
-function help() {
-  return `Usage: node pull-email.mjs --project <P> --alias <A> --entity <folder-name>
-  [--mailbox <upn>] [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]`;
-}
-
-function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
-function configError(msg) { const e = new Error(msg); e.exitCode = 2; return e; }
-function authError(msg)   { const e = new Error(msg); e.exitCode = 3; return e; }
-
-async function buildClient({ mailbox, fixture }) {
-  if (fixture) {
-    const data = JSON.parse(await fs.readFile(fixture, 'utf8'));
-    return makeFixtureClient(data);
-  }
-  const token = await getToken(SCOPES.graph).catch(e => { throw authError(`token fetch failed: ${e.message}`); });
-  const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' };
-  const base = mailbox
-    ? `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(mailbox)}`
-    : `https://graph.microsoft.com/v1.0/me`;
-  return {
-    async findFolder(displayName) {
-      const q = `${encodeODataOp('$filter')}=displayName eq '${displayName.replace(/'/g, "''")}'`;
-      const url = `${base}/mailFolders?${q}&${encodeODataOp('$top')}=2`;
-      const res = await fetchWithRetry(url, { headers });
-      const body = await res.json();
-      return (body.value && body.value[0]) || null;
-    },
-    async listMessages(folderId, fromIso, toIso) {
-      const filter = `receivedDateTime ge ${fromIso} and receivedDateTime lt ${toIso}`;
-      const url = `${base}/mailFolders/${encodeURIComponent(folderId)}/messages?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}&${encodeODataOp('$orderby')}=receivedDateTime desc&${encodeODataOp('$top')}=50`;
-      const { items } = await fetchAllPages(url, { headers });
-      return items;
-    },
-  };
-}
-
-function makeFixtureClient(data) {
-  const foldersByName = new Map();
-  for (const f of (data.folders || [])) foldersByName.set(f.displayName, f);
-  return {
-    async findFolder(name) { return foldersByName.get(name) || null; },
-    async listMessages(folderId, fromIso, toIso) {
-      if (data.throwOnListMessages) {
-        const t = data.throwOnListMessages;
-        const e = new Error(t.message || 'fixture-throw');
-        if (t.status) e.status = t.status;
-        throw e;
-      }
-      const all = (data.messagesByFolder && data.messagesByFolder[folderId]) || [];
-      return all.filter(m => m.receivedDateTime >= fromIso && m.receivedDateTime < toIso);
-    },
-  };
-}
-
-function weekBounds(weekStartYmd) {
-  const start = parseYmd(weekStartYmd);
-  const end = new Date(start);
-  end.setDate(end.getDate() + 7);
-  return { fromIso: start.toISOString(), toIso: end.toISOString() };
-}
-
-async function main() {
-  const args = parseArgs(process.argv.slice(2));
-  if (args.help) { console.log(help()); return 0; }
-  if (!args.project || !args.alias || !args.entity) {
-    console.error(help());
-    emit({ source: SOURCE, status: 'failed', errors: [{ signature: 'bad-args', message: 'required: --project --alias --entity' }] });
-    return 2;
-  }
-  const projectRoot = await assertProject(args.project).catch(e => { throw configError(e.message); });
-  await loadConfig(projectRoot, args.alias);
-  const weekStart = args.week || ymd(currentIsoMonday());
-  const { fromIso, toIso } = weekBounds(weekStart);
-
-  const client = await buildClient({ mailbox: args.mailbox, fixture: args.fixture });
-  const startedAt = new Date().toISOString();
-
-  const folder = await client.findFolder(args.entity).catch(e => { throw e; });
-  if (!folder) {
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
-      last_status: 'failed', last_error: `folder not found: ${args.entity}`,
-    });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'failed', errors: [{ signature: 'folder-not-found', message: args.entity }] });
-    return 0;
-  }
-
-  let messages;
-  try {
-    messages = await client.listMessages(folder.id, fromIso, toIso);
-  } catch (e) {
-    const retryable = !e.status || [429, 502, 503, 504].includes(e.status);
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: retryable ? 'deferred' : 'failed', last_error: e.message });
-    if (retryable && !args.dryRun) await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'fetch-failed', reason: e.message });
-    if (!retryable && !args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'fetch-failed', message: e.message, status: e.status }, context: { runner: 'pull-email' } });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: retryable ? 'deferred' : 'failed', errors: [{ message: e.message, status: e.status }] });
-    return retryable ? 1 : 0;
-  }
-
-  if (messages.length === 0) {
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'no-activity', items_pulled: 0, folder_id: folder.id, folder_name: folder.displayName });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'no-activity', items_pulled: 0, files_written: [] });
-    return 0;
-  }
-
-  const outDir = path.join(sourceDir(projectRoot, args.alias, SOURCE), safeSegment(args.entity), weekStart);
-  const filesWritten = [];
-  if (!args.dryRun) {
-    const r1 = await writeAtomic(path.join(outDir, 'messages.yml'), YAML.stringify(messages));
-    const r2 = await writeAtomic(path.join(outDir, 'index.md'), renderIndexMd({ folder, messages, weekStart, pulledAt: startedAt }), { skipIfUnchanged: false });
-    for (const r of [r1, r2]) if (r.written !== false) filesWritten.push(path.relative(projectRoot, r.path));
-  }
-
-  await clear(projectRoot, args.alias, SOURCE, args.entity).catch(() => {});
-
-  await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
-    last_status: 'captured',
-    items_pulled: messages.length,
-    folder_id: folder.id,
-    folder_name: folder.displayName,
-  });
-
-  if (!args.dryRun) {
-    await appendRunLog(projectRoot, { runner: 'pull-email', alias: args.alias, entity: args.entity, week: weekStart, status: 'captured', items_pulled: messages.length });
-  }
-
-  emit({
-    source: SOURCE, entity: args.entity, week: weekStart, status: 'captured',
-    items_pulled: messages.length, folder_id: folder.id,
-    files_written: filesWritten, ledger_key: `email::${args.entity}::${weekStart}`,
-  });
-  return 0;
-}
-
-function renderIndexMd({ folder, messages, weekStart, pulledAt }) {
-  const lines = [
-    `# Email — ${folder.displayName} — week ${weekStart}`,
-    '',
-    `- folder_id: ${folder.id}`,
-    `- week_start: ${weekStart}`,
-    `- messages: ${messages.length}`,
-    `- pulled_at: ${pulledAt}`,
-    '',
-    '## Messages',
-  ];
-  for (const m of messages) {
-    const from = (m.from && m.from.emailAddress && m.from.emailAddress.address) || '?';
-    lines.push(`- [${m.receivedDateTime}] **${m.subject || '(no subject)'}** — ${from}`);
-  }
-  lines.push('');
-  return lines.join('\n');
-}
-
-main().then(code => { process.exitCode = code; }).catch(e => {
-  emit({ source: SOURCE, status: 'failed', errors: [{ message: e.message, code: e.exitCode || 'unknown' }] });
-  process.exitCode = e.exitCode || 1;
-});
+runCli('email').then(code => { process.exitCode = code; });

package/plugin/runners/pull-meetings.mjs

@@ -1,224 +1,8 @@
 #!/usr/bin/env node
 // plugin/runners/pull-meetings.mjs
-// Deterministic meeting pull: one occurrence per call.
-// Verbatim transcript required at meetings/<dir>/verbatim/transcript.txt.
-// BODY_UNAVAILABLE (no transcript yet) defers 4h (per RETRY_MIN_AGE_MIN.meetings).
-//
-// Usage:
-//   node plugin/runners/pull-meetings.mjs --project <P> --alias <A> --entity <joinUrl-or-meetingId>
-//     [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]
+// Deterministic meetings pull via WorkIQ. CSC weekly file emitted via runCli.
+// Verbatim transcript capture is handled inside pullSource (meetings-only branch).
 
-import path from 'node:path';
-import { promises as fs } from 'node:fs';
-import YAML from 'yaml';
-import { assertProject, loadConfig } from './lib/config.mjs';
-import { sourceDir } from './lib/layout.mjs';
-import { writeAtomic } from './lib/evidence.mjs';
-import { fetchWithRetry, encodeODataOp } from './lib/http.mjs';
-import { getToken, SCOPES } from './lib/identity.mjs';
-import { updateCell } from './lib/ledger.mjs';
-import { appendRunLog } from './lib/runlog.mjs';
-import { enqueue, clear } from './lib/deferred.mjs';
-import { shortHash } from './lib/dedup.mjs';
-import { currentIsoMonday, ymd } from './lib/weeks.mjs';
-import { emitLearningCandidate } from './lib/learnings.mjs';
+import { runCli } from './lib/csc-pull.mjs';
 
-const SOURCE = 'meetings';
-
-function parseArgs(argv) {
-  const args = { dryRun: false, force: false };
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (a === '--project') args.project = argv[++i];
-    else if (a === '--alias') args.alias = argv[++i];
-    else if (a === '--entity') args.entity = argv[++i];
-    else if (a === '--week') args.week = argv[++i];
-    else if (a === '--dry-run') args.dryRun = true;
-    else if (a === '--force') args.force = true;
-    else if (a === '--fixture') args.fixture = argv[++i];
-    else if (a === '--help' || a === '-h') args.help = true;
-  }
-  return args;
-}
-
-function help() {
-  return `Usage: node pull-meetings.mjs --project <P> --alias <A> --entity <joinUrl-or-meetingId>
-  [--week YYYY-MM-DD] [--dry-run] [--force] [--fixture <path>]`;
-}
-
-function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
-function configError(msg) { const e = new Error(msg); e.exitCode = 2; return e; }
-function authError(msg)   { const e = new Error(msg); e.exitCode = 3; return e; }
-
-async function buildClient({ fixture }) {
-  if (fixture) {
-    const data = JSON.parse(await fs.readFile(fixture, 'utf8'));
-    return makeFixtureClient(data);
-  }
-  const token = await getToken(SCOPES.graph).catch(e => { throw authError(`token fetch failed: ${e.message}`); });
-  const headers = { Authorization: `Bearer ${token}`, Accept: 'application/json' };
-  return {
-    async resolveMeeting(entity) {
-      // Treat joinUrl-shaped entities specially; otherwise treat as onlineMeeting id.
-      if (/^https?:\/\//i.test(entity)) {
-        const filter = `JoinWebUrl eq '${entity.replace(/'/g, "''")}'`;
-        const url = `https://graph.microsoft.com/v1.0/me/onlineMeetings?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}`;
-        const res = await fetchWithRetry(url, { headers });
-        const body = await res.json();
-        return (body.value && body.value[0]) || null;
-      }
-      const res = await fetchWithRetry(`https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(entity)}`, { headers });
-      return await res.json();
-    },
-    async getTranscript(meetingId) {
-      // List transcripts, fetch most recent .vtt content.
-      const listUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(meetingId)}/transcripts`;
-      let res;
-      try { res = await fetchWithRetry(listUrl, { headers }); } catch (e) { if (e.status === 404) return null; throw e; }
-      const body = await res.json();
-      const list = body.value || [];
-      if (list.length === 0) return null;
-      list.sort((a, b) => String(b.createdDateTime || '').localeCompare(String(a.createdDateTime || '')));
-      const tx = list[0];
-      const contentUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings/${encodeURIComponent(meetingId)}/transcripts/${encodeURIComponent(tx.id)}/content?$format=text/vtt`;
-      const res2 = await fetchWithRetry(contentUrl, { headers: { ...headers, Accept: 'text/vtt' } });
-      return await res2.text();
-    },
-  };
-}
-
-function makeFixtureClient(data) {
-  return {
-    async resolveMeeting(_entity) { return data.meeting || null; },
-    async getTranscript(_id) {
-      if (data.transcript === null) return null;
-      return data.transcript || null;
-    },
-  };
-}
-
-async function main() {
-  const args = parseArgs(process.argv.slice(2));
-  if (args.help) { console.log(help()); return 0; }
-  if (!args.project || !args.alias || !args.entity) {
-    console.error(help());
-    emit({ source: SOURCE, status: 'failed', errors: [{ signature: 'bad-args', message: 'required: --project --alias --entity' }] });
-    return 2;
-  }
-  const projectRoot = await assertProject(args.project).catch(e => { throw configError(e.message); });
-  await loadConfig(projectRoot, args.alias);
-  const weekStart = args.week || ymd(currentIsoMonday());
-
-  // Validate entity is a Teams meeting join URL or meeting ID (not a meeting subject
-  // from discover fallback). Subjects produce 403/404 from Graph onlineMeetings.
-  // Fixture mode bypasses validation.
-  if (!args.fixture && !/^https?:\/\//i.test(args.entity) && !/^MS[a-z0-9_\-=]+$/i.test(args.entity)) {
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
-      last_status: 'failed',
-      last_error: 'entity is not a Teams meeting join URL or meeting ID (subject fallback from discover)',
-    });
-    if (!args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'entity-not-join-url', message: 'expected meetup-join URL or meeting ID', status: null }, context: { runner: 'pull-meetings' } });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'failed', errors: [{ signature: 'entity-not-join-url', message: `entity '${args.entity}' is not a Teams meeting join URL. Discover stored a meeting-subject fallback. Replace with a join URL like 'https://teams.microsoft.com/l/meetup-join/...' in boundaries.yml meetings.joinUrls.` }] });
-    return 0;
-  }
-
-  const client = await buildClient({ fixture: args.fixture });
-  const startedAt = new Date().toISOString();
-
-  let meeting;
-  try { meeting = await client.resolveMeeting(args.entity); }
-  catch (e) {
-    const retryable = !e.status || [429, 502, 503, 504].includes(e.status);
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: retryable ? 'deferred' : 'failed', last_error: e.message });
-    if (retryable && !args.dryRun) await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'fetch-failed', reason: e.message });
-    if (!retryable && !args.dryRun) await emitLearningCandidate({ projectRoot, alias: args.alias, source: SOURCE, entity: args.entity, week: weekStart, error: { signature: 'fetch-failed', message: e.message, status: e.status }, context: { runner: 'pull-meetings' } });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: retryable ? 'deferred' : 'failed', errors: [{ message: e.message, status: e.status }] });
-    return retryable ? 1 : 0;
-  }
-
-  if (!meeting) {
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, { last_status: 'no-activity', items_pulled: 0 });
-    emit({ source: SOURCE, entity: args.entity, week: weekStart, status: 'no-activity', items_pulled: 0, files_written: [] });
-    return 0;
-  }
-
-  const meetingId = meeting.id || shortHash(args.entity);
-  const dirHash = shortHash(args.entity);
-  const outDir = path.join(sourceDir(projectRoot, args.alias, SOURCE), dirHash);
-
-  let transcript;
-  try { transcript = await client.getTranscript(meetingId); }
-  catch (e) { transcript = null; }
-
-  const filesWritten = [];
-  if (!args.dryRun) {
-    const r1 = await writeAtomic(path.join(outDir, 'meeting.yml'), YAML.stringify(meeting));
-    if (r1.written !== false) filesWritten.push(path.relative(projectRoot, r1.path));
-  }
-
-  if (transcript == null) {
-    // BODY_UNAVAILABLE — defer 4h
-    if (!args.dryRun) {
-      await enqueue(projectRoot, args.alias, { source: SOURCE, entity: args.entity, weekStart, signature: 'body-unavailable', reason: 'transcript not yet available' });
-    }
-    await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
-      last_status: 'body-unavailable',
-      meeting_id: meetingId,
-      transcript_available: false,
-    });
-    emit({
-      source: SOURCE, entity: args.entity, week: weekStart, status: 'body-unavailable',
-      meeting_id: meetingId, files_written: filesWritten,
-      errors: [{ signature: 'body-unavailable', action_taken: 'deferred-retry-marker-written', retry_after_min: 240 }],
-    });
-    return 0;
-  }
-
-  if (!args.dryRun) {
-    const r2 = await writeAtomic(path.join(outDir, 'verbatim', 'transcript.txt'), transcript);
-    const r3 = await writeAtomic(path.join(outDir, 'index.md'), renderIndexMd({ meeting, transcriptChars: transcript.length, pulledAt: startedAt }), { skipIfUnchanged: false });
-    for (const r of [r2, r3]) if (r.written !== false) filesWritten.push(path.relative(projectRoot, r.path));
-  }
-
-  await clear(projectRoot, args.alias, SOURCE, args.entity).catch(() => {});
-
-  await updateCell(projectRoot, args.alias, SOURCE, args.entity, weekStart, {
-    last_status: 'captured',
-    items_pulled: 1,
-    meeting_id: meetingId,
-    transcript_available: true,
-    transcript_chars: transcript.length,
-  });
-
-  if (!args.dryRun) {
-    await appendRunLog(projectRoot, { runner: 'pull-meetings', alias: args.alias, entity: args.entity, week: weekStart, status: 'captured', transcript_chars: transcript.length });
-  }
-
-  emit({
-    source: SOURCE, entity: args.entity, week: weekStart, status: 'captured',
-    items_pulled: 1, meeting_id: meetingId, transcript_chars: transcript.length,
-    files_written: filesWritten, ledger_key: `meetings::${args.entity}::${weekStart}`,
-  });
-  return 0;
-}
-
-function renderIndexMd({ meeting, transcriptChars, pulledAt }) {
-  const lines = [
-    `# Meeting — ${meeting.subject || '(no subject)'}`,
-    '',
-    `- meeting_id: ${meeting.id || '?'}`,
-    `- start: ${(meeting.startDateTime) || '?'}`,
-    `- end: ${(meeting.endDateTime) || '?'}`,
-    `- transcript_chars: ${transcriptChars}`,
-    `- pulled_at: ${pulledAt}`,
-    '',
-    'Verbatim transcript at `verbatim/transcript.txt`.',
-    '',
-  ];
-  return lines.join('\n');
-}
-
-main().then(code => { process.exitCode = code; }).catch(e => {
-  emit({ source: SOURCE, status: 'failed', errors: [{ message: e.message, code: e.exitCode || 'unknown' }] });
-  process.exitCode = e.exitCode || 1;
-});
+runCli('meetings').then(code => { process.exitCode = code; });