kushi-agents 3.4.2 → 3.13.0

package/plugin/skills/pull-misc/runner.mjs

@@ -0,0 +1,280 @@
+#!/usr/bin/env node
+// pull-misc runner — read external-links.txt, classify by type, route to runner branches.
+// Per pull-misc SKILL.md v0.1.0 (kushi v3.9.0).
+//
+// Usage:
+//   node runner.mjs --project HCA --links-file "<path-to-external-links.txt>"
+//   node runner.mjs --project HCA --links-file "..." --headless
+//   node runner.mjs --project HCA --links-file "..." --types "loop,web"           # only fetch some
+//   node runner.mjs --project HCA --links-file "..." --titles "ABN Workshop,..."  # only specific titles
+//
+// Output is a single JSON object on stdout containing:
+//   { project, linksFile, links: [<perLinkRecord>...], runStatus, counts }
+
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import process from 'node:process';
+
+// ----- arg parse -----
+const args = Object.fromEntries(
+  process.argv.slice(2).reduce((acc, cur, idx, arr) => {
+    if (cur.startsWith('--')) {
+      const key = cur.replace(/^--/, '');
+      const next = arr[idx + 1];
+      acc.push([key, next && !next.startsWith('--') ? next : true]);
+    }
+    return acc;
+  }, [])
+);
+
+const PROJECT = args.project || null;
+const LINKS_FILE = args['links-file'] || null;
+const ENGAGEMENT_ROOT = args['engagement-root'] || null;
+const HEADLESS = !!args.headless;
+const TYPES_FILTER = args.types ? String(args.types).split(',').map(s => s.trim().toLowerCase()) : null;
+const TITLES_FILTER = args.titles ? String(args.titles).split(',').map(s => s.trim()) : null;
+const TIMEOUT_MS = parseInt(args.timeout || '45000', 10);
+const PROFILE_DIR = path.join(os.homedir(), '.copilot', 'playwright-profile', 'onenote');
+
+if (!PROJECT || !LINKS_FILE) {
+  console.error('Usage: --project <name> --links-file <path> [--headless] [--types t1,t2] [--titles "t1,t2"]');
+  process.exit(2);
+}
+
+// ----- constants -----
+const DELEGATED_TYPES = new Set(['onenote', 'sharepoint', 'ado']);
+const BROWSER_TYPES = new Set(['loop']);
+const PLACEHOLDER_RE = /<\s*(PASTE_[A-Z_]*_URL|TODO[^>]*)\s*>/i;
+
+// ----- parse external-links.txt -----
+function parseLinksFile(p) {
+  const txt = fs.readFileSync(p, 'utf-8');
+  const out = [];
+  for (const raw of txt.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (!line || line.startsWith('#')) continue;
+    const parts = line.split('|').map(s => s.trim());
+    if (parts.length < 4) continue; // malformed
+    const [type, owner, title, url, ...noteParts] = parts;
+    out.push({
+      type: type.toLowerCase(),
+      owner,
+      title,
+      url,
+      notes: noteParts.join('|') || ''
+    });
+  }
+  return out;
+}
+
+function safeName(s) {
+  return s.replace(/[\\/:*?"<>|]/g, '-').replace(/\s+/g, '-').slice(0, 120);
+}
+
+function classify(link) {
+  if (PLACEHOLDER_RE.test(link.url)) return { route: 'placeholder' };
+  if (DELEGATED_TYPES.has(link.type)) return { route: 'delegated', delegated_to: `pull-${link.type}` };
+  if (BROWSER_TYPES.has(link.type)) return { route: 'browser' };
+  if (link.type === 'file') return { route: 'file' };
+  return { route: 'http' };
+}
+
+// ----- HTTP fetch branch -----
+async function fetchHttp(url) {
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      headers: { 'User-Agent': 'kushi-pull-misc/0.1', 'Accept': '*/*' },
+      signal: AbortSignal.timeout(TIMEOUT_MS)
+    });
+    const ct = res.headers.get('content-type') || '';
+    const etag = res.headers.get('etag') || null;
+    const lm = res.headers.get('last-modified') || null;
+
+    if (res.status === 401 || res.status === 403) {
+      return { last_status: 'auth-required', http_status: res.status, content_type: ct };
+    }
+    if (!res.ok) {
+      return { last_status: 'fetch-failed', http_status: res.status, content_type: ct };
+    }
+
+    if (ct.startsWith('application/pdf')) {
+      const buf = Buffer.from(await res.arrayBuffer());
+      // Defer text extract to a separate step (pdfjs not bundled to keep dep weight low).
+      return {
+        last_status: 'captured',
+        http_status: res.status,
+        content_type: ct,
+        etag, last_modified_http: lm,
+        body: `[binary PDF — ${buf.length} bytes; text extraction not yet implemented in v0.1]`,
+        char_count: 0
+      };
+    }
+
+    const body = await res.text();
+
+    // Heuristic: HTML → readability extract; plain → keep raw.
+    let extracted = body;
+    if (ct.includes('html')) {
+      try {
+        const { JSDOM } = await import('jsdom');
+        const { Readability } = await import('@mozilla/readability');
+        const dom = new JSDOM(body, { url });
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        if (article && article.textContent && article.textContent.length > 100) {
+          extracted = `# ${article.title || ''}\n\n${article.textContent.trim()}`;
+        }
+      } catch (e) {
+        // jsdom or readability not available — fall through to raw
+        extracted = `[readability unavailable: ${e.message}]\n\n${body.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()}`;
+      }
+    }
+
+    return {
+      last_status: 'captured',
+      http_status: res.status,
+      content_type: ct,
+      etag, last_modified_http: lm,
+      body: extracted,
+      char_count: extracted.length
+    };
+  } catch (e) {
+    return { last_status: 'fetch-failed', error: String(e.message || e) };
+  }
+}
+
+// ----- File read branch -----
+async function readFileBranch(link) {
+  const root = ENGAGEMENT_ROOT || process.cwd();
+  const candidate1 = path.resolve(root, link.url);
+  const candidate2 = path.resolve(root, PROJECT, link.url);
+  const tryPath = fs.existsSync(candidate1) ? candidate1 : (fs.existsSync(candidate2) ? candidate2 : null);
+  if (!tryPath) return { last_status: 'fetch-failed', error: `not found at ${candidate1} or ${candidate2}` };
+  try {
+    const buf = await fsp.readFile(tryPath);
+    if (buf.includes(0)) return { last_status: 'skipped-binary', char_count: buf.length };
+    const txt = buf.toString('utf-8');
+    return { last_status: 'captured', body: txt, char_count: txt.length, resolved_path: tryPath };
+  } catch (e) {
+    return { last_status: 'fetch-failed', error: String(e.message || e) };
+  }
+}
+
+// ----- Browser (Loop) branch -----
+async function fetchBrowser(link, ctx) {
+  const page = await ctx.newPage();
+  try {
+    const resp = await page.goto(link.url, { waitUntil: 'domcontentloaded', timeout: TIMEOUT_MS });
+    if (/login\.microsoftonline\.com|login\.live\.com/.test(page.url())) {
+      return { last_status: 'auth-required', captured_via: 'browser' };
+    }
+    // Wait for Loop content. Loop uses Fluid; the main content settles in a few seconds.
+    await page.waitForTimeout(4000);
+    // Try Loop-specific selectors first, then fallback to body
+    const body = await page.evaluate(() => {
+      const sels = [
+        '[data-loop-canvas]',
+        '.fui-FluentProvider',
+        '[role="main"]',
+        'body'
+      ];
+      for (const s of sels) {
+        const el = document.querySelector(s);
+        if (el && el.innerText && el.innerText.length > 100) return el.innerText;
+      }
+      return document.body ? document.body.innerText : '';
+    });
+    if (!body || body.length < 50) {
+      return { last_status: 'fetch-failed', captured_via: 'browser', error: 'empty or near-empty body', char_count: body?.length || 0 };
+    }
+    return { last_status: 'captured', captured_via: 'browser', body, char_count: body.length };
+  } catch (e) {
+    return { last_status: 'fetch-failed', captured_via: 'browser', error: String(e.message || e) };
+  } finally {
+    await page.close().catch(() => {});
+  }
+}
+
+// ----- main -----
+(async () => {
+  const links = parseLinksFile(LINKS_FILE);
+  const filtered = links.filter(l => {
+    if (TYPES_FILTER && !TYPES_FILTER.includes(l.type)) return false;
+    if (TITLES_FILTER && !TITLES_FILTER.some(t => l.title.includes(t))) return false;
+    return true;
+  });
+
+  const needsBrowser = filtered.some(l => classify(l).route === 'browser');
+  let ctx = null;
+  if (needsBrowser) {
+    if (!fs.existsSync(PROFILE_DIR)) {
+      console.error('[pull-misc] Playwright profile missing — loop links will be marked auth-required.');
+    } else {
+      const { chromium } = await import('playwright');
+      ctx = await chromium.launchPersistentContext(PROFILE_DIR, { headless: HEADLESS, viewport: { width: 1400, height: 900 } });
+    }
+  }
+
+  const results = [];
+  const now = new Date().toISOString();
+
+  for (const link of filtered) {
+    const cls = classify(link);
+    let rec = {
+      type: link.type,
+      owner: link.owner,
+      title: link.title,
+      url: link.url,
+      notes: link.notes,
+      attempts: 1,
+      last_attempt_at: now,
+      snapshot_path: `Evidence/<alias>/misc/snapshot/${safeName(link.type)}__${safeName(link.title)}.md`
+    };
+
+    if (cls.route === 'placeholder') {
+      Object.assign(rec, { last_status: 'placeholder', captured_via: null });
+    } else if (cls.route === 'delegated') {
+      Object.assign(rec, { last_status: 'delegated', captured_via: 'delegated', delegated_to: cls.delegated_to });
+    } else if (cls.route === 'http') {
+      const r = await fetchHttp(link.url);
+      Object.assign(rec, r, { captured_via: 'http' });
+      if (r.last_status === 'captured') rec.captured_at = now;
+    } else if (cls.route === 'file') {
+      const r = await readFileBranch(link);
+      Object.assign(rec, r, { captured_via: 'file' });
+      if (r.last_status === 'captured') rec.captured_at = now;
+    } else if (cls.route === 'browser') {
+      if (!ctx) {
+        Object.assign(rec, { last_status: 'auth-required', captured_via: 'browser', error: 'no playwright profile' });
+      } else {
+        const r = await fetchBrowser(link, ctx);
+        Object.assign(rec, r);
+        if (r.last_status === 'captured') rec.captured_at = now;
+      }
+    }
+    results.push(rec);
+  }
+
+  if (ctx) await ctx.close();
+
+  const counts = {
+    total: results.length,
+    captured: results.filter(r => r.last_status === 'captured').length,
+    placeholder: results.filter(r => r.last_status === 'placeholder').length,
+    delegated: results.filter(r => r.last_status === 'delegated').length,
+    auth_required: results.filter(r => r.last_status === 'auth-required').length,
+    fetch_failed: results.filter(r => r.last_status === 'fetch-failed').length,
+    skipped_binary: results.filter(r => r.last_status === 'skipped-binary').length
+  };
+  const runStatus = counts.auth_required > 0 ? 'auth-required'
+    : counts.fetch_failed > 0 ? 'partial'
+    : 'ok';
+
+  console.log(JSON.stringify({ project: PROJECT, linksFile: LINKS_FILE, runStatus, counts, links: results }, null, 2));
+})().catch(e => {
+  console.error('[pull-misc] runner crashed:', e);
+  process.exit(1);
+});

package/plugin/skills/pull-onenote/README.md

@@ -0,0 +1,90 @@
+# pull-onenote — runner README
+
+## What this is
+
+A Playwright-driven browser scraper for OneNote-for-Web that returns verbatim page bodies for an entire section. Used by the `pull-onenote` skill (v2.6.0+) as the PRIMARY capture path.
+
+## One-time setup
+
+```pwsh
+# 1. Install Node + Playwright if you don't have them
+npm install -g playwright
+npx playwright install chromium
+
+# 2. Seed the persistent browser profile (interactive — one-time)
+node plugin/skills/pull-onenote/runner.mjs --bootstrap
+```
+
+When the bootstrap command opens a Chromium window:
+- Sign in with your work account at `onenote.cloud.microsoft`.
+- Check **Keep me signed in**.
+- Wait for the OneNote home page to load.
+- Close the window.
+
+The profile is persisted at `~/.copilot/playwright-profile/onenote/`. Subsequent runs reuse it.
+
+## Per-section runs
+
+```pwsh
+node plugin/skills/pull-onenote/runner.mjs `
+  --project "HCA" `
+  --section-url "https://microsoft-my.sharepoint-df.com/personal/<upn>/_layouts/15/Doc.aspx?sourcedoc=%7B<notebookSourceDoc>%7D&action=edit&wd=target%28<sectionName>.one%7C<sectionFileId>%2F%29" `
+  --headless
+```
+
+The runner emits a single JSON object on stdout with this shape:
+
+```json
+{
+  "project": "HCA",
+  "sectionUrl": "...",
+  "runStatus": "ok | auth-required | partial",
+  "exitedEarly": false,
+  "authRequiredCount": 0,
+  "pages": [
+    {
+      "title": "4/3 - HCA with Jay and Martin",
+      "pos": 12, "total": 16,
+      "webPageId": "78aac9b5-0629-4daa-ada3-f2436cb2381c",
+      "last_status": "captured | short-suspect | auth-required | browser-error",
+      "captured_via": "browser",
+      "body": "...verbatim text...",
+      "bodyLen": 6904,
+      "capturedAt": "2026-05-14T03:55:00Z"
+    }
+  ]
+}
+```
+
+The skill driver merges this into `m365-mutable.json#knownSections.<projectKey>.one_pages[]` and writes the snapshot pages.
+
+## Filtering
+
+For retry-only runs, pass `--titles` with a comma-separated list:
+
+```pwsh
+node runner.mjs --project HCA --section-url "..." --titles "4/3 - HCA with Jay and Martin,5/7" --headless
+```
+
+## Scheduled / unattended runs
+
+For Monday-9am-style schedules, always pass `--headless`. The runner detects redirects to `login.microsoftonline.com` and exits early with `runStatus: "auth-required"` so the driver can mark all queued pages as `auth-required` and surface in the run report.
+
+There is currently no way to satisfy a Conditional Access / MFA challenge unattended. When that happens (rough estimate: every 1–4 weeks under a Microsoft tenant CA policy), do one interactive `--bootstrap` to refresh the profile and the next scheduled run resumes silently.
+
+## Constructing the section URL
+
+Once a project's `knownSections` entry is populated in `m365-mutable.json`, the URL is:
+
+```
+<one_notebookSpoBaseUrl>/_layouts/15/Doc.aspx?sourcedoc=%7B<one_notebookSourceDoc>%7D&action=edit&wd=target(<one_sectionName>%7C<one_sectionFileId>%2F)
+```
+
+URL-encode the section name; double-encode the pipe (`%7C`) and trailing slash (`%2F`).
+
+## Troubleshooting
+
+- **Picker shown every run.** First-run only, and only if multiple work accounts are listed. After picking once it sticks.
+- **"OneNote canvas frame did not attach."** Almost always auth — check the headless output's `runStatus`.
+- **Bodies look short.** The `last_status: short-suspect` flag is for this. Verify in OneNote desktop; if genuinely sparse, accept; otherwise re-run.
+- **Pages enumerate fine but no `webPageId` returned.** The `wd=target(...)` URL fragment shape may have changed. Update `parseWebPageIdFromUrl` regex in `runner.mjs`.