kushi-agents 5.8.4 → 5.9.1

package/bin/cli.mjs

@@ -103,22 +103,47 @@ if (args.length > 0 && args[0] === 'lint') {
   process.exit(0);
 }
 
+// ── one-shot wiki verb (v5.9.1+) ─────────────────────────────────────────────
+// `kushi wiki` — deterministic do-the-wiki: resolve root, init if missing,
+// print path + status + clickable file:// URL. Idempotent across
+// "create wiki" / "do wiki" / "refresh wiki" / "update wiki".
+if (args.length > 0 && args[0] === 'wiki') {
+  const cliMod = await import('../src/global-wiki-cli.mjs');
+  await cliMod.runWiki();
+  process.exit(0);
+}
+
 // ── global verb (v5.3.0+) ────────────────────────────────────────────────────
 if (args.length > 0 && args[0] === 'global') {
   const sub = args[1] || '';
-  const validSubs = ['init', 'status', 'ask', 'lint'];
+  const validSubs = ['init', 'status', 'ask', 'lint', 'migrate', 'set-root', 'show-root'];
   if (!validSubs.includes(sub)) {
-    console.error('\n  Usage: kushi global init           Scaffold ~/.kushi-global/State/');
-    console.error('         kushi global status         Show counts + freshness');
-    console.error('         kushi global ask <question> Ask the global wiki');
-    console.error('         kushi global lint           Lint the global wiki\n');
+    console.error('\n  Usage: kushi global init                    Scaffold State/ at the resolved root');
+    console.error('         kushi global status                  Show counts + freshness');
+    console.error('         kushi global ask <question>          Ask the global wiki');
+    console.error('         kushi global lint                    Lint the global wiki');
+    console.error('         kushi global show-root               Show how the root path is resolved');
+    console.error('         kushi global set-root <path> [--scope workspace|home]');
+    console.error('                                              Persist root (workspace shared by default,');
+    console.error('                                              falls back to ~/.kushi/config.json)');
+    console.error('         kushi global migrate <new-path>      Copy wiki to a new root + persist setting\n');
+    console.error('  Tip: Set the root to a OneDrive-synced SharePoint folder to share');
+    console.error('       the wiki across a team. See docs/how-to/wiki-on-sharepoint.md.\n');
     process.exit(1);
   }
-  const { runGlobalInit, runGlobalStatus, runGlobalAsk, runGlobalLint } = await import('../src/global-wiki-cli.mjs');
-  if (sub === 'init') await runGlobalInit();
-  else if (sub === 'status') await runGlobalStatus();
-  else if (sub === 'ask') await runGlobalAsk(args.slice(2).join(' '));
-  else if (sub === 'lint') await runGlobalLint();
+  const cliMod = await import('../src/global-wiki-cli.mjs');
+  if (sub === 'init') await cliMod.runGlobalInit();
+  else if (sub === 'status') await cliMod.runGlobalStatus();
+  else if (sub === 'ask') await cliMod.runGlobalAsk(args.slice(2).join(' '));
+  else if (sub === 'lint') await cliMod.runGlobalLint();
+  else if (sub === 'migrate') await cliMod.runGlobalMigrate(args[2]);
+  else if (sub === 'set-root') {
+    const scopeIdx = args.indexOf('--scope');
+    const scope = scopeIdx > 0 && args[scopeIdx + 1] ? args[scopeIdx + 1] : null;
+    const target = args.find((a, i) => i >= 2 && !a.startsWith('--') && args[i - 1] !== '--scope');
+    await cliMod.runGlobalSetRoot(target, { scope });
+  }
+  else if (sub === 'show-root') await cliMod.runGlobalShowRoot();
   process.exit(0);
 }
 
@@ -252,17 +277,105 @@ if (args.includes('--help') || args.includes('-h')) {
   After install, talk to Kushi:
     bootstrap <project>     First-time setup
     refresh <project>       Incremental refresh + rebuild State/
-    state <project>         Re-render State/ from existing Evidence
+    state <project>         Re-render State/ from existing Evidence (deterministic
+                            inventory; LLM build-state skill does narrative synthesis)
+    references <project>    Scan Evidence for URLs and refresh the shared
+                            references pool (Evidence/_shared/references/)
     consolidate <project>   Merge per-user evidence
     status <project>        Show run-log
     ask <project> <q>       Cited Q&A over Evidence/ (auto-routes, --file-back to save)
     lint <project>          Run wiki-lint checks on State/
 
+  Workspace lifecycle (v5.9.0+):
+    wiki                           One-shot: resolve global wiki root, scaffold
+                                   if missing, print status + clickable file://
+                                   link to the wiki index. Idempotent —
+                                   "do wiki" / "refresh wiki" / "create wiki"
+                                   all map here.
+    uninstall [--keep-config]      Remove <cwd>/.kushi/ (preserves Evidence/, State/).
+                                   --keep-config preserves config/user/ identity files.
+    upgrade                        npm i -g kushi-agents@latest then re-seed assets
+                                   in cwd (config preserved).
+
   In VS Code Chat the prefix is "@Kushi". In Clawpilot just say "kushi <verb>".
 `);
   process.exit(0);
 }
 
+// ── state / refresh / bootstrap verbs (v5.9.0+) ─────────────────────────────
+// Thin shells that exec the deterministic runners. Keeps `kushi state HCA` etc.
+// runnable from the global bin without users having to know the runner paths.
+if (args.length > 0 && ['state', 'refresh-runner', 'bootstrap-runner', 'discover', 'references'].includes(args[0])) {
+  const verb = args[0];
+  const project = args[1];
+  if (!project) {
+    console.error(`\n  Usage: kushi ${verb} <project> [options]\n`);
+    process.exit(1);
+  }
+  const { spawnSync } = await import('node:child_process');
+  const pathMod = await import('node:path');
+  const urlMod = await import('node:url');
+  const here = pathMod.dirname(urlMod.fileURLToPath(import.meta.url));
+  const runnerMap = {
+    state: 'pull-state.mjs',
+    references: 'pull-references.mjs',
+    discover: 'discover.mjs',
+    'refresh-runner': 'refresh.mjs',
+    'bootstrap-runner': 'bootstrap.mjs',
+  };
+  const runner = pathMod.resolve(here, '..', 'plugin', 'runners', runnerMap[verb]);
+  const passthrough = args.slice(2);
+  const r = spawnSync(process.execPath, [runner, '--project', project, ...passthrough], { stdio: 'inherit' });
+  process.exit(r.status ?? 1);
+}
+
+// ── workspace uninstall / upgrade verbs (v5.9.0+) ───────────────────────────
+if (args.length > 0 && args[0] === 'uninstall' && !args.includes('--clawpilot') && !args.includes('--vscode') && !args.includes('--all-hosts')) {
+  // Workspace uninstall: remove .kushi/ from cwd (preserves Evidence/, State/).
+  const fsMod = await import('node:fs');
+  const pathMod = await import('node:path');
+  const dest = pathMod.resolve(process.cwd(), '.kushi');
+  const keepConfig = args.includes('--keep-config');
+  if (!fsMod.existsSync(dest)) {
+    console.error(`\n  No .kushi/ directory found at ${dest}\n`);
+    process.exit(1);
+  }
+  if (keepConfig) {
+    const assetDirs = ['agents', 'instructions', 'prompts', 'skills', 'templates', 'reference-packs', 'lib', 'runners'];
+    let removed = 0;
+    for (const d of assetDirs) {
+      const p = pathMod.join(dest, d);
+      if (fsMod.existsSync(p)) { fsMod.rmSync(p, { recursive: true, force: true }); removed++; }
+    }
+    console.log(`\n  Removed ${removed} asset dir(s) from ${dest} (config/user/ preserved).\n`);
+  } else {
+    fsMod.rmSync(dest, { recursive: true, force: true });
+    console.log(`\n  Removed ${dest}\n  Evidence/ and State/ left untouched.\n`);
+  }
+  process.exit(0);
+}
+
+if (args.length > 0 && args[0] === 'upgrade') {
+  // Upgrade: npm i -g @latest, then re-seed assets in cwd preserving config.
+  const { spawnSync } = await import('node:child_process');
+  console.log('\n  Upgrading kushi-agents globally via npm...\n');
+  const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+  const r1 = spawnSync(npm, ['install', '-g', 'kushi-agents@latest'], { stdio: 'inherit' });
+  if (r1.status !== 0) {
+    console.error('\n  npm install failed.\n');
+    process.exit(r1.status ?? 1);
+  }
+  console.log('\n  Refreshing assets in cwd (config preserved)...\n');
+  const fsMod = await import('node:fs');
+  if (fsMod.existsSync('.kushi')) {
+    const r2 = spawnSync(npm, ['exec', '--', 'kushi-agents', '--no-prompt', '--force'], { stdio: 'inherit' });
+    process.exit(r2.status ?? 0);
+  } else {
+    console.log('\n  No .kushi/ in cwd — global upgrade complete; cd into a project and run `kushi` to install.\n');
+    process.exit(0);
+  }
+}
+
 // ── multi-host mode (v5.0.2+) ───────────────────────────────────────────────
 // Trigger when the user passes any of: --vscode, --all-hosts, --uninstall.
 // --clawpilot ALONE continues to route through the legacy main.mjs path so

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "kushi-agents",
-  "version": "5.8.4",
+  "version": "5.9.1",
   "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
   "type": "module",
   "bin": {
+    "kushi": "./bin/cli.mjs",
     "kushi-agents": "./bin/cli.mjs"
   },
   "files": [

package/plugin/instructions/global-wiki.instructions.md

@@ -17,8 +17,13 @@ The global wiki is a personal, cross-engagement knowledge base that lives **outs
 ### 1. Location
 
 - **Default:** `~/.kushi-global/State/` (`$HOME` on POSIX, `$env:USERPROFILE` on Windows).
-- **Override:** `$KUSHI_GLOBAL_ROOT` environment variable (absolute path; tilde-expanded). Tests MUST set this to `.testtmp/.kushi-global/` — never touch the real path.
-- The global root is **per-user**, never per-project, never per-host.
+- **Resolution chain (v5.9.1+, first non-empty wins):**
+  1. `$KUSHI_GLOBAL_ROOT` env var (absolute path; tilde-expanded). Tests MUST set this to `.testtmp/.kushi-global/` — never touch the real path.
+  2. **`<workspace>/.kushi/config/shared/kushi.yml` → `globalRoot`** (workspace-shared, team-shareable, committed)
+  3. `~/.kushi/config.json` → `globalRoot` (per-user, per-machine)
+  4. Default `~/.kushi-global/`
+- The global root is **per-user OR per-team** (workspace-shared scope), never per-project, never per-host.
+- Inspect with `kushi global show-root`. Set with `kushi global set-root <path> [--scope workspace|home]` (default scope = workspace if a `.kushi/` is found above cwd, else home).
 
 ### 2. Shape
 

package/plugin/runners/discover.mjs

@@ -201,12 +201,14 @@ function applyRows(source, rows, currentBounds, currentInteg) {
     return { boundariesPatch: added.length ? { onenote: { section_file_ids: merged } } : null, accepted };
   }
   if (source === 'sharepoint') {
-    const existing = currentBounds.sharepoint?.sites || [];
+    // v5.9.0: SP sites are project-wide, not per-alias. Write into integrations.yml.
+    const existing = currentInteg.sharepoint?.sites || [];
     const incoming = rows.map(r => r.site_url).filter(Boolean);
     const merged = dedup([...existing, ...incoming]);
     const added = merged.filter(v => !existing.includes(v));
     if (added.length) accepted.push(...added);
-    return { boundariesPatch: added.length ? { sharepoint: { sites: merged } } : null, accepted };
+    const cur = currentInteg.sharepoint || {};
+    return { integrationsPatch: added.length ? { sharepoint: { ...cur, sites: merged } } : null, accepted };
   }
   if (source === 'crm') {
     const cur = currentInteg.crm || {};

package/plugin/runners/lib/references.mjs

@@ -0,0 +1,164 @@
+// plugin/runners/lib/references.mjs
+// Deterministic URL extraction, classification, and lightweight HTTP snapshot
+// for the unified references pool. No LLM. v5.9.0.
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+/** Permissive URL regex. Captures http(s) URLs in markdown / yaml / plain text. */
+const URL_RE = /\bhttps?:\/\/[^\s<>"'`)\]}|\\]+/gi;
+
+/** Trailing punctuation that is almost never part of a URL. */
+const TRAILING_TRIM = /[)\].,;:!?>'"]+$/;
+
+export function extractUrls(text) {
+  if (!text || typeof text !== 'string') return [];
+  const out = new Set();
+  const matches = text.match(URL_RE) || [];
+  for (let m of matches) {
+    m = m.replace(TRAILING_TRIM, '');
+    if (m.length > 8) out.add(m);
+  }
+  return [...out];
+}
+
+/** Stable sha1 of normalized URL (for filenames + index keys). */
+export function urlHash(url) {
+  return crypto.createHash('sha1').update(normalizeUrl(url)).digest('hex').slice(0, 16);
+}
+
+/** Strip fragment + common tracking query params. Keep path/query semantically. */
+export function normalizeUrl(url) {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    const drop = ['utm_source','utm_medium','utm_campaign','utm_term','utm_content','wt.mc_id'];
+    for (const k of drop) u.searchParams.delete(k);
+    return u.toString();
+  } catch {
+    return url;
+  }
+}
+
+const HOST_RULES = [
+  { match: /(^|\.)sharepoint\.com$/i, host: 'sharepoint.com', kind: 'sharepoint', authRequired: true },
+  { match: /(^|\.)loop\.microsoft\.com$/i, host: 'loop.microsoft.com', kind: 'loop', authRequired: true },
+  { match: /loop\.cloud\.microsoft$/i, host: 'loop.cloud.microsoft', kind: 'loop', authRequired: true },
+  { match: /loop-api\.cloud\.microsoft$/i, host: 'loop-api.cloud.microsoft', kind: 'loop', authRequired: true },
+  { match: /(^|\.)fluidpreview\.office\.net$/i, host: 'fluidpreview.office.net', kind: 'loop', authRequired: true },
+  { match: /(^|\.)teams\.microsoft\.com$/i, host: 'teams.microsoft.com', kind: 'teams', authRequired: true },
+  { match: /(^|\.)office\.com$/i, host: 'office.com', kind: 'office', authRequired: true },
+  { match: /(^|\.)dev\.azure\.com$/i, host: 'dev.azure.com', kind: 'ado', authRequired: true },
+  { match: /(^|\.)visualstudio\.com$/i, host: 'visualstudio.com', kind: 'ado', authRequired: true },
+  { match: /(^|\.)dynamics\.com$/i, host: 'dynamics.com', kind: 'crm', authRequired: true },
+  { match: /(^|\.)learn\.microsoft\.com$/i, host: 'learn.microsoft.com', kind: 'docs', authRequired: false },
+  { match: /(^|\.)docs\.microsoft\.com$/i, host: 'docs.microsoft.com', kind: 'docs', authRequired: false },
+  { match: /(^|\.)github\.com$/i, host: 'github.com', kind: 'repo', authRequired: false },
+];
+
+/** Classify a URL into { host, kind, authRequired }. Falls back to "external". */
+export function classify(url) {
+  let host = '';
+  try { host = new URL(url).hostname.toLowerCase(); } catch { return { host: 'unknown', kind: 'invalid', authRequired: false }; }
+  for (const r of HOST_RULES) {
+    if (r.match.test(host)) return { host: r.host, kind: r.kind, authRequired: r.authRequired };
+  }
+  return { host, kind: 'external', authRequired: false };
+}
+
+/** Safe filename segment for hosts. */
+export function safeHost(host) {
+  return (host || 'unknown').toLowerCase().replace(/[^a-z0-9.-]/g, '_').slice(0, 80);
+}
+
+/**
+ * Fetch a URL and extract a small content snapshot.
+ * Returns { ok, status, title, description, h1, contentType, bytes, snippet }.
+ * Keeps payload bounded (<= maxBytes default 64KB).
+ */
+export async function fetchSnapshot(url, { timeoutMs = 15000, maxBytes = 64 * 1024 } = {}) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      signal: ctrl.signal,
+      headers: { 'user-agent': 'kushi-references/1.0 (+https://github.com/ushakrishnan/kushi)' },
+    });
+    const contentType = res.headers.get('content-type') || '';
+    const reader = res.body?.getReader();
+    let received = 0;
+    const chunks = [];
+    if (reader) {
+      while (received < maxBytes) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+        received += value.byteLength;
+      }
+      try { reader.cancel(); } catch {}
+    }
+    const buf = Buffer.concat(chunks.map(c => Buffer.from(c)));
+    const text = buf.toString('utf8');
+    const html = /html|xml/i.test(contentType) || /^\s*<!doctype html|<html/i.test(text);
+    const title = html ? extractTag(text, 'title') : '';
+    const description = html ? extractMeta(text, 'description') : '';
+    const h1 = html ? extractTag(text, 'h1') : '';
+    const snippet = html ? stripHtml(text).slice(0, 600) : text.slice(0, 600);
+    return {
+      ok: res.ok,
+      status: res.status,
+      title: clean(title),
+      description: clean(description),
+      h1: clean(h1),
+      contentType,
+      bytes: received,
+      snippet: clean(snippet),
+    };
+  } catch (e) {
+    return { ok: false, status: 0, error: e.name === 'AbortError' ? 'timeout' : (e.code || e.message) };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function extractTag(html, tag) {
+  const m = html.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, 'i'));
+  return m ? m[1] : '';
+}
+function extractMeta(html, name) {
+  const m = html.match(new RegExp(`<meta[^>]+name=["']${name}["'][^>]*content=["']([^"']+)["']`, 'i'))
+    || html.match(new RegExp(`<meta[^>]+property=["']og:${name}["'][^>]*content=["']([^"']+)["']`, 'i'));
+  return m ? m[1] : '';
+}
+function stripHtml(s) {
+  return s.replace(/<script[\s\S]*?<\/script>/gi, ' ')
+          .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+          .replace(/<[^>]+>/g, ' ')
+          .replace(/\s+/g, ' ')
+          .trim();
+}
+function clean(s) {
+  return (s || '').replace(/\s+/g, ' ').trim().slice(0, 800);
+}
+
+/** Walk a directory recursively, returning files matching `extensions`. */
+export async function walkFiles(dir, { extensions = ['.yml','.yaml','.md','.txt','.json'], skipDirs = ['_shared/references','node_modules','.git'] } = {}) {
+  const out = [];
+  async function walk(d) {
+    let entries;
+    try { entries = await fs.readdir(d, { withFileTypes: true }); } catch { return; }
+    for (const e of entries) {
+      const full = path.join(d, e.name);
+      if (e.isDirectory()) {
+        const skip = skipDirs.some(s => full.replaceAll('\\','/').includes(s));
+        if (!skip) await walk(full);
+      } else if (e.isFile()) {
+        if (extensions.includes(path.extname(e.name).toLowerCase())) out.push(full);
+      }
+    }
+  }
+  await walk(dir);
+  return out;
+}

package/plugin/runners/pull-references.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+// plugin/runners/pull-references.mjs
+// Unified references pool. Scans Evidence/ for URLs, dedupes against an index,
+// classifies by host, snapshots external URLs via HTTP. Auth-protected URLs
+// (SP/Loop/Teams/ADO/CRM) are recorded with metadata only and marked
+// `pending-auth-fetch` for follow-up by source-specific pulls.
+//
+// Project-shared. Not dated. One snapshot per URL. Re-crawl with --refresh.
+//
+// Usage:
+//   node plugin/runners/pull-references.mjs --project <P> [--refresh] [--dry-run]
+//     [--timeout-ms N] [--max-fetch N] [--only-host <h>]
+
+import path from 'node:path';
+import { promises as fs } from 'node:fs';
+import YAML from 'yaml';
+import { evidenceRoot, sharedRoot, projectRoot } from './lib/layout.mjs';
+import { writeAtomic, pathExists } from './lib/evidence.mjs';
+import { extractUrls, urlHash, normalizeUrl, classify, safeHost, fetchSnapshot, walkFiles } from './lib/references.mjs';
+
+function parseArgs(argv) {
+  const args = { dryRun: false, refresh: false, timeoutMs: 15000, maxFetch: 50 };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--project') args.project = argv[++i];
+    else if (a === '--refresh') args.refresh = true;
+    else if (a === '--dry-run') args.dryRun = true;
+    else if (a === '--timeout-ms') args.timeoutMs = Number(argv[++i]) || 15000;
+    else if (a === '--max-fetch') args.maxFetch = Number(argv[++i]) || 50;
+    else if (a === '--only-host') args.onlyHost = argv[++i];
+    else if (a === '--help' || a === '-h') args.help = true;
+  }
+  return args;
+}
+
+function help() {
+  return `Usage: node pull-references.mjs --project <P> [--refresh] [--dry-run] [--timeout-ms N] [--max-fetch N] [--only-host <h>]`;
+}
+
+function emit(obj) { process.stdout.write(JSON.stringify(obj) + '\n'); }
+function log(msg) { process.stderr.write(`[references] ${msg}\n`); }
+
+function refsRoot(project) { return path.join(sharedRoot(project), 'references'); }
+function indexPath(project) { return path.join(refsRoot(project), 'index.yml'); }
+function recordPath(project, host, hash) {
+  return path.join(refsRoot(project), 'by-host', safeHost(host), `${hash}.md`);
+}
+
+async function loadIndex(p) {
+  if (!await pathExists(p)) return { version: 1, entries: {} };
+  try { return YAML.parse(await fs.readFile(p, 'utf8')) || { version: 1, entries: {} }; }
+  catch { return { version: 1, entries: {} }; }
+}
+
+function recordTemplate({ url, host, kind, authRequired, firstSeen, sourceFiles, snapshot }) {
+  const fm = {
+    url,
+    normalized_url: normalizeUrl(url),
+    host,
+    kind,
+    auth_required: authRequired,
+    first_seen: firstSeen,
+    last_crawled: snapshot?.crawledAt || null,
+    fetch_status: snapshot?.fetch_status || (authRequired ? 'pending-auth-fetch' : 'unfetched'),
+    http_status: snapshot?.status ?? null,
+    title: snapshot?.title || '',
+    description: snapshot?.description || '',
+    source_files: sourceFiles.slice(0, 20),
+  };
+  const yamlFm = YAML.stringify(fm).trimEnd();
+  const body = snapshot?.snippet
+    ? `\n## Snippet\n\n${snapshot.snippet}\n`
+    : (authRequired
+      ? `\n_Content fetch deferred — this URL requires Microsoft 365 authentication. The matching source-specific pull (sharepoint/loop/teams/ado/crm) will populate richer evidence._\n`
+      : `\n_No snapshot captured._\n`);
+  return `---\n${yamlFm}\n---\n${body}`;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) { console.log(help()); return 0; }
+  if (!args.project) { console.error(help()); emit({ status: 'failed', error: 'required: --project' }); return 2; }
+
+  const root = projectRoot(args.project);
+  if (!await pathExists(root)) { emit({ status: 'failed', error: `project-not-bootstrapped: ${root}` }); return 2; }
+
+  const evRoot = evidenceRoot(root);
+  if (!await pathExists(evRoot)) { emit({ status: 'failed', error: `evidence-missing: ${evRoot}` }); return 2; }
+
+  log(`scanning ${evRoot} for URLs...`);
+  const files = await walkFiles(evRoot);
+  log(`scanning ${files.length} file(s)...`);
+
+  /** url(normalized) → { url, sourceFiles:Set } */
+  const found = new Map();
+  for (const f of files) {
+    let txt = '';
+    try { txt = await fs.readFile(f, 'utf8'); } catch { continue; }
+    for (const u of extractUrls(txt)) {
+      const k = normalizeUrl(u);
+      const rel = path.relative(root, f);
+      if (!found.has(k)) found.set(k, { url: u, sourceFiles: new Set() });
+      found.get(k).sourceFiles.add(rel);
+    }
+  }
+  log(`found ${found.size} unique URL(s)`);
+
+  const idxFile = indexPath(root);
+  const index = await loadIndex(idxFile);
+  index.entries ??= {};
+
+  const today = new Date().toISOString().slice(0, 10);
+  const tasks = [];
+  let newCount = 0, refreshCount = 0, skippedCount = 0;
+
+  for (const [normalized, { url, sourceFiles }] of found.entries()) {
+    if (args.onlyHost) {
+      const c = classify(url);
+      if (c.host !== args.onlyHost) { skippedCount++; continue; }
+    }
+    const hash = urlHash(url);
+    const existing = index.entries[hash];
+    const isNew = !existing;
+    const needsRefresh = !isNew && args.refresh;
+    if (!isNew && !needsRefresh) {
+      // Update source_files only.
+      const merged = new Set([...(existing.source_files || []), ...sourceFiles]);
+      existing.source_files = [...merged].slice(0, 20);
+      existing.last_seen = today;
+      skippedCount++;
+      continue;
+    }
+    const c = classify(url);
+    tasks.push({ url, normalized, hash, sourceFiles: [...sourceFiles], cls: c, isNew });
+    if (isNew) newCount++; else refreshCount++;
+  }
+
+  // Cap fetches per run.
+  const fetchable = tasks.filter(t => !t.cls.authRequired && t.cls.kind !== 'invalid').slice(0, args.maxFetch);
+  const fetchKeys = new Set(fetchable.map(t => t.hash));
+  log(`new: ${newCount}, refresh: ${refreshCount}, skip-existing: ${skippedCount}, will fetch: ${fetchable.length}`);
+
+  let written = 0, fetched = 0, fetchOk = 0;
+  for (const t of tasks) {
+    let snap = null;
+    if (fetchKeys.has(t.hash)) {
+      log(`  fetch ${t.cls.host}: ${t.url.slice(0, 100)}`);
+      snap = await fetchSnapshot(t.url, { timeoutMs: args.timeoutMs });
+      snap.crawledAt = today;
+      snap.fetch_status = snap.ok ? 'fetched' : `fetch-failed:${snap.error || snap.status}`;
+      fetched++;
+      if (snap.ok) fetchOk++;
+    }
+    const firstSeen = index.entries[t.hash]?.first_seen || today;
+    const record = recordTemplate({
+      url: t.url,
+      host: t.cls.host,
+      kind: t.cls.kind,
+      authRequired: t.cls.authRequired,
+      firstSeen,
+      sourceFiles: t.sourceFiles,
+      snapshot: snap,
+    });
+    const recPath = recordPath(root, t.cls.host, t.hash);
+    if (!args.dryRun) {
+      const r = await writeAtomic(recPath, record, { skipIfUnchanged: true });
+      if (r.written) written++;
+    }
+    index.entries[t.hash] = {
+      url: t.url,
+      normalized_url: t.normalized,
+      host: t.cls.host,
+      kind: t.cls.kind,
+      auth_required: t.cls.authRequired,
+      first_seen: firstSeen,
+      last_seen: today,
+      last_crawled: snap?.crawledAt || index.entries[t.hash]?.last_crawled || null,
+      fetch_status: snap?.fetch_status || index.entries[t.hash]?.fetch_status || (t.cls.authRequired ? 'pending-auth-fetch' : 'unfetched'),
+      http_status: snap?.status ?? index.entries[t.hash]?.http_status ?? null,
+      title: snap?.title || index.entries[t.hash]?.title || '',
+      record_path: path.relative(root, recPath).replaceAll('\\', '/'),
+      source_files: t.sourceFiles.slice(0, 20),
+    };
+  }
+
+  if (!args.dryRun) {
+    await writeAtomic(idxFile, YAML.stringify(index), { skipIfUnchanged: true });
+  }
+
+  emit({
+    status: 'ok',
+    project: root,
+    dry_run: args.dryRun,
+    scanned_files: files.length,
+    urls_total: found.size,
+    new: newCount,
+    refresh: refreshCount,
+    fetched,
+    fetch_ok: fetchOk,
+    written,
+    index: path.relative(root, idxFile).replaceAll('\\', '/'),
+  });
+  return 0;
+}
+
+main().then(code => process.exit(code || 0)).catch(e => {
+  emit({ status: 'failed', error: e.message || String(e) });
+  process.exit(1);
+});