futura-scion 0.2.3 → 0.2.4

package/bin/scion.js

@@ -791,6 +791,98 @@ switch (cmd || '') {
     console.log(JSON.stringify(out, null, 2));
     break;
   }
+  case 'scaffold': {
+    // CONSTRUCTION: compose a template into a project skeleton, gated by
+    // the stack's real verifier recipe before success is reported.
+    const { scaffold, listTemplates, loadTemplate } = await import('../src/mind/scaffold.js');
+    if (!arg || arg === 'list' || arg === '--list') {
+      const tpls = listTemplates();
+      if (tpls.length === 0) { console.log('no templates in templates/ — add YAML manifests'); break; }
+      for (const t of tpls) console.log(`${t.id.padEnd(16)} ${(t.stack ?? '-').toString().padEnd(8)} ${t.description}`);
+      break;
+    }
+    const flags = restArgs.filter((a) => a.startsWith('--'));
+    const positional = restArgs.filter((a) => !a.startsWith('--'));
+    const target = positional[0];   // output directory
+    if (!target) {
+      console.error('usage: scion scaffold <template|list> <target-dir> [--name x] [--title y] [--port N] [--param k=v] [--no-verify]');
+      process.exitCode = 2;
+      break;
+    }
+    const tpl = loadTemplate(arg);
+    const params = {};
+    for (const f of flags) {
+      const m = f.match(/^--([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
+      if (m) params[m[1]] = m[2];
+    }
+    // Default missing params to sensible template-agnostic values instead
+    // of a raw throw — the CLI user should see WHICH param is missing in a
+    // friendly line, not a stack trace.
+    for (const req of ['name', 'title', 'port', 'description']) {
+      if (!(req in params)) {
+        if (req === 'port') params.port = '3000';
+        else if (req === 'title') params.title = params.name ?? arg;
+        else if (req === 'name') params.name = params.name ?? 'app';
+        else params.description ??= `built with FS scaffold (${arg})`;
+      }
+    }
+    let r;
+    try {
+      r = await scaffold(arg, { params, cwd: resolve(target), verify: !flags.includes('--no-verify') });
+    } catch (err) {
+      console.error(`scaffold failed: ${err.message}`);
+      process.exitCode = 1;
+      break;
+    }
+    if (r.ok === false && r.error) {
+      console.error(`scaffold failed: ${r.error}`);
+      process.exitCode = 1;
+      break;
+    }
+    console.log(`scaffold ${r.template} → ${target}`);
+    for (const f of r.files) console.log(`  + ${f}`);
+    if (r.gate) {
+      console.log(`gate: ${r.gate.verified ? 'VERIFIED' : 'FAILED'} — [${r.gate.evidence.exit_codes.join(',')}]`);
+      if (!r.gate.verified) process.exitCode = 1;
+    } else {
+      console.log('gate: skipped (no stack recipe)');
+    }
+    break;
+  }
+  case 'research': {
+    // The research rung, driven from the CLI (webresearcher lineage):
+    // deterministic-first web research with a completion gate.
+    const q = [arg, ...restArgs.filter((a) => !a.startsWith('--'))].filter(Boolean).join(' ').trim();
+    const flags = restArgs.filter((a) => a.startsWith('--')) || [];
+    const { runResearch } = await import('../src/mind/research/index.js');
+    if (!q) {
+      console.error('usage: scion research "<question>" [--no-persist] [--pages N] [--deep] [--learn]');
+      process.exitCode = 2;
+      break;
+    }
+    const recipe = flags.includes('--deep')
+      ? { id: 'deep-dive', parameters: { max_pages: 16 }, verify: { min_pages: 2, min_items: 5, min_corroborated: 1 } }
+      : undefined;
+    const report = await runResearch(q, {
+      recipe,
+      persist: !flags.includes('--no-persist'),
+      max_pages: Number((flags.find((f) => f.startsWith('--pages')) ?? '').split('=')[1]) || (flags.includes('--pages') ? Number(restArgs[restArgs.indexOf('--pages') + 1]) : undefined) || undefined,
+    });
+    console.log(`research: ${report.pages} pages → ${report.items.length} items ` +
+      `(${report.census.SUPPORTED} corroborated, ${report.census.CONFLICTING} conflicting)`);
+    console.log(`gate: ${report.gate.verdict} — ${report.gate.checks.map((c) => `${c.check}:${c.pass ? 'ok' : 'fail'}(${c.actual})`).join(' ')}`);
+    for (const f of report.summary.KEY_FACT.slice(0, 5)) console.log(`  ✓ ${f.slice(0, 140)}`);
+    for (const c of report.summary.CONTRADICTION.slice(0, 3)) console.log(`  ⚡ ${c.slice(0, 140)}`);
+    for (const e of report.summary.EVIDENCE.slice(0, 3)) console.log(`  · ${e.slice(0, 140)}`);
+    if (report.fallback) console.log(`  [llm-fallback] ${report.fallback.synthesis.slice(0, 200)}`);
+    if (flags.includes('--learn')) {
+      const { learnResearchLoops } = await import('../src/mind/research/loop-learner.js');
+      const learned = learnResearchLoops();
+      console.log(`loop-learner: ${learned.length} lesson(s) persisted`);
+    }
+    process.exitCode = report.gate.verdict === 'PASS' ? 0 : 1;
+    break;
+  }
   case 'conventions': {
     // Measure + persist project conventions (cortex convention-detector
     // lineage, as EVIDENCE): discovered, never enforced.
@@ -884,6 +976,10 @@ switch (cmd || '') {
       leaseMs, pollMs,
       primaryUrl: followerMode || primaryUrl ? (primaryUrl || null) : null,
     });
+    // Lever 3: the capability tools (web-search, scholar-search, scaffold-*)
+    // register at boot so serve/UI/MCP all share the widened hands.
+    const { registerBuiltinCapabilities } = await import('../src/mind/builtins-register.js');
+    registerBuiltinCapabilities();
     const api = await startApi({ port: Number(arg) || undefined, leader });
     leader.start();
     // FS Desktop support: provider-backed generator (rung G) from config,

package/config/scion.config.yaml

@@ -11,12 +11,8 @@ ladder:
   min_insight_confidence: 0.55  # reasoner rung admission floor
 
 llm:
-  daily_tokens: 500
+  daily_tokens: 0               # 0 = LLM rung off (zero-LLM doctrine)
 
-  apiKey: null
-  model: "qwen2.5-coder:7b"
-  baseUrl: "http://localhost:11434/v1"
-  provider: "ollama"
 bounds:
   max_turns: 25
   timeout_ms: 300000

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "futura-scion",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "The fused scion of cortex-os-agent + persona: one zero-LLM-dependent agent stack — Mind proposes, Muscle executes, Gate disposes.",
   "type": "module",
   "main": "src/index.js",
@@ -13,6 +13,7 @@
     "bin/",
     "src/",
     "recipes/",
+    "templates/",
     "config/",
     "knowledge/",
     "README.md",

package/src/brain/memory.js

@@ -368,6 +368,30 @@ export function list(opts = {}) {
   return rows.map(r => ({ ...r, tags: JSON.parse(r.tags || '[]') }));
 }
 
+/**
+ * reinforce — DEDUP-REINFORCE (webresearcher cortex_bridge lineage): when
+ * new evidence matches an existing memory, STRENGTHEN it instead of
+ * duplicating. Confidence rises by delta (capped at 100), importance to at
+ * least the floor, and the access counter ticks. Repeated corroboration is
+ * a trust signal — the memory-graph weights read it on every recall.
+ * @param {number} id — existing memory id
+ * @param {object} [opts] — { confidence_delta=5, importance_floor=null }
+ * @returns {{ id, confidence, importance, reinforced: true }}
+ */
+export function reinforce(id, opts = {}) {
+  const db = getDb();
+  const row = db.prepare('SELECT confidence, importance FROM memories WHERE id = ?').get(id);
+  if (!row) throw new Error(`memory.reinforce: no memory ${id}`);
+  const delta = opts.confidence_delta ?? 5;
+  const confidence = Math.min(100, row.confidence + delta);
+  const importance = opts.importance_floor != null
+    ? Math.max(row.importance, opts.importance_floor)
+    : Math.min(10, row.importance + (delta >= 5 ? 1 : 0));
+  db.prepare('UPDATE memories SET confidence = ?, importance = ?, access_count = access_count + 1 WHERE id = ?')
+    .run(confidence, importance, id);
+  return { id, confidence, importance, reinforced: true };
+}
+
 export function stats() {
   const db = getDb();
   const row = db.prepare(

package/src/config.js

@@ -59,6 +59,22 @@ function parseScalar(raw) {
   return v.replace(/^['"]|['"]$/g, '');
 }
 
+/**
+ * Strip a block scalar's BASE indentation while preserving relative
+ * indentation (Python code in templates lives or dies by its indents).
+ * The base is the indentation of the FIRST content line; subsequent lines
+ * are dedented by the same amount. `|` literal blocks are exact; `>` folded
+ * blocks joined with spaces downstream, so per-line dedent is harmless.
+ */
+function dedentBlockLine(line, blockScalar) {
+  if (blockScalar.folded) return line.trim();
+  if (blockScalar.baseIndent === undefined) {
+    blockScalar.baseIndent = line.match(/^\s*/)[0].length;
+  }
+  const cut = Math.min(blockScalar.baseIndent, line.match(/^\s*/)[0].length);
+  return line.slice(cut);
+}
+
 /**
  * Minimal YAML-subset parser: nested maps by indentation, scalars, inline
  * arrays, `#` comments. Deliberately NOT full YAML — unsupported syntax
@@ -144,11 +160,23 @@ export function parseYaml(text, source = CONFIG_PATH) {
     if (!match) {
       // Continuation line of a block scalar ("key: >" folded text)?
       if (blockScalar !== null && /^\s+\S/.test(line)) {
-        blockScalar.text.push(line.trim());
+        blockScalar.text.push(dedentBlockLine(line, blockScalar));
         continue;
       }
       throw new Error(`config: ${source}:${lineno} unsupported syntax — ${JSON.stringify(line.trim())}`);
     }
+    // A block scalar is OPEN and this line is deeper-indented than its key:
+    // it is SCALAR TEXT, never a new key — even when it looks like
+    // "word: value" (template manifests embed whole files in block scalars;
+    // their `port: 3000`-shaped lines are content, not structure). Leading
+    // indentation is PRESERVED relative to the block's first content line —
+    // Python files in templates live or die by their indentation.
+    if (blockScalar !== null
+        && match[1].length > (blockScalar.indent ?? 0)
+        && !/^\s*-\s/.test(line)) {
+      blockScalar.text.push(dedentBlockLine(line, blockScalar));
+      continue;
+    }
     const [, ws, key, rest] = match;
     const indent = ws.length;
 

package/src/mind/builtins-register.js

@@ -0,0 +1,88 @@
+/**
+ * mind/builtins-register.js — LEVER 3 (tool-calling breadth): registers the
+ * network + construction capabilities as FIRST-CLASS registry tools so the
+ * research rung (`research()`) and the swarm can invoke them by name like
+ * any shell tool. Import once at boot/serve; registration is idempotent.
+ *
+ * These are ASYNC capability tools (functions, not argv templates) — the
+ * registry stores their runner and runTool dispatches to it. Same rails
+ * apply: journaled, redacted, bounded.
+ *
+ * @module mind/builtins-register
+ */
+
+'use strict';
+
+import { registerTool } from './tools.js';
+import { searchWeb } from './research/search.js';
+import { listTemplates, loadTemplate } from './scaffold.js';
+
+let registered = false;
+
+/** Idempotently register the capability tools. */
+export function registerBuiltinCapabilities() {
+  if (registered) return listTemplates();
+  registered = true;
+
+  registerTool({
+    name: 'web-search',
+    description: 'search the web (DuckDuckGo keyless / Tavily keyed); {topic} is the query',
+    tags: ['web', 'research', 'network'],
+    capability: async (opts = {}) => {
+      const r = await searchWeb(opts.topic ?? '', { max_results: opts.max_results ?? 5 });
+      return {
+        ok: r.results.length > 0,
+        stdout: r.results.map((x) => `- ${x.title}\n  ${x.url}\n  ${String(x.snippet ?? '').slice(0, 160)}`).join('\n').slice(0, 8000),
+        meta: { provider: r.provider, count: r.results.length, attempts: r.attempts.length },
+      };
+    },
+  });
+
+  registerTool({
+    name: 'scholar-search',
+    description: 'search scholarship (arXiv/Semantic Scholar/CrossRef); {topic} is the query',
+    tags: ['academic', 'research', 'network'],
+    capability: async (opts = {}) => {
+      const { searchScholar } = await import('./research/academic.js');
+      const papers = await searchScholar(opts.topic ?? '', { max_results: 5 });
+      return {
+        ok: papers.length > 0,
+        stdout: papers.map((p) => `- ${p.title} (${p.year ?? 'n.d.'}, cites: ${p.citation_count ?? '?'})\n  ${p.url}`).join('\n').slice(0, 8000),
+        meta: { count: papers.length },
+      };
+    },
+  });
+
+  registerTool({
+    name: 'scaffold-list',
+    description: 'list constructable project templates (what FS can BUILD)',
+    tags: ['construction'],
+    capability: async () => {
+      const tpls = listTemplates();
+      return {
+        ok: tpls.length > 0,
+        stdout: tpls.map((t) => `${t.id.padEnd(16)} ${(t.stack ?? '-').padEnd(8)} ${t.description}`).join('\n'),
+        meta: { count: tpls.length },
+      };
+    },
+  });
+
+  registerTool({
+    name: 'scaffold-inspect',
+    description: 'show a template manifest (files, post commands, stack) for {topic}=template id',
+    tags: ['construction'],
+    capability: async (opts = {}) => {
+      try {
+        const t = loadTemplate(opts.topic ?? '');
+        return {
+          ok: true,
+          stdout: `${t.name} (stack: ${t.stack ?? 'none'})\n${t.description}\nfiles:\n${t.files.map((f) => `  + ${f.path}`).join('\n')}\npost:\n${(t.post ?? []).map((c) => `  $ ${c}`).join('\n') || '  (none)'}`,
+        };
+      } catch (err) {
+        return { ok: false, error: String(err?.message || err).slice(0, 200) };
+      }
+    },
+  });
+
+  return listTemplates();
+}

package/src/mind/research/academic.js

@@ -0,0 +1,183 @@
+/**
+ * mind/research/academic.js — ACADEMIC SEARCH (webresearcher
+ * `academic_search.py` ARCANE lineage): unified access to arXiv, Semantic
+ * Scholar, and CrossRef with smart ID resolution (DOI <-> arXivID <-> S2 ID).
+ *
+ * All APIs are FREE and require NO authentication. Deterministic parsing
+ * (regex over XML/JSON), bounded fetches, graceful degradation — a provider
+ * that fails returns fewer results, never a throw.
+ *
+ * @module mind/research/academic
+ */
+
+'use strict';
+
+import { fetchBounded } from './search.js';
+import * as trail from '../../kernel/trail.js';
+
+const S2_API = 'https://api.semanticscholar.org/graph/v1';
+const CROSSREF_API = 'https://api.crossref.org/works';
+
+function stripXml(s) {
+  return String(s ?? '').replace(/<[^>]+>/g, '').trim();
+}
+
+/** One paper record: { id, doi?, arxiv_id?, title, abstract, year, venue, authors, url, source } */
+function paper(p) {
+  return { ...p, authors: p.authors ?? [], citation_count: p.citation_count ?? null };
+}
+
+/* ------------------------------- arXiv ------------------------------- */
+
+function parseArxivXml(xml, max) {
+  const papers = [];
+  const entries = xml.split(/<entry>/).slice(1);
+  for (const e of entries) {
+    if (papers.length >= max) break;
+    const id = stripXml(e.match(/<id>([\s\S]*?)<\/id>/)?.[1]);
+    const title = stripXml(e.match(/<title>([\s\S]*?)<\/title>/)?.[1]).replace(/\s+/g, ' ');
+    const abstract = stripXml(e.match(/<summary>([\s\S]*?)<\/summary>/)?.[1]).replace(/\s+/g, ' ');
+    const published = stripXml(e.match(/<published>([\s\S]*?)<\/published>/)?.[1]);
+    const authors = [...e.matchAll(/<name>([\s\S]*?)<\/name>/g)].map((m) => stripXml(m[1]));
+    if (!id || !title) continue;
+    papers.push(paper({
+      id, arxiv_id: id.replace(/^https?:\/\/arxiv\.org\/abs\//, ''),
+      title, abstract, year: published ? Number(published.slice(0, 4)) : null,
+      venue: 'arXiv', authors, url: id, source: 'arxiv',
+    }));
+  }
+  return papers;
+}
+
+async function arxivSearch(query, opts = {}) {
+  const max = opts.max_results ?? 5;
+  const url = `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&max_results=${max}&sortBy=relevance`;
+  const r = await fetchBounded(url, { timeout_ms: opts.timeout_ms });
+  if (!r.ok) return [];
+  return parseArxivXml(r.text, max);
+}
+
+/* ------------------------- Semantic Scholar -------------------------- */
+
+async function s2Search(query, opts = {}) {
+  const max = opts.max_results ?? 5;
+  const fields = 'title,abstract,year,venue,authors,externalIds,citationCount,url';
+  const url = `${S2_API}/paper/search?query=${encodeURIComponent(query)}&limit=${max}&fields=${encodeURIComponent(fields)}`;
+  const r = await fetchBounded(url, { timeout_ms: opts.timeout_ms });
+  if (!r.ok) return [];
+  let body;
+  try { body = JSON.parse(r.text); } catch { return []; }
+  return (body.data ?? []).map((p) => paper({
+    id: p.paperId, doi: p.externalIds?.DOI ?? null, arxiv_id: p.externalIds?.ArXiv ?? null,
+    title: p.title, abstract: p.abstract, year: p.year, venue: p.venue || null,
+    authors: (p.authors ?? []).map((a) => a.name), citation_count: p.citation_count,
+    url: p.url ?? `https://www.semanticscholar.org/paper/${p.paperId}`, source: 's2',
+  }));
+}
+
+/* ------------------------------ CrossRef ----------------------------- */
+
+async function crossrefSearch(query, opts = {}) {
+  const max = opts.max_results ?? 5;
+  const url = `${CROSSREF_API}?query=${encodeURIComponent(query)}&rows=${max}&select=DOI,title,author,published,container-title,URL,is-referenced-by-count,abstract`;
+  const r = await fetchBounded(url, { timeout_ms: opts.timeout_ms });
+  if (!r.ok) return [];
+  let body;
+  try { body = JSON.parse(r.text); } catch { return []; }
+  return ((body.message ?? {}).items ?? []).map((p) => paper({
+    id: p.DOI, doi: p.DOI,
+    title: Array.isArray(p.title) ? p.title[0] : p.title,
+    abstract: typeof p.abstract === 'string' ? stripXml(p.abstract).replace(/\s+/g, ' ').slice(0, 2000) : null,
+    year: p.published ? Number(String(p.published['date-parts']?.[0]?.[0] ?? '').slice(0, 4)) || null : null,
+    venue: Array.isArray(p['container-title']) ? p['container-title'][0] : null,
+    authors: (p.author ?? []).map((a) => [a.given, a.family].filter(Boolean).join(' ')),
+    citation_count: p['is-referenced-by-count'] ?? null,
+    url: p.URL ?? `https://doi.org/${p.DOI}`, source: 'crossref',
+  })).filter((p) => p.title);
+}
+
+/* ---------------------------- ID resolution --------------------------- */
+
+const DOI_RE = /^10\.\d{4,}\//;
+const ARXIV_RE = /^\d{4}\.\d{4,5}(v\d+)?$/;
+
+/**
+ * Resolve any identifier to a canonical paper record. Accepts a DOI
+ * (10.xxxx/...), an arXiv ID (2401.12345), or an S2 paper ID.
+ */
+export async function resolveId(id, opts = {}) {
+  const raw = String(id).trim();
+  if (DOI_RE.test(raw)) {
+    const r = await fetchBounded(`${CROSSREF_API}/${encodeURIComponent(raw)}`, { timeout_ms: opts.timeout_ms });
+    if (r.ok) {
+      try {
+        const m = JSON.parse(r.text).message;
+        return paper({
+          id: m.DOI, doi: m.DOI, title: Array.isArray(m.title) ? m.title[0] : m.title,
+          year: m.published ? Number(String(m.published['date-parts']?.[0]?.[0] ?? '').slice(0, 4)) || null : null,
+          venue: Array.isArray(m['container-title']) ? m['container-title'][0] : null,
+          authors: (m.author ?? []).map((a) => [a.given, a.family].filter(Boolean).join(' ')),
+          citation_count: m['is-referenced-by-count'] ?? null,
+          url: m.URL ?? `https://doi.org/${m.DOI}`, source: 'crossref',
+        });
+      } catch { /* fall through */ }
+    }
+  }
+  if (ARXIV_RE.test(raw)) {
+    const r = await fetchBounded(`https://export.arxiv.org/api/query?id_list=${encodeURIComponent(raw)}`, { timeout_ms: opts.timeout_ms });
+    if (r.ok) {
+      const papers = parseArxivXml(r.text, 1);
+      if (papers.length) return papers[0];
+    }
+  }
+  // S2 fallback (handles S2 IDs and hashes DOIs/arXiv too).
+  const s2 = await fetchBounded(
+    `${S2_API}/paper/${encodeURIComponent(raw)}?fields=title,abstract,year,venue,authors,externalIds,citationCount,url`,
+    { timeout_ms: opts.timeout_ms }
+  );
+  if (s2.ok) {
+    try {
+      const p = JSON.parse(s2.text);
+      return paper({
+        id: p.paperId, doi: p.externalIds?.DOI ?? null, arxiv_id: p.externalIds?.ArXiv ?? null,
+        title: p.title, abstract: p.abstract, year: p.year, venue: p.venue || null,
+        authors: (p.authors ?? []).map((a) => a.name), citation_count: p.citation_count,
+        url: p.url ?? `https://www.semanticscholar.org/paper/${p.paperId}`, source: 's2',
+      });
+    } catch { /* fall through */ }
+  }
+  return null;
+}
+
+/* ------------------------------- search ------------------------------- */
+
+/**
+ * Search scholarship across providers. Order: arXiv (preprints) → S2 →
+ * CrossRef; merges and dedups by DOI/arXiv ID/title, then by citation count
+ * (descending) — deterministic.
+ * @param {string} query
+ * @param {object} [opts] — { max_results?, timeout_ms? }
+ */
+export async function searchScholar(query, opts = {}) {
+  const max = opts.max_results ?? 5;
+  const started = Date.now();
+  const settled = await Promise.allSettled([
+    arxivSearch(query, opts), s2Search(query, opts), crossrefSearch(query, opts),
+  ]);
+  const byKey = new Map();
+  for (const s of settled) {
+    if (s.status !== 'fulfilled') continue;
+    for (const p of s.value) {
+      const key = p.doi ?? (p.arxiv_id ? `arxiv:${p.arxiv_id}` : `t:${String(p.title).toLowerCase().slice(0, 90)}`);
+      const prev = byKey.get(key);
+      if (!prev || (p.citation_count ?? 0) > (prev.citation_count ?? 0)) byKey.set(key, p);
+    }
+  }
+  const papers = [...byKey.values()]
+    .sort((a, b) => (b.citation_count ?? 0) - (a.citation_count ?? 0))
+    .slice(0, max);
+  trail.journal('research.scholar', {
+    query: String(query).slice(0, 120), papers: papers.length, duration_ms: Date.now() - started,
+  });
+  return papers;
+}

package/src/mind/research/corroborate.js

@@ -0,0 +1,110 @@
+/**
+ * mind/research/corroborate.js — CLAIM CORROBORATION WITH EPISTEMIC HONESTY
+ * (webresearcher `claim_verifier.py` / `validation_loop.py` lineage).
+ *
+ * Most validators just say yes/no. This says "we don't have enough evidence"
+ * and "sources disagree" as FIRST-CLASS outcomes. A claim is corroborated
+ * when INDEPENDENT sources (different registrable domains) agree on it;
+ * conflicting numbers or explicit negation produce CONFLICTING; a single
+ * source is INSUFFICIENT_EVIDENCE — never silently treated as truth.
+ *
+ * Agreement is deterministic: Jaccard token overlap over the claim's
+ * normalized content words, plus numeric equality for statistics.
+ *
+ * @module mind/research/corroborate
+ */
+
+'use strict';
+
+const STOP = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'of', 'to', 'in', 'on', 'for', 'and', 'or', 'it', 'its', 'this', 'that', 'with', 'as', 'by', 'at', 'be', 'can', 'has', 'have', 'had']);
+
+function words(s) {
+  return String(s).toLowerCase().normalize('NFKD').replace(/[^a-z0-9\s]/g, ' ')
+    .split(/\s+/).filter((w) => w.length > 2 && !STOP.has(w));
+}
+
+function jaccard(a, b) {
+  const A = new Set(a);
+  const B = new Set(b);
+  if (A.size === 0 || B.size === 0) return 0;
+  let inter = 0;
+  for (const w of A) if (B.has(w)) inter += 1;
+  return inter / (A.size + B.size - inter);
+}
+
+function numbers(s) {
+  return [...String(s).matchAll(/\d+(?:\.\d+)?/g)].map((m) => Number(m[0]));
+}
+
+/** Registrable domain (best-effort, no PSL): last two labels. */
+export function domainOf(url) {
+  try {
+    const host = new URL(url).hostname.replace(/^www\./, '');
+    const parts = host.split('.');
+    return parts.length >= 2 ? parts.slice(-2).join('.') : host;
+  } catch { return String(url); }
+}
+
+/**
+ * Corroborate one extracted item against the others from INDEPENDENT sources.
+ * @param {object} item — an extract.js item ({ kind, text, source_url })
+ * @param {Array} all — every extracted item
+ * @param {object} [opts] — { threshold=0.45 }
+ * @returns {{ verdict, agreement_count, domains, numeric_match, reason }}
+ */
+export function corroborateItem(item, all, opts = {}) {
+  const threshold = opts.threshold ?? 0.45;
+  const w = words(item.text);
+  const nums = numbers(item.text);
+  const ownDomain = item.source_url ? domainOf(item.source_url) : null;
+  const agreeing = new Map();   // domain → item
+  const disagreeing = [];
+
+  for (const other of all) {
+    if (other === item) continue;
+    if (other.kind !== item.kind) continue;
+    const od = other.source_url ? domainOf(other.source_url) : `__${words(other.text).join('+')}`;
+    if (ownDomain && od === ownDomain) continue;        // same domain ≠ independence
+    const sim = jaccard(w, words(other.text));
+    if (sim < threshold) continue;
+    const on = numbers(other.text);
+    if (nums.length > 0 && on.length > 0) {
+      const match = nums.every((n, i) => on[i] === n);
+      if (!match) { disagreeing.push({ url: other.source_url, sim, number: on[0] }); continue; }
+    }
+    if (!agreeing.has(od)) agreeing.set(od, other);
+  }
+
+  const domains = [...agreeing.keys()];
+  if (disagreeing.length > 0 && agreeing.size > 0) {
+    return { verdict: 'CONFLICTING', agreement_count: agreeing.size, domains, numeric_match: false, reason: `${agreeing.size} source(s) agree but ${disagreeing.length} report different values` };
+  }
+  if (disagreeing.length > 0) {
+    return { verdict: 'CONFLICTING', agreement_count: 0, domains, numeric_match: false, reason: `${disagreeing.length} independent source(s) report different values` };
+  }
+  if (agreeing.size >= 1) {
+    return { verdict: 'SUPPORTED', agreement_count: agreeing.size + 1, domains, numeric_match: true, reason: `${agreeing.size + 1} independent source(s) agree` };
+  }
+  return { verdict: 'INSUFFICIENT_EVIDENCE', agreement_count: 1, domains: ownDomain ? [ownDomain] : [], numeric_match: null, reason: 'single source only' };
+}
+
+/**
+ * Corroborate all items; annotate each with its verdict.
+ * @param {Array} items — extract.js output
+ * @returns {Array} items, each with `.corroboration`
+ */
+export function corroborateAll(items, opts = {}) {
+  for (const it of items ?? []) {
+    it.corroboration = corroborateItem(it, items, opts);
+  }
+  return items ?? [];
+}
+
+/** Verdict census for a run report. */
+export function verdictCensus(items) {
+  const census = { SUPPORTED: 0, CONFLICTING: 0, INSUFFICIENT_EVIDENCE: 0 };
+  for (const it of items ?? []) {
+    if (census[it.corroboration?.verdict] !== undefined) census[it.corroboration.verdict] += 1;
+  }
+  return census;
+}