@agentproto/corpus-cli 0.2.0 → 0.4.0

package/dist/cli.mjs

@@ -12,7 +12,7 @@ import matter2 from 'gray-matter';
 import { execFile } from 'child_process';
 import { promisify } from 'util';
 import { parseClaudeJsonOutput, spawnWithStdin } from '@agentproto/cli-exec';
-import { stitchReport, buildPacks, reportConfigSchema } from '@agentproto/corpus/report';
+import { stitchReport, buildPacks, lintReportConfig, reportConfigSchema } from '@agentproto/corpus/report';
 
 /**
  * @agentproto/corpus-cli v0.1.0-alpha
@@ -104,9 +104,9 @@ function resolvePackageJson(packageName) {
   }
   return null;
 }
-async function readJsonIfExists(path5) {
+async function readJsonIfExists(path6) {
   try {
-    const raw = await readFile(path5, "utf8");
+    const raw = await readFile(path6, "utf8");
     return JSON.parse(raw);
   } catch (err) {
     if (err.code === "ENOENT") return void 0;
@@ -115,9 +115,9 @@ async function readJsonIfExists(path5) {
 }
 async function readConfiguredPresetPackages() {
   const base = process.env["AGENTPROTO_HOME"] ?? join(homedir(), ".agentproto");
-  const path5 = join(base, "config.json");
+  const path6 = join(base, "config.json");
   try {
-    const raw = await readFile(path5, "utf8");
+    const raw = await readFile(path6, "utf8");
     const parsed = JSON.parse(raw);
     const list = parsed.corpusPresetPackages;
     if (Array.isArray(list) && list.length > 0) return list;
@@ -404,8 +404,8 @@ function resolveWorkspacePath(arg) {
   if (!arg) return process.cwd();
   return path.resolve(process.cwd(), arg);
 }
-function fail(msg4, code = 1) {
-  process.stderr.write(`corpus: ${msg4}
+function fail(msg5, code = 1) {
+  process.stderr.write(`corpus: ${msg5}
 `);
   return code;
 }
@@ -631,6 +631,160 @@ async function runLint(args) {
   );
   return report.errorCount > 0 ? 1 : 0;
 }
+var FLAG_PATTERNS = [
+  /does not match/i,
+  /content is about/i,
+  /no substantive content/i,
+  /content mismatch/i,
+  /appears to be a (?:newsletter|navigation|login|error) page/i,
+  /stated title/i,
+  /placeholder page/i,
+  /cookie consent/i,
+  /page could not be/i
+];
+function parse(args) {
+  const out = {
+    workspace: void 0,
+    facets: [],
+    thin: 8,
+    apply: false,
+    contaminated: false
+  };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    const next = () => args[++i];
+    switch (a) {
+      case "--facets": {
+        const v = next();
+        if (v) out.facets = v.split(",").map((s) => s.trim()).filter(Boolean);
+        break;
+      }
+      case "--thin": {
+        const v = next();
+        if (v) out.thin = Number(v);
+        break;
+      }
+      case "--apply":
+        out.apply = true;
+        break;
+      case "--contaminated":
+        out.contaminated = true;
+        break;
+      default:
+        if (!a.startsWith("-") && out.workspace === void 0) out.workspace = a;
+    }
+  }
+  return out;
+}
+function firstFlag(haystack) {
+  for (const re of FLAG_PATTERNS) {
+    const m = haystack.match(re);
+    if (m) return m[0];
+  }
+  return void 0;
+}
+function entryStatus(entry) {
+  const corpus = entry.frontmatter.metadata?.corpus;
+  return corpus?.status ?? "active";
+}
+function tagsOf(entry) {
+  const tags = entry.frontmatter.tags;
+  return Array.isArray(tags) ? tags.map(String) : [];
+}
+function sourcesOf(entry) {
+  const sources = entry.frontmatter.sources;
+  return Array.isArray(sources) ? sources.map(String) : [];
+}
+async function runVerify(args) {
+  const parsed = parse(args);
+  if (parsed.facets.length === 0) {
+    return fail(
+      "verify requires --facets a,b,c. Usage: corpus verify <workspace> --facets landscape,daemons,... [--thin 8] [--apply] [--contaminated]",
+      2
+    );
+  }
+  const target = resolveWorkspacePath(parsed.workspace);
+  const fs = new NodeFsAdapter({ root: target });
+  if (!await fs.exists("entries")) {
+    return fail(`verify: no entries/ under ${target}`, 1);
+  }
+  const snapshot = await new CorpusWorkspaceReader({ fs }).read("");
+  const entries = snapshot.entries.filter((e) => entryStatus(e) === "active");
+  const perFacet = /* @__PURE__ */ new Map();
+  for (const facet of parsed.facets) perFacet.set(facet, 0);
+  for (const entry of entries) {
+    for (const tag of tagsOf(entry)) {
+      if (perFacet.has(tag)) perFacet.set(tag, (perFacet.get(tag) ?? 0) + 1);
+    }
+  }
+  process.stdout.write(`== coverage (${entries.length} entries) ==
+`);
+  const thin = [];
+  for (const facet of parsed.facets) {
+    const n = perFacet.get(facet) ?? 0;
+    if (n < parsed.thin) thin.push(facet);
+    const mark = n < parsed.thin ? "  THIN \u26A0" : "";
+    process.stdout.write(`  ${facet.padEnd(16)} ${String(n).padStart(4)}${mark}
+`);
+  }
+  const flagged = [];
+  for (const entry of entries) {
+    const title = String(entry.frontmatter.title ?? "");
+    const why = firstFlag(`${title}
+${entry.body}`);
+    if (why) flagged.push({ entry, why });
+  }
+  process.stdout.write(`
+== self-flagged entries (${flagged.length}) ==
+`);
+  const poisonedSources = /* @__PURE__ */ new Set();
+  for (const { entry, why } of flagged) {
+    process.stdout.write(`  ${entry.path}  [${why}]
+`);
+    for (const s of sourcesOf(entry)) poisonedSources.add(s);
+  }
+  const flaggedPaths = new Set(flagged.map((f) => f.entry.path));
+  const contaminated = [];
+  if (parsed.contaminated && poisonedSources.size > 0) {
+    for (const entry of entries) {
+      if (flaggedPaths.has(entry.path)) continue;
+      if (sourcesOf(entry).some((s) => poisonedSources.has(s))) {
+        contaminated.push(entry);
+      }
+    }
+    process.stdout.write(
+      `
+== contaminated siblings (${contaminated.length}) from ${poisonedSources.size} poisoned source(s) ==
+`
+    );
+    for (const entry of contaminated) {
+      process.stdout.write(`  ${entry.path}
+`);
+    }
+  }
+  if (parsed.apply) {
+    const toMove = [...flagged.map((f) => f.entry), ...contaminated];
+    for (const entry of toMove) {
+      const rel = path.relative("entries", entry.path);
+      const dest = path.join(target, "demoted", rel);
+      await mkdir(path.dirname(dest), { recursive: true });
+      await rename(path.join(target, entry.path), dest);
+    }
+    process.stdout.write(`
+moved ${toMove.length} entries \u2192 demoted/
+`);
+  } else if (flagged.length > 0) {
+    process.stdout.write(
+      "\n(re-run with --apply to quarantine; --contaminated to also demote siblings)\n"
+    );
+  }
+  if (thin.length > 0) {
+    process.stdout.write(`
+thin facets \u2192 loop back to \u2461 DISCOVER: ${thin.join(", ")}
+`);
+  }
+  return 0;
+}
 var OsIdentityAdapter = class {
   cached;
   constructor(opts) {
@@ -1040,6 +1194,146 @@ function defaultYtDlpDownloader(bin, opts) {
     }
   };
 }
+var execFileAsync2 = promisify(execFile);
+var YTDLP_INFO2 = z.object({ title: z.string().optional(), language: z.string().optional() }).loose();
+var YtDlpCaptionsFetcher = class {
+  download;
+  extraHosts;
+  minChars;
+  constructor(opts = {}) {
+    this.minChars = opts.minChars ?? 400;
+    this.download = opts.download ?? defaultYtDlpCaptionDownloader(opts.ytDlpBin ?? "yt-dlp", {
+      ...opts.subLangs ? { subLangs: opts.subLangs } : {},
+      ...opts.preferLang ? { preferLang: opts.preferLang } : {},
+      ...opts.maxDurationSec ? { maxDurationSec: opts.maxDurationSec } : {},
+      ...opts.cookiesFromBrowser ? { cookiesFromBrowser: opts.cookiesFromBrowser } : {},
+      ...opts.cookiesFile ? { cookiesFile: opts.cookiesFile } : {}
+    });
+    this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
+  }
+  async fetch(url) {
+    if (!isVideoUrl(url, this.extraHosts)) return null;
+    let dl;
+    try {
+      dl = await this.download(url);
+    } catch (e) {
+      process.stderr.write(
+        `corpus: caption fetch failed for ${url} \u2014 falling back (${msg2(e)})
+`
+      );
+      return null;
+    }
+    try {
+      if (!dl.vttPath) return null;
+      const text = parseVttToText(await readFile(dl.vttPath, "utf-8"));
+      if (text.length < this.minChars) return null;
+      return {
+        title: dl.title || url,
+        text,
+        kind: "video",
+        ...dl.language ? { language: dl.language } : {},
+        via: "captions"
+      };
+    } catch {
+      return null;
+    } finally {
+      await dl.cleanup().catch(() => {
+      });
+    }
+  }
+};
+function msg2(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+function parseVttToText(vtt) {
+  const out = [];
+  for (const raw of vtt.split(/\r?\n/)) {
+    if (/^WEBVTT/.test(raw)) continue;
+    if (/^(Kind|Language|NOTE|STYLE):/i.test(raw)) continue;
+    if (raw.includes("-->")) continue;
+    const line = raw.replace(/<[^>]+>/g, "").replace(/&nbsp;/gi, " ").replace(/ /g, " ").trim();
+    if (!line) continue;
+    if (out.length && out[out.length - 1] === line) continue;
+    out.push(line);
+  }
+  const cleaned = [];
+  for (let i = 0; i < out.length; i++) {
+    const next = out[i + 1];
+    if (next && next.startsWith(out[i])) continue;
+    cleaned.push(out[i]);
+  }
+  return cleaned.join(" ").replace(/\s+/g, " ").trim();
+}
+function buildYtDlpCaptionArgs(url, dir, opts) {
+  const subLangs = opts?.subLangs ?? (opts?.preferLang ? `${opts.preferLang}.*` : "en.*,fr.*,es.*,de.*,it.*,pt.*,nl.*");
+  const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+  const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+  return [
+    "--skip-download",
+    "--write-auto-subs",
+    "--write-subs",
+    "--sub-langs",
+    subLangs,
+    "--sub-format",
+    "vtt",
+    "--no-playlist",
+    "--no-progress",
+    "--sleep-requests",
+    "1",
+    ...durationGuard,
+    ...cookieArgs,
+    "--write-info-json",
+    "-o",
+    join(dir, "%(id)s.%(ext)s"),
+    url
+  ];
+}
+function pickVtt(files, preferLang) {
+  const vtts = files.filter((f) => f.endsWith(".vtt"));
+  if (vtts.length === 0) return void 0;
+  const bySuffix = (suffix) => vtts.find((f) => f.toLowerCase().endsWith(suffix.toLowerCase()));
+  if (preferLang) {
+    return bySuffix(`.${preferLang}-orig.vtt`) ?? bySuffix(`.${preferLang}.vtt`) ?? vtts.find((f) => new RegExp(`\\.${preferLang}[.-]`, "i").test(f)) ?? vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+  }
+  return vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+}
+function defaultYtDlpCaptionDownloader(bin, opts) {
+  return async (url) => {
+    const dir = await mkdtemp(join(tmpdir(), "corpus-captions-"));
+    const cleanup = () => rm(dir, { recursive: true, force: true });
+    try {
+      await execFileAsync2(bin, buildYtDlpCaptionArgs(url, dir, opts), {
+        maxBuffer: 16 * 1024 * 1024
+      }).catch(() => {
+      });
+      const files = await readdir(dir);
+      const vtt = pickVtt(files, opts?.preferLang);
+      let title = url;
+      let language;
+      const infoName = files.find((f) => f.endsWith(".info.json"));
+      if (infoName) {
+        try {
+          const info = YTDLP_INFO2.parse(
+            JSON.parse(await readFile(join(dir, infoName), "utf-8"))
+          );
+          if (info.title) title = info.title;
+          if (info.language) language = info.language;
+        } catch {
+        }
+      }
+      return {
+        vttPath: vtt ? join(dir, vtt) : null,
+        title,
+        ...language ? { language } : {},
+        cleanup
+      };
+    } catch (e) {
+      await cleanup().catch(() => {
+      });
+      throw e;
+    }
+  };
+}
 var WHISPER_RESPONSE = z.object({ text: z.string().optional(), language: z.string().optional() }).loose();
 var WHISPER_MAX_BYTES = 25 * 1024 * 1024;
 var OpenAiWhisperStt = class {
@@ -1157,18 +1451,18 @@ var AssemblyAiStt = class {
       ...job.language_code ? { language: job.language_code } : {}
     };
   }
-  async post(path5, body) {
-    const r = await fetch(`${this.baseUrl}${path5}`, {
+  async post(path6, body) {
+    const r = await fetch(`${this.baseUrl}${path6}`, {
       method: "POST",
       headers: { authorization: this.apiKey, "content-type": "application/json" },
       body: JSON.stringify(body)
     });
-    if (!r.ok) throw new Error(`AssemblyAI POST ${path5} ${r.status}: ${(await r.text()).slice(0, 200)}`);
+    if (!r.ok) throw new Error(`AssemblyAI POST ${path6} ${r.status}: ${(await r.text()).slice(0, 200)}`);
     return AAI_TRANSCRIPT.parse(await r.json());
   }
-  async get(path5) {
-    const r = await fetch(`${this.baseUrl}${path5}`, { headers: { authorization: this.apiKey } });
-    if (!r.ok) throw new Error(`AssemblyAI GET ${path5} ${r.status}: ${(await r.text()).slice(0, 200)}`);
+  async get(path6) {
+    const r = await fetch(`${this.baseUrl}${path6}`, { headers: { authorization: this.apiKey } });
+    if (!r.ok) throw new Error(`AssemblyAI GET ${path6} ${r.status}: ${(await r.text()).slice(0, 200)}`);
     return AAI_TRANSCRIPT.parse(await r.json());
   }
 };
@@ -1178,7 +1472,7 @@ function formatDiarized(job) {
   }
   return (job.text ?? "").trim();
 }
-var execFileAsync2 = promisify(execFile);
+var execFileAsync3 = promisify(execFile);
 var DEFAULT_MAX_BYTES = 24 * 1024 * 1024;
 var DEFAULT_SEGMENT_SECONDS = 1200;
 var ChunkedStt = class {
@@ -1237,7 +1531,7 @@ var ChunkedStt = class {
 function defaultFfmpegSplitter(bin) {
   return async (audioPath, segmentSeconds, outDir) => {
     const ext = extname(audioPath) || ".mp3";
-    await execFileAsync2(
+    await execFileAsync3(
       bin,
       [
         "-hide_banner",
@@ -1453,7 +1747,7 @@ var SOURCE_URL_FRONTMATTER = z.object({
     }).loose().optional().catch(void 0)
   }).loose().optional().catch(void 0)
 }).loose();
-function parse(args) {
+function parse2(args) {
   const out = {
     workspace: void 0,
     urlsFile: void 0,
@@ -1468,6 +1762,7 @@ function parse(args) {
     throttleMs: 2e3,
     force: false,
     diarize: false,
+    noCaptions: false,
     browserMcp: process.env.BROWSER_MCP_URL,
     scrapeMcp: process.env.SCRAPE_MCP_URL,
     importerId: "web",
@@ -1523,6 +1818,9 @@ function parse(args) {
       case "--diarize":
         out.diarize = true;
         break;
+      case "--no-captions":
+        out.noCaptions = true;
+        break;
       case "--browser-mcp":
         out.browserMcp = next();
         break;
@@ -1579,7 +1877,7 @@ async function scanIngestedUrls(workspaceRoot) {
   return seen;
 }
 async function runImportWeb(args) {
-  const parsed = parse(args);
+  const parsed = parse2(args);
   const target = resolveWorkspacePath(parsed.workspace);
   const urls = await readUrls(parsed);
   if (urls.length === 0) {
@@ -1589,10 +1887,11 @@ async function runImportWeb(args) {
   const todo = urls.filter((u) => !done.has(u));
   const batch = parsed.max !== void 0 ? todo.slice(0, parsed.max) : todo;
   const remaining = todo.length - batch.length;
+  const videoLabel = parsed.noCaptions ? parsed.diarize ? "AssemblyAI" : "Whisper" : `captions-first \u2192 ${parsed.diarize ? "AssemblyAI" : "Whisper"} fallback`;
   const plan = `  workspace: ${target}
   urls:      ${urls.length} total \xB7 ${done.size && !parsed.force ? `${urls.length - todo.length} already ingested \xB7 ` : ""}${todo.length} to do
   this run:  ${batch.length}${parsed.max !== void 0 ? ` (--max ${parsed.max})` : ""} \xB7 ${remaining} remaining after
-  video stt: ${parsed.diarize ? "AssemblyAI (speaker-labelled)" : "Whisper"}
+  video:     ${videoLabel}
   throttle:  ${parsed.throttleMs} ms between fetches
 `;
   if (batch.length === 0) {
@@ -1613,6 +1912,16 @@ ${plan}`);
   process.stdout.write(`import-web
 ${plan}`);
   const chain = [];
+  if (!parsed.noCaptions) {
+    chain.push(
+      new YtDlpCaptionsFetcher({
+        ...parsed.lang ? { preferLang: parsed.lang } : {},
+        ...parsed.maxDurationSec ? { maxDurationSec: parsed.maxDurationSec } : {},
+        ...parsed.cookiesFromBrowser ? { cookiesFromBrowser: parsed.cookiesFromBrowser } : {},
+        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {}
+      })
+    );
+  }
   let stt;
   if (parsed.diarize) {
     const aaiKey = process.env.ASSEMBLYAI_API_KEY;
@@ -1627,7 +1936,7 @@ ${plan}`);
       stt = new ChunkedStt({ base: new OpenAiWhisperStt({ apiKey: openaiKey }) });
     else
       process.stderr.write(
-        "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n"
+        parsed.noCaptions ? "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n" : "corpus: OPENAI_API_KEY not set \u2014 caption-less videos will be skipped (captioned videos still import).\n"
       );
   }
   if (stt)
@@ -1645,7 +1954,7 @@ ${plan}`);
       const client = await connectBrowserMcp({ endpoint: parsed.scrapeMcp });
       chain.push(new ScrapeMcpFetcher({ client }));
     } catch (e) {
-      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg3(e)}`, 1);
     }
   }
   if (parsed.browserMcp) {
@@ -1654,7 +1963,7 @@ ${plan}`);
       browser = await connectBrowserMcp({ endpoint: parsed.browserMcp });
       chain.push(new BrowserMcpFetcher({ browser }));
     } catch (e) {
-      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg3(e)}`, 1);
     }
   }
   chain.push(new HttpReadabilityFetcher());
@@ -1685,7 +1994,7 @@ ${plan}`);
       }
     });
   } catch (e) {
-    return fail(`import failed: ${msg2(e)}`, 1);
+    return fail(`import failed: ${msg3(e)}`, 1);
   }
   process.stdout.write(
     `import-web \u2192 ${target}
@@ -1705,10 +2014,10 @@ knowledge engine (RAG) via the WriterPort. Import only stages the sources.
   );
   return 0;
 }
-function msg2(e) {
+function msg3(e) {
   return e instanceof Error ? e.message : String(e);
 }
-function parse2(args) {
+function parse3(args) {
   const out = {
     topic: void 0,
     outputPath: void 0,
@@ -1716,7 +2025,8 @@ function parse2(args) {
     channels: /* @__PURE__ */ new Set(["web"]),
     lang: void 0,
     tags: [],
-    doImport: false
+    doImport: false,
+    fresh: false
   };
   for (let i = 0; i < args.length; i++) {
     const a = args[i];
@@ -1743,6 +2053,9 @@ function parse2(args) {
       case "--import":
         out.doImport = true;
         break;
+      case "--fresh":
+        out.fresh = true;
+        break;
       default:
         if (!a.startsWith("-")) {
           if (out.topic === void 0) out.topic = a;
@@ -1870,10 +2183,18 @@ async function discoverSocial(_topic, _lang, _max) {
   process.stdout.write("  social channel needs a Bureau session \u2014 skipped.\n");
   return [];
 }
+async function readExistingUrls(outFile) {
+  try {
+    const prior = await readFile(outFile, "utf-8");
+    return prior.split("\n").map((l) => l.trim()).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
 async function runDiscover(args) {
-  const parsed = parse2(args);
+  const parsed = parse3(args);
   if (!parsed.topic) {
-    return fail("discover requires a <topic> argument. Usage: corpus discover <topic> [path] [--max N] [--channels web,youtube,social] [--lang fr] [--tags t] [--import]", 2);
+    return fail("discover requires a <topic> argument. Usage: corpus discover <topic> [path] [--max N] [--channels web,youtube,social] [--lang fr] [--tags t] [--import] [--fresh]", 2);
   }
   const outputPath = parsed.outputPath ? resolveWorkspacePath(parsed.outputPath) : process.cwd();
   process.stdout.write(`discover "${parsed.topic}"
@@ -1916,10 +2237,22 @@ async function runDiscover(args) {
   }
   await mkdir(outputPath, { recursive: true });
   const outFile = join(outputPath, "urls.discovered.txt");
-  await writeFile(outFile, allUrls.join("\n") + "\n", "utf-8");
-  process.stdout.write(`
-  written \u2192 ${outFile}
-`);
+  const existing = parsed.fresh ? [] : await readExistingUrls(outFile);
+  const merged = [...existing];
+  const known = new Set(existing);
+  for (const url of allUrls) {
+    if (!known.has(url)) {
+      known.add(url);
+      merged.push(url);
+    }
+  }
+  const added = merged.length - existing.length;
+  await writeFile(outFile, merged.join("\n") + "\n", "utf-8");
+  process.stdout.write(
+    `
+  written \u2192 ${outFile} (${merged.length} URLs` + (parsed.fresh ? ", --fresh overwrite" : `, ${added} new, ${existing.length} kept from previous runs`) + `)
+`
+  );
   if (parsed.doImport) {
     process.stdout.write("\n  --import: chaining import-web\u2026\n");
     const importArgs = [
@@ -2290,7 +2623,7 @@ var DISTILLER_ENGINES = {
     ])
   )
 };
-function parse3(args) {
+function parse4(args) {
   const out = {
     workspace: void 0,
     sourceId: void 0,
@@ -2345,13 +2678,13 @@ async function readSources(root) {
   const out = [];
   for (const e of entries) {
     if (!e.isFile() || !e.name.endsWith(".md")) continue;
-    const path5 = join(e.parentPath, e.name);
+    const path6 = join(e.parentPath, e.name);
     try {
-      const parsed = matter2(await readFile(path5, "utf-8"));
+      const parsed = matter2(await readFile(path6, "utf-8"));
       const fm = SOURCE_FRONTMATTER.parse(parsed.data);
       if (!fm.id || !parsed.content.trim()) continue;
       out.push({
-        path: path5,
+        path: path6,
         id: fm.id,
         title: fm.title ?? fm.id,
         body: parsed.content.trim(),
@@ -2387,7 +2720,7 @@ async function scanDistilledSourceIds(root) {
 }
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 async function runDistill(args) {
-  const parsed = parse3(args);
+  const parsed = parse4(args);
   const target = resolveWorkspacePath(parsed.workspace);
   const engine = DISTILLER_ENGINES[parsed.engine];
   if (!engine) {
@@ -2526,7 +2859,12 @@ async function runReport(args) {
 }
 function loadReportConfig(configPath) {
   const raw = readFileSync(path.resolve(process.cwd(), configPath), "utf8");
-  return reportConfigSchema.parse(JSON.parse(raw));
+  const parsed = JSON.parse(raw);
+  for (const warning of lintReportConfig(parsed)) {
+    process.stderr.write(`corpus report: ${warning}
+`);
+  }
+  return reportConfigSchema.parse(parsed);
 }
 async function runReportPacks(args) {
   let dataset;
@@ -2786,10 +3124,10 @@ var SseMcpClient = class {
     try {
       const parsed = RPC_MESSAGE.safeParse(JSON.parse(payload));
       if (!parsed.success) return;
-      const msg4 = parsed.data;
-      if (typeof msg4.id === "number" && this.pending.has(msg4.id)) {
-        this.pending.get(msg4.id)(msg4);
-        this.pending.delete(msg4.id);
+      const msg5 = parsed.data;
+      if (typeof msg5.id === "number" && this.pending.has(msg5.id)) {
+        this.pending.get(msg5.id)(msg5);
+        this.pending.delete(msg5.id);
       }
     } catch {
     }
@@ -2858,7 +3196,7 @@ var McpSink = class {
     try {
       client = await this.ensure();
     } catch (e) {
-      return { uri: item.uri, ok: false, error: `connect: ${msg3(e)}` };
+      return { uri: item.uri, ok: false, error: `connect: ${msg4(e)}` };
     }
     const args = template(this.config.args, item);
     const call = this.config.importedAlias ? { name: "mcp_imported_call", args: { alias: this.config.importedAlias, toolName: this.config.tool, args } } : { name: this.config.tool, args };
@@ -2867,7 +3205,7 @@ var McpSink = class {
       if (res.isError) return { uri: item.uri, ok: false, error: "tool returned isError" };
       return { uri: item.uri, ok: true };
     } catch (e) {
-      return { uri: item.uri, ok: false, error: msg3(e) };
+      return { uri: item.uri, ok: false, error: msg4(e) };
     }
   }
 };
@@ -2911,7 +3249,7 @@ function stringify(v) {
   if (Array.isArray(v)) return v.join(", ");
   return v == null ? "" : String(v);
 }
-function msg3(e) {
+function msg4(e) {
   return e instanceof Error ? e.message : String(e);
 }
 
@@ -3007,12 +3345,20 @@ Commands:
                                          (--preset seeds a full vertical; --list shows them)
   validate [path]                        JSON Schema check across every AIP file
   lint [path]                            Run lints declared in KNOWLEDGE.md
+  verify [path] --facets a,b,... [--thin 8] [--apply] [--contaminated]
+                                         SOP \u2464b: coverage per facet tag +
+                                         quarantine of self-flagged bad scrapes.
+                                         --apply moves flagged entries to
+                                         demoted/ (reversible); --contaminated
+                                         (with --apply) also demotes siblings
+                                         sharing a poisoned source.
   events:emit <kind> --payload <json> [path]
                                          Append an event to _log.md
   events:tail [path]                     Print _log.md
-  discover <topic> [path] [--max N --channels web,youtube,social --lang l --tags t --import]
+  discover <topic> [path] [--max N --channels web,youtube,social --lang l --tags t --import --fresh]
                                          Fan out across channels (web search + YouTube +
-                                         social), dedup, write urls.discovered.txt.
+                                         social), dedup, merge into urls.discovered.txt
+                                         (union with previous runs; --fresh overwrites).
                                          Web: auto-picks first available key (SERPER /
                                          EXA / TAVILY / GOOGLE_SEARCH). YouTube: yt-dlp
                                          flat-playlist. --import chains import-web.
@@ -3074,6 +3420,8 @@ async function main(argv2) {
       return await runValidate(rest);
     case "lint":
       return await runLint(rest);
+    case "verify":
+      return await runVerify(rest);
     case "events:emit":
       return await runEventsEmit(rest);
     case "events:tail":