@agentproto/corpus-cli 0.1.1 → 0.3.0

package/dist/cli.mjs

@@ -404,8 +404,8 @@ function resolveWorkspacePath(arg) {
   if (!arg) return process.cwd();
   return path.resolve(process.cwd(), arg);
 }
-function fail(msg4, code = 1) {
-  process.stderr.write(`corpus: ${msg4}
+function fail(msg5, code = 1) {
+  process.stderr.write(`corpus: ${msg5}
 `);
   return code;
 }
@@ -926,7 +926,8 @@ var YtDlpWhisperFetcher = class {
     this.download = opts.download ?? defaultYtDlpDownloader(opts.ytDlpBin ?? "yt-dlp", {
       maxDurationSec: opts.maxDurationSec,
       cookiesFromBrowser: opts.cookiesFromBrowser,
-      cookiesFile: opts.cookiesFile
+      cookiesFile: opts.cookiesFile,
+      ffmpegLocation: opts.ffmpegLocation
     });
     this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
   }
@@ -970,32 +971,44 @@ function isAuthError(e) {
   const m = msg(e).toLowerCase();
   return m.includes(" 401") || m.includes(" 403") || m.includes("unauthorized") || m.includes("forbidden") || m.includes("api key") || m.includes("api_key");
 }
+function buildYtDlpArgs(url, dir, opts) {
+  const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+  const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+  const ffmpegArgs = opts?.ffmpegLocation ? ["--ffmpeg-location", opts.ffmpegLocation] : [];
+  const client = cookieArgs.length === 0 ? "android" : "web_creator";
+  const extractorArgs = ["--extractor-args", `youtube:player_client=${client}`];
+  return [
+    "-f",
+    "bestaudio/best",
+    // /best fallback: many videos lack a standalone bestaudio
+    "-x",
+    "--audio-format",
+    "mp3",
+    "--audio-quality",
+    "64K",
+    "--no-playlist",
+    "--no-progress",
+    "--remote-components",
+    "ejs:github",
+    // solves YouTube nsig "n challenge"
+    ...extractorArgs,
+    ...durationGuard,
+    ...cookieArgs,
+    ...ffmpegArgs,
+    "--write-info-json",
+    "-o",
+    join(dir, "track.%(ext)s"),
+    url
+  ];
+}
 function defaultYtDlpDownloader(bin, opts) {
   return async (url) => {
     const dir = await mkdtemp(join(tmpdir(), "corpus-ytdlp-"));
     const cleanup = () => rm(dir, { recursive: true, force: true });
     try {
-      const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
-      const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
       await execFileAsync(
         bin,
-        [
-          "-f",
-          "bestaudio",
-          "-x",
-          "--audio-format",
-          "mp3",
-          "--audio-quality",
-          "64K",
-          "--no-playlist",
-          "--no-progress",
-          ...durationGuard,
-          ...cookieArgs,
-          "--write-info-json",
-          "-o",
-          join(dir, "track.%(ext)s"),
-          url
-        ],
+        buildYtDlpArgs(url, dir, opts),
         { maxBuffer: 16 * 1024 * 1024 }
       );
       const files = await readdir(dir);
@@ -1027,6 +1040,146 @@ function defaultYtDlpDownloader(bin, opts) {
     }
   };
 }
+var execFileAsync2 = promisify(execFile);
+var YTDLP_INFO2 = z.object({ title: z.string().optional(), language: z.string().optional() }).loose();
+var YtDlpCaptionsFetcher = class {
+  download;
+  extraHosts;
+  minChars;
+  constructor(opts = {}) {
+    this.minChars = opts.minChars ?? 400;
+    this.download = opts.download ?? defaultYtDlpCaptionDownloader(opts.ytDlpBin ?? "yt-dlp", {
+      ...opts.subLangs ? { subLangs: opts.subLangs } : {},
+      ...opts.preferLang ? { preferLang: opts.preferLang } : {},
+      ...opts.maxDurationSec ? { maxDurationSec: opts.maxDurationSec } : {},
+      ...opts.cookiesFromBrowser ? { cookiesFromBrowser: opts.cookiesFromBrowser } : {},
+      ...opts.cookiesFile ? { cookiesFile: opts.cookiesFile } : {}
+    });
+    this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
+  }
+  async fetch(url) {
+    if (!isVideoUrl(url, this.extraHosts)) return null;
+    let dl;
+    try {
+      dl = await this.download(url);
+    } catch (e) {
+      process.stderr.write(
+        `corpus: caption fetch failed for ${url} \u2014 falling back (${msg2(e)})
+`
+      );
+      return null;
+    }
+    try {
+      if (!dl.vttPath) return null;
+      const text = parseVttToText(await readFile(dl.vttPath, "utf-8"));
+      if (text.length < this.minChars) return null;
+      return {
+        title: dl.title || url,
+        text,
+        kind: "video",
+        ...dl.language ? { language: dl.language } : {},
+        via: "captions"
+      };
+    } catch {
+      return null;
+    } finally {
+      await dl.cleanup().catch(() => {
+      });
+    }
+  }
+};
+function msg2(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+function parseVttToText(vtt) {
+  const out = [];
+  for (const raw of vtt.split(/\r?\n/)) {
+    if (/^WEBVTT/.test(raw)) continue;
+    if (/^(Kind|Language|NOTE|STYLE):/i.test(raw)) continue;
+    if (raw.includes("-->")) continue;
+    const line = raw.replace(/<[^>]+>/g, "").replace(/&nbsp;/gi, " ").replace(/ /g, " ").trim();
+    if (!line) continue;
+    if (out.length && out[out.length - 1] === line) continue;
+    out.push(line);
+  }
+  const cleaned = [];
+  for (let i = 0; i < out.length; i++) {
+    const next = out[i + 1];
+    if (next && next.startsWith(out[i])) continue;
+    cleaned.push(out[i]);
+  }
+  return cleaned.join(" ").replace(/\s+/g, " ").trim();
+}
+function buildYtDlpCaptionArgs(url, dir, opts) {
+  const subLangs = opts?.subLangs ?? (opts?.preferLang ? `${opts.preferLang}.*` : "en.*,fr.*,es.*,de.*,it.*,pt.*,nl.*");
+  const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+  const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+  return [
+    "--skip-download",
+    "--write-auto-subs",
+    "--write-subs",
+    "--sub-langs",
+    subLangs,
+    "--sub-format",
+    "vtt",
+    "--no-playlist",
+    "--no-progress",
+    "--sleep-requests",
+    "1",
+    ...durationGuard,
+    ...cookieArgs,
+    "--write-info-json",
+    "-o",
+    join(dir, "%(id)s.%(ext)s"),
+    url
+  ];
+}
+function pickVtt(files, preferLang) {
+  const vtts = files.filter((f) => f.endsWith(".vtt"));
+  if (vtts.length === 0) return void 0;
+  const bySuffix = (suffix) => vtts.find((f) => f.toLowerCase().endsWith(suffix.toLowerCase()));
+  if (preferLang) {
+    return bySuffix(`.${preferLang}-orig.vtt`) ?? bySuffix(`.${preferLang}.vtt`) ?? vtts.find((f) => new RegExp(`\\.${preferLang}[.-]`, "i").test(f)) ?? vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+  }
+  return vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+}
+function defaultYtDlpCaptionDownloader(bin, opts) {
+  return async (url) => {
+    const dir = await mkdtemp(join(tmpdir(), "corpus-captions-"));
+    const cleanup = () => rm(dir, { recursive: true, force: true });
+    try {
+      await execFileAsync2(bin, buildYtDlpCaptionArgs(url, dir, opts), {
+        maxBuffer: 16 * 1024 * 1024
+      }).catch(() => {
+      });
+      const files = await readdir(dir);
+      const vtt = pickVtt(files, opts?.preferLang);
+      let title = url;
+      let language;
+      const infoName = files.find((f) => f.endsWith(".info.json"));
+      if (infoName) {
+        try {
+          const info = YTDLP_INFO2.parse(
+            JSON.parse(await readFile(join(dir, infoName), "utf-8"))
+          );
+          if (info.title) title = info.title;
+          if (info.language) language = info.language;
+        } catch {
+        }
+      }
+      return {
+        vttPath: vtt ? join(dir, vtt) : null,
+        title,
+        ...language ? { language } : {},
+        cleanup
+      };
+    } catch (e) {
+      await cleanup().catch(() => {
+      });
+      throw e;
+    }
+  };
+}
 var WHISPER_RESPONSE = z.object({ text: z.string().optional(), language: z.string().optional() }).loose();
 var WHISPER_MAX_BYTES = 25 * 1024 * 1024;
 var OpenAiWhisperStt = class {
@@ -1165,7 +1318,7 @@ function formatDiarized(job) {
   }
   return (job.text ?? "").trim();
 }
-var execFileAsync2 = promisify(execFile);
+var execFileAsync3 = promisify(execFile);
 var DEFAULT_MAX_BYTES = 24 * 1024 * 1024;
 var DEFAULT_SEGMENT_SECONDS = 1200;
 var ChunkedStt = class {
@@ -1224,7 +1377,7 @@ var ChunkedStt = class {
 function defaultFfmpegSplitter(bin) {
   return async (audioPath, segmentSeconds, outDir) => {
     const ext = extname(audioPath) || ".mp3";
-    await execFileAsync2(
+    await execFileAsync3(
       bin,
       [
         "-hide_banner",
@@ -1451,9 +1604,11 @@ function parse(args) {
     maxDurationSec: void 0,
     cookiesFromBrowser: void 0,
     cookiesFile: void 0,
+    ffmpegLocation: void 0,
     throttleMs: 2e3,
     force: false,
     diarize: false,
+    noCaptions: false,
     browserMcp: process.env.BROWSER_MCP_URL,
     scrapeMcp: process.env.SCRAPE_MCP_URL,
     importerId: "web",
@@ -1495,6 +1650,9 @@ function parse(args) {
       case "--cookies":
         out.cookiesFile = next();
         break;
+      case "--ffmpeg-location":
+        out.ffmpegLocation = next();
+        break;
       case "--throttle": {
         const v = next();
         if (v) out.throttleMs = Number(v);
@@ -1506,6 +1664,9 @@ function parse(args) {
       case "--diarize":
         out.diarize = true;
         break;
+      case "--no-captions":
+        out.noCaptions = true;
+        break;
       case "--browser-mcp":
         out.browserMcp = next();
         break;
@@ -1572,10 +1733,11 @@ async function runImportWeb(args) {
   const todo = urls.filter((u) => !done.has(u));
   const batch = parsed.max !== void 0 ? todo.slice(0, parsed.max) : todo;
   const remaining = todo.length - batch.length;
+  const videoLabel = parsed.noCaptions ? parsed.diarize ? "AssemblyAI" : "Whisper" : `captions-first \u2192 ${parsed.diarize ? "AssemblyAI" : "Whisper"} fallback`;
   const plan = `  workspace: ${target}
   urls:      ${urls.length} total \xB7 ${done.size && !parsed.force ? `${urls.length - todo.length} already ingested \xB7 ` : ""}${todo.length} to do
   this run:  ${batch.length}${parsed.max !== void 0 ? ` (--max ${parsed.max})` : ""} \xB7 ${remaining} remaining after
-  video stt: ${parsed.diarize ? "AssemblyAI (speaker-labelled)" : "Whisper"}
+  video:     ${videoLabel}
   throttle:  ${parsed.throttleMs} ms between fetches
 `;
   if (batch.length === 0) {
@@ -1596,6 +1758,16 @@ ${plan}`);
   process.stdout.write(`import-web
 ${plan}`);
   const chain = [];
+  if (!parsed.noCaptions) {
+    chain.push(
+      new YtDlpCaptionsFetcher({
+        ...parsed.lang ? { preferLang: parsed.lang } : {},
+        ...parsed.maxDurationSec ? { maxDurationSec: parsed.maxDurationSec } : {},
+        ...parsed.cookiesFromBrowser ? { cookiesFromBrowser: parsed.cookiesFromBrowser } : {},
+        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {}
+      })
+    );
+  }
   let stt;
   if (parsed.diarize) {
     const aaiKey = process.env.ASSEMBLYAI_API_KEY;
@@ -1610,7 +1782,7 @@ ${plan}`);
       stt = new ChunkedStt({ base: new OpenAiWhisperStt({ apiKey: openaiKey }) });
     else
       process.stderr.write(
-        "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n"
+        parsed.noCaptions ? "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n" : "corpus: OPENAI_API_KEY not set \u2014 caption-less videos will be skipped (captioned videos still import).\n"
       );
   }
   if (stt)
@@ -1619,7 +1791,8 @@ ${plan}`);
         stt,
         ...parsed.maxDurationSec ? { maxDurationSec: parsed.maxDurationSec } : {},
         ...parsed.cookiesFromBrowser ? { cookiesFromBrowser: parsed.cookiesFromBrowser } : {},
-        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {}
+        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {},
+        ...parsed.ffmpegLocation ? { ffmpegLocation: parsed.ffmpegLocation } : {}
       })
     );
   if (parsed.scrapeMcp) {
@@ -1627,7 +1800,7 @@ ${plan}`);
       const client = await connectBrowserMcp({ endpoint: parsed.scrapeMcp });
       chain.push(new ScrapeMcpFetcher({ client }));
     } catch (e) {
-      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg3(e)}`, 1);
     }
   }
   if (parsed.browserMcp) {
@@ -1636,7 +1809,7 @@ ${plan}`);
       browser = await connectBrowserMcp({ endpoint: parsed.browserMcp });
       chain.push(new BrowserMcpFetcher({ browser }));
     } catch (e) {
-      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg3(e)}`, 1);
     }
   }
   chain.push(new HttpReadabilityFetcher());
@@ -1667,7 +1840,7 @@ ${plan}`);
       }
     });
   } catch (e) {
-    return fail(`import failed: ${msg2(e)}`, 1);
+    return fail(`import failed: ${msg3(e)}`, 1);
   }
   process.stdout.write(
     `import-web \u2192 ${target}
@@ -1687,9 +1860,234 @@ knowledge engine (RAG) via the WriterPort. Import only stages the sources.
   );
   return 0;
 }
-function msg2(e) {
+function msg3(e) {
   return e instanceof Error ? e.message : String(e);
 }
+function parse2(args) {
+  const out = {
+    topic: void 0,
+    outputPath: void 0,
+    max: 20,
+    channels: /* @__PURE__ */ new Set(["web"]),
+    lang: void 0,
+    tags: [],
+    doImport: false
+  };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    const next = () => args[++i];
+    switch (a) {
+      case "--max": {
+        const v = next();
+        if (v) out.max = Number(v);
+        break;
+      }
+      case "--channels": {
+        const v = next();
+        if (v) out.channels = new Set(v.split(",").map((s) => s.trim()).filter(Boolean));
+        break;
+      }
+      case "--lang":
+        out.lang = next();
+        break;
+      case "--tags": {
+        const v = next();
+        if (v) out.tags.push(...v.split(",").map((s) => s.trim()).filter(Boolean));
+        break;
+      }
+      case "--import":
+        out.doImport = true;
+        break;
+      default:
+        if (!a.startsWith("-")) {
+          if (out.topic === void 0) out.topic = a;
+          else if (out.outputPath === void 0) out.outputPath = a;
+        }
+    }
+  }
+  return out;
+}
+async function searchSerper(query, lang, max) {
+  const apiKey = process.env.SERPER_API_KEY;
+  const gl = lang === "fr" ? "fr" : lang ?? "us";
+  const hl = lang ?? "en";
+  const body = JSON.stringify({ q: query, gl, hl, num: max });
+  const res = await fetch("https://google.serper.dev/search", {
+    method: "POST",
+    headers: { "X-API-KEY": apiKey, "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Serper ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.organic ?? []).flatMap((r) => r.link ? [{ url: r.link, title: r.title }] : []);
+}
+async function searchExa(query, _lang, max) {
+  const apiKey = process.env.EXA_API_KEY;
+  const body = JSON.stringify({ query, numResults: max, type: "neural", contents: { text: false } });
+  const res = await fetch("https://api.exa.ai/search", {
+    method: "POST",
+    headers: { "x-api-key": apiKey, "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Exa ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.results ?? []).flatMap((r) => r.url ? [{ url: r.url, title: r.title }] : []);
+}
+async function searchTavily(query, _lang, max) {
+  const apiKey = process.env.TAVILY_API_KEY;
+  const body = JSON.stringify({ api_key: apiKey, query, max_results: max });
+  const res = await fetch("https://api.tavily.com/search", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Tavily ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.results ?? []).flatMap((r) => r.url ? [{ url: r.url, title: r.title }] : []);
+}
+async function searchGoogle(query, lang, max) {
+  const apiKey = process.env.GOOGLE_SEARCH_API_KEY;
+  const cx = process.env.GOOGLE_SEARCH_CX ?? "";
+  const params = new URLSearchParams({
+    key: apiKey,
+    q: query,
+    num: String(Math.min(max, 10)),
+    ...cx ? { cx } : {},
+    ...lang ? { lr: `lang_${lang}`, hl: lang } : {}
+  });
+  const res = await fetch(`https://www.googleapis.com/customsearch/v1?${params}`);
+  if (!res.ok) throw new Error(`Google ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.items ?? []).flatMap((r) => r.link ? [{ url: r.link, title: r.title }] : []);
+}
+function resolveWebProvider() {
+  if (process.env.SERPER_API_KEY) return { name: "serper", fn: searchSerper };
+  if (process.env.EXA_API_KEY) return { name: "exa", fn: searchExa };
+  if (process.env.TAVILY_API_KEY) return { name: "tavily", fn: searchTavily };
+  if (process.env.GOOGLE_SEARCH_API_KEY) return { name: "google", fn: searchGoogle };
+  return null;
+}
+function buildWebQueries(topic, lang) {
+  const langSuffix = lang && lang !== "en" ? ` ${lang}` : "";
+  const q1 = topic;
+  const q2 = `${topic} guide tutorial${langSuffix}`;
+  const q3 = lang && lang !== "en" ? `${topic} ressources ${lang}` : `${topic} best practices resources`;
+  return [q1, q2, q3];
+}
+async function discoverWeb(topic, lang, max) {
+  const provider = resolveWebProvider();
+  if (!provider) {
+    process.stderr.write("corpus discover: no web search API key found (SERPER_API_KEY / EXA_API_KEY / TAVILY_API_KEY / GOOGLE_SEARCH_API_KEY) \u2014 web channel skipped.\n");
+    return { urls: [], provider: "none" };
+  }
+  const queries = buildWebQueries(topic, lang);
+  const seen = /* @__PURE__ */ new Set();
+  const urls = [];
+  for (const q of queries) {
+    if (urls.length >= max) break;
+    try {
+      const results = await provider.fn(q, lang, max);
+      for (const r of results) {
+        if (urls.length >= max) break;
+        if (!seen.has(r.url)) {
+          seen.add(r.url);
+          urls.push(r.url);
+        }
+      }
+    } catch (e) {
+      process.stderr.write(`corpus discover: web search query "${q}" failed \u2014 ${e instanceof Error ? e.message : String(e)}
+`);
+    }
+  }
+  return { urls, provider: provider.name };
+}
+async function discoverYoutube(topic, max) {
+  return new Promise((resolve2) => {
+    execFile(
+      "yt-dlp",
+      [`ytsearch${max}:${topic}`, "--flat-playlist", "--print", "%(url)s", "--no-warnings"],
+      { timeout: 6e4 },
+      (err, stdout, _stderr) => {
+        if (err) {
+          process.stderr.write(`corpus discover: youtube channel failed \u2014 ${err.message}
+`);
+          resolve2([]);
+          return;
+        }
+        resolve2(
+          stdout.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("http"))
+        );
+      }
+    );
+  });
+}
+async function discoverSocial(_topic, _lang, _max) {
+  process.stdout.write("  social channel needs a Bureau session \u2014 skipped.\n");
+  return [];
+}
+async function runDiscover(args) {
+  const parsed = parse2(args);
+  if (!parsed.topic) {
+    return fail("discover requires a <topic> argument. Usage: corpus discover <topic> [path] [--max N] [--channels web,youtube,social] [--lang fr] [--tags t] [--import]", 2);
+  }
+  const outputPath = parsed.outputPath ? resolveWorkspacePath(parsed.outputPath) : process.cwd();
+  process.stdout.write(`discover "${parsed.topic}"
+`);
+  process.stdout.write(`  channels: ${[...parsed.channels].join(", ")} \xB7 max: ${parsed.max}${parsed.lang ? ` \xB7 lang: ${parsed.lang}` : ""}
+`);
+  const webUrls = [];
+  const ytUrls = [];
+  const socialUrls = [];
+  if (parsed.channels.has("web")) {
+    const { urls, provider } = await discoverWeb(parsed.topic, parsed.lang, parsed.max);
+    webUrls.push(...urls);
+    if (urls.length > 0) process.stdout.write(`  web (${provider}): ${urls.length} URLs
+`);
+  }
+  if (parsed.channels.has("youtube")) {
+    const urls = await discoverYoutube(parsed.topic, parsed.max);
+    ytUrls.push(...urls);
+    if (urls.length > 0) process.stdout.write(`  youtube: ${urls.length} URLs
+`);
+  }
+  if (parsed.channels.has("social")) {
+    const urls = await discoverSocial();
+    socialUrls.push(...urls);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const allUrls = [];
+  for (const url of [...webUrls, ...ytUrls, ...socialUrls]) {
+    if (!seen.has(url)) {
+      seen.add(url);
+      allUrls.push(url);
+    }
+  }
+  process.stdout.write(`
+  web: ${webUrls.length} \xB7 youtube: ${ytUrls.length} \xB7 social: ${socialUrls.length} \xB7 total: ${allUrls.length} unique
+`);
+  if (allUrls.length === 0) {
+    process.stderr.write("corpus discover: no URLs found across any channel.\n");
+    return 1;
+  }
+  await mkdir(outputPath, { recursive: true });
+  const outFile = join(outputPath, "urls.discovered.txt");
+  await writeFile(outFile, allUrls.join("\n") + "\n", "utf-8");
+  process.stdout.write(`
+  written \u2192 ${outFile}
+`);
+  if (parsed.doImport) {
+    process.stdout.write("\n  --import: chaining import-web\u2026\n");
+    const importArgs = [
+      outputPath,
+      "--urls-file",
+      outFile,
+      ...parsed.tags.flatMap((t) => ["--tags", t]),
+      ...parsed.lang ? ["--lang", parsed.lang] : []
+    ];
+    return runImportWeb(importArgs);
+  }
+  return 0;
+}
 
 // src/ports/anthropic-distiller.ts
 var ANTHROPIC_RESPONSE = z.object({
@@ -1703,15 +2101,17 @@ var AnthropicDistiller = class {
   baseUrl;
   maxItems;
   onUsage;
+  lang;
   constructor(opts) {
     this.apiKey = opts.apiKey;
     this.model = opts.model ?? "claude-sonnet-4-6";
     this.baseUrl = (opts.baseUrl ?? "https://api.anthropic.com/v1").replace(/\/+$/, "");
     this.maxItems = opts.maxItems ?? 8;
     this.onUsage = opts.onUsage;
+    this.lang = opts.lang;
   }
   async distill(input) {
-    const prompt = buildDistillPrompt(input, this.maxItems);
+    const prompt = buildDistillPrompt(input, this.maxItems, { lang: this.lang });
     const startedAt = (/* @__PURE__ */ new Date()).toISOString();
     const res = await fetch(`${this.baseUrl}/messages`, {
       method: "POST",
@@ -1750,18 +2150,20 @@ var CliAgentDistiller = class {
   maxItems;
   timeoutMs;
   cwd;
+  lang;
   constructor(opts) {
     this.engine = opts.engine;
     this.model = opts.model;
     this.maxItems = opts.maxItems ?? 8;
     this.timeoutMs = opts.timeoutMs ?? 12e4;
     this.cwd = opts.cwd ?? tmpdir();
+    this.lang = opts.lang;
   }
   async distill(input) {
-    const prompt = buildDistillPrompt(input, this.maxItems);
+    const prompt = buildDistillPrompt(input, this.maxItems, { lang: this.lang });
     const stdout = await spawnWithStdin({
       command: this.engine.command,
-      args: this.engine.buildArgs({ ...this.model ? { model: this.model } : {} }),
+      args: this.engine.buildArgs({ ...this.model ? { model: this.model } : {}, prompt }),
       stdin: prompt,
       cwd: this.cwd,
       timeoutMs: this.timeoutMs
@@ -1836,11 +2238,23 @@ var OPENCODE = {
   buildArgs: ({ model }) => ["run", ...model ? ["-m", model] : []],
   parseOutput: plainTextOutput
 };
+var HERMES = {
+  id: "hermes",
+  subscriptionBilled: true,
+  command: "hermes",
+  buildArgs: ({ model, prompt }) => [
+    "--oneshot",
+    prompt ?? "",
+    ...model ? ["-m", model] : []
+  ],
+  parseOutput: plainTextOutput
+};
 var CLI_ENGINES = {
   [CLAUDE_CODE.id]: CLAUDE_CODE,
   [GEMINI.id]: GEMINI,
   [CODEX.id]: CODEX,
-  [OPENCODE.id]: OPENCODE
+  [OPENCODE.id]: OPENCODE,
+  [HERMES.id]: HERMES
 };
 var PRICING = [
   { match: "opus", inPerM: 15, outPerM: 75 },
@@ -2009,10 +2423,11 @@ var DISTILLER_ENGINES = {
   [DEFAULT_ENGINE]: {
     id: DEFAULT_ENGINE,
     needsApiKey: true,
-    create: ({ apiKey, model, onUsage }) => new AnthropicDistiller({
+    create: ({ apiKey, model, onUsage, lang }) => new AnthropicDistiller({
       apiKey,
       ...model ? { model } : {},
-      ...onUsage ? { onUsage } : {}
+      ...onUsage ? { onUsage } : {},
+      ...lang ? { lang } : {}
     })
   },
   ...Object.fromEntries(
@@ -2021,19 +2436,24 @@ var DISTILLER_ENGINES = {
       {
         id: engine.id,
         needsApiKey: false,
-        create: ({ model }) => new CliAgentDistiller({ engine, ...model ? { model } : {} })
+        create: ({ model, lang }) => new CliAgentDistiller({
+          engine,
+          ...model ? { model } : {},
+          ...lang ? { lang } : {}
+        })
       }
     ])
   )
 };
-function parse2(args) {
+function parse3(args) {
   const out = {
     workspace: void 0,
     sourceId: void 0,
     max: void 0,
     throttleMs: 1e3,
     model: void 0,
-    engine: DEFAULT_ENGINE
+    engine: DEFAULT_ENGINE,
+    lang: void 0
   };
   for (let i = 0; i < args.length; i++) {
     const a = args[i];
@@ -2060,6 +2480,9 @@ function parse2(args) {
         if (v) out.engine = v;
         break;
       }
+      case "--lang":
+        out.lang = next();
+        break;
       default:
         if (!a.startsWith("-") && out.workspace === void 0) out.workspace = a;
     }
@@ -2119,7 +2542,7 @@ async function scanDistilledSourceIds(root) {
 }
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 async function runDistill(args) {
-  const parsed = parse2(args);
+  const parsed = parse3(args);
   const target = resolveWorkspacePath(parsed.workspace);
   const engine = DISTILLER_ENGINES[parsed.engine];
   if (!engine) {
@@ -2157,7 +2580,8 @@ async function runDistill(args) {
     distiller: engine.create({
       ...apiKey ? { apiKey } : {},
       ...parsed.model ? { model: parsed.model } : {},
-      onUsage: usage.record
+      onUsage: usage.record,
+      ...parsed.lang ? { lang: parsed.lang } : {}
     })
   });
   let totalEntries = 0;
@@ -2517,10 +2941,10 @@ var SseMcpClient = class {
     try {
       const parsed = RPC_MESSAGE.safeParse(JSON.parse(payload));
       if (!parsed.success) return;
-      const msg4 = parsed.data;
-      if (typeof msg4.id === "number" && this.pending.has(msg4.id)) {
-        this.pending.get(msg4.id)(msg4);
-        this.pending.delete(msg4.id);
+      const msg5 = parsed.data;
+      if (typeof msg5.id === "number" && this.pending.has(msg5.id)) {
+        this.pending.get(msg5.id)(msg5);
+        this.pending.delete(msg5.id);
       }
     } catch {
     }
@@ -2589,7 +3013,7 @@ var McpSink = class {
     try {
       client = await this.ensure();
     } catch (e) {
-      return { uri: item.uri, ok: false, error: `connect: ${msg3(e)}` };
+      return { uri: item.uri, ok: false, error: `connect: ${msg4(e)}` };
     }
     const args = template(this.config.args, item);
     const call = this.config.importedAlias ? { name: "mcp_imported_call", args: { alias: this.config.importedAlias, toolName: this.config.tool, args } } : { name: this.config.tool, args };
@@ -2598,7 +3022,7 @@ var McpSink = class {
       if (res.isError) return { uri: item.uri, ok: false, error: "tool returned isError" };
       return { uri: item.uri, ok: true };
     } catch (e) {
-      return { uri: item.uri, ok: false, error: msg3(e) };
+      return { uri: item.uri, ok: false, error: msg4(e) };
     }
   }
 };
@@ -2642,7 +3066,7 @@ function stringify(v) {
   if (Array.isArray(v)) return v.join(", ");
   return v == null ? "" : String(v);
 }
-function msg3(e) {
+function msg4(e) {
   return e instanceof Error ? e.message : String(e);
 }
 
@@ -2741,6 +3165,12 @@ Commands:
   events:emit <kind> --payload <json> [path]
                                          Append an event to _log.md
   events:tail [path]                     Print _log.md
+  discover <topic> [path] [--max N --channels web,youtube,social --lang l --tags t --import]
+                                         Fan out across channels (web search + YouTube +
+                                         social), dedup, write urls.discovered.txt.
+                                         Web: auto-picks first available key (SERPER /
+                                         EXA / TAVILY / GOOGLE_SEARCH). YouTube: yt-dlp
+                                         flat-playlist. --import chains import-web.
   import-web [path] --urls-file <f> [--max n --max-duration s --throttle ms --tags t --lang l --force --diarize]
                                          Import URLs as sources (video\u2192Whisper, article\u2192
                                          readability). Resumable: skips already-ingested
@@ -2755,7 +3185,7 @@ Commands:
                                          scrape MCP server (stealth + clean Markdown) for
                                          walled/JS pages, ahead of plain readability.
                                          --diarize: AssemblyAI speaker labels (interviews).
-  distill [path] [--source id --max n --throttle ms --model m]
+  distill [path] [--source id --max n --throttle ms --model m --lang l --engine e]
                                          Distill raw sources \u2192 refined entries
                                          (principle/pattern/\u2026) with sources:[id] provenance.
                                          Resumable: skips already-distilled sources.
@@ -2805,6 +3235,8 @@ async function main(argv2) {
       return await runEventsTail(rest);
     case "import-web":
       return await runImportWeb(rest);
+    case "discover":
+      return await runDiscover(rest);
     case "distill":
       return await runDistill(rest);
     case "knowledge":