@agentproto/corpus-cli 0.1.1 → 0.2.0

package/dist/cli.mjs

@@ -926,7 +926,8 @@ var YtDlpWhisperFetcher = class {
     this.download = opts.download ?? defaultYtDlpDownloader(opts.ytDlpBin ?? "yt-dlp", {
       maxDurationSec: opts.maxDurationSec,
       cookiesFromBrowser: opts.cookiesFromBrowser,
-      cookiesFile: opts.cookiesFile
+      cookiesFile: opts.cookiesFile,
+      ffmpegLocation: opts.ffmpegLocation
     });
     this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
   }
@@ -970,32 +971,44 @@ function isAuthError(e) {
   const m = msg(e).toLowerCase();
   return m.includes(" 401") || m.includes(" 403") || m.includes("unauthorized") || m.includes("forbidden") || m.includes("api key") || m.includes("api_key");
 }
+function buildYtDlpArgs(url, dir, opts) {
+  const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+  const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+  const ffmpegArgs = opts?.ffmpegLocation ? ["--ffmpeg-location", opts.ffmpegLocation] : [];
+  const client = cookieArgs.length === 0 ? "android" : "web_creator";
+  const extractorArgs = ["--extractor-args", `youtube:player_client=${client}`];
+  return [
+    "-f",
+    "bestaudio/best",
+    // /best fallback: many videos lack a standalone bestaudio
+    "-x",
+    "--audio-format",
+    "mp3",
+    "--audio-quality",
+    "64K",
+    "--no-playlist",
+    "--no-progress",
+    "--remote-components",
+    "ejs:github",
+    // solves YouTube nsig "n challenge"
+    ...extractorArgs,
+    ...durationGuard,
+    ...cookieArgs,
+    ...ffmpegArgs,
+    "--write-info-json",
+    "-o",
+    join(dir, "track.%(ext)s"),
+    url
+  ];
+}
 function defaultYtDlpDownloader(bin, opts) {
   return async (url) => {
     const dir = await mkdtemp(join(tmpdir(), "corpus-ytdlp-"));
     const cleanup = () => rm(dir, { recursive: true, force: true });
     try {
-      const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
-      const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
       await execFileAsync(
         bin,
-        [
-          "-f",
-          "bestaudio",
-          "-x",
-          "--audio-format",
-          "mp3",
-          "--audio-quality",
-          "64K",
-          "--no-playlist",
-          "--no-progress",
-          ...durationGuard,
-          ...cookieArgs,
-          "--write-info-json",
-          "-o",
-          join(dir, "track.%(ext)s"),
-          url
-        ],
+        buildYtDlpArgs(url, dir, opts),
         { maxBuffer: 16 * 1024 * 1024 }
       );
       const files = await readdir(dir);
@@ -1451,6 +1464,7 @@ function parse(args) {
     maxDurationSec: void 0,
     cookiesFromBrowser: void 0,
     cookiesFile: void 0,
+    ffmpegLocation: void 0,
     throttleMs: 2e3,
     force: false,
     diarize: false,
@@ -1495,6 +1509,9 @@ function parse(args) {
       case "--cookies":
         out.cookiesFile = next();
         break;
+      case "--ffmpeg-location":
+        out.ffmpegLocation = next();
+        break;
       case "--throttle": {
         const v = next();
         if (v) out.throttleMs = Number(v);
@@ -1619,7 +1636,8 @@ ${plan}`);
         stt,
         ...parsed.maxDurationSec ? { maxDurationSec: parsed.maxDurationSec } : {},
         ...parsed.cookiesFromBrowser ? { cookiesFromBrowser: parsed.cookiesFromBrowser } : {},
-        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {}
+        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {},
+        ...parsed.ffmpegLocation ? { ffmpegLocation: parsed.ffmpegLocation } : {}
       })
     );
   if (parsed.scrapeMcp) {
@@ -1690,6 +1708,231 @@ knowledge engine (RAG) via the WriterPort. Import only stages the sources.
 function msg2(e) {
   return e instanceof Error ? e.message : String(e);
 }
+function parse2(args) {
+  const out = {
+    topic: void 0,
+    outputPath: void 0,
+    max: 20,
+    channels: /* @__PURE__ */ new Set(["web"]),
+    lang: void 0,
+    tags: [],
+    doImport: false
+  };
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    const next = () => args[++i];
+    switch (a) {
+      case "--max": {
+        const v = next();
+        if (v) out.max = Number(v);
+        break;
+      }
+      case "--channels": {
+        const v = next();
+        if (v) out.channels = new Set(v.split(",").map((s) => s.trim()).filter(Boolean));
+        break;
+      }
+      case "--lang":
+        out.lang = next();
+        break;
+      case "--tags": {
+        const v = next();
+        if (v) out.tags.push(...v.split(",").map((s) => s.trim()).filter(Boolean));
+        break;
+      }
+      case "--import":
+        out.doImport = true;
+        break;
+      default:
+        if (!a.startsWith("-")) {
+          if (out.topic === void 0) out.topic = a;
+          else if (out.outputPath === void 0) out.outputPath = a;
+        }
+    }
+  }
+  return out;
+}
+async function searchSerper(query, lang, max) {
+  const apiKey = process.env.SERPER_API_KEY;
+  const gl = lang === "fr" ? "fr" : lang ?? "us";
+  const hl = lang ?? "en";
+  const body = JSON.stringify({ q: query, gl, hl, num: max });
+  const res = await fetch("https://google.serper.dev/search", {
+    method: "POST",
+    headers: { "X-API-KEY": apiKey, "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Serper ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.organic ?? []).flatMap((r) => r.link ? [{ url: r.link, title: r.title }] : []);
+}
+async function searchExa(query, _lang, max) {
+  const apiKey = process.env.EXA_API_KEY;
+  const body = JSON.stringify({ query, numResults: max, type: "neural", contents: { text: false } });
+  const res = await fetch("https://api.exa.ai/search", {
+    method: "POST",
+    headers: { "x-api-key": apiKey, "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Exa ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.results ?? []).flatMap((r) => r.url ? [{ url: r.url, title: r.title }] : []);
+}
+async function searchTavily(query, _lang, max) {
+  const apiKey = process.env.TAVILY_API_KEY;
+  const body = JSON.stringify({ api_key: apiKey, query, max_results: max });
+  const res = await fetch("https://api.tavily.com/search", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body
+  });
+  if (!res.ok) throw new Error(`Tavily ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.results ?? []).flatMap((r) => r.url ? [{ url: r.url, title: r.title }] : []);
+}
+async function searchGoogle(query, lang, max) {
+  const apiKey = process.env.GOOGLE_SEARCH_API_KEY;
+  const cx = process.env.GOOGLE_SEARCH_CX ?? "";
+  const params = new URLSearchParams({
+    key: apiKey,
+    q: query,
+    num: String(Math.min(max, 10)),
+    ...cx ? { cx } : {},
+    ...lang ? { lr: `lang_${lang}`, hl: lang } : {}
+  });
+  const res = await fetch(`https://www.googleapis.com/customsearch/v1?${params}`);
+  if (!res.ok) throw new Error(`Google ${res.status}: ${await res.text()}`);
+  const json = await res.json();
+  return (json.items ?? []).flatMap((r) => r.link ? [{ url: r.link, title: r.title }] : []);
+}
+function resolveWebProvider() {
+  if (process.env.SERPER_API_KEY) return { name: "serper", fn: searchSerper };
+  if (process.env.EXA_API_KEY) return { name: "exa", fn: searchExa };
+  if (process.env.TAVILY_API_KEY) return { name: "tavily", fn: searchTavily };
+  if (process.env.GOOGLE_SEARCH_API_KEY) return { name: "google", fn: searchGoogle };
+  return null;
+}
+function buildWebQueries(topic, lang) {
+  const langSuffix = lang && lang !== "en" ? ` ${lang}` : "";
+  const q1 = topic;
+  const q2 = `${topic} guide tutorial${langSuffix}`;
+  const q3 = lang && lang !== "en" ? `${topic} ressources ${lang}` : `${topic} best practices resources`;
+  return [q1, q2, q3];
+}
+async function discoverWeb(topic, lang, max) {
+  const provider = resolveWebProvider();
+  if (!provider) {
+    process.stderr.write("corpus discover: no web search API key found (SERPER_API_KEY / EXA_API_KEY / TAVILY_API_KEY / GOOGLE_SEARCH_API_KEY) \u2014 web channel skipped.\n");
+    return { urls: [], provider: "none" };
+  }
+  const queries = buildWebQueries(topic, lang);
+  const seen = /* @__PURE__ */ new Set();
+  const urls = [];
+  for (const q of queries) {
+    if (urls.length >= max) break;
+    try {
+      const results = await provider.fn(q, lang, max);
+      for (const r of results) {
+        if (urls.length >= max) break;
+        if (!seen.has(r.url)) {
+          seen.add(r.url);
+          urls.push(r.url);
+        }
+      }
+    } catch (e) {
+      process.stderr.write(`corpus discover: web search query "${q}" failed \u2014 ${e instanceof Error ? e.message : String(e)}
+`);
+    }
+  }
+  return { urls, provider: provider.name };
+}
+async function discoverYoutube(topic, max) {
+  return new Promise((resolve2) => {
+    execFile(
+      "yt-dlp",
+      [`ytsearch${max}:${topic}`, "--flat-playlist", "--print", "%(url)s", "--no-warnings"],
+      { timeout: 6e4 },
+      (err, stdout, _stderr) => {
+        if (err) {
+          process.stderr.write(`corpus discover: youtube channel failed \u2014 ${err.message}
+`);
+          resolve2([]);
+          return;
+        }
+        resolve2(
+          stdout.split("\n").map((l) => l.trim()).filter((l) => l.startsWith("http"))
+        );
+      }
+    );
+  });
+}
+async function discoverSocial(_topic, _lang, _max) {
+  process.stdout.write("  social channel needs a Bureau session \u2014 skipped.\n");
+  return [];
+}
+async function runDiscover(args) {
+  const parsed = parse2(args);
+  if (!parsed.topic) {
+    return fail("discover requires a <topic> argument. Usage: corpus discover <topic> [path] [--max N] [--channels web,youtube,social] [--lang fr] [--tags t] [--import]", 2);
+  }
+  const outputPath = parsed.outputPath ? resolveWorkspacePath(parsed.outputPath) : process.cwd();
+  process.stdout.write(`discover "${parsed.topic}"
+`);
+  process.stdout.write(`  channels: ${[...parsed.channels].join(", ")} \xB7 max: ${parsed.max}${parsed.lang ? ` \xB7 lang: ${parsed.lang}` : ""}
+`);
+  const webUrls = [];
+  const ytUrls = [];
+  const socialUrls = [];
+  if (parsed.channels.has("web")) {
+    const { urls, provider } = await discoverWeb(parsed.topic, parsed.lang, parsed.max);
+    webUrls.push(...urls);
+    if (urls.length > 0) process.stdout.write(`  web (${provider}): ${urls.length} URLs
+`);
+  }
+  if (parsed.channels.has("youtube")) {
+    const urls = await discoverYoutube(parsed.topic, parsed.max);
+    ytUrls.push(...urls);
+    if (urls.length > 0) process.stdout.write(`  youtube: ${urls.length} URLs
+`);
+  }
+  if (parsed.channels.has("social")) {
+    const urls = await discoverSocial();
+    socialUrls.push(...urls);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const allUrls = [];
+  for (const url of [...webUrls, ...ytUrls, ...socialUrls]) {
+    if (!seen.has(url)) {
+      seen.add(url);
+      allUrls.push(url);
+    }
+  }
+  process.stdout.write(`
+  web: ${webUrls.length} \xB7 youtube: ${ytUrls.length} \xB7 social: ${socialUrls.length} \xB7 total: ${allUrls.length} unique
+`);
+  if (allUrls.length === 0) {
+    process.stderr.write("corpus discover: no URLs found across any channel.\n");
+    return 1;
+  }
+  await mkdir(outputPath, { recursive: true });
+  const outFile = join(outputPath, "urls.discovered.txt");
+  await writeFile(outFile, allUrls.join("\n") + "\n", "utf-8");
+  process.stdout.write(`
+  written \u2192 ${outFile}
+`);
+  if (parsed.doImport) {
+    process.stdout.write("\n  --import: chaining import-web\u2026\n");
+    const importArgs = [
+      outputPath,
+      "--urls-file",
+      outFile,
+      ...parsed.tags.flatMap((t) => ["--tags", t]),
+      ...parsed.lang ? ["--lang", parsed.lang] : []
+    ];
+    return runImportWeb(importArgs);
+  }
+  return 0;
+}
 
 // src/ports/anthropic-distiller.ts
 var ANTHROPIC_RESPONSE = z.object({
@@ -1703,15 +1946,17 @@ var AnthropicDistiller = class {
   baseUrl;
   maxItems;
   onUsage;
+  lang;
   constructor(opts) {
     this.apiKey = opts.apiKey;
     this.model = opts.model ?? "claude-sonnet-4-6";
     this.baseUrl = (opts.baseUrl ?? "https://api.anthropic.com/v1").replace(/\/+$/, "");
     this.maxItems = opts.maxItems ?? 8;
     this.onUsage = opts.onUsage;
+    this.lang = opts.lang;
   }
   async distill(input) {
-    const prompt = buildDistillPrompt(input, this.maxItems);
+    const prompt = buildDistillPrompt(input, this.maxItems, { lang: this.lang });
     const startedAt = (/* @__PURE__ */ new Date()).toISOString();
     const res = await fetch(`${this.baseUrl}/messages`, {
       method: "POST",
@@ -1750,18 +1995,20 @@ var CliAgentDistiller = class {
   maxItems;
   timeoutMs;
   cwd;
+  lang;
   constructor(opts) {
     this.engine = opts.engine;
     this.model = opts.model;
     this.maxItems = opts.maxItems ?? 8;
     this.timeoutMs = opts.timeoutMs ?? 12e4;
     this.cwd = opts.cwd ?? tmpdir();
+    this.lang = opts.lang;
   }
   async distill(input) {
-    const prompt = buildDistillPrompt(input, this.maxItems);
+    const prompt = buildDistillPrompt(input, this.maxItems, { lang: this.lang });
     const stdout = await spawnWithStdin({
       command: this.engine.command,
-      args: this.engine.buildArgs({ ...this.model ? { model: this.model } : {} }),
+      args: this.engine.buildArgs({ ...this.model ? { model: this.model } : {}, prompt }),
       stdin: prompt,
       cwd: this.cwd,
       timeoutMs: this.timeoutMs
@@ -1836,11 +2083,23 @@ var OPENCODE = {
   buildArgs: ({ model }) => ["run", ...model ? ["-m", model] : []],
   parseOutput: plainTextOutput
 };
+var HERMES = {
+  id: "hermes",
+  subscriptionBilled: true,
+  command: "hermes",
+  buildArgs: ({ model, prompt }) => [
+    "--oneshot",
+    prompt ?? "",
+    ...model ? ["-m", model] : []
+  ],
+  parseOutput: plainTextOutput
+};
 var CLI_ENGINES = {
   [CLAUDE_CODE.id]: CLAUDE_CODE,
   [GEMINI.id]: GEMINI,
   [CODEX.id]: CODEX,
-  [OPENCODE.id]: OPENCODE
+  [OPENCODE.id]: OPENCODE,
+  [HERMES.id]: HERMES
 };
 var PRICING = [
   { match: "opus", inPerM: 15, outPerM: 75 },
@@ -2009,10 +2268,11 @@ var DISTILLER_ENGINES = {
   [DEFAULT_ENGINE]: {
     id: DEFAULT_ENGINE,
     needsApiKey: true,
-    create: ({ apiKey, model, onUsage }) => new AnthropicDistiller({
+    create: ({ apiKey, model, onUsage, lang }) => new AnthropicDistiller({
       apiKey,
       ...model ? { model } : {},
-      ...onUsage ? { onUsage } : {}
+      ...onUsage ? { onUsage } : {},
+      ...lang ? { lang } : {}
     })
   },
   ...Object.fromEntries(
@@ -2021,19 +2281,24 @@ var DISTILLER_ENGINES = {
       {
         id: engine.id,
         needsApiKey: false,
-        create: ({ model }) => new CliAgentDistiller({ engine, ...model ? { model } : {} })
+        create: ({ model, lang }) => new CliAgentDistiller({
+          engine,
+          ...model ? { model } : {},
+          ...lang ? { lang } : {}
+        })
       }
     ])
   )
 };
-function parse2(args) {
+function parse3(args) {
   const out = {
     workspace: void 0,
     sourceId: void 0,
     max: void 0,
     throttleMs: 1e3,
     model: void 0,
-    engine: DEFAULT_ENGINE
+    engine: DEFAULT_ENGINE,
+    lang: void 0
   };
   for (let i = 0; i < args.length; i++) {
     const a = args[i];
@@ -2060,6 +2325,9 @@ function parse2(args) {
         if (v) out.engine = v;
         break;
       }
+      case "--lang":
+        out.lang = next();
+        break;
       default:
         if (!a.startsWith("-") && out.workspace === void 0) out.workspace = a;
     }
@@ -2119,7 +2387,7 @@ async function scanDistilledSourceIds(root) {
 }
 var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 async function runDistill(args) {
-  const parsed = parse2(args);
+  const parsed = parse3(args);
   const target = resolveWorkspacePath(parsed.workspace);
   const engine = DISTILLER_ENGINES[parsed.engine];
   if (!engine) {
@@ -2157,7 +2425,8 @@ async function runDistill(args) {
     distiller: engine.create({
       ...apiKey ? { apiKey } : {},
       ...parsed.model ? { model: parsed.model } : {},
-      onUsage: usage.record
+      onUsage: usage.record,
+      ...parsed.lang ? { lang: parsed.lang } : {}
     })
   });
   let totalEntries = 0;
@@ -2741,6 +3010,12 @@ Commands:
   events:emit <kind> --payload <json> [path]
                                          Append an event to _log.md
   events:tail [path]                     Print _log.md
+  discover <topic> [path] [--max N --channels web,youtube,social --lang l --tags t --import]
+                                         Fan out across channels (web search + YouTube +
+                                         social), dedup, write urls.discovered.txt.
+                                         Web: auto-picks first available key (SERPER /
+                                         EXA / TAVILY / GOOGLE_SEARCH). YouTube: yt-dlp
+                                         flat-playlist. --import chains import-web.
   import-web [path] --urls-file <f> [--max n --max-duration s --throttle ms --tags t --lang l --force --diarize]
                                          Import URLs as sources (video\u2192Whisper, article\u2192
                                          readability). Resumable: skips already-ingested
@@ -2755,7 +3030,7 @@ Commands:
                                          scrape MCP server (stealth + clean Markdown) for
                                          walled/JS pages, ahead of plain readability.
                                          --diarize: AssemblyAI speaker labels (interviews).
-  distill [path] [--source id --max n --throttle ms --model m]
+  distill [path] [--source id --max n --throttle ms --model m --lang l --engine e]
                                          Distill raw sources \u2192 refined entries
                                          (principle/pattern/\u2026) with sources:[id] provenance.
                                          Resumable: skips already-distilled sources.
@@ -2805,6 +3080,8 @@ async function main(argv2) {
       return await runEventsTail(rest);
     case "import-web":
       return await runImportWeb(rest);
+    case "discover":
+      return await runDiscover(rest);
     case "distill":
       return await runDistill(rest);
     case "knowledge":