@agentproto/corpus-cli 0.2.0 → 0.3.0

package/dist/cli.mjs

@@ -404,8 +404,8 @@ function resolveWorkspacePath(arg) {
   if (!arg) return process.cwd();
   return path.resolve(process.cwd(), arg);
 }
-function fail(msg4, code = 1) {
-  process.stderr.write(`corpus: ${msg4}
+function fail(msg5, code = 1) {
+  process.stderr.write(`corpus: ${msg5}
 `);
   return code;
 }
@@ -1040,6 +1040,146 @@ function defaultYtDlpDownloader(bin, opts) {
     }
   };
 }
+var execFileAsync2 = promisify(execFile);
+var YTDLP_INFO2 = z.object({ title: z.string().optional(), language: z.string().optional() }).loose();
+var YtDlpCaptionsFetcher = class {
+  download;
+  extraHosts;
+  minChars;
+  constructor(opts = {}) {
+    this.minChars = opts.minChars ?? 400;
+    this.download = opts.download ?? defaultYtDlpCaptionDownloader(opts.ytDlpBin ?? "yt-dlp", {
+      ...opts.subLangs ? { subLangs: opts.subLangs } : {},
+      ...opts.preferLang ? { preferLang: opts.preferLang } : {},
+      ...opts.maxDurationSec ? { maxDurationSec: opts.maxDurationSec } : {},
+      ...opts.cookiesFromBrowser ? { cookiesFromBrowser: opts.cookiesFromBrowser } : {},
+      ...opts.cookiesFile ? { cookiesFile: opts.cookiesFile } : {}
+    });
+    this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
+  }
+  async fetch(url) {
+    if (!isVideoUrl(url, this.extraHosts)) return null;
+    let dl;
+    try {
+      dl = await this.download(url);
+    } catch (e) {
+      process.stderr.write(
+        `corpus: caption fetch failed for ${url} \u2014 falling back (${msg2(e)})
+`
+      );
+      return null;
+    }
+    try {
+      if (!dl.vttPath) return null;
+      const text = parseVttToText(await readFile(dl.vttPath, "utf-8"));
+      if (text.length < this.minChars) return null;
+      return {
+        title: dl.title || url,
+        text,
+        kind: "video",
+        ...dl.language ? { language: dl.language } : {},
+        via: "captions"
+      };
+    } catch {
+      return null;
+    } finally {
+      await dl.cleanup().catch(() => {
+      });
+    }
+  }
+};
+function msg2(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+function parseVttToText(vtt) {
+  const out = [];
+  for (const raw of vtt.split(/\r?\n/)) {
+    if (/^WEBVTT/.test(raw)) continue;
+    if (/^(Kind|Language|NOTE|STYLE):/i.test(raw)) continue;
+    if (raw.includes("-->")) continue;
+    const line = raw.replace(/<[^>]+>/g, "").replace(/&nbsp;/gi, " ").replace(/ /g, " ").trim();
+    if (!line) continue;
+    if (out.length && out[out.length - 1] === line) continue;
+    out.push(line);
+  }
+  const cleaned = [];
+  for (let i = 0; i < out.length; i++) {
+    const next = out[i + 1];
+    if (next && next.startsWith(out[i])) continue;
+    cleaned.push(out[i]);
+  }
+  return cleaned.join(" ").replace(/\s+/g, " ").trim();
+}
+function buildYtDlpCaptionArgs(url, dir, opts) {
+  const subLangs = opts?.subLangs ?? (opts?.preferLang ? `${opts.preferLang}.*` : "en.*,fr.*,es.*,de.*,it.*,pt.*,nl.*");
+  const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+  const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+  return [
+    "--skip-download",
+    "--write-auto-subs",
+    "--write-subs",
+    "--sub-langs",
+    subLangs,
+    "--sub-format",
+    "vtt",
+    "--no-playlist",
+    "--no-progress",
+    "--sleep-requests",
+    "1",
+    ...durationGuard,
+    ...cookieArgs,
+    "--write-info-json",
+    "-o",
+    join(dir, "%(id)s.%(ext)s"),
+    url
+  ];
+}
+function pickVtt(files, preferLang) {
+  const vtts = files.filter((f) => f.endsWith(".vtt"));
+  if (vtts.length === 0) return void 0;
+  const bySuffix = (suffix) => vtts.find((f) => f.toLowerCase().endsWith(suffix.toLowerCase()));
+  if (preferLang) {
+    return bySuffix(`.${preferLang}-orig.vtt`) ?? bySuffix(`.${preferLang}.vtt`) ?? vtts.find((f) => new RegExp(`\\.${preferLang}[.-]`, "i").test(f)) ?? vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+  }
+  return vtts.find((f) => /-orig\.vtt$/i.test(f)) ?? vtts[0];
+}
+function defaultYtDlpCaptionDownloader(bin, opts) {
+  return async (url) => {
+    const dir = await mkdtemp(join(tmpdir(), "corpus-captions-"));
+    const cleanup = () => rm(dir, { recursive: true, force: true });
+    try {
+      await execFileAsync2(bin, buildYtDlpCaptionArgs(url, dir, opts), {
+        maxBuffer: 16 * 1024 * 1024
+      }).catch(() => {
+      });
+      const files = await readdir(dir);
+      const vtt = pickVtt(files, opts?.preferLang);
+      let title = url;
+      let language;
+      const infoName = files.find((f) => f.endsWith(".info.json"));
+      if (infoName) {
+        try {
+          const info = YTDLP_INFO2.parse(
+            JSON.parse(await readFile(join(dir, infoName), "utf-8"))
+          );
+          if (info.title) title = info.title;
+          if (info.language) language = info.language;
+        } catch {
+        }
+      }
+      return {
+        vttPath: vtt ? join(dir, vtt) : null,
+        title,
+        ...language ? { language } : {},
+        cleanup
+      };
+    } catch (e) {
+      await cleanup().catch(() => {
+      });
+      throw e;
+    }
+  };
+}
 var WHISPER_RESPONSE = z.object({ text: z.string().optional(), language: z.string().optional() }).loose();
 var WHISPER_MAX_BYTES = 25 * 1024 * 1024;
 var OpenAiWhisperStt = class {
@@ -1178,7 +1318,7 @@ function formatDiarized(job) {
   }
   return (job.text ?? "").trim();
 }
-var execFileAsync2 = promisify(execFile);
+var execFileAsync3 = promisify(execFile);
 var DEFAULT_MAX_BYTES = 24 * 1024 * 1024;
 var DEFAULT_SEGMENT_SECONDS = 1200;
 var ChunkedStt = class {
@@ -1237,7 +1377,7 @@ var ChunkedStt = class {
 function defaultFfmpegSplitter(bin) {
   return async (audioPath, segmentSeconds, outDir) => {
     const ext = extname(audioPath) || ".mp3";
-    await execFileAsync2(
+    await execFileAsync3(
       bin,
       [
         "-hide_banner",
@@ -1468,6 +1608,7 @@ function parse(args) {
     throttleMs: 2e3,
     force: false,
     diarize: false,
+    noCaptions: false,
     browserMcp: process.env.BROWSER_MCP_URL,
     scrapeMcp: process.env.SCRAPE_MCP_URL,
     importerId: "web",
@@ -1523,6 +1664,9 @@ function parse(args) {
       case "--diarize":
         out.diarize = true;
         break;
+      case "--no-captions":
+        out.noCaptions = true;
+        break;
       case "--browser-mcp":
         out.browserMcp = next();
         break;
@@ -1589,10 +1733,11 @@ async function runImportWeb(args) {
   const todo = urls.filter((u) => !done.has(u));
   const batch = parsed.max !== void 0 ? todo.slice(0, parsed.max) : todo;
   const remaining = todo.length - batch.length;
+  const videoLabel = parsed.noCaptions ? parsed.diarize ? "AssemblyAI" : "Whisper" : `captions-first \u2192 ${parsed.diarize ? "AssemblyAI" : "Whisper"} fallback`;
   const plan = `  workspace: ${target}
   urls:      ${urls.length} total \xB7 ${done.size && !parsed.force ? `${urls.length - todo.length} already ingested \xB7 ` : ""}${todo.length} to do
   this run:  ${batch.length}${parsed.max !== void 0 ? ` (--max ${parsed.max})` : ""} \xB7 ${remaining} remaining after
-  video stt: ${parsed.diarize ? "AssemblyAI (speaker-labelled)" : "Whisper"}
+  video:     ${videoLabel}
   throttle:  ${parsed.throttleMs} ms between fetches
 `;
   if (batch.length === 0) {
@@ -1613,6 +1758,16 @@ ${plan}`);
   process.stdout.write(`import-web
 ${plan}`);
   const chain = [];
+  if (!parsed.noCaptions) {
+    chain.push(
+      new YtDlpCaptionsFetcher({
+        ...parsed.lang ? { preferLang: parsed.lang } : {},
+        ...parsed.maxDurationSec ? { maxDurationSec: parsed.maxDurationSec } : {},
+        ...parsed.cookiesFromBrowser ? { cookiesFromBrowser: parsed.cookiesFromBrowser } : {},
+        ...parsed.cookiesFile ? { cookiesFile: parsed.cookiesFile } : {}
+      })
+    );
+  }
   let stt;
   if (parsed.diarize) {
     const aaiKey = process.env.ASSEMBLYAI_API_KEY;
@@ -1627,7 +1782,7 @@ ${plan}`);
       stt = new ChunkedStt({ base: new OpenAiWhisperStt({ apiKey: openaiKey }) });
     else
       process.stderr.write(
-        "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n"
+        parsed.noCaptions ? "corpus: OPENAI_API_KEY not set \u2014 video URLs will be skipped (no transcription).\n" : "corpus: OPENAI_API_KEY not set \u2014 caption-less videos will be skipped (captioned videos still import).\n"
       );
   }
   if (stt)
@@ -1645,7 +1800,7 @@ ${plan}`);
       const client = await connectBrowserMcp({ endpoint: parsed.scrapeMcp });
       chain.push(new ScrapeMcpFetcher({ client }));
     } catch (e) {
-      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to scrape MCP at ${parsed.scrapeMcp}: ${msg3(e)}`, 1);
     }
   }
   if (parsed.browserMcp) {
@@ -1654,7 +1809,7 @@ ${plan}`);
       browser = await connectBrowserMcp({ endpoint: parsed.browserMcp });
       chain.push(new BrowserMcpFetcher({ browser }));
     } catch (e) {
-      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg2(e)}`, 1);
+      return fail(`could not connect to browser MCP at ${parsed.browserMcp}: ${msg3(e)}`, 1);
     }
   }
   chain.push(new HttpReadabilityFetcher());
@@ -1685,7 +1840,7 @@ ${plan}`);
       }
     });
   } catch (e) {
-    return fail(`import failed: ${msg2(e)}`, 1);
+    return fail(`import failed: ${msg3(e)}`, 1);
   }
   process.stdout.write(
     `import-web \u2192 ${target}
@@ -1705,7 +1860,7 @@ knowledge engine (RAG) via the WriterPort. Import only stages the sources.
   );
   return 0;
 }
-function msg2(e) {
+function msg3(e) {
   return e instanceof Error ? e.message : String(e);
 }
 function parse2(args) {
@@ -2786,10 +2941,10 @@ var SseMcpClient = class {
     try {
       const parsed = RPC_MESSAGE.safeParse(JSON.parse(payload));
       if (!parsed.success) return;
-      const msg4 = parsed.data;
-      if (typeof msg4.id === "number" && this.pending.has(msg4.id)) {
-        this.pending.get(msg4.id)(msg4);
-        this.pending.delete(msg4.id);
+      const msg5 = parsed.data;
+      if (typeof msg5.id === "number" && this.pending.has(msg5.id)) {
+        this.pending.get(msg5.id)(msg5);
+        this.pending.delete(msg5.id);
       }
     } catch {
     }
@@ -2858,7 +3013,7 @@ var McpSink = class {
     try {
       client = await this.ensure();
     } catch (e) {
-      return { uri: item.uri, ok: false, error: `connect: ${msg3(e)}` };
+      return { uri: item.uri, ok: false, error: `connect: ${msg4(e)}` };
     }
     const args = template(this.config.args, item);
     const call = this.config.importedAlias ? { name: "mcp_imported_call", args: { alias: this.config.importedAlias, toolName: this.config.tool, args } } : { name: this.config.tool, args };
@@ -2867,7 +3022,7 @@ var McpSink = class {
       if (res.isError) return { uri: item.uri, ok: false, error: "tool returned isError" };
       return { uri: item.uri, ok: true };
     } catch (e) {
-      return { uri: item.uri, ok: false, error: msg3(e) };
+      return { uri: item.uri, ok: false, error: msg4(e) };
     }
   }
 };
@@ -2911,7 +3066,7 @@ function stringify(v) {
   if (Array.isArray(v)) return v.join(", ");
   return v == null ? "" : String(v);
 }
-function msg3(e) {
+function msg4(e) {
   return e instanceof Error ? e.message : String(e);
 }