@agentproto/corpus-cli 0.1.0-alpha.1

package/dist/ports/index.mjs

@@ -0,0 +1,1129 @@
+import { randomBytes } from 'crypto';
+import { stat, readFile, mkdir, writeFile, rename, appendFile, readdir, rm, mkdtemp } from 'fs/promises';
+import path, { join, basename, extname } from 'path';
+import { existsSync } from 'fs';
+import os, { tmpdir } from 'os';
+import { z } from 'zod';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+import { normalizeLanguageTag, buildDistillPrompt, parseItems } from '@agentproto/corpus';
+export { buildDistillPrompt, parseItems } from '@agentproto/corpus';
+import { parseClaudeJsonOutput, spawnWithStdin } from '@agentproto/cli-exec';
+
+/**
+ * @agentproto/corpus-cli v0.1.0-alpha
+ * The `corpus` binary — local-topology host for AIP-10 corpora.
+ */
+
+var NodeFsAdapter = class {
+  root;
+  constructor(opts) {
+    this.root = opts.root;
+  }
+  async exists(p) {
+    try {
+      await stat(this.resolve(p));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  async readFile(p) {
+    return await readFile(this.resolve(p), "utf8");
+  }
+  /**
+   * Atomic write: write to `${target}.tmp-{nonce}`, fsync, rename.
+   * Rename is atomic on the same filesystem. Creates parent dirs as
+   * needed (most callers expect mkdir -p semantics).
+   */
+  async writeFile(p, content) {
+    const target = this.resolve(p);
+    await mkdir(path.dirname(target), { recursive: true });
+    const tmp = `${target}.tmp-${randomBytes(8).toString("hex")}`;
+    await writeFile(tmp, content, "utf8");
+    await rename(tmp, target);
+  }
+  async appendFile(p, content) {
+    const target = this.resolve(p);
+    await mkdir(path.dirname(target), { recursive: true });
+    await appendFile(target, content, "utf8");
+  }
+  async readdir(p) {
+    const target = this.resolve(p);
+    const entries = await readdir(target);
+    return entries;
+  }
+  async walk(p) {
+    const out = [];
+    const start = this.resolve(p);
+    const rootForRelative = start;
+    const visit = async (dir) => {
+      let entries = [];
+      try {
+        entries = await readdir(dir);
+      } catch {
+        return;
+      }
+      for (const ent of entries) {
+        if (ent.startsWith(".")) continue;
+        const full = path.join(dir, ent);
+        const s = await stat(full);
+        if (s.isDirectory()) {
+          await visit(full);
+        } else if (s.isFile()) {
+          const rel = path.relative(rootForRelative, full).split(path.sep).join("/");
+          out.push(rel);
+        }
+      }
+    };
+    await visit(start);
+    return out;
+  }
+  async stat(p) {
+    try {
+      const s = await stat(this.resolve(p));
+      return {
+        kind: s.isDirectory() ? "directory" : "file",
+        bytes: s.isFile() ? s.size : void 0,
+        modifiedAt: s.mtime
+      };
+    } catch {
+      return null;
+    }
+  }
+  /**
+   * Advisory lock via a lockfile. Spins waiting for the lockfile to
+   * disappear, then creates it with exclusive flag. Released by
+   * deleting it. Single-process safe; multi-process semi-safe (the
+   * write is best-effort exclusive — adequate for the corpus's
+   * promote-entry transaction which is rare and short-lived).
+   */
+  async lock(p) {
+    const target = this.resolve(p);
+    const lockDir = path.join(path.dirname(target), ".corpus-locks");
+    await mkdir(lockDir, { recursive: true });
+    const lockFile = path.join(lockDir, path.basename(target) + ".lock");
+    const myToken = `${process.pid}-${randomBytes(4).toString("hex")}`;
+    while (true) {
+      if (!existsSync(lockFile)) {
+        try {
+          await writeFile(lockFile, myToken, { encoding: "utf8", flag: "wx" });
+          const written = await readFile(lockFile, "utf8").catch(() => "");
+          if (written === myToken) break;
+        } catch {
+        }
+      }
+      await new Promise((r) => setTimeout(r, 25));
+    }
+    return {
+      release: async () => {
+        try {
+          await rm(lockFile, { force: true });
+        } catch {
+        }
+      }
+    };
+  }
+  // ── Helpers ────────────────────────────────────────────────────
+  /**
+   * Resolve a workspace-relative path to an absolute host path.
+   * Refuses to climb out of the workspace root (defense against
+   * `../../etc/passwd`-style abuse) — the corpus kit's paths must
+   * stay inside the workspace.
+   */
+  resolve(p) {
+    const normalized = path.posix.normalize("/" + p.replace(/^\/+/, ""));
+    const rel = normalized.slice(1);
+    if (rel.startsWith("..")) {
+      throw new Error(
+        `NodeFsAdapter: path "${p}" escapes the workspace root "${this.root}"`
+      );
+    }
+    return path.join(this.root, rel);
+  }
+};
+var OsIdentityAdapter = class {
+  cached;
+  constructor(opts) {
+    const user = opts.userOverride ?? process.env.USER ?? process.env.USERNAME ?? safeGetUser() ?? "anonymous";
+    const wsName = path.basename(opts.workspaceRoot) || "default";
+    this.cached = Object.freeze({
+      principal: `ws://users/${slugify(user)}`,
+      identityTree: Object.freeze([
+        `ws://users/${slugify(user)}`,
+        `ws://workspaces/${slugify(wsName)}`
+      ]),
+      displayName: user
+    });
+  }
+  async resolve() {
+    return this.cached;
+  }
+};
+function safeGetUser() {
+  try {
+    return os.userInfo().username;
+  } catch {
+    return void 0;
+  }
+}
+function slugify(s) {
+  return s.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").replace(/-{2,}/g, "-").slice(0, 64) || "anonymous";
+}
+
+// src/ports/video-hosts.ts
+var VIDEO_HOSTS = /* @__PURE__ */ new Set([
+  "youtube.com",
+  "www.youtube.com",
+  "m.youtube.com",
+  "music.youtube.com",
+  "youtu.be",
+  "vimeo.com"
+]);
+function isVideoUrl(url, extraHosts) {
+  let host;
+  try {
+    host = new URL(url).hostname;
+  } catch {
+    return false;
+  }
+  const bare = host.replace(/^www\./, "");
+  return VIDEO_HOSTS.has(host) || VIDEO_HOSTS.has(bare) || (extraHosts?.has(host) ?? false) || (extraHosts?.has(bare) ?? false);
+}
+
+// src/ports/browser-fetcher.adapter.ts
+var FETCHED_PAYLOAD = z.object({
+  title: z.string(),
+  text: z.string(),
+  kind: z.enum(["video", "article", "page", "unknown"]).catch("unknown"),
+  language: z.string().optional().catch(void 0),
+  via: z.string().optional().catch(void 0)
+}).loose();
+var MCP_TEXT_BLOCK = z.object({ type: z.literal("text"), text: z.string() }).loose();
+var READABILITY_FN = `() => {
+  const title = document.title || "";
+  const article = document.querySelector("article") || document.querySelector("main");
+  const root = article || document.body;
+  const text = (root?.innerText || "").replace(/\\n{3,}/g, "\\n\\n").trim().slice(0, 200000);
+  const lang = document.documentElement.getAttribute("lang") || undefined;
+  return { title, text, kind: "article", language: lang, via: "readability" };
+}`;
+var BrowserMcpFetcher = class {
+  browser;
+  navigateTool;
+  evaluateTool;
+  constructor(opts) {
+    this.browser = opts.browser;
+    this.navigateTool = opts.navigateToolName ?? "navigate_page";
+    this.evaluateTool = opts.evaluateToolName ?? "evaluate_script";
+  }
+  async fetch(url) {
+    if (isVideoUrl(url)) return null;
+    await this.browser.callTool({
+      name: this.navigateTool,
+      arguments: { url }
+    });
+    const result = await this.browser.callTool({
+      name: this.evaluateTool,
+      arguments: { function: READABILITY_FN }
+    });
+    const extracted = parseEvaluateResult(result);
+    if (!extracted || !extracted.text.trim()) return null;
+    return extracted;
+  }
+};
+function parseEvaluateResult(result) {
+  return coerceFetched(result.structuredContent) ?? coerceFetched(extractText(result.content));
+}
+function coerceFetched(value) {
+  let obj = value;
+  if (typeof obj === "string") {
+    try {
+      obj = JSON.parse(obj);
+    } catch {
+      return null;
+    }
+  }
+  const parsed = FETCHED_PAYLOAD.safeParse(obj);
+  if (!parsed.success) return null;
+  const { title, text, kind, language, via } = parsed.data;
+  return {
+    title,
+    text,
+    kind,
+    ...language ? { language } : {},
+    ...via ? { via } : {}
+  };
+}
+function extractText(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    for (const block of content) {
+      const parsed = MCP_TEXT_BLOCK.safeParse(block);
+      if (parsed.success) return parsed.data.text;
+    }
+  }
+  return null;
+}
+var SCRAPE_PAYLOAD = z.object({
+  title: z.string().optional().catch(void 0),
+  markdown: z.string().optional().catch(void 0),
+  html: z.string().optional().catch(void 0),
+  language: z.string().optional().catch(void 0),
+  tierUsed: z.union([z.string(), z.number()]).optional().catch(void 0)
+}).loose();
+var MCP_TEXT_BLOCK2 = z.object({ type: z.literal("text"), text: z.string() }).loose();
+var ScrapeMcpFetcher = class {
+  client;
+  toolName;
+  constructor(opts) {
+    this.client = opts.client;
+    this.toolName = opts.scrapeToolName ?? "scrape";
+  }
+  async fetch(url) {
+    if (isVideoUrl(url)) return null;
+    let res;
+    try {
+      res = await this.client.callTool({
+        name: this.toolName,
+        arguments: { url }
+      });
+    } catch {
+      return null;
+    }
+    if (res.isError) return null;
+    const payload = extractPayload(res);
+    if (!payload) return null;
+    const text = (payload.markdown?.trim() || stripTags(payload.html)).trim();
+    if (!text) return null;
+    return {
+      title: payload.title?.trim() || url,
+      text,
+      kind: "article",
+      ...payload.language ? { language: payload.language } : {},
+      via: payload.tierUsed ? `scrape:${payload.tierUsed}` : "scrape"
+    };
+  }
+};
+function extractPayload(res) {
+  if (res.structuredContent && typeof res.structuredContent === "object") {
+    const parsed = SCRAPE_PAYLOAD.safeParse(res.structuredContent);
+    if (parsed.success) return parsed.data;
+  }
+  if (Array.isArray(res.content)) {
+    for (const block of res.content) {
+      const text = MCP_TEXT_BLOCK2.safeParse(block);
+      if (!text.success) continue;
+      try {
+        const parsed = SCRAPE_PAYLOAD.safeParse(JSON.parse(text.data.text));
+        if (parsed.success) return parsed.data;
+      } catch {
+        return { markdown: text.data.text };
+      }
+    }
+  }
+  return null;
+}
+function stripTags(html) {
+  if (!html) return "";
+  return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+}
+var execFileAsync = promisify(execFile);
+var YTDLP_INFO = z.object({ title: z.string().optional(), language: z.string().optional() }).loose();
+var YtDlpWhisperFetcher = class {
+  stt;
+  download;
+  extraHosts;
+  constructor(opts) {
+    this.stt = opts.stt;
+    this.download = opts.download ?? defaultYtDlpDownloader(opts.ytDlpBin ?? "yt-dlp", {
+      maxDurationSec: opts.maxDurationSec,
+      cookiesFromBrowser: opts.cookiesFromBrowser,
+      cookiesFile: opts.cookiesFile
+    });
+    this.extraHosts = opts.extraVideoHosts ? new Set(opts.extraVideoHosts) : void 0;
+  }
+  async fetch(url) {
+    if (!isVideoUrl(url, this.extraHosts)) return null;
+    let dl;
+    try {
+      dl = await this.download(url);
+    } catch (e) {
+      process.stderr.write(`corpus: download failed for ${url} \u2014 skipped (${msg(e)})
+`);
+      return null;
+    }
+    try {
+      const t = await this.stt.transcribe(dl.audioPath);
+      if (!t.text.trim()) return null;
+      return {
+        title: dl.title || url,
+        text: t.text,
+        kind: "video",
+        ...t.language ?? dl.language ? { language: t.language ?? dl.language } : {},
+        via: "transcription"
+      };
+    } catch (e) {
+      if (isAuthError(e)) throw e;
+      process.stderr.write(
+        `corpus: transcription failed for ${url} \u2014 skipped (${msg(e)})
+`
+      );
+      return null;
+    } finally {
+      await dl.cleanup().catch(() => {
+      });
+    }
+  }
+};
+function msg(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+function isAuthError(e) {
+  const m = msg(e).toLowerCase();
+  return m.includes(" 401") || m.includes(" 403") || m.includes("unauthorized") || m.includes("forbidden") || m.includes("api key") || m.includes("api_key");
+}
+function defaultYtDlpDownloader(bin, opts) {
+  return async (url) => {
+    const dir = await mkdtemp(join(tmpdir(), "corpus-ytdlp-"));
+    const cleanup = () => rm(dir, { recursive: true, force: true });
+    try {
+      const durationGuard = opts?.maxDurationSec && opts.maxDurationSec > 0 ? ["--match-filter", `duration <= ${Math.floor(opts.maxDurationSec)}`] : [];
+      const cookieArgs = opts?.cookiesFromBrowser ? ["--cookies-from-browser", opts.cookiesFromBrowser] : opts?.cookiesFile ? ["--cookies", opts.cookiesFile] : [];
+      await execFileAsync(
+        bin,
+        [
+          "-f",
+          "bestaudio",
+          "-x",
+          "--audio-format",
+          "mp3",
+          "--audio-quality",
+          "64K",
+          "--no-playlist",
+          "--no-progress",
+          ...durationGuard,
+          ...cookieArgs,
+          "--write-info-json",
+          "-o",
+          join(dir, "track.%(ext)s"),
+          url
+        ],
+        { maxBuffer: 16 * 1024 * 1024 }
+      );
+      const files = await readdir(dir);
+      const audio = files.find((f) => f === "track.mp3") ?? files.find((f) => /\.(mp3|m4a|opus|webm|ogg|wav)$/i.test(f));
+      if (!audio) throw new Error("yt-dlp produced no audio file");
+      let title = url;
+      let language;
+      const infoName = files.find((f) => f.endsWith(".info.json"));
+      if (infoName) {
+        try {
+          const info = YTDLP_INFO.parse(
+            JSON.parse(await readFile(join(dir, infoName), "utf-8"))
+          );
+          if (info.title) title = info.title;
+          if (info.language) language = info.language;
+        } catch {
+        }
+      }
+      return {
+        audioPath: join(dir, audio),
+        title,
+        ...language ? { language } : {},
+        cleanup
+      };
+    } catch (e) {
+      await cleanup().catch(() => {
+      });
+      throw e;
+    }
+  };
+}
+var WHISPER_RESPONSE = z.object({ text: z.string().optional(), language: z.string().optional() }).loose();
+var WHISPER_MAX_BYTES = 25 * 1024 * 1024;
+var OpenAiWhisperStt = class {
+  apiKey;
+  model;
+  baseUrl;
+  constructor(opts) {
+    this.apiKey = opts.apiKey;
+    this.model = opts.model ?? "whisper-1";
+    this.baseUrl = (opts.baseUrl ?? "https://api.openai.com/v1").replace(/\/+$/, "");
+  }
+  async transcribe(audioPath) {
+    const bytes = await readFile(audioPath);
+    if (bytes.byteLength > WHISPER_MAX_BYTES) {
+      throw new Error(
+        `audio ${basename(audioPath)} is ${(bytes.byteLength / 1024 / 1024).toFixed(1)} MB \u2014 over Whisper's 25 MB cap. Lower the bitrate or chunk the media.`
+      );
+    }
+    const form = new FormData();
+    form.append("file", new Blob([bytes]), basename(audioPath));
+    form.append("model", this.model);
+    form.append("response_format", "verbose_json");
+    const res = await this.postWithRetry(form);
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      throw new Error(`Whisper STT ${res.status} ${res.statusText}: ${body.slice(0, 200)}`);
+    }
+    const data = WHISPER_RESPONSE.parse(await res.json());
+    const language = normalizeLanguageTag(data.language);
+    return {
+      text: (data.text ?? "").trim(),
+      ...language ? { language } : {}
+    };
+  }
+  /**
+   * POST the audio with bounded retry. Transcribing a long talk fans out
+   * into many segment uploads (see ChunkedStt), so a single transient
+   * network blip ("fetch failed") or a 429 / 5xx must not abort the whole
+   * batch. Network errors and retryable statuses back off and retry;
+   * client errors (401 auth, 400 bad request) return immediately so a real
+   * misconfiguration still surfaces loudly.
+   */
+  async postWithRetry(form, attempts = 4) {
+    const url = `${this.baseUrl}/audio/transcriptions`;
+    let lastErr;
+    for (let i = 0; i < attempts; i++) {
+      try {
+        const res = await fetch(url, {
+          method: "POST",
+          headers: { authorization: `Bearer ${this.apiKey}` },
+          body: form
+        });
+        if (res.status !== 429 && res.status < 500) return res;
+        lastErr = new Error(`Whisper STT ${res.status} ${res.statusText}`);
+      } catch (e) {
+        lastErr = e;
+      }
+      if (i < attempts - 1) await delay(500 * 2 ** i);
+    }
+    throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+  }
+};
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var UPLOAD_RESPONSE = z.object({ upload_url: z.string() });
+var AAI_TRANSCRIPT = z.object({
+  id: z.string(),
+  status: z.enum(["queued", "processing", "completed", "error"]),
+  text: z.string().optional(),
+  language_code: z.string().optional(),
+  error: z.string().optional(),
+  utterances: z.array(z.object({ speaker: z.string(), text: z.string() })).nullish()
+}).loose();
+var AssemblyAiStt = class {
+  apiKey;
+  baseUrl;
+  pollIntervalMs;
+  maxWaitMs;
+  sleep;
+  constructor(opts) {
+    this.apiKey = opts.apiKey;
+    this.baseUrl = (opts.baseUrl ?? "https://api.assemblyai.com/v2").replace(/\/+$/, "");
+    this.pollIntervalMs = opts.pollIntervalMs ?? 3e3;
+    this.maxWaitMs = opts.maxWaitMs ?? 15 * 60 * 1e3;
+    this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  }
+  async transcribe(audioPath) {
+    const bytes = await readFile(audioPath);
+    const up = await fetch(`${this.baseUrl}/upload`, {
+      method: "POST",
+      headers: { authorization: this.apiKey, "content-type": "application/octet-stream" },
+      body: bytes
+    });
+    if (!up.ok) throw new Error(`AssemblyAI upload ${up.status}: ${(await up.text()).slice(0, 200)}`);
+    const { upload_url } = UPLOAD_RESPONSE.parse(await up.json());
+    const created = await this.post("/transcript", {
+      audio_url: upload_url,
+      speaker_labels: true,
+      language_detection: true
+    });
+    const id = created.id;
+    const deadline = this.maxWaitMs;
+    let waited = 0;
+    let job = created;
+    while (job.status !== "completed" && job.status !== "error") {
+      if (waited >= deadline) throw new Error(`AssemblyAI transcript ${id} timed out after ${deadline} ms`);
+      await this.sleep(this.pollIntervalMs);
+      waited += this.pollIntervalMs;
+      job = await this.get(`/transcript/${id}`);
+    }
+    if (job.status === "error") throw new Error(`AssemblyAI transcript failed: ${job.error ?? "unknown"}`);
+    return {
+      text: formatDiarized(job),
+      ...job.language_code ? { language: job.language_code } : {}
+    };
+  }
+  async post(path3, body) {
+    const r = await fetch(`${this.baseUrl}${path3}`, {
+      method: "POST",
+      headers: { authorization: this.apiKey, "content-type": "application/json" },
+      body: JSON.stringify(body)
+    });
+    if (!r.ok) throw new Error(`AssemblyAI POST ${path3} ${r.status}: ${(await r.text()).slice(0, 200)}`);
+    return AAI_TRANSCRIPT.parse(await r.json());
+  }
+  async get(path3) {
+    const r = await fetch(`${this.baseUrl}${path3}`, { headers: { authorization: this.apiKey } });
+    if (!r.ok) throw new Error(`AssemblyAI GET ${path3} ${r.status}: ${(await r.text()).slice(0, 200)}`);
+    return AAI_TRANSCRIPT.parse(await r.json());
+  }
+};
+function formatDiarized(job) {
+  if (job.utterances && job.utterances.length > 0) {
+    return job.utterances.map((u) => `Speaker ${u.speaker}: ${u.text.trim()}`).join("\n\n").trim();
+  }
+  return (job.text ?? "").trim();
+}
+var execFileAsync2 = promisify(execFile);
+var DEFAULT_MAX_BYTES = 24 * 1024 * 1024;
+var DEFAULT_SEGMENT_SECONDS = 1200;
+var ChunkedStt = class {
+  base;
+  maxBytes;
+  segmentSeconds;
+  split;
+  constructor(opts) {
+    this.base = opts.base;
+    this.maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+    this.segmentSeconds = opts.segmentSeconds ?? DEFAULT_SEGMENT_SECONDS;
+    this.split = opts.split ?? defaultFfmpegSplitter(opts.ffmpegBin ?? "ffmpeg");
+  }
+  async transcribe(audioPath) {
+    const { size } = await stat(audioPath);
+    if (size <= this.maxBytes) return this.base.transcribe(audioPath);
+    const dir = await mkdtemp(join(tmpdir(), "corpus-stt-chunk-"));
+    try {
+      const parts = await this.split(audioPath, this.segmentSeconds, dir);
+      if (parts.length === 0) {
+        return this.base.transcribe(audioPath);
+      }
+      const texts = [];
+      let language;
+      let failed = 0;
+      let firstErr;
+      for (const part of parts) {
+        try {
+          const t = await this.base.transcribe(part);
+          if (t.text.trim()) texts.push(t.text.trim());
+          language ??= t.language;
+        } catch (e) {
+          failed++;
+          firstErr ??= e;
+        }
+      }
+      if (texts.length === 0) {
+        throw firstErr ?? new Error("all audio segments failed transcription");
+      }
+      if (failed > 0) {
+        process.stderr.write(
+          `corpus: ${failed}/${parts.length} audio segments failed \u2014 partial transcript
+`
+        );
+      }
+      return {
+        text: texts.join("\n\n"),
+        ...language ? { language } : {}
+      };
+    } finally {
+      await rm(dir, { recursive: true, force: true }).catch(() => {
+      });
+    }
+  }
+};
+function defaultFfmpegSplitter(bin) {
+  return async (audioPath, segmentSeconds, outDir) => {
+    const ext = extname(audioPath) || ".mp3";
+    await execFileAsync2(
+      bin,
+      [
+        "-hide_banner",
+        "-loglevel",
+        "error",
+        "-i",
+        audioPath,
+        "-f",
+        "segment",
+        "-segment_time",
+        String(segmentSeconds),
+        "-c",
+        "copy",
+        join(outDir, `part%03d${ext}`)
+      ],
+      { maxBuffer: 16 * 1024 * 1024 }
+    );
+    const files = (await readdir(outDir)).filter((f) => f.startsWith("part") && f.endsWith(ext)).sort();
+    return files.map((f) => join(outDir, f));
+  };
+}
+
+// src/ports/composite-fetcher.ts
+var CompositeFetcher = class {
+  children;
+  constructor(children) {
+    this.children = children;
+  }
+  async fetch(url) {
+    for (const child of this.children) {
+      const result = await child.fetch(url);
+      if (result) return result;
+    }
+    return null;
+  }
+};
+
+// src/ports/throttle-fetcher.adapter.ts
+var ThrottleFetcher = class {
+  inner;
+  minIntervalMs;
+  now;
+  sleep;
+  lastStart = 0;
+  started = false;
+  constructor(inner, opts) {
+    this.inner = inner;
+    this.minIntervalMs = Math.max(0, opts.minIntervalMs);
+    this.now = opts.now ?? (() => Date.now());
+    this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  }
+  async fetch(url) {
+    if (this.minIntervalMs > 0 && this.started) {
+      const wait = this.minIntervalMs - (this.now() - this.lastStart);
+      if (wait > 0) await this.sleep(wait);
+    }
+    this.started = true;
+    this.lastStart = this.now();
+    return this.inner.fetch(url);
+  }
+};
+
+// src/ports/http-readability-fetcher.adapter.ts
+var DEFAULT_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0 Safari/537.36";
+var HttpReadabilityFetcher = class {
+  ua;
+  maxChars;
+  constructor(opts = {}) {
+    this.ua = opts.userAgent ?? DEFAULT_UA;
+    this.maxChars = opts.maxChars ?? 2e5;
+  }
+  async fetch(url) {
+    if (isVideoUrl(url)) return null;
+    let res;
+    try {
+      res = await fetch(url, {
+        headers: { "user-agent": this.ua, "accept-language": "en,fr;q=0.8" },
+        redirect: "follow"
+      });
+    } catch {
+      return null;
+    }
+    if (!res.ok) return null;
+    const ct = res.headers.get("content-type") ?? "";
+    if (!ct.includes("text/html") && !ct.includes("application/xhtml")) return null;
+    const html = await res.text();
+    const title = decodeEntities(
+      (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").trim()
+    );
+    const lang = html.match(/<html[^>]*\blang=["']?([a-zA-Z-]+)/i)?.[1];
+    const text = extractReadable(html).slice(0, this.maxChars);
+    if (!text.trim()) return null;
+    return {
+      title: title || url,
+      text,
+      kind: "article",
+      ...lang ? { language: lang } : {},
+      via: "readability"
+    };
+  }
+};
+function extractReadable(html) {
+  const region = html.match(/<article[\s\S]*?<\/article>/i)?.[0] ?? html.match(/<main[\s\S]*?<\/main>/i)?.[0] ?? html.match(/<body[\s\S]*?<\/body>/i)?.[0] ?? html;
+  return decodeEntities(
+    region.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<(nav|header|footer|aside|form)[\s\S]*?<\/\1>/gi, " ").replace(/<[^>]+>/g, " ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n")
+  ).trim();
+}
+function decodeEntities(s) {
+  return s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)));
+}
+
+// src/ports/anthropic-distiller.ts
+var ANTHROPIC_RESPONSE = z.object({
+  model: z.string().optional(),
+  content: z.array(z.object({ type: z.string(), text: z.string().optional() }).loose()).optional(),
+  usage: z.object({ input_tokens: z.number(), output_tokens: z.number() }).loose().optional()
+}).loose();
+var AnthropicDistiller = class {
+  apiKey;
+  model;
+  baseUrl;
+  maxItems;
+  onUsage;
+  constructor(opts) {
+    this.apiKey = opts.apiKey;
+    this.model = opts.model ?? "claude-sonnet-4-6";
+    this.baseUrl = (opts.baseUrl ?? "https://api.anthropic.com/v1").replace(/\/+$/, "");
+    this.maxItems = opts.maxItems ?? 8;
+    this.onUsage = opts.onUsage;
+  }
+  async distill(input) {
+    const prompt = buildDistillPrompt(input, this.maxItems);
+    const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+    const res = await fetch(`${this.baseUrl}/messages`, {
+      method: "POST",
+      headers: {
+        "x-api-key": this.apiKey,
+        "anthropic-version": "2023-06-01",
+        "content-type": "application/json"
+      },
+      body: JSON.stringify({
+        model: this.model,
+        max_tokens: 4096,
+        messages: [{ role: "user", content: prompt }]
+      })
+    });
+    if (!res.ok) {
+      throw new Error(`Anthropic distill ${res.status}: ${(await res.text()).slice(0, 200)}`);
+    }
+    const parsed = ANTHROPIC_RESPONSE.safeParse(await res.json());
+    const text = parsed.success ? (parsed.data.content ?? []).find((c) => c.type === "text")?.text ?? "" : "";
+    if (parsed.success && parsed.data.usage && this.onUsage) {
+      this.onUsage({
+        model: parsed.data.model ?? this.model,
+        inputTokens: parsed.data.usage.input_tokens,
+        outputTokens: parsed.data.usage.output_tokens,
+        label: input.title,
+        startedAt,
+        endedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+    }
+    return parseItems(text);
+  }
+};
+var CliAgentDistiller = class {
+  engine;
+  model;
+  maxItems;
+  timeoutMs;
+  cwd;
+  constructor(opts) {
+    this.engine = opts.engine;
+    this.model = opts.model;
+    this.maxItems = opts.maxItems ?? 8;
+    this.timeoutMs = opts.timeoutMs ?? 12e4;
+    this.cwd = opts.cwd ?? tmpdir();
+  }
+  async distill(input) {
+    const prompt = buildDistillPrompt(input, this.maxItems);
+    const stdout = await spawnWithStdin({
+      command: this.engine.command,
+      args: this.engine.buildArgs({ ...this.model ? { model: this.model } : {} }),
+      stdin: prompt,
+      cwd: this.cwd,
+      timeoutMs: this.timeoutMs
+    });
+    const text = this.engine.parseOutput(stdout) ?? stdout;
+    return parseItems(text);
+  }
+};
+var CLAUDE_CODE = {
+  id: "claude-code",
+  subscriptionBilled: true,
+  command: "claude",
+  buildArgs: ({ model }) => [
+    "-p",
+    "--output-format",
+    "json",
+    "--strict-mcp-config",
+    ...model ? ["--model", model] : []
+  ],
+  parseOutput: parseClaudeJsonOutput
+};
+var CLI_ENGINES = {
+  [CLAUDE_CODE.id]: CLAUDE_CODE
+};
+var RPC_RESPONSE = z.object({ result: z.unknown().optional(), error: z.unknown().optional() }).loose();
+var RPC_MESSAGE = z.object({
+  id: z.number().optional(),
+  result: z.unknown().optional(),
+  error: z.unknown().optional()
+}).loose();
+var CALL_RESULT = z.object({
+  content: z.unknown().optional(),
+  structuredContent: z.unknown().optional(),
+  isError: z.boolean().optional()
+}).loose();
+async function connectMcpHttp(opts) {
+  if (opts.transport === "sse") {
+    const sse = new SseMcpClient(opts);
+    await sse.connect();
+    return sse;
+  }
+  const client = new StreamableHttpMcpClient(opts);
+  await client.initialize();
+  return client;
+}
+var StreamableHttpMcpClient = class {
+  id = 0;
+  sessionId;
+  endpoint;
+  headers;
+  protocolVersion;
+  constructor(opts) {
+    this.endpoint = opts.endpoint;
+    this.headers = opts.headers ?? {};
+    this.protocolVersion = opts.protocolVersion ?? "2025-06-18";
+  }
+  async initialize() {
+    const res = await this.rpc({
+      jsonrpc: "2.0",
+      id: ++this.id,
+      method: "initialize",
+      params: {
+        protocolVersion: this.protocolVersion,
+        capabilities: {},
+        clientInfo: { name: "corpus-cli", version: "0.1.0" }
+      }
+    });
+    await this.rpc({ jsonrpc: "2.0", method: "notifications/initialized", params: {} }, false);
+    if (res?.error) throw new Error(`initialize failed: ${JSON.stringify(res.error)}`);
+  }
+  async callTool(name, args) {
+    const res = await this.rpc({
+      jsonrpc: "2.0",
+      id: ++this.id,
+      method: "tools/call",
+      params: { name, arguments: args }
+    });
+    if (res?.error) throw new Error(`tools/call ${name}: ${JSON.stringify(res.error)}`);
+    const parsed = CALL_RESULT.safeParse(res?.result ?? {});
+    return parsed.success ? parsed.data : {};
+  }
+  async rpc(body, expectResponse = true) {
+    const res = await fetch(this.endpoint, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        accept: "application/json, text/event-stream",
+        ...this.sessionId ? { "mcp-session-id": this.sessionId } : {},
+        ...this.headers
+      },
+      body: JSON.stringify(body)
+    });
+    const sid = res.headers.get("mcp-session-id");
+    if (sid) this.sessionId = sid;
+    if (!expectResponse) return null;
+    if (!res.ok) throw new Error(`MCP HTTP ${res.status} ${res.statusText}`);
+    return parseRpc(await res.text(), res.headers.get("content-type"));
+  }
+};
+var SseMcpClient = class {
+  constructor(opts) {
+    this.opts = opts;
+    this.base = new URL(opts.endpoint);
+  }
+  id = 0;
+  messagesUrl;
+  pending = /* @__PURE__ */ new Map();
+  readyResolve;
+  ready = new Promise((r) => this.readyResolve = r);
+  base;
+  async connect() {
+    const res = await fetch(this.opts.endpoint, {
+      headers: { accept: "text/event-stream", ...this.opts.headers ?? {} }
+    });
+    if (!res.ok || !res.body) throw new Error(`SSE connect ${res.status} ${res.statusText}`);
+    void this.readLoop(res.body.getReader());
+    await withTimeout(this.ready, 1e4, "SSE endpoint event");
+    const init = await this.send("initialize", {
+      protocolVersion: this.opts.protocolVersion ?? "2025-06-18",
+      capabilities: {},
+      clientInfo: { name: "corpus-cli", version: "0.1.0" }
+    });
+    if (init?.error) throw new Error(`initialize failed: ${JSON.stringify(init.error)}`);
+    await this.post({ jsonrpc: "2.0", method: "notifications/initialized", params: {} });
+  }
+  async callTool(name, args) {
+    const res = await this.send("tools/call", { name, arguments: args });
+    if (res?.error) throw new Error(`tools/call ${name}: ${JSON.stringify(res.error)}`);
+    const parsed = CALL_RESULT.safeParse(res?.result ?? {});
+    return parsed.success ? parsed.data : {};
+  }
+  async send(method, params) {
+    const id = ++this.id;
+    const wait = new Promise((resolve) => this.pending.set(id, resolve));
+    await this.post({ jsonrpc: "2.0", id, method, params });
+    return withTimeout(wait, 6e4, `${method} response`);
+  }
+  async post(body) {
+    if (!this.messagesUrl) throw new Error("SSE messages endpoint not ready");
+    const res = await fetch(this.messagesUrl, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...this.opts.headers ?? {} },
+      body: JSON.stringify(body)
+    });
+    if (!res.ok && res.status !== 202) throw new Error(`SSE POST ${res.status}`);
+  }
+  async readLoop(reader) {
+    const dec = new TextDecoder();
+    let buf = "";
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buf += dec.decode(value, { stream: true });
+      let nl;
+      while ((nl = buf.indexOf("\n\n")) >= 0) {
+        const raw = buf.slice(0, nl);
+        buf = buf.slice(nl + 2);
+        this.handleEvent(raw);
+      }
+    }
+  }
+  handleEvent(raw) {
+    let event = "message";
+    const data = [];
+    for (const line of raw.split("\n")) {
+      if (line.startsWith("event:")) event = line.slice(6).trim();
+      else if (line.startsWith("data:")) data.push(line.slice(5).trim());
+    }
+    const payload = data.join("\n");
+    if (event === "endpoint") {
+      this.messagesUrl = new URL(payload, this.base).toString();
+      this.readyResolve();
+      return;
+    }
+    if (!payload) return;
+    try {
+      const parsed = RPC_MESSAGE.safeParse(JSON.parse(payload));
+      if (!parsed.success) return;
+      const msg3 = parsed.data;
+      if (typeof msg3.id === "number" && this.pending.has(msg3.id)) {
+        this.pending.get(msg3.id)(msg3);
+        this.pending.delete(msg3.id);
+      }
+    } catch {
+    }
+  }
+};
+function withTimeout(p, ms, what) {
+  return Promise.race([
+    p,
+    new Promise((_, rej) => setTimeout(() => rej(new Error(`timeout waiting for ${what}`)), ms))
+  ]);
+}
+function parseRpc(text, contentType) {
+  if (!text.trim()) return null;
+  if (!(contentType ?? "").includes("text/event-stream")) {
+    const parsed = RPC_RESPONSE.safeParse(JSON.parse(text));
+    return parsed.success ? parsed.data : null;
+  }
+  let last = null;
+  for (const line of text.split("\n")) {
+    const t = line.trim();
+    if (!t.startsWith("data:")) continue;
+    const p = t.slice(5).trim();
+    if (!p || p === "[DONE]") continue;
+    try {
+      last = JSON.parse(p);
+    } catch {
+    }
+  }
+  return last;
+}
+z.object({
+  endpoint: z.string().min(1),
+  tool: z.string().min(1),
+  /** Arg template — substituted per entry. */
+  args: z.record(z.string(), z.unknown()),
+  headers: z.record(z.string(), z.string()).optional(),
+  /** MCP transport: "streamable-http" (default) or "sse". */
+  transport: z.enum(["streamable-http", "sse"]).optional(),
+  /**
+   * If set, calls go through `mcp_imported_call({ alias, toolName, args })`
+   * (the agentproto daemon's imported-MCP proxy) instead of the tool directly.
+   */
+  importedAlias: z.string().optional()
+});
+var McpSink = class {
+  constructor(config, client) {
+    this.config = config;
+    this.client = client;
+  }
+  client;
+  async ensure() {
+    if (!this.client) {
+      this.client = await connectMcpHttp({
+        endpoint: this.config.endpoint,
+        ...this.config.headers ? { headers: this.config.headers } : {},
+        ...this.config.transport ? { transport: this.config.transport } : {}
+      });
+    }
+    return this.client;
+  }
+  async push(item) {
+    let client;
+    try {
+      client = await this.ensure();
+    } catch (e) {
+      return { uri: item.uri, ok: false, error: `connect: ${msg2(e)}` };
+    }
+    const args = template(this.config.args, item);
+    const call = this.config.importedAlias ? { name: "mcp_imported_call", args: { alias: this.config.importedAlias, toolName: this.config.tool, args } } : { name: this.config.tool, args };
+    try {
+      const res = await client.callTool(call.name, call.args);
+      if (res.isError) return { uri: item.uri, ok: false, error: "tool returned isError" };
+      return { uri: item.uri, ok: true };
+    } catch (e) {
+      return { uri: item.uri, ok: false, error: msg2(e) };
+    }
+  }
+};
+var PLACEHOLDER_RE = /\$\{(\w+)\}/g;
+function template(args, item) {
+  const fields = {
+    slug: item.slug,
+    title: item.title,
+    body: item.body,
+    sources: item.sources,
+    tags: item.tags,
+    access: item.access ?? "",
+    uri: item.uri,
+    kind: item.kind,
+    confidence: item.confidence
+  };
+  const out = {};
+  for (const [key, value] of Object.entries(args)) {
+    out[key] = substitute(value, fields);
+  }
+  return out;
+}
+function substitute(value, fields) {
+  if (typeof value === "string") {
+    const whole = value.match(/^\$\{(\w+)\}$/);
+    if (whole && whole[1] in fields) return fields[whole[1]];
+    return value.replace(
+      PLACEHOLDER_RE,
+      (_, k) => k in fields ? stringify(fields[k]) : `\${${k}}`
+    );
+  }
+  if (Array.isArray(value)) return value.map((v) => substitute(v, fields));
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = substitute(v, fields);
+    return out;
+  }
+  return value;
+}
+function stringify(v) {
+  if (Array.isArray(v)) return v.join(", ");
+  return v == null ? "" : String(v);
+}
+function msg2(e) {
+  return e instanceof Error ? e.message : String(e);
+}
+
+export { AnthropicDistiller, AssemblyAiStt, BrowserMcpFetcher, CLI_ENGINES, ChunkedStt, CliAgentDistiller, CompositeFetcher, HttpReadabilityFetcher, McpSink, NodeFsAdapter, OpenAiWhisperStt, OsIdentityAdapter, ScrapeMcpFetcher, ThrottleFetcher, YtDlpWhisperFetcher, connectMcpHttp };
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map