imperium-crawl 2.4.0 → 2.5.1

package/dist/tools/watch.js

@@ -0,0 +1,224 @@
+/**
+ * watch — one-shot change detector for URLs.
+ *
+ * v2.5.0: one-shot mode only. Snapshots content, hashes it, diffs against
+ * the previous snapshot for the same URL. Fires a webhook on change.
+ *
+ * Daemon mode (SIGINT loop) is deferred to v2.6.0 — use cron externally:
+ *   * /30 * * * *  imperium-crawl watch --url X --output-dir /var/watch
+ */
+import { z } from "zod";
+import { createHash } from "node:crypto";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { join, resolve as resolvePath } from "node:path";
+import { JSDOM } from "jsdom";
+import { Readability } from "@mozilla/readability";
+import { fetchPage } from "../utils/fetcher.js";
+import { htmlToMarkdown } from "../utils/markdown.js";
+import { toolResult, errorResult } from "../utils/tool-response.js";
+import { debugLog } from "../utils/debug.js";
+import { MAX_URL_LENGTH } from "../constants.js";
+export const name = "watch";
+export const description = "One-shot change detector: scrape a URL, hash its content, and compare against the last snapshot. Fires a webhook on change. Run via cron for periodic monitoring.";
+export const schema = z.object({
+    url: z
+        .string()
+        .max(MAX_URL_LENGTH)
+        .describe("URL to watch"),
+    output_dir: z
+        .string()
+        .default("./data/watch")
+        .describe("Directory to persist snapshots and state"),
+    hash_on: z
+        .enum(["content", "readability", "markdown"])
+        .default("readability")
+        .describe("What to hash: full HTML, readability main content, or markdown"),
+    webhook: z
+        .string()
+        .max(MAX_URL_LENGTH)
+        .optional()
+        .describe("If set, POST a JSON payload to this URL on detected change"),
+    diff_format: z
+        .enum(["unified", "json"])
+        .default("unified")
+        .describe("Diff representation in the result"),
+    one_shot: z
+        .boolean()
+        .default(true)
+        .describe("v2.5.0: always true. Daemon mode lands in v2.6.0."),
+});
+function slugify(url) {
+    return createHash("sha1").update(url).digest("hex").slice(0, 16);
+}
+function hashString(s) {
+    return createHash("sha256").update(s).digest("hex");
+}
+function simpleUnifiedDiff(prev, next, maxLines = 200) {
+    const prevLines = prev.split("\n");
+    const nextLines = next.split("\n");
+    const prevSet = new Set(prevLines);
+    const nextSet = new Set(nextLines);
+    const out = [];
+    let removed = 0;
+    let added = 0;
+    for (const line of prevLines) {
+        if (!nextSet.has(line)) {
+            out.push(`- ${line}`);
+            removed++;
+        }
+    }
+    for (const line of nextLines) {
+        if (!prevSet.has(line)) {
+            out.push(`+ ${line}`);
+            added++;
+        }
+    }
+    const header = `@@ -${prevLines.length} +${nextLines.length} @@ (${removed} removed, ${added} added)`;
+    const body = out.slice(0, maxLines).join("\n");
+    const truncated = out.length > maxLines ? `\n... (${out.length - maxLines} more lines)` : "";
+    return `${header}\n${body}${truncated}`;
+}
+function jsonDiff(prev, next) {
+    const prevLines = prev.split("\n");
+    const nextLines = next.split("\n");
+    const prevSet = new Set(prevLines);
+    const nextSet = new Set(nextLines);
+    const removed = prevLines.filter((l) => !nextSet.has(l));
+    const added = nextLines.filter((l) => !prevSet.has(l));
+    return JSON.stringify({ removed, added, prev_lines: prevLines.length, next_lines: nextLines.length });
+}
+export async function computeSignature(html, url, hashOn) {
+    if (hashOn === "content")
+        return html;
+    if (hashOn === "markdown")
+        return htmlToMarkdown(html);
+    // readability
+    try {
+        const dom = new JSDOM(html, { url });
+        const reader = new Readability(dom.window.document);
+        const article = reader.parse();
+        if (article?.textContent)
+            return article.textContent.trim();
+        return htmlToMarkdown(html);
+    }
+    catch {
+        return htmlToMarkdown(html);
+    }
+}
+async function loadState(stateFile) {
+    if (!existsSync(stateFile))
+        return {};
+    try {
+        const raw = await readFile(stateFile, "utf-8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return {};
+    }
+}
+async function saveState(stateFile, state) {
+    await writeFile(stateFile, JSON.stringify(state, null, 2), "utf-8");
+}
+async function fireWebhook(webhook, payload) {
+    try {
+        const res = await fetch(webhook, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(payload),
+        });
+        return { fired: true, status: res.status };
+    }
+    catch (err) {
+        debugLog("watch", "webhook failed", err);
+        return { fired: false };
+    }
+}
+export async function runWatchOnce(input) {
+    const outDir = resolvePath(input.output_dir);
+    await mkdir(outDir, { recursive: true });
+    const stateFile = join(outDir, ".state.json");
+    const state = await loadState(stateFile);
+    const fetched = await fetchPage(input.url);
+    const signature = await computeSignature(fetched.html, input.url, input.hash_on);
+    const currentHash = hashString(signature);
+    const slug = slugify(input.url);
+    const snapshotFile = join(outDir, `${slug}.snapshot.txt`);
+    const prevSnapshotFile = join(outDir, `${slug}.previous.txt`);
+    const existing = state[input.url];
+    const firstRun = !existing;
+    const changed = !firstRun && existing.last_hash !== currentHash;
+    let previousSig = null;
+    if (existsSync(snapshotFile)) {
+        try {
+            previousSig = await readFile(snapshotFile, "utf-8");
+        }
+        catch {
+            previousSig = null;
+        }
+    }
+    // Rotate previous snapshot only when content changed
+    if (changed && previousSig !== null) {
+        await writeFile(prevSnapshotFile, previousSig, "utf-8");
+    }
+    await writeFile(snapshotFile, signature, "utf-8");
+    const nowIso = new Date().toISOString();
+    const newState = {
+        url: input.url,
+        last_hash: currentHash,
+        last_checked: nowIso,
+        last_changed: changed ? nowIso : existing?.last_changed ?? null,
+        hash_on: input.hash_on,
+        check_count: (existing?.check_count ?? 0) + 1,
+        change_count: (existing?.change_count ?? 0) + (changed ? 1 : 0),
+    };
+    state[input.url] = newState;
+    await saveState(stateFile, state);
+    let diff = null;
+    if (changed && previousSig !== null) {
+        diff =
+            input.diff_format === "unified"
+                ? simpleUnifiedDiff(previousSig, signature)
+                : jsonDiff(previousSig, signature);
+    }
+    let webhookFired = false;
+    let webhookStatus;
+    if (changed && input.webhook) {
+        const payload = {
+            event: "watch.change",
+            url: input.url,
+            previous_hash: existing?.last_hash ?? null,
+            current_hash: currentHash,
+            detected_at: nowIso,
+            diff,
+        };
+        const res = await fireWebhook(input.webhook, payload);
+        webhookFired = res.fired;
+        webhookStatus = res.status;
+    }
+    return {
+        url: input.url,
+        changed,
+        first_run: firstRun,
+        previous_hash: existing?.last_hash ?? null,
+        current_hash: currentHash,
+        hash_on: input.hash_on,
+        snapshot_file: snapshotFile,
+        diff,
+        webhook_fired: webhookFired,
+        webhook_status: webhookStatus,
+        state: newState,
+        checked_at: nowIso,
+    };
+}
+export async function execute(input) {
+    try {
+        const result = await runWatchOnce(input);
+        return toolResult(result);
+    }
+    catch (err) {
+        debugLog("watch", "failed", err);
+        return errorResult(err instanceof Error ? err.message : String(err));
+    }
+}
+//# sourceMappingURL=watch.js.map

package/dist/tools/watch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watch.js","sourceRoot":"","sources":["../../src/tools/watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC;AAE5B,MAAM,CAAC,MAAM,WAAW,GACtB,mKAAmK,CAAC;AAEtK,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,CAAC,cAAc,CAAC;IAC3B,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,OAAO,CAAC,cAAc,CAAC;SACvB,QAAQ,CAAC,0CAA0C,CAAC;IACvD,OAAO,EAAE,CAAC;SACP,IAAI,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;SAC5C,OAAO,CAAC,aAAa,CAAC;SACtB,QAAQ,CAAC,gEAAgE,CAAC;IAC7E,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;IACzE,WAAW,EAAE,CAAC;SACX,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;SACzB,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CAAC,mCAAmC,CAAC;IAChD,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,mDAAmD,CAAC;CACjE,CAAC,CAAC;AA6BH,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,IAAY,EAAE,QAAQ,GAAG,GAAG;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAEnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,QAAQ,OAAO,aAAa,KAAK,SAAS,CAAC;IACtG,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,MAAM,GAAG,QAAQ,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7F,OAAO,GAAG,MAAM,KAAK,IAAI,GAAG,SAAS,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAY;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;AACxG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,GAAW,EACX,MAA6B;IAE7B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAEvD,cAAc;IACd,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,OAAO,EAAE,WAAW;YAAE,OAAO,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAC5D,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB;IACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA+B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB,EAAE,KAAiC;IAC3E,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAe,EACf,OAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB;IAClD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,WAAW,CAAC;IAEhE,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,SAAS,CAAC,gBAAgB,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,SAAS,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAe;QAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,YAAY,IAAI,IAAI;QAC/D,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,WAAW,EAAE,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;QAC7C,YAAY,EAAE,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAChE,CAAC;IACF,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;IAC5B,MAAM,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAElC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,IAAI;YACF,KAAK,CAAC,WAAW,KAAK,SAAS;gBAC7B,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE,SAAS,CAAC;gBAC3C,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,aAAiC,CAAC;IACtC,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,cAAc;YACrB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;YAC1C,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,IAAI;SACL,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtD,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC;QACzB,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,OAAO;QACP,SAAS,EAAE,QAAQ;QACnB,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QAC1C,YAAY,EAAE,WAAW;QACzB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,aAAa,EAAE,YAAY;QAC3B,IAAI;QACJ,aAAa,EAAE,YAAY;QAC3B,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACjC,OAAO,WAAW,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}

package/dist/utils/fetcher.d.ts

@@ -20,10 +20,19 @@ declare const CIRCUIT_OPEN_DURATION_MS = 60000;
 declare const CIRCUIT_PROBE_SUCCESSES = 3;
 declare const circuits: Map<string, CircuitBreaker>;
 declare const CIRCUIT_STALE_MS = 3600000;
-declare function getCircuit(domain: string): CircuitBreaker;
-declare function recordSuccess(domain: string): void;
-declare function recordFailure(domain: string): void;
-export { circuits, getCircuit, recordSuccess, recordFailure, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
+/**
+ * Get circuit breaker key for a URL.
+ * Uses domain + first 2 path segments for endpoint-level granularity.
+ */
+declare function getCircuitKey(url: string): string;
+declare function getCircuit(key: string): CircuitBreaker;
+declare function recordSuccess(key: string): void;
+declare function recordFailure(key: string): void;
+/**
+ * Check domain-level circuit: opens when 3+ endpoint circuits are open for this domain.
+ */
+declare function isDomainCircuitOpen(domain: string): boolean;
+export { circuits, getCircuit, getCircuitKey, recordSuccess, recordFailure, isDomainCircuitOpen, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
 export interface SmartFetchOptions extends StealthOptions {
     respectRobots?: boolean;
     retries?: number;

package/dist/utils/fetcher.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAgD9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAsD9F"}
+{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAqB,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQzH,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAIlC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC;;;GAGG;AACH,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAS1C;AAED,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAgB/C;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAYxC;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAcxC;AAED;;GAEG;AACH,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAQpD;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAsDlM,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA+BD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiG9F"}

package/dist/utils/fetcher.js

@@ -1,7 +1,7 @@
 import { smartFetch, StealthError } from "../stealth/index.js";
 import { isAllowed } from "./robots.js";
 import { getDomain } from "./url.js";
-import { DEFAULT_CONCURRENCY } from "../constants.js";
+import { DEFAULT_CONCURRENCY, DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS } from "../constants.js";
 import { getKnowledgeEngine } from "../knowledge/index.js";
 // ── Concurrency Limiter ──
 export class ConcurrencyLimiter {
@@ -31,22 +31,39 @@ export const defaultLimiter = new ConcurrencyLimiter();
 const CIRCUIT_FAILURE_THRESHOLD = 5;
 const CIRCUIT_OPEN_DURATION_MS = 60_000;
 const CIRCUIT_PROBE_SUCCESSES = 3;
+// Domain-level circuit: higher threshold — only opens when multiple endpoints fail
+const DOMAIN_CIRCUIT_FAILURE_THRESHOLD = 10;
 const circuits = new Map();
 // Periodic cleanup: remove closed circuits idle for >1 hour
 const CIRCUIT_STALE_MS = 3_600_000;
 setInterval(() => {
     const now = Date.now();
-    for (const [domain, circuit] of circuits) {
+    for (const [key, circuit] of circuits) {
         if (now - circuit.lastAccessed > CIRCUIT_STALE_MS) {
-            circuits.delete(domain);
+            circuits.delete(key);
         }
     }
 }, 300_000).unref();
-function getCircuit(domain) {
-    let circuit = circuits.get(domain);
+/**
+ * Get circuit breaker key for a URL.
+ * Uses domain + first 2 path segments for endpoint-level granularity.
+ */
+function getCircuitKey(url) {
+    try {
+        const parsed = new URL(url);
+        const pathParts = parsed.pathname.split("/").filter(Boolean);
+        const pathPrefix = pathParts.slice(0, 2).join("/");
+        return pathPrefix ? `${parsed.hostname}/${pathPrefix}` : parsed.hostname;
+    }
+    catch {
+        return getDomain(url);
+    }
+}
+function getCircuit(key) {
+    let circuit = circuits.get(key);
     if (!circuit) {
         circuit = { state: "closed", failures: 0, openedAt: 0, probeSuccesses: 0, lastAccessed: Date.now() };
-        circuits.set(domain, circuit);
+        circuits.set(key, circuit);
     }
     circuit.lastAccessed = Date.now();
     // Check if open circuit should transition to half-open
@@ -56,8 +73,8 @@ function getCircuit(domain) {
     }
     return circuit;
 }
-function recordSuccess(domain) {
-    const circuit = getCircuit(domain);
+function recordSuccess(key) {
+    const circuit = getCircuit(key);
     if (circuit.state === "half-open") {
         circuit.probeSuccesses++;
         if (circuit.probeSuccesses >= CIRCUIT_PROBE_SUCCESSES) {
@@ -70,8 +87,8 @@ function recordSuccess(domain) {
         circuit.failures = 0;
     }
 }
-function recordFailure(domain) {
-    const circuit = getCircuit(domain);
+function recordFailure(key) {
+    const circuit = getCircuit(key);
     // Half-open probe failed → immediately reopen circuit
     if (circuit.state === "half-open") {
         circuit.state = "open";
@@ -85,14 +102,32 @@ function recordFailure(domain) {
         circuit.openedAt = Date.now();
     }
 }
+/**
+ * Check domain-level circuit: opens when 3+ endpoint circuits are open for this domain.
+ */
+function isDomainCircuitOpen(domain) {
+    let openEndpoints = 0;
+    for (const [key, circuit] of circuits) {
+        if (key.startsWith(domain) && circuit.state === "open") {
+            openEndpoints++;
+        }
+    }
+    return openEndpoints >= 3;
+}
 // Exported for testing
-export { circuits, getCircuit, recordSuccess, recordFailure, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
+export { circuits, getCircuit, getCircuitKey, recordSuccess, recordFailure, isDomainCircuitOpen, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
 // ── Exponential Backoff with Full Jitter (AWS pattern) ──
 const BACKOFF_BASE_MS = 1000;
 const BACKOFF_CAP_MS = 30_000;
-function fullJitterBackoff(attempt) {
+const RATE_LIMIT_EXTRA_JITTER_MS = 10_000; // Extra jitter for 429 responses
+function fullJitterBackoff(attempt, is429 = false) {
     const expDelay = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * Math.pow(2, attempt));
-    return Math.random() * expDelay;
+    const baseJitter = Math.random() * expDelay;
+    // On 429, add extra random jitter to avoid thundering herd
+    if (is429) {
+        return baseJitter + 5000 + Math.random() * RATE_LIMIT_EXTRA_JITTER_MS;
+    }
+    return baseJitter;
 }
 // ── Per-Domain Rate Limiter ──
 const DEFAULT_DOMAIN_RATE_MS = parseInt(process.env.DOMAIN_RATE_LIMIT_MS || "500", 10);
@@ -120,6 +155,31 @@ class DomainThrottle {
     }
 }
 const domainThrottle = new DomainThrottle();
+/**
+ * Compute adaptive timeout based on knowledge engine data.
+ * Uses avg_response_time * 3 with a floor of DEFAULT_TIMEOUT_MS and ceiling of MAX_TIMEOUT_MS.
+ */
+function computeAdaptiveTimeout(avgResponseTimeMs) {
+    if (!avgResponseTimeMs || avgResponseTimeMs <= 0)
+        return DEFAULT_TIMEOUT_MS;
+    return Math.min(MAX_TIMEOUT_MS, Math.max(DEFAULT_TIMEOUT_MS, avgResponseTimeMs * 3));
+}
+/**
+ * Determine the escalated stealth level for a retry attempt.
+ * attempt 0: user's level, attempt 1: level+1, attempt 2: L3
+ */
+function getEscalatedLevel(baseLevel, attempt, lastError) {
+    // If last failure was a StealthError with detected anti-bot, jump to L3
+    if (lastError instanceof StealthError && lastError.antiBotSystem) {
+        return 3;
+    }
+    const base = baseLevel || 1;
+    if (attempt === 0)
+        return base;
+    if (attempt === 1)
+        return Math.min(base + 1, 3);
+    return 3; // attempt >= 2 → always L3
+}
 export async function fetchPage(url, options) {
     const respectRobots = options?.respectRobots ?? (process.env.RESPECT_ROBOTS !== "false");
     if (respectRobots) {
@@ -128,40 +188,77 @@ export async function fetchPage(url, options) {
             throw new Error(`URL blocked by robots.txt: ${url}`);
         }
     }
-    // Circuit breaker check
+    // Per-endpoint circuit breaker check
     const domain = getDomain(url);
-    const circuit = getCircuit(domain);
+    const circuitKey = getCircuitKey(url);
+    const circuit = getCircuit(circuitKey);
     if (circuit.state === "open") {
-        throw new Error(`Circuit breaker open for ${domain} — too many consecutive failures. Retry after cooldown.`);
+        throw new Error(`Circuit breaker open for endpoint ${circuitKey} — too many consecutive failures. Retry after cooldown.`);
+    }
+    // Domain-level circuit check (opens when 3+ endpoints are broken)
+    if (isDomainCircuitOpen(domain)) {
+        throw new Error(`Circuit breaker open for domain ${domain} — multiple endpoints failing. Retry after cooldown.`);
     }
     // Per-domain rate limiting — use knowledge engine's safe_rate_limit if available
-    const knowledge = await getKnowledgeEngine().get(domain);
+    const engine = getKnowledgeEngine();
+    const knowledge = await engine.get(domain);
     const knowledgeDelayMs = knowledge?.safe_rate_limit
         ? Math.round(60_000 / knowledge.safe_rate_limit)
         : undefined;
     await domainThrottle.throttle(domain, knowledgeDelayMs);
+    // ── Adaptive timeout from knowledge engine ──
+    const adaptiveTimeout = computeAdaptiveTimeout(knowledge?.avg_response_time_ms);
+    const timeout = options?.timeout || adaptiveTimeout;
     const retries = options?.retries ?? 2;
     let lastError;
+    let lastHttpStatus = 0;
     for (let attempt = 0; attempt <= retries; attempt++) {
         try {
-            const result = await smartFetch(url, options);
-            recordSuccess(domain);
+            // ── Smart retry: escalate stealth level on each attempt ──
+            const escalatedLevel = getEscalatedLevel(options?.forceLevel, attempt, lastError);
+            const attemptOptions = {
+                ...options,
+                timeout,
+                // On retry, escalate stealth level (unless user forced a specific level)
+                forceLevel: attempt > 0 && !options?.forceLevel ? escalatedLevel : options?.forceLevel,
+            };
+            const result = await smartFetch(url, attemptOptions);
+            recordSuccess(circuitKey);
+            // Feed successful strategy back to knowledge engine
+            if (attempt > 0 && result.level > 1) {
+                engine.record({
+                    url, domain,
+                    levelUsed: result.level,
+                    success: true,
+                    responseTimeMs: 0, // Already recorded by smartFetch
+                    antiBotSystem: result.antiBotSystem || null,
+                    captchaType: result.captchaSolved ? "detected" : null,
+                    proxyUsed: !!result.proxyUsed,
+                    blocked: false,
+                    httpStatus: result.status,
+                });
+            }
             return result;
         }
         catch (err) {
             lastError = err instanceof Error ? err : new Error(String(err));
-            recordFailure(domain);
-            // Check if circuit just opened
-            const updatedCircuit = getCircuit(domain);
+            recordFailure(circuitKey);
+            // Track HTTP status for backoff decisions
+            if (err instanceof StealthError) {
+                lastHttpStatus = err.httpStatus;
+            }
+            // Check if endpoint circuit just opened
+            const updatedCircuit = getCircuit(circuitKey);
             if (updatedCircuit.state === "open") {
                 // Enrich error message with StealthError info if available
                 const detail = err instanceof StealthError
                     ? `L${err.lastLevel} HTTP ${err.httpStatus}${err.antiBotSystem ? ` [${err.antiBotSystem}]` : ""}`
                     : "";
-                throw new Error(`Circuit breaker opened for ${domain}${detail ? ` (${detail})` : ""}: ${lastError.message}`);
+                throw new Error(`Circuit breaker opened for ${circuitKey}${detail ? ` (${detail})` : ""}: ${lastError.message}`);
             }
             if (attempt < retries) {
-                const delay = fullJitterBackoff(attempt);
+                const is429 = lastHttpStatus === 429;
+                const delay = fullJitterBackoff(attempt, is429);
                 await new Promise((r) => setTimeout(r, delay));
             }
         }

package/dist/utils/fetcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AAED,gCAAgC;AAEhC,MAAM,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AAEvF,MAAM,cAAc;IACV,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,YAAY,CAAS;IAE7B,YAAY,iBAAyB,sBAAsB;QACzD,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,gBAAyB;QACtD,MAAM,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QACpD,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAE3B,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAS5C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,iFAAiF;IACjF,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,SAAS,EAAE,eAAe;QACjD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAExD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAA4D,MAAM,qBAAqB,CAAC;AACzH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,mFAAmF;AACnF,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAE5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB;;;GAGG;AACH,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAc;IACzC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACvD,aAAa,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,aAAa,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAElM,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,0BAA0B,GAAG,MAAM,CAAC,CAAC,iCAAiC;AAE5E,SAAS,iBAAiB,CAAC,OAAe,EAAE,KAAK,GAAG,KAAK;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;IAC5C,2DAA2D;IAC3D,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,UAAU,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,0BAA0B,CAAC;IACxE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,gCAAgC;AAEhC,MAAM,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AAEvF,MAAM,cAAc;IACV,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,YAAY,CAAS;IAE7B,YAAY,iBAAyB,sBAAsB;QACzD,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,gBAAyB;QACtD,MAAM,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QACpD,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAE3B,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAS5C;;;GAGG;AACH,SAAS,sBAAsB,CAAC,iBAAqC;IACnE,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5E,OAAO,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC;AACvF,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CACxB,SAAmC,EACnC,OAAe,EACf,SAA4B;IAE5B,wEAAwE;IACxE,IAAI,SAAS,YAAY,YAAY,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;QACjE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,IAAI,CAAC,CAAC;IAC5B,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAiB,CAAC;IAChE,OAAO,CAAC,CAAC,CAAC,2BAA2B;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,yDAAyD,CAAC,CAAC;IAC5H,CAAC;IAED,kEAAkE;IAClE,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,sDAAsD,CAAC,CAAC;IACnH,CAAC;IAED,iFAAiF;IACjF,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,gBAAgB,GAAG,SAAS,EAAE,eAAe;QACjD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAExD,+CAA+C;IAC/C,MAAM,eAAe,GAAG,sBAAsB,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;IAChF,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,eAAe,CAAC;IAEpD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IACjC,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,4DAA4D;YAC5D,MAAM,cAAc,GAAG,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAClF,MAAM,cAAc,GAAmB;gBACrC,GAAG,OAAO;gBACV,OAAO;gBACP,yEAAyE;gBACzE,UAAU,EAAE,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,EAAE,UAAU;aACvF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;YACrD,aAAa,CAAC,UAAU,CAAC,CAAC;YAE1B,oDAAoD;YACpD,IAAI,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,CAAC,MAAM,CAAC;oBACZ,GAAG,EAAE,MAAM;oBACX,SAAS,EAAE,MAAM,CAAC,KAAK;oBACvB,OAAO,EAAE,IAAI;oBACb,cAAc,EAAE,CAAC,EAAE,iCAAiC;oBACpD,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,IAAI;oBAC3C,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;oBACrD,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS;oBAC7B,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,MAAM,CAAC,MAAM;iBAC1B,CAAC,CAAC;YACL,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,UAAU,CAAC,CAAC;YAE1B,0CAA0C;YAC1C,IAAI,GAAG,YAAY,YAAY,EAAE,CAAC;gBAChC,cAAc,GAAG,GAAG,CAAC,UAAU,CAAC;YAClC,CAAC;YAED,wCAAwC;YACxC,MAAM,cAAc,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAC9C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACnH,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,cAAc,KAAK,GAAG,CAAC;gBACrC,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAChD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "imperium-crawl",
-  "version": "2.4.0",
-  "description": "Open-source CLI tool for web scraping, crawling, search, and custom skills",
+  "version": "2.5.1",
+  "description": "39-tool open-source CLI for web scraping, PDF extraction, content monitoring, reusable browser flows, RSS aggregation, and custom skills. Zero API keys for core tools.",
   "type": "module",
   "bin": {
     "imperium-crawl": "dist/index.js",
@@ -26,7 +26,9 @@
     "start": "node dist/index.js",
     "test": "vitest run",
     "test:watch": "vitest",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "autoresearch": "tsx autoresearch/eval.ts",
+    "autoresearch:baseline": "tsx autoresearch/eval.ts --baseline --verbose"
   },
   "keywords": [
     "scraping",
@@ -34,7 +36,15 @@
     "web-search",
     "brave-search",
     "firecrawl",
-    "cli"
+    "cli",
+    "pdf-extract",
+    "web-monitoring",
+    "url-watch",
+    "content-diff",
+    "intelligence-digest",
+    "browser-workflows",
+    "workflow-recorder",
+    "flow-api"
   ],
   "author": "ImperiumTech",
   "license": "MIT",
@@ -55,6 +65,7 @@
     "normalize-url": "^8.1.1",
     "ora": "^8.2.0",
     "p-queue": "^8.1.1",
+    "pdfjs-dist": "^4.0.379",
     "playwright": "1.52",
     "robots-parser": "^3.0.1",
     "rss-parser": "^3.13.0",