@m4ike1/ion-coding-agent 0.1.0

package/dist/core/webfetch/backend-firecrawl.js

@@ -0,0 +1,339 @@
+/**
+ * Firecrawl backend for webfetch: thin `undici` POSTs to the hosted Firecrawl
+ * API. The SDK is never vendored (extra `axios` dep) and the server is never
+ * vendored (AGPL-3.0).
+ *
+ * Auth is an optional `Bearer` key, omitted entirely on the keyless tier.
+ * Resolve it with `resolveFirecrawlApiKey` (explicit settings value first,
+ * `FIRECRAWL_API_KEY` env second).
+ */
+import { request as undiciRequest } from "undici";
+import { clampLimit, WEBFETCH_FIRECRAWL_MAX_LIMIT, WEBFETCH_REQUEST_TIMEOUT_MS, WebfetchError, } from "./guardrails.js";
+export const FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
+export const FIRECRAWL_API_KEY_ENV = "FIRECRAWL_API_KEY";
+const FIRECRAWL_502_MAX_ATTEMPTS = 3;
+const FIRECRAWL_HTTP_HEADROOM_MS = 5_000;
+/** Explicit settings value wins; `FIRECRAWL_API_KEY` env is the fallback. Keyless (undefined) works with lower limits. */
+export function resolveFirecrawlApiKey(explicit, env = process.env) {
+    const trimmed = explicit?.trim() || env[FIRECRAWL_API_KEY_ENV]?.trim();
+    return trimmed || undefined;
+}
+export class FirecrawlError extends WebfetchError {
+    status;
+    /** True when the caller should try the `undici` direct-fetch fallback (transport/auth failure, never content failure). */
+    fallback;
+    constructor(message, options) {
+        super(message, options.code);
+        this.name = "FirecrawlError";
+        this.status = options.status;
+        this.fallback = options.fallback;
+    }
+}
+/**
+ * The keyless Firecrawl tier refused this network (403 "your IP address looks
+ * suspicious", 401 without a key, or 429 on the shared anonymous pool). This is
+ * provider-side and often IP-dependent, so the message names the exact fix
+ * instead of leaking the raw status. Direct page fetch still works through the
+ * `undici` fallback; only search is unavailable.
+ */
+export function keylessUnavailableMessage(status, path) {
+    const reason = status === 429
+        ? "the keyless rate limit was exhausted"
+        : status === 401
+            ? "no API key was sent"
+            : "this network is not allowed on the anonymous tier";
+    return [
+        `Firecrawl ${path} is unavailable: ${reason} (status ${status}).`,
+        "Set a Firecrawl API key to fix this: add the `firecrawlApiKey` setting or the",
+        "`FIRECRAWL_API_KEY` environment variable, then retry.",
+        "`webextract` still works without a key: it fetches pages directly.",
+    ].join(" ");
+}
+/** True when a failed Firecrawl call should fall back to direct `undici` fetch instead of surfacing the error. */
+export function isFallbackEligible(error) {
+    if (error instanceof FirecrawlError)
+        return error.fallback;
+    if (error instanceof WebfetchError) {
+        return error.code === "timeout" || error.code === "transport" || error.code === "empty";
+    }
+    return false;
+}
+async function defaultPost(path, body, init) {
+    const timeoutSignal = AbortSignal.timeout(init.timeoutMs + FIRECRAWL_HTTP_HEADROOM_MS);
+    const signal = init.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal;
+    let response;
+    try {
+        response = await undiciRequest(`${FIRECRAWL_BASE_URL}${path}`, {
+            method: "POST",
+            headers: { "content-type": "application/json", ...init.headers },
+            body: JSON.stringify(body),
+            signal,
+        });
+    }
+    catch (error) {
+        if (timeoutSignal.aborted && !init.signal?.aborted) {
+            throw new WebfetchError(`Firecrawl request to ${path} timed out`, "timeout");
+        }
+        if (init.signal?.aborted) {
+            throw new Error("Operation aborted");
+        }
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new WebfetchError(`Firecrawl request to ${path} failed: ${reason}`, "transport");
+    }
+    let text;
+    try {
+        text = await response.body.text();
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new WebfetchError(`Firecrawl request to ${path} failed: ${reason}`, "transport");
+    }
+    let json;
+    try {
+        json = text ? JSON.parse(text) : undefined;
+    }
+    catch {
+        throw new WebfetchError(`Firecrawl request to ${path} returned invalid JSON`, "transport");
+    }
+    return { status: response.statusCode, json };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function asString(value) {
+    return typeof value === "string" ? value : undefined;
+}
+function asNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+function firecrawlErrorMessage(json) {
+    if (!isRecord(json))
+        return undefined;
+    const direct = asString(json.error);
+    if (direct)
+        return direct;
+    if (isRecord(json.data)) {
+        const nested = asString(json.data.error);
+        if (nested)
+            return nested;
+    }
+    return undefined;
+}
+function readMeta(json) {
+    if (!isRecord(json))
+        return undefined;
+    const data = isRecord(json.data) ? json.data : undefined;
+    const warning = asString(json.warning) ?? (data ? asString(data.warning) : undefined);
+    const creditsUsed = asNumber(json.creditsUsed) ?? (data ? asNumber(data.creditsUsed) : undefined);
+    if (warning === undefined && creditsUsed === undefined)
+        return undefined;
+    return { warning, creditsUsed };
+}
+function normalizeSearchEntry(entry, source) {
+    if (!isRecord(entry))
+        return undefined;
+    const url = asString(entry.url);
+    if (!url)
+        return undefined;
+    const markdown = asString(entry.markdown);
+    const description = asString(entry.description);
+    const snippet = source === "news" ? (asString(entry.snippet) ?? description ?? "") : description || markdown?.slice(0, 300) || "";
+    return {
+        title: asString(entry.title) || url,
+        url,
+        snippet,
+        publishedDate: asString(entry.publishedDate) ?? asString(entry.date),
+        score: asNumber(entry.score),
+        source,
+        ...(source === "images"
+            ? {
+                imageUrl: asString(entry.imageUrl) ?? asString(entry.image),
+                imageWidth: asNumber(entry.imageWidth),
+                imageHeight: asNumber(entry.imageHeight),
+            }
+            : {}),
+    };
+}
+/**
+ * Flatten Firecrawl's per-source buckets into one ordered list, each bucket
+ * trimmed to its requested budget. Web hits keep their inline markdown as the
+ * snippet; image hits point at the image and its source page.
+ */
+function normalizeSearchResponse(json, budgets) {
+    if (!isRecord(json))
+        return [];
+    const data = json.data;
+    const results = [];
+    for (const source of ["web", "images", "news"]) {
+        const budget = budgets[source];
+        if (budget === undefined)
+            continue;
+        const bucket = isRecord(data) ? data[source] : undefined;
+        const entries = Array.isArray(bucket) ? bucket : source === "web" && Array.isArray(data) ? data : undefined;
+        if (!entries)
+            continue;
+        let taken = 0;
+        for (const entry of entries) {
+            if (taken >= budget)
+                break;
+            const normalized = normalizeSearchEntry(entry, source);
+            if (!normalized)
+                continue;
+            results.push(normalized);
+            taken++;
+        }
+    }
+    return results;
+}
+function normalizeScrapeResponse(url, json) {
+    const data = isRecord(json) ? (isRecord(json.data) ? json.data : undefined) : undefined;
+    const markdown = (data ? asString(data.markdown) : undefined) ?? "";
+    const metadata = data && isRecord(data.metadata) ? data.metadata : undefined;
+    const title = (metadata ? asString(metadata.title) : undefined) ?? (data ? asString(data.title) : undefined) ?? url;
+    if (!markdown.trim()) {
+        throw new FirecrawlError(`Firecrawl scrape of ${url} returned no usable text`, {
+            code: "empty",
+            fallback: true,
+        });
+    }
+    return { url, title, markdown, fetchedAt: new Date().toISOString() };
+}
+export function createFirecrawlBackend(options = {}) {
+    const timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;
+    const post = options.post ?? defaultPost;
+    const headers = {};
+    if (options.apiKey) {
+        headers.authorization = `Bearer ${options.apiKey}`;
+    }
+    async function callApi(path, body, signal) {
+        let lastError;
+        for (let attempt = 1; attempt <= FIRECRAWL_502_MAX_ATTEMPTS; attempt++) {
+            if (signal?.aborted) {
+                throw new Error("Operation aborted");
+            }
+            try {
+                const response = await post(path, body, { headers, timeoutMs, signal });
+                if (response.status === 502 && attempt < FIRECRAWL_502_MAX_ATTEMPTS) {
+                    lastError = new FirecrawlError(`Firecrawl request to ${path} failed with status 502`, {
+                        code: "transport",
+                        status: 502,
+                        fallback: true,
+                    });
+                    continue;
+                }
+                return response;
+            }
+            catch (error) {
+                if (error instanceof FirecrawlError && error.status === 502 && attempt < FIRECRAWL_502_MAX_ATTEMPTS) {
+                    lastError = error;
+                    continue;
+                }
+                throw error;
+            }
+        }
+        throw lastError;
+    }
+    function throwForStatus(path, status, json) {
+        const detail = firecrawlErrorMessage(json);
+        // Keyless 401/403/429 is provider-side (anonymous pool or blocked network),
+        // not a transport blip: name the fix instead of the raw status.
+        if (!options.apiKey && (status === 401 || status === 403 || status === 429)) {
+            throw new FirecrawlError(keylessUnavailableMessage(status, path), {
+                code: "unavailable",
+                status,
+                fallback: true,
+            });
+        }
+        const message = `Firecrawl request to ${path} failed with status ${status}${detail ? `: ${detail}` : ""}`;
+        if (status === 401 || status === 403) {
+            throw new FirecrawlError(message, { code: "transport", status, fallback: true });
+        }
+        if (status === 402 || status === 429 || status >= 500) {
+            throw new FirecrawlError(message, { code: "transport", status, fallback: true });
+        }
+        throw new FirecrawlError(message, { code: "transport", status, fallback: false });
+    }
+    const backend = {
+        id: "firecrawl",
+        lastMeta: undefined,
+        async search(intent, searchOptions) {
+            if (!intent.query) {
+                throw new Error("Search query must not be empty");
+            }
+            // Per-modality limits: sum the requested counts, then slice each bucket
+            // back to its own budget so one modality cannot starve another.
+            const sources = [];
+            if (intent.results.web !== undefined)
+                sources.push("web");
+            if (intent.results.images !== undefined)
+                sources.push("images");
+            if (intent.results.news !== undefined)
+                sources.push("news");
+            const budgets = {};
+            let total = 0;
+            for (const source of sources) {
+                const count = clampLimit(intent.results[source] ?? 0, {
+                    fallback: 1,
+                    min: 1,
+                    max: WEBFETCH_FIRECRAWL_MAX_LIMIT,
+                });
+                budgets[source] = count;
+                total += count;
+            }
+            total = clampLimit(total, { fallback: 10, min: 1, max: WEBFETCH_FIRECRAWL_MAX_LIMIT });
+            const body = {
+                query: intent.query,
+                limit: total,
+                sources,
+                highlights: true,
+                timeout: timeoutMs,
+            };
+            if (intent.filters.domains) {
+                // Firecrawl treats includeDomains/excludeDomains as mutually exclusive;
+                // a request for both is narrowed to the include list.
+                body.includeDomains = intent.filters.domains;
+            }
+            if (intent.filters.recencyMs !== undefined) {
+                body.tbs = `qdr:d${Math.max(1, Math.round(intent.filters.recencyMs / (24 * 60 * 60 * 1000)))}`;
+            }
+            if (intent.filters.language) {
+                body.country = intent.filters.language;
+            }
+            // Page text is only useful for web hits; image and news hits carry their
+            // own metadata and are cheaper without a scrape.
+            if (intent.results.web !== undefined) {
+                body.scrapeOptions = { formats: ["markdown"], onlyMainContent: true };
+            }
+            const response = await callApi("/v2/search", body, searchOptions.signal);
+            if (response.status !== 200) {
+                throwForStatus("/v2/search", response.status, response.json);
+            }
+            if (isRecord(response.json) && response.json.success === false) {
+                throw new FirecrawlError(firecrawlErrorMessage(response.json) ?? "Firecrawl search failed", {
+                    code: "transport",
+                    status: response.status,
+                    fallback: false,
+                });
+            }
+            backend.lastMeta = readMeta(response.json);
+            return normalizeSearchResponse(response.json, budgets);
+        },
+        async fetch(url, fetchOptions) {
+            const response = await callApi("/v2/scrape", { url, formats: ["markdown"], onlyMainContent: true, timeout: timeoutMs }, fetchOptions.signal);
+            if (response.status !== 200) {
+                throwForStatus("/v2/scrape", response.status, response.json);
+            }
+            if (isRecord(response.json) && response.json.success === false) {
+                throw new FirecrawlError(firecrawlErrorMessage(response.json) ?? `Firecrawl scrape of ${url} failed`, {
+                    code: "transport",
+                    status: response.status,
+                    fallback: false,
+                });
+            }
+            backend.lastMeta = readMeta(response.json);
+            return normalizeScrapeResponse(url, response.json);
+        },
+    };
+    return backend;
+}
+//# sourceMappingURL=backend-firecrawl.js.map

package/dist/core/webfetch/backend-firecrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend-firecrawl.js","sourceRoot":"","sources":["../../../src/core/webfetch/backend-firecrawl.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,IAAI,aAAa,EAAE,MAAM,QAAQ,CAAC;AAElD,OAAO,EACN,UAAU,EACV,4BAA4B,EAC5B,2BAA2B,EAC3B,aAAa,GAEb,MAAM,iBAAiB,CAAC;AAGzB,MAAM,CAAC,MAAM,kBAAkB,GAAG,2BAA2B,CAAC;AAC9D,MAAM,CAAC,MAAM,qBAAqB,GAAG,mBAAmB,CAAC;AACzD,MAAM,0BAA0B,GAAG,CAAC,CAAC;AACrC,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC,0HAA0H;AAC1H,MAAM,UAAU,sBAAsB,CAAC,QAAiB,EAAE,GAAG,GAAsB,OAAO,CAAC,GAAG,EAAsB;IACnH,MAAM,OAAO,GAAG,QAAQ,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,CAAC;IACvE,OAAO,OAAO,IAAI,SAAS,CAAC;AAAA,CAC5B;AAED,MAAM,OAAO,cAAe,SAAQ,aAAa;IACvC,MAAM,CAAU;IACzB,0HAA0H;IACjH,QAAQ,CAAU;IAC3B,YAAY,OAAe,EAAE,OAAwE,EAAE;QACtG,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAAA,CACjC;CACD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CAAC,MAAc,EAAE,IAAY,EAAU;IAC/E,MAAM,MAAM,GACX,MAAM,KAAK,GAAG;QACb,CAAC,CAAC,sCAAsC;QACxC,CAAC,CAAC,MAAM,KAAK,GAAG;YACf,CAAC,CAAC,qBAAqB;YACvB,CAAC,CAAC,mDAAmD,CAAC;IACzD,OAAO;QACN,aAAa,IAAI,oBAAoB,MAAM,YAAY,MAAM,IAAI;QACjE,+EAA+E;QAC/E,uDAAuD;QACvD,oEAAoE;KACpE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED,kHAAkH;AAClH,MAAM,UAAU,kBAAkB,CAAC,KAAc,EAAW;IAC3D,IAAI,KAAK,YAAY,cAAc;QAAE,OAAO,KAAK,CAAC,QAAQ,CAAC;IAC3D,IAAI,KAAK,YAAY,aAAa,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC;IACzF,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAmBD,KAAK,UAAU,WAAW,CACzB,IAAY,EACZ,IAA6B,EAC7B,IAAkF,EACrC;IAC7C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,0BAA0B,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;IAC3F,IAAI,QAAmD,CAAC;IACxD,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,kBAAkB,GAAG,IAAI,EAAE,EAAE;YAC9D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE;YAChE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM;SACN,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,aAAa,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YACpD,MAAM,IAAI,aAAa,CAAC,wBAAwB,IAAI,YAAY,EAAE,SAAS,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,aAAa,CAAC,wBAAwB,IAAI,YAAY,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,aAAa,CAAC,wBAAwB,IAAI,YAAY,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACJ,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,aAAa,CAAC,wBAAwB,IAAI,wBAAwB,EAAE,WAAW,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;AAAA,CAC7C;AAQD,SAAS,QAAQ,CAAC,KAAc,EAAoC;IACnE,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAAA,CAC5E;AAED,SAAS,QAAQ,CAAC,KAAc,EAAsB;IACrD,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACrD;AAED,SAAS,QAAQ,CAAC,KAAc,EAAsB;IACrD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CAC/E;AAED,SAAS,qBAAqB,CAAC,IAAa,EAAsB;IACjE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC3B,CAAC;IACD,OAAO,SAAS,CAAC;AAAA,CACjB;AAED,SAAS,QAAQ,CAAC,IAAa,EAAqC;IACnE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACtC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtF,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClG,IAAI,OAAO,KAAK,SAAS,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AAAA,CAChC;AAED,SAAS,oBAAoB,CAAC,KAAc,EAAE,MAAiC,EAA+B;IAC7G,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,OAAO,GACZ,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IACnH,OAAO;QACN,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG;QACnC,GAAG;QACH,OAAO;QACP,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QACpE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;QAC5B,MAAM;QACN,GAAG,CAAC,MAAM,KAAK,QAAQ;YACtB,CAAC,CAAC;gBACA,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;gBAC3D,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC;gBACtC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;aACxC;YACF,CAAC,CAAC,EAAE,CAAC;KACN,CAAC;AAAA,CACF;AAED;;;;GAIG;AACH,SAAS,uBAAuB,CAAC,IAAa,EAAE,OAA+B,EAAqB;IACnG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAU,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS;YAAE,SAAS;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5G,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,KAAK,IAAI,MAAM;gBAAE,MAAM;YAC3B,MAAM,UAAU,GAAG,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACvD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAC1B,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,KAAK,EAAE,CAAC;QACT,CAAC;IACF,CAAC;IACD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,SAAS,uBAAuB,CAAC,GAAW,EAAE,IAAa,EAAe;IACzE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACxF,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;IACpE,MAAM,QAAQ,GAAG,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7E,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC;IACpH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACtB,MAAM,IAAI,cAAc,CAAC,uBAAuB,GAAG,0BAA0B,EAAE;YAC9E,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE,IAAI;SACd,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;AAAA,CACrE;AAED,MAAM,UAAU,sBAAsB,CAAC,OAAO,GAA4B,EAAE,EAAoB;IAC/F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,2BAA2B,CAAC;IACnE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,WAAW,CAAC;IACzC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,aAAa,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;IACpD,CAAC;IAED,KAAK,UAAU,OAAO,CACrB,IAAY,EACZ,IAA6B,EAC7B,MAAoB,EACyB;QAC7C,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,0BAA0B,EAAE,OAAO,EAAE,EAAE,CAAC;YACxE,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACtC,CAAC;YACD,IAAI,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,GAAG,0BAA0B,EAAE,CAAC;oBACrE,SAAS,GAAG,IAAI,cAAc,CAAC,wBAAwB,IAAI,yBAAyB,EAAE;wBACrF,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,GAAG;wBACX,QAAQ,EAAE,IAAI;qBACd,CAAC,CAAC;oBACH,SAAS;gBACV,CAAC;gBACD,OAAO,QAAQ,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,GAAG,0BAA0B,EAAE,CAAC;oBACrG,SAAS,GAAG,KAAK,CAAC;oBAClB,SAAS;gBACV,CAAC;gBACD,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;QACD,MAAM,SAAS,CAAC;IAAA,CAChB;IAED,SAAS,cAAc,CAAC,IAAY,EAAE,MAAc,EAAE,IAAa,EAAS;QAC3E,MAAM,MAAM,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC3C,4EAA4E;QAC5E,gEAAgE;QAChE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,cAAc,CAAC,yBAAyB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE;gBACjE,IAAI,EAAE,aAAa;gBACnB,MAAM;gBACN,QAAQ,EAAE,IAAI;aACd,CAAC,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,wBAAwB,IAAI,uBAAuB,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1G,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACtC,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACvD,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAAA,CAClF;IAED,MAAM,OAAO,GAAqB;QACjC,EAAE,EAAE,WAAW;QACf,QAAQ,EAAE,SAAS;QACnB,KAAK,CAAC,MAAM,CACX,MAA4B,EAC5B,aAAsD,EACzB;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACnD,CAAC;YACD,wEAAwE;YACxE,gEAAgE;YAChE,MAAM,OAAO,GAAqC,EAAE,CAAC;YACrD,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChE,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,OAAO,GAA2B,EAAE,CAAC;YAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;oBACrD,QAAQ,EAAE,CAAC;oBACX,GAAG,EAAE,CAAC;oBACN,GAAG,EAAE,4BAA4B;iBACjC,CAAC,CAAC;gBACH,OAAO,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;gBACxB,KAAK,IAAI,KAAK,CAAC;YAChB,CAAC;YACD,KAAK,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,4BAA4B,EAAE,CAAC,CAAC;YAEvF,MAAM,IAAI,GAA4B;gBACrC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,KAAK;gBACZ,OAAO;gBACP,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,SAAS;aAClB,CAAC;YACF,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC5B,wEAAwE;gBACxE,sDAAsD;gBACtD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;YAC9C,CAAC;YACD,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5C,IAAI,CAAC,GAAG,GAAG,QAAQ,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,CAAC;YACD,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;YACxC,CAAC;YACD,yEAAyE;YACzE,iDAAiD;YACjD,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtC,IAAI,CAAC,aAAa,GAAG,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YACzE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7B,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;gBAChE,MAAM,IAAI,cAAc,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,yBAAyB,EAAE;oBAC3F,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,QAAQ,EAAE,KAAK;iBACf,CAAC,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAAA,CACvD;QACD,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,YAAsC,EAAwB;YACtF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAC7B,YAAY,EACZ,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,EACzE,YAAY,CAAC,MAAM,CACnB,CAAC;YACF,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7B,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;gBAChE,MAAM,IAAI,cAAc,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,uBAAuB,GAAG,SAAS,EAAE;oBACrG,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,QAAQ,EAAE,KAAK;iBACf,CAAC,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,uBAAuB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAA,CACnD;KACD,CAAC;IACF,OAAO,OAAO,CAAC;AAAA,CACf","sourcesContent":["/**\n * Firecrawl backend for webfetch: thin `undici` POSTs to the hosted Firecrawl\n * API. The SDK is never vendored (extra `axios` dep) and the server is never\n * vendored (AGPL-3.0).\n *\n * Auth is an optional `Bearer` key, omitted entirely on the keyless tier.\n * Resolve it with `resolveFirecrawlApiKey` (explicit settings value first,\n * `FIRECRAWL_API_KEY` env second).\n */\n\nimport { request as undiciRequest } from \"undici\";\nimport type { WebDocument, WebSearchBackend, WebSearchResult } from \"./backend.ts\";\nimport {\n\tclampLimit,\n\tWEBFETCH_FIRECRAWL_MAX_LIMIT,\n\tWEBFETCH_REQUEST_TIMEOUT_MS,\n\tWebfetchError,\n\ttype WebfetchErrorCode,\n} from \"./guardrails.ts\";\nimport type { ResolvedSearchIntent } from \"./search-intent.ts\";\n\nexport const FIRECRAWL_BASE_URL = \"https://api.firecrawl.dev\";\nexport const FIRECRAWL_API_KEY_ENV = \"FIRECRAWL_API_KEY\";\nconst FIRECRAWL_502_MAX_ATTEMPTS = 3;\nconst FIRECRAWL_HTTP_HEADROOM_MS = 5_000;\n\n/** Explicit settings value wins; `FIRECRAWL_API_KEY` env is the fallback. Keyless (undefined) works with lower limits. */\nexport function resolveFirecrawlApiKey(explicit?: string, env: NodeJS.ProcessEnv = process.env): string | undefined {\n\tconst trimmed = explicit?.trim() || env[FIRECRAWL_API_KEY_ENV]?.trim();\n\treturn trimmed || undefined;\n}\n\nexport class FirecrawlError extends WebfetchError {\n\treadonly status?: number;\n\t/** True when the caller should try the `undici` direct-fetch fallback (transport/auth failure, never content failure). */\n\treadonly fallback: boolean;\n\tconstructor(message: string, options: { code: WebfetchErrorCode; status?: number; fallback: boolean }) {\n\t\tsuper(message, options.code);\n\t\tthis.name = \"FirecrawlError\";\n\t\tthis.status = options.status;\n\t\tthis.fallback = options.fallback;\n\t}\n}\n\n/**\n * The keyless Firecrawl tier refused this network (403 \"your IP address looks\n * suspicious\", 401 without a key, or 429 on the shared anonymous pool). This is\n * provider-side and often IP-dependent, so the message names the exact fix\n * instead of leaking the raw status. Direct page fetch still works through the\n * `undici` fallback; only search is unavailable.\n */\nexport function keylessUnavailableMessage(status: number, path: string): string {\n\tconst reason =\n\t\tstatus === 429\n\t\t\t? \"the keyless rate limit was exhausted\"\n\t\t\t: status === 401\n\t\t\t\t? \"no API key was sent\"\n\t\t\t\t: \"this network is not allowed on the anonymous tier\";\n\treturn [\n\t\t`Firecrawl ${path} is unavailable: ${reason} (status ${status}).`,\n\t\t\"Set a Firecrawl API key to fix this: add the `firecrawlApiKey` setting or the\",\n\t\t\"`FIRECRAWL_API_KEY` environment variable, then retry.\",\n\t\t\"`webextract` still works without a key: it fetches pages directly.\",\n\t].join(\" \");\n}\n\n/** True when a failed Firecrawl call should fall back to direct `undici` fetch instead of surfacing the error. */\nexport function isFallbackEligible(error: unknown): boolean {\n\tif (error instanceof FirecrawlError) return error.fallback;\n\tif (error instanceof WebfetchError) {\n\t\treturn error.code === \"timeout\" || error.code === \"transport\" || error.code === \"empty\";\n\t}\n\treturn false;\n}\n\nexport interface FirecrawlResponseMeta {\n\twarning?: string;\n\tcreditsUsed?: number;\n}\n\nexport interface FirecrawlBackend extends WebSearchBackend {\n\treadonly id: \"firecrawl\";\n\t/** `warning`/`creditsUsed` from the most recent search or scrape, for cost observability in tool details. */\n\tlastMeta: FirecrawlResponseMeta | undefined;\n}\n\nexport type FirecrawlPostFn = (\n\tpath: string,\n\tbody: Record<string, unknown>,\n\tinit: { headers: Record<string, string>; timeoutMs: number; signal?: AbortSignal },\n) => Promise<{ status: number; json: unknown }>;\n\nasync function defaultPost(\n\tpath: string,\n\tbody: Record<string, unknown>,\n\tinit: { headers: Record<string, string>; timeoutMs: number; signal?: AbortSignal },\n): Promise<{ status: number; json: unknown }> {\n\tconst timeoutSignal = AbortSignal.timeout(init.timeoutMs + FIRECRAWL_HTTP_HEADROOM_MS);\n\tconst signal = init.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal;\n\tlet response: Awaited<ReturnType<typeof undiciRequest>>;\n\ttry {\n\t\tresponse = await undiciRequest(`${FIRECRAWL_BASE_URL}${path}`, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: { \"content-type\": \"application/json\", ...init.headers },\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal,\n\t\t});\n\t} catch (error) {\n\t\tif (timeoutSignal.aborted && !init.signal?.aborted) {\n\t\t\tthrow new WebfetchError(`Firecrawl request to ${path} timed out`, \"timeout\");\n\t\t}\n\t\tif (init.signal?.aborted) {\n\t\t\tthrow new Error(\"Operation aborted\");\n\t\t}\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\tthrow new WebfetchError(`Firecrawl request to ${path} failed: ${reason}`, \"transport\");\n\t}\n\tlet text: string;\n\ttry {\n\t\ttext = await response.body.text();\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\tthrow new WebfetchError(`Firecrawl request to ${path} failed: ${reason}`, \"transport\");\n\t}\n\tlet json: unknown;\n\ttry {\n\t\tjson = text ? (JSON.parse(text) as unknown) : undefined;\n\t} catch {\n\t\tthrow new WebfetchError(`Firecrawl request to ${path} returned invalid JSON`, \"transport\");\n\t}\n\treturn { status: response.statusCode, json };\n}\n\nexport interface FirecrawlBackendOptions {\n\tapiKey?: string;\n\ttimeoutMs?: number;\n\tpost?: FirecrawlPostFn;\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n\treturn typeof value === \"object\" && value !== null && !Array.isArray(value);\n}\n\nfunction asString(value: unknown): string | undefined {\n\treturn typeof value === \"string\" ? value : undefined;\n}\n\nfunction asNumber(value: unknown): number | undefined {\n\treturn typeof value === \"number\" && Number.isFinite(value) ? value : undefined;\n}\n\nfunction firecrawlErrorMessage(json: unknown): string | undefined {\n\tif (!isRecord(json)) return undefined;\n\tconst direct = asString(json.error);\n\tif (direct) return direct;\n\tif (isRecord(json.data)) {\n\t\tconst nested = asString(json.data.error);\n\t\tif (nested) return nested;\n\t}\n\treturn undefined;\n}\n\nfunction readMeta(json: unknown): FirecrawlResponseMeta | undefined {\n\tif (!isRecord(json)) return undefined;\n\tconst data = isRecord(json.data) ? json.data : undefined;\n\tconst warning = asString(json.warning) ?? (data ? asString(data.warning) : undefined);\n\tconst creditsUsed = asNumber(json.creditsUsed) ?? (data ? asNumber(data.creditsUsed) : undefined);\n\tif (warning === undefined && creditsUsed === undefined) return undefined;\n\treturn { warning, creditsUsed };\n}\n\nfunction normalizeSearchEntry(entry: unknown, source: \"web\" | \"images\" | \"news\"): WebSearchResult | undefined {\n\tif (!isRecord(entry)) return undefined;\n\tconst url = asString(entry.url);\n\tif (!url) return undefined;\n\tconst markdown = asString(entry.markdown);\n\tconst description = asString(entry.description);\n\tconst snippet =\n\t\tsource === \"news\" ? (asString(entry.snippet) ?? description ?? \"\") : description || markdown?.slice(0, 300) || \"\";\n\treturn {\n\t\ttitle: asString(entry.title) || url,\n\t\turl,\n\t\tsnippet,\n\t\tpublishedDate: asString(entry.publishedDate) ?? asString(entry.date),\n\t\tscore: asNumber(entry.score),\n\t\tsource,\n\t\t...(source === \"images\"\n\t\t\t? {\n\t\t\t\t\timageUrl: asString(entry.imageUrl) ?? asString(entry.image),\n\t\t\t\t\timageWidth: asNumber(entry.imageWidth),\n\t\t\t\t\timageHeight: asNumber(entry.imageHeight),\n\t\t\t\t}\n\t\t\t: {}),\n\t};\n}\n\n/**\n * Flatten Firecrawl's per-source buckets into one ordered list, each bucket\n * trimmed to its requested budget. Web hits keep their inline markdown as the\n * snippet; image hits point at the image and its source page.\n */\nfunction normalizeSearchResponse(json: unknown, budgets: Record<string, number>): WebSearchResult[] {\n\tif (!isRecord(json)) return [];\n\tconst data = json.data;\n\tconst results: WebSearchResult[] = [];\n\tfor (const source of [\"web\", \"images\", \"news\"] as const) {\n\t\tconst budget = budgets[source];\n\t\tif (budget === undefined) continue;\n\t\tconst bucket = isRecord(data) ? data[source] : undefined;\n\t\tconst entries = Array.isArray(bucket) ? bucket : source === \"web\" && Array.isArray(data) ? data : undefined;\n\t\tif (!entries) continue;\n\t\tlet taken = 0;\n\t\tfor (const entry of entries) {\n\t\t\tif (taken >= budget) break;\n\t\t\tconst normalized = normalizeSearchEntry(entry, source);\n\t\t\tif (!normalized) continue;\n\t\t\tresults.push(normalized);\n\t\t\ttaken++;\n\t\t}\n\t}\n\treturn results;\n}\n\nfunction normalizeScrapeResponse(url: string, json: unknown): WebDocument {\n\tconst data = isRecord(json) ? (isRecord(json.data) ? json.data : undefined) : undefined;\n\tconst markdown = (data ? asString(data.markdown) : undefined) ?? \"\";\n\tconst metadata = data && isRecord(data.metadata) ? data.metadata : undefined;\n\tconst title = (metadata ? asString(metadata.title) : undefined) ?? (data ? asString(data.title) : undefined) ?? url;\n\tif (!markdown.trim()) {\n\t\tthrow new FirecrawlError(`Firecrawl scrape of ${url} returned no usable text`, {\n\t\t\tcode: \"empty\",\n\t\t\tfallback: true,\n\t\t});\n\t}\n\treturn { url, title, markdown, fetchedAt: new Date().toISOString() };\n}\n\nexport function createFirecrawlBackend(options: FirecrawlBackendOptions = {}): FirecrawlBackend {\n\tconst timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;\n\tconst post = options.post ?? defaultPost;\n\tconst headers: Record<string, string> = {};\n\tif (options.apiKey) {\n\t\theaders.authorization = `Bearer ${options.apiKey}`;\n\t}\n\n\tasync function callApi(\n\t\tpath: string,\n\t\tbody: Record<string, unknown>,\n\t\tsignal?: AbortSignal,\n\t): Promise<{ status: number; json: unknown }> {\n\t\tlet lastError: unknown;\n\t\tfor (let attempt = 1; attempt <= FIRECRAWL_502_MAX_ATTEMPTS; attempt++) {\n\t\t\tif (signal?.aborted) {\n\t\t\t\tthrow new Error(\"Operation aborted\");\n\t\t\t}\n\t\t\ttry {\n\t\t\t\tconst response = await post(path, body, { headers, timeoutMs, signal });\n\t\t\t\tif (response.status === 502 && attempt < FIRECRAWL_502_MAX_ATTEMPTS) {\n\t\t\t\t\tlastError = new FirecrawlError(`Firecrawl request to ${path} failed with status 502`, {\n\t\t\t\t\t\tcode: \"transport\",\n\t\t\t\t\t\tstatus: 502,\n\t\t\t\t\t\tfallback: true,\n\t\t\t\t\t});\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\treturn response;\n\t\t\t} catch (error) {\n\t\t\t\tif (error instanceof FirecrawlError && error.status === 502 && attempt < FIRECRAWL_502_MAX_ATTEMPTS) {\n\t\t\t\t\tlastError = error;\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tthrow lastError;\n\t}\n\n\tfunction throwForStatus(path: string, status: number, json: unknown): never {\n\t\tconst detail = firecrawlErrorMessage(json);\n\t\t// Keyless 401/403/429 is provider-side (anonymous pool or blocked network),\n\t\t// not a transport blip: name the fix instead of the raw status.\n\t\tif (!options.apiKey && (status === 401 || status === 403 || status === 429)) {\n\t\t\tthrow new FirecrawlError(keylessUnavailableMessage(status, path), {\n\t\t\t\tcode: \"unavailable\",\n\t\t\t\tstatus,\n\t\t\t\tfallback: true,\n\t\t\t});\n\t\t}\n\t\tconst message = `Firecrawl request to ${path} failed with status ${status}${detail ? `: ${detail}` : \"\"}`;\n\t\tif (status === 401 || status === 403) {\n\t\t\tthrow new FirecrawlError(message, { code: \"transport\", status, fallback: true });\n\t\t}\n\t\tif (status === 402 || status === 429 || status >= 500) {\n\t\t\tthrow new FirecrawlError(message, { code: \"transport\", status, fallback: true });\n\t\t}\n\t\tthrow new FirecrawlError(message, { code: \"transport\", status, fallback: false });\n\t}\n\n\tconst backend: FirecrawlBackend = {\n\t\tid: \"firecrawl\",\n\t\tlastMeta: undefined,\n\t\tasync search(\n\t\t\tintent: ResolvedSearchIntent,\n\t\t\tsearchOptions: { limit: number; signal?: AbortSignal },\n\t\t): Promise<WebSearchResult[]> {\n\t\t\tif (!intent.query) {\n\t\t\t\tthrow new Error(\"Search query must not be empty\");\n\t\t\t}\n\t\t\t// Per-modality limits: sum the requested counts, then slice each bucket\n\t\t\t// back to its own budget so one modality cannot starve another.\n\t\t\tconst sources: Array<\"web\" | \"images\" | \"news\"> = [];\n\t\t\tif (intent.results.web !== undefined) sources.push(\"web\");\n\t\t\tif (intent.results.images !== undefined) sources.push(\"images\");\n\t\t\tif (intent.results.news !== undefined) sources.push(\"news\");\n\t\t\tconst budgets: Record<string, number> = {};\n\t\t\tlet total = 0;\n\t\t\tfor (const source of sources) {\n\t\t\t\tconst count = clampLimit(intent.results[source] ?? 0, {\n\t\t\t\t\tfallback: 1,\n\t\t\t\t\tmin: 1,\n\t\t\t\t\tmax: WEBFETCH_FIRECRAWL_MAX_LIMIT,\n\t\t\t\t});\n\t\t\t\tbudgets[source] = count;\n\t\t\t\ttotal += count;\n\t\t\t}\n\t\t\ttotal = clampLimit(total, { fallback: 10, min: 1, max: WEBFETCH_FIRECRAWL_MAX_LIMIT });\n\n\t\t\tconst body: Record<string, unknown> = {\n\t\t\t\tquery: intent.query,\n\t\t\t\tlimit: total,\n\t\t\t\tsources,\n\t\t\t\thighlights: true,\n\t\t\t\ttimeout: timeoutMs,\n\t\t\t};\n\t\t\tif (intent.filters.domains) {\n\t\t\t\t// Firecrawl treats includeDomains/excludeDomains as mutually exclusive;\n\t\t\t\t// a request for both is narrowed to the include list.\n\t\t\t\tbody.includeDomains = intent.filters.domains;\n\t\t\t}\n\t\t\tif (intent.filters.recencyMs !== undefined) {\n\t\t\t\tbody.tbs = `qdr:d${Math.max(1, Math.round(intent.filters.recencyMs / (24 * 60 * 60 * 1000)))}`;\n\t\t\t}\n\t\t\tif (intent.filters.language) {\n\t\t\t\tbody.country = intent.filters.language;\n\t\t\t}\n\t\t\t// Page text is only useful for web hits; image and news hits carry their\n\t\t\t// own metadata and are cheaper without a scrape.\n\t\t\tif (intent.results.web !== undefined) {\n\t\t\t\tbody.scrapeOptions = { formats: [\"markdown\"], onlyMainContent: true };\n\t\t\t}\n\n\t\t\tconst response = await callApi(\"/v2/search\", body, searchOptions.signal);\n\t\t\tif (response.status !== 200) {\n\t\t\t\tthrowForStatus(\"/v2/search\", response.status, response.json);\n\t\t\t}\n\t\t\tif (isRecord(response.json) && response.json.success === false) {\n\t\t\t\tthrow new FirecrawlError(firecrawlErrorMessage(response.json) ?? \"Firecrawl search failed\", {\n\t\t\t\t\tcode: \"transport\",\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\tfallback: false,\n\t\t\t\t});\n\t\t\t}\n\t\t\tbackend.lastMeta = readMeta(response.json);\n\t\t\treturn normalizeSearchResponse(response.json, budgets);\n\t\t},\n\t\tasync fetch(url: string, fetchOptions: { signal?: AbortSignal }): Promise<WebDocument> {\n\t\t\tconst response = await callApi(\n\t\t\t\t\"/v2/scrape\",\n\t\t\t\t{ url, formats: [\"markdown\"], onlyMainContent: true, timeout: timeoutMs },\n\t\t\t\tfetchOptions.signal,\n\t\t\t);\n\t\t\tif (response.status !== 200) {\n\t\t\t\tthrowForStatus(\"/v2/scrape\", response.status, response.json);\n\t\t\t}\n\t\t\tif (isRecord(response.json) && response.json.success === false) {\n\t\t\t\tthrow new FirecrawlError(firecrawlErrorMessage(response.json) ?? `Firecrawl scrape of ${url} failed`, {\n\t\t\t\t\tcode: \"transport\",\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\tfallback: false,\n\t\t\t\t});\n\t\t\t}\n\t\t\tbackend.lastMeta = readMeta(response.json);\n\t\t\treturn normalizeScrapeResponse(url, response.json);\n\t\t},\n\t};\n\treturn backend;\n}\n"]}

package/dist/core/webfetch/backend-undici.d.ts

@@ -0,0 +1,21 @@
+/**
+ * `undici` fallback backend for webfetch: direct fetch of the target URL under
+ * the shared guardrails, with minimal inline HTML→text cleanup (no readability
+ * dep, no new packages).
+ *
+ * Only `fetch` is supported. `search` has no direct-fetch equivalent, so it
+ * throws a named error instead of fabricating results.
+ */
+import type { WebFetchBackend } from "./backend.ts";
+import { type DnsLookup, type LowLevelRequestFn } from "./guardrails.ts";
+export interface UndiciBackendOptions {
+    timeoutMs?: number;
+    maxBytes?: number;
+    maxRedirects?: number;
+    lookup?: DnsLookup;
+    request?: LowLevelRequestFn;
+}
+/** Minimal HTML→text cleanup: drop scripts/styles/comments/tags, decode entities, collapse whitespace. */
+export declare function htmlToText(html: string): string;
+export declare function createUndiciBackend(options?: UndiciBackendOptions): WebFetchBackend;
+//# sourceMappingURL=backend-undici.d.ts.map

package/dist/core/webfetch/backend-undici.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend-undici.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/backend-undici.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAe,eAAe,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAEN,KAAK,SAAS,EAEd,KAAK,iBAAiB,EAKtB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,oBAAoB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAwBD,4GAA0G;AAC1G,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAe/C;AAoBD,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,eAAe,CAgDvF","sourcesContent":["/**\n * `undici` fallback backend for webfetch: direct fetch of the target URL under\n * the shared guardrails, with minimal inline HTML→text cleanup (no readability\n * dep, no new packages).\n *\n * Only `fetch` is supported. `search` has no direct-fetch equivalent, so it\n * throws a named error instead of fabricating results.\n */\n\nimport type { WebDocument, WebFetchBackend } from \"./backend.ts\";\nimport {\n\tassertBytesWithinBudget,\n\ttype DnsLookup,\n\tguardedRequest,\n\ttype LowLevelRequestFn,\n\tWEBFETCH_MAX_BYTES,\n\tWEBFETCH_MAX_REDIRECTS,\n\tWEBFETCH_REQUEST_TIMEOUT_MS,\n\tWebfetchError,\n} from \"./guardrails.ts\";\n\nexport interface UndiciBackendOptions {\n\ttimeoutMs?: number;\n\tmaxBytes?: number;\n\tmaxRedirects?: number;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n}\n\nfunction decodeEntities(text: string): string {\n\treturn text\n\t\t.replace(/&#(\\d+);/g, (_match, digits: string) => String.fromCodePoint(Number.parseInt(digits, 10)))\n\t\t.replace(/&#x([0-9a-fA-F]+);/g, (_match, digits: string) => String.fromCodePoint(Number.parseInt(digits, 16)))\n\t\t.replace(/&(amp|lt|gt|quot|apos|nbsp);/g, (_match, name: string) => {\n\t\t\tswitch (name) {\n\t\t\t\tcase \"amp\":\n\t\t\t\t\treturn \"&\";\n\t\t\t\tcase \"lt\":\n\t\t\t\t\treturn \"<\";\n\t\t\t\tcase \"gt\":\n\t\t\t\t\treturn \">\";\n\t\t\t\tcase \"quot\":\n\t\t\t\t\treturn '\"';\n\t\t\t\tcase \"apos\":\n\t\t\t\t\treturn \"'\";\n\t\t\t\tdefault:\n\t\t\t\t\treturn \" \";\n\t\t\t}\n\t\t});\n}\n\n/** Minimal HTML→text cleanup: drop scripts/styles/comments/tags, decode entities, collapse whitespace. */\nexport function htmlToText(html: string): string {\n\tconst withoutScripts = html\n\t\t.replace(/<head[\\s\\S]*?<\\/head\\s*>/gi, \" \")\n\t\t.replace(/<script[\\s\\S]*?<\\/script\\s*>/gi, \" \")\n\t\t.replace(/<style[\\s\\S]*?<\\/style\\s*>/gi, \" \")\n\t\t.replace(/<!--[\\s\\S]*?-->/g, \" \");\n\tconst withBreaks = withoutScripts\n\t\t.replace(/<br[^>]*>/gi, \"\\n\")\n\t\t.replace(/<\\/(p|div|h[1-6]|li|tr|table|section|article|header|footer|blockquote|pre)[^>]*>/gi, \"\\n\");\n\tconst withoutTags = withBreaks.replace(/<[^>]+>/g, \" \");\n\tconst lines = decodeEntities(withoutTags)\n\t\t.split(\"\\n\")\n\t\t.map((line) => line.replace(/[ \\t\\r\\f\\v]+/g, \" \").trim())\n\t\t.filter((line) => line.length > 0);\n\treturn lines.join(\"\\n\");\n}\n\nfunction extractHtmlTitle(html: string): string | undefined {\n\tconst match = /<title[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n\tconst title = match?.[1] ? decodeEntities(match[1].replace(/\\s+/g, \" \").trim()) : undefined;\n\treturn title || undefined;\n}\n\nfunction isSupportedContentType(contentType: string): boolean {\n\tif (contentType === \"\") return true;\n\tconst type = contentType.split(\";\")[0]?.trim().toLowerCase() ?? \"\";\n\treturn (\n\t\ttype.startsWith(\"text/\") ||\n\t\ttype.includes(\"html\") ||\n\t\ttype.includes(\"json\") ||\n\t\ttype.includes(\"xml\") ||\n\t\ttype.includes(\"javascript\")\n\t);\n}\n\nexport function createUndiciBackend(options: UndiciBackendOptions = {}): WebFetchBackend {\n\tconst timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;\n\tconst maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;\n\tconst maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;\n\treturn {\n\t\tid: \"undici\",\n\t\tasync search(): Promise<never> {\n\t\t\tthrow new WebfetchError(\n\t\t\t\t\"The undici backend does not support search; configure Firecrawl for search\",\n\t\t\t\t\"transport\",\n\t\t\t);\n\t\t},\n\t\tasync fetch(url: string, fetchOptions: { signal?: AbortSignal }): Promise<WebDocument> {\n\t\t\tconst response = await guardedRequest(url, {\n\t\t\t\ttimeoutMs,\n\t\t\t\tmaxBytes,\n\t\t\t\tmaxRedirects,\n\t\t\t\tsignal: fetchOptions.signal,\n\t\t\t\tlookup: options.lookup,\n\t\t\t\trequest: options.request,\n\t\t\t});\n\t\t\tconst contentType = response.headers[\"content-type\"] ?? \"\";\n\t\t\tif (!isSupportedContentType(contentType)) {\n\t\t\t\tthrow new WebfetchError(`Unsupported content type \"${contentType}\" for ${response.finalUrl}`, \"empty\");\n\t\t\t}\n\t\t\tconst chunks: Uint8Array[] = [];\n\t\t\tlet totalBytes = 0;\n\t\t\tfor await (const chunk of response.body) {\n\t\t\t\ttotalBytes += chunk.length;\n\t\t\t\tassertBytesWithinBudget(totalBytes, maxBytes);\n\t\t\t\tchunks.push(chunk);\n\t\t\t}\n\t\t\tconst merged = new Uint8Array(totalBytes);\n\t\t\tlet offset = 0;\n\t\t\tfor (const chunk of chunks) {\n\t\t\t\tmerged.set(chunk, offset);\n\t\t\t\toffset += chunk.length;\n\t\t\t}\n\t\t\tconst raw = new TextDecoder().decode(merged);\n\t\t\tconst looksLikeHtml = contentType.toLowerCase().includes(\"html\") || /^\\s*</.test(raw.slice(0, 1024));\n\t\t\tconst markdown = (looksLikeHtml ? htmlToText(raw) : raw.trim()).trim();\n\t\t\tif (!markdown) {\n\t\t\t\tthrow new WebfetchError(`Fetch of ${response.finalUrl} returned no usable text`, \"empty\");\n\t\t\t}\n\t\t\tconst title = (looksLikeHtml ? extractHtmlTitle(raw) : undefined) ?? response.finalUrl;\n\t\t\treturn { url: response.finalUrl, title, markdown, fetchedAt: new Date().toISOString() };\n\t\t},\n\t};\n}\n"]}

package/dist/core/webfetch/backend-undici.js

@@ -0,0 +1,109 @@
+/**
+ * `undici` fallback backend for webfetch: direct fetch of the target URL under
+ * the shared guardrails, with minimal inline HTML→text cleanup (no readability
+ * dep, no new packages).
+ *
+ * Only `fetch` is supported. `search` has no direct-fetch equivalent, so it
+ * throws a named error instead of fabricating results.
+ */
+import { assertBytesWithinBudget, guardedRequest, WEBFETCH_MAX_BYTES, WEBFETCH_MAX_REDIRECTS, WEBFETCH_REQUEST_TIMEOUT_MS, WebfetchError, } from "./guardrails.js";
+function decodeEntities(text) {
+    return text
+        .replace(/&#(\d+);/g, (_match, digits) => String.fromCodePoint(Number.parseInt(digits, 10)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_match, digits) => String.fromCodePoint(Number.parseInt(digits, 16)))
+        .replace(/&(amp|lt|gt|quot|apos|nbsp);/g, (_match, name) => {
+        switch (name) {
+            case "amp":
+                return "&";
+            case "lt":
+                return "<";
+            case "gt":
+                return ">";
+            case "quot":
+                return '"';
+            case "apos":
+                return "'";
+            default:
+                return " ";
+        }
+    });
+}
+/** Minimal HTML→text cleanup: drop scripts/styles/comments/tags, decode entities, collapse whitespace. */
+export function htmlToText(html) {
+    const withoutScripts = html
+        .replace(/<head[\s\S]*?<\/head\s*>/gi, " ")
+        .replace(/<script[\s\S]*?<\/script\s*>/gi, " ")
+        .replace(/<style[\s\S]*?<\/style\s*>/gi, " ")
+        .replace(/<!--[\s\S]*?-->/g, " ");
+    const withBreaks = withoutScripts
+        .replace(/<br[^>]*>/gi, "\n")
+        .replace(/<\/(p|div|h[1-6]|li|tr|table|section|article|header|footer|blockquote|pre)[^>]*>/gi, "\n");
+    const withoutTags = withBreaks.replace(/<[^>]+>/g, " ");
+    const lines = decodeEntities(withoutTags)
+        .split("\n")
+        .map((line) => line.replace(/[ \t\r\f\v]+/g, " ").trim())
+        .filter((line) => line.length > 0);
+    return lines.join("\n");
+}
+function extractHtmlTitle(html) {
+    const match = /<title[^>]*>([\s\S]*?)<\/title\s*>/i.exec(html);
+    const title = match?.[1] ? decodeEntities(match[1].replace(/\s+/g, " ").trim()) : undefined;
+    return title || undefined;
+}
+function isSupportedContentType(contentType) {
+    if (contentType === "")
+        return true;
+    const type = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+    return (type.startsWith("text/") ||
+        type.includes("html") ||
+        type.includes("json") ||
+        type.includes("xml") ||
+        type.includes("javascript"));
+}
+export function createUndiciBackend(options = {}) {
+    const timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;
+    const maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;
+    const maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;
+    return {
+        id: "undici",
+        async search() {
+            throw new WebfetchError("The undici backend does not support search; configure Firecrawl for search", "transport");
+        },
+        async fetch(url, fetchOptions) {
+            const response = await guardedRequest(url, {
+                timeoutMs,
+                maxBytes,
+                maxRedirects,
+                signal: fetchOptions.signal,
+                lookup: options.lookup,
+                request: options.request,
+            });
+            const contentType = response.headers["content-type"] ?? "";
+            if (!isSupportedContentType(contentType)) {
+                throw new WebfetchError(`Unsupported content type "${contentType}" for ${response.finalUrl}`, "empty");
+            }
+            const chunks = [];
+            let totalBytes = 0;
+            for await (const chunk of response.body) {
+                totalBytes += chunk.length;
+                assertBytesWithinBudget(totalBytes, maxBytes);
+                chunks.push(chunk);
+            }
+            const merged = new Uint8Array(totalBytes);
+            let offset = 0;
+            for (const chunk of chunks) {
+                merged.set(chunk, offset);
+                offset += chunk.length;
+            }
+            const raw = new TextDecoder().decode(merged);
+            const looksLikeHtml = contentType.toLowerCase().includes("html") || /^\s*</.test(raw.slice(0, 1024));
+            const markdown = (looksLikeHtml ? htmlToText(raw) : raw.trim()).trim();
+            if (!markdown) {
+                throw new WebfetchError(`Fetch of ${response.finalUrl} returned no usable text`, "empty");
+            }
+            const title = (looksLikeHtml ? extractHtmlTitle(raw) : undefined) ?? response.finalUrl;
+            return { url: response.finalUrl, title, markdown, fetchedAt: new Date().toISOString() };
+        },
+    };
+}
+//# sourceMappingURL=backend-undici.js.map

package/dist/core/webfetch/backend-undici.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend-undici.js","sourceRoot":"","sources":["../../../src/core/webfetch/backend-undici.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACN,uBAAuB,EAEvB,cAAc,EAEd,kBAAkB,EAClB,sBAAsB,EACtB,2BAA2B,EAC3B,aAAa,GACb,MAAM,iBAAiB,CAAC;AAUzB,SAAS,cAAc,CAAC,IAAY,EAAU;IAC7C,OAAO,IAAI;SACT,OAAO,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;SACnG,OAAO,CAAC,qBAAqB,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;SAC7G,OAAO,CAAC,+BAA+B,EAAE,CAAC,MAAM,EAAE,IAAY,EAAE,EAAE,CAAC;QACnE,QAAQ,IAAI,EAAE,CAAC;YACd,KAAK,KAAK;gBACT,OAAO,GAAG,CAAC;YACZ,KAAK,IAAI;gBACR,OAAO,GAAG,CAAC;YACZ,KAAK,IAAI;gBACR,OAAO,GAAG,CAAC;YACZ,KAAK,MAAM;gBACV,OAAO,GAAG,CAAC;YACZ,KAAK,MAAM;gBACV,OAAO,GAAG,CAAC;YACZ;gBACC,OAAO,GAAG,CAAC;QACb,CAAC;IAAA,CACD,CAAC,CAAC;AAAA,CACJ;AAED,4GAA0G;AAC1G,MAAM,UAAU,UAAU,CAAC,IAAY,EAAU;IAChD,MAAM,cAAc,GAAG,IAAI;SACzB,OAAO,CAAC,4BAA4B,EAAE,GAAG,CAAC;SAC1C,OAAO,CAAC,gCAAgC,EAAE,GAAG,CAAC;SAC9C,OAAO,CAAC,8BAA8B,EAAE,GAAG,CAAC;SAC5C,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACnC,MAAM,UAAU,GAAG,cAAc;SAC/B,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC;SAC5B,OAAO,CAAC,oFAAoF,EAAE,IAAI,CAAC,CAAC;IACtG,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,cAAc,CAAC,WAAW,CAAC;SACvC,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;SACxD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAsB;IAC3D,MAAM,KAAK,GAAG,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5F,OAAO,KAAK,IAAI,SAAS,CAAC;AAAA,CAC1B;AAED,SAAS,sBAAsB,CAAC,WAAmB,EAAW;IAC7D,IAAI,WAAW,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACnE,OAAO,CACN,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAC3B,CAAC;AAAA,CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAO,GAAyB,EAAE,EAAmB;IACxF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,2BAA2B,CAAC;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,kBAAkB,CAAC;IACxD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC;IACpE,OAAO;QACN,EAAE,EAAE,QAAQ;QACZ,KAAK,CAAC,MAAM,GAAmB;YAC9B,MAAM,IAAI,aAAa,CACtB,4EAA4E,EAC5E,WAAW,CACX,CAAC;QAAA,CACF;QACD,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,YAAsC,EAAwB;YACtF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;gBAC1C,SAAS;gBACT,QAAQ;gBACR,YAAY;gBACZ,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,OAAO,EAAE,OAAO,CAAC,OAAO;aACxB,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC3D,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC1C,MAAM,IAAI,aAAa,CAAC,6BAA6B,WAAW,SAAS,QAAQ,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YACxG,CAAC;YACD,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACzC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3B,uBAAuB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;YACrG,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,aAAa,CAAC,YAAY,QAAQ,CAAC,QAAQ,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,KAAK,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;YACvF,OAAO,EAAE,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAAA,CACxF;KACD,CAAC;AAAA,CACF","sourcesContent":["/**\n * `undici` fallback backend for webfetch: direct fetch of the target URL under\n * the shared guardrails, with minimal inline HTML→text cleanup (no readability\n * dep, no new packages).\n *\n * Only `fetch` is supported. `search` has no direct-fetch equivalent, so it\n * throws a named error instead of fabricating results.\n */\n\nimport type { WebDocument, WebFetchBackend } from \"./backend.ts\";\nimport {\n\tassertBytesWithinBudget,\n\ttype DnsLookup,\n\tguardedRequest,\n\ttype LowLevelRequestFn,\n\tWEBFETCH_MAX_BYTES,\n\tWEBFETCH_MAX_REDIRECTS,\n\tWEBFETCH_REQUEST_TIMEOUT_MS,\n\tWebfetchError,\n} from \"./guardrails.ts\";\n\nexport interface UndiciBackendOptions {\n\ttimeoutMs?: number;\n\tmaxBytes?: number;\n\tmaxRedirects?: number;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n}\n\nfunction decodeEntities(text: string): string {\n\treturn text\n\t\t.replace(/&#(\\d+);/g, (_match, digits: string) => String.fromCodePoint(Number.parseInt(digits, 10)))\n\t\t.replace(/&#x([0-9a-fA-F]+);/g, (_match, digits: string) => String.fromCodePoint(Number.parseInt(digits, 16)))\n\t\t.replace(/&(amp|lt|gt|quot|apos|nbsp);/g, (_match, name: string) => {\n\t\t\tswitch (name) {\n\t\t\t\tcase \"amp\":\n\t\t\t\t\treturn \"&\";\n\t\t\t\tcase \"lt\":\n\t\t\t\t\treturn \"<\";\n\t\t\t\tcase \"gt\":\n\t\t\t\t\treturn \">\";\n\t\t\t\tcase \"quot\":\n\t\t\t\t\treturn '\"';\n\t\t\t\tcase \"apos\":\n\t\t\t\t\treturn \"'\";\n\t\t\t\tdefault:\n\t\t\t\t\treturn \" \";\n\t\t\t}\n\t\t});\n}\n\n/** Minimal HTML→text cleanup: drop scripts/styles/comments/tags, decode entities, collapse whitespace. */\nexport function htmlToText(html: string): string {\n\tconst withoutScripts = html\n\t\t.replace(/<head[\\s\\S]*?<\\/head\\s*>/gi, \" \")\n\t\t.replace(/<script[\\s\\S]*?<\\/script\\s*>/gi, \" \")\n\t\t.replace(/<style[\\s\\S]*?<\\/style\\s*>/gi, \" \")\n\t\t.replace(/<!--[\\s\\S]*?-->/g, \" \");\n\tconst withBreaks = withoutScripts\n\t\t.replace(/<br[^>]*>/gi, \"\\n\")\n\t\t.replace(/<\\/(p|div|h[1-6]|li|tr|table|section|article|header|footer|blockquote|pre)[^>]*>/gi, \"\\n\");\n\tconst withoutTags = withBreaks.replace(/<[^>]+>/g, \" \");\n\tconst lines = decodeEntities(withoutTags)\n\t\t.split(\"\\n\")\n\t\t.map((line) => line.replace(/[ \\t\\r\\f\\v]+/g, \" \").trim())\n\t\t.filter((line) => line.length > 0);\n\treturn lines.join(\"\\n\");\n}\n\nfunction extractHtmlTitle(html: string): string | undefined {\n\tconst match = /<title[^>]*>([\\s\\S]*?)<\\/title\\s*>/i.exec(html);\n\tconst title = match?.[1] ? decodeEntities(match[1].replace(/\\s+/g, \" \").trim()) : undefined;\n\treturn title || undefined;\n}\n\nfunction isSupportedContentType(contentType: string): boolean {\n\tif (contentType === \"\") return true;\n\tconst type = contentType.split(\";\")[0]?.trim().toLowerCase() ?? \"\";\n\treturn (\n\t\ttype.startsWith(\"text/\") ||\n\t\ttype.includes(\"html\") ||\n\t\ttype.includes(\"json\") ||\n\t\ttype.includes(\"xml\") ||\n\t\ttype.includes(\"javascript\")\n\t);\n}\n\nexport function createUndiciBackend(options: UndiciBackendOptions = {}): WebFetchBackend {\n\tconst timeoutMs = options.timeoutMs ?? WEBFETCH_REQUEST_TIMEOUT_MS;\n\tconst maxBytes = options.maxBytes ?? WEBFETCH_MAX_BYTES;\n\tconst maxRedirects = options.maxRedirects ?? WEBFETCH_MAX_REDIRECTS;\n\treturn {\n\t\tid: \"undici\",\n\t\tasync search(): Promise<never> {\n\t\t\tthrow new WebfetchError(\n\t\t\t\t\"The undici backend does not support search; configure Firecrawl for search\",\n\t\t\t\t\"transport\",\n\t\t\t);\n\t\t},\n\t\tasync fetch(url: string, fetchOptions: { signal?: AbortSignal }): Promise<WebDocument> {\n\t\t\tconst response = await guardedRequest(url, {\n\t\t\t\ttimeoutMs,\n\t\t\t\tmaxBytes,\n\t\t\t\tmaxRedirects,\n\t\t\t\tsignal: fetchOptions.signal,\n\t\t\t\tlookup: options.lookup,\n\t\t\t\trequest: options.request,\n\t\t\t});\n\t\t\tconst contentType = response.headers[\"content-type\"] ?? \"\";\n\t\t\tif (!isSupportedContentType(contentType)) {\n\t\t\t\tthrow new WebfetchError(`Unsupported content type \"${contentType}\" for ${response.finalUrl}`, \"empty\");\n\t\t\t}\n\t\t\tconst chunks: Uint8Array[] = [];\n\t\t\tlet totalBytes = 0;\n\t\t\tfor await (const chunk of response.body) {\n\t\t\t\ttotalBytes += chunk.length;\n\t\t\t\tassertBytesWithinBudget(totalBytes, maxBytes);\n\t\t\t\tchunks.push(chunk);\n\t\t\t}\n\t\t\tconst merged = new Uint8Array(totalBytes);\n\t\t\tlet offset = 0;\n\t\t\tfor (const chunk of chunks) {\n\t\t\t\tmerged.set(chunk, offset);\n\t\t\t\toffset += chunk.length;\n\t\t\t}\n\t\t\tconst raw = new TextDecoder().decode(merged);\n\t\t\tconst looksLikeHtml = contentType.toLowerCase().includes(\"html\") || /^\\s*</.test(raw.slice(0, 1024));\n\t\t\tconst markdown = (looksLikeHtml ? htmlToText(raw) : raw.trim()).trim();\n\t\t\tif (!markdown) {\n\t\t\t\tthrow new WebfetchError(`Fetch of ${response.finalUrl} returned no usable text`, \"empty\");\n\t\t\t}\n\t\t\tconst title = (looksLikeHtml ? extractHtmlTitle(raw) : undefined) ?? response.finalUrl;\n\t\t\treturn { url: response.finalUrl, title, markdown, fetchedAt: new Date().toISOString() };\n\t\t},\n\t};\n}\n"]}

package/dist/core/webfetch/backend.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Shared shapes for harness-native web search and extraction.
+ *
+ * Normalization keeps backends interchangeable: Firecrawl `web[]`/`images[]`/
+ * `news[]` entries and raw HTML all collapse to these shapes, and every backend
+ * receives the same `ResolvedSearchIntent` (see search-intent.ts).
+ */
+import type { ResolvedSearchIntent } from "./search-intent.ts";
+export interface WebSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+    publishedDate?: string;
+    score?: number;
+    /** Which modality produced this hit (`web`, `images`, `news`). */
+    source?: "web" | "images" | "news";
+    /** Image hits: the image URL and its source page. */
+    imageUrl?: string;
+    imageWidth?: number;
+    imageHeight?: number;
+}
+export interface WebDocument {
+    url: string;
+    title: string;
+    markdown: string;
+    fetchedAt: string;
+}
+export interface WebSearchBackend {
+    readonly id: "firecrawl" | "exa" | "undici";
+    search(intent: ResolvedSearchIntent, options: {
+        limit: number;
+        signal?: AbortSignal;
+    }): Promise<WebSearchResult[]>;
+    fetch(url: string, options: {
+        signal?: AbortSignal;
+    }): Promise<WebDocument>;
+}
+/**
+ * Backends that can actually return a page. Exa is search-only in ion, so a
+ * document store entry can only ever come from Firecrawl or the undici fallback.
+ */
+export type WebFetchBackend = WebSearchBackend & {
+    readonly id: "firecrawl" | "undici";
+};
+//# sourceMappingURL=backend.d.ts.map

package/dist/core/webfetch/backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/backend.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE/D,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,EAAE,EAAE,WAAW,GAAG,KAAK,GAAG,QAAQ,CAAC;IAC5C,MAAM,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACnH,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CAC5E;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,WAAW,GAAG,QAAQ,CAAA;CAAE,CAAC","sourcesContent":["/**\n * Shared shapes for harness-native web search and extraction.\n *\n * Normalization keeps backends interchangeable: Firecrawl `web[]`/`images[]`/\n * `news[]` entries and raw HTML all collapse to these shapes, and every backend\n * receives the same `ResolvedSearchIntent` (see search-intent.ts).\n */\n\nimport type { ResolvedSearchIntent } from \"./search-intent.ts\";\n\nexport interface WebSearchResult {\n\ttitle: string;\n\turl: string;\n\tsnippet: string;\n\tpublishedDate?: string;\n\tscore?: number;\n\t/** Which modality produced this hit (`web`, `images`, `news`). */\n\tsource?: \"web\" | \"images\" | \"news\";\n\t/** Image hits: the image URL and its source page. */\n\timageUrl?: string;\n\timageWidth?: number;\n\timageHeight?: number;\n}\n\nexport interface WebDocument {\n\turl: string;\n\ttitle: string;\n\tmarkdown: string;\n\tfetchedAt: string;\n}\n\nexport interface WebSearchBackend {\n\treadonly id: \"firecrawl\" | \"exa\" | \"undici\";\n\tsearch(intent: ResolvedSearchIntent, options: { limit: number; signal?: AbortSignal }): Promise<WebSearchResult[]>;\n\tfetch(url: string, options: { signal?: AbortSignal }): Promise<WebDocument>;\n}\n\n/**\n * Backends that can actually return a page. Exa is search-only in ion, so a\n * document store entry can only ever come from Firecrawl or the undici fallback.\n */\nexport type WebFetchBackend = WebSearchBackend & { readonly id: \"firecrawl\" | \"undici\" };\n"]}

package/dist/core/webfetch/backend.js

@@ -0,0 +1,9 @@
+/**
+ * Shared shapes for harness-native web search and extraction.
+ *
+ * Normalization keeps backends interchangeable: Firecrawl `web[]`/`images[]`/
+ * `news[]` entries and raw HTML all collapse to these shapes, and every backend
+ * receives the same `ResolvedSearchIntent` (see search-intent.ts).
+ */
+export {};
+//# sourceMappingURL=backend.js.map

package/dist/core/webfetch/backend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backend.js","sourceRoot":"","sources":["../../../src/core/webfetch/backend.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG","sourcesContent":["/**\n * Shared shapes for harness-native web search and extraction.\n *\n * Normalization keeps backends interchangeable: Firecrawl `web[]`/`images[]`/\n * `news[]` entries and raw HTML all collapse to these shapes, and every backend\n * receives the same `ResolvedSearchIntent` (see search-intent.ts).\n */\n\nimport type { ResolvedSearchIntent } from \"./search-intent.ts\";\n\nexport interface WebSearchResult {\n\ttitle: string;\n\turl: string;\n\tsnippet: string;\n\tpublishedDate?: string;\n\tscore?: number;\n\t/** Which modality produced this hit (`web`, `images`, `news`). */\n\tsource?: \"web\" | \"images\" | \"news\";\n\t/** Image hits: the image URL and its source page. */\n\timageUrl?: string;\n\timageWidth?: number;\n\timageHeight?: number;\n}\n\nexport interface WebDocument {\n\turl: string;\n\ttitle: string;\n\tmarkdown: string;\n\tfetchedAt: string;\n}\n\nexport interface WebSearchBackend {\n\treadonly id: \"firecrawl\" | \"exa\" | \"undici\";\n\tsearch(intent: ResolvedSearchIntent, options: { limit: number; signal?: AbortSignal }): Promise<WebSearchResult[]>;\n\tfetch(url: string, options: { signal?: AbortSignal }): Promise<WebDocument>;\n}\n\n/**\n * Backends that can actually return a page. Exa is search-only in ion, so a\n * document store entry can only ever come from Firecrawl or the undici fallback.\n */\nexport type WebFetchBackend = WebSearchBackend & { readonly id: \"firecrawl\" | \"undici\" };\n"]}