@m4ike1/ion-coding-agent 0.1.0

package/dist/core/tools/webextract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webextract.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webextract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,KAAK,EAAoB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAO/E,OAAO,EAAwB,KAAK,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAE/E,QAAA,MAAM,gBAAgB;;;;;EASpB,CAAC;AAEH,eAAO,MAAM,sCAAsC;;;CAKzC,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAElE,MAAM,WAAW,qBAAqB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,qBAAsB,SAAQ,mBAAmB;CAAG;AAkBrE,wBAAgB,mBAAmB,CAClC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjG,MAAM,CAaR;AAED,wBAAgB,8BAA8B,CAC7C,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,qBAAqB,GAC7B,cAAc,CAAC,OAAO,gBAAgB,EAAE,qBAAqB,CAAC,CAsGhE;AAED,wBAAgB,oBAAoB,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,qBAAqB,GAC7B,SAAS,CAAC,OAAO,gBAAgB,CAAC,CAEpC","sourcesContent":["import type { AgentTool } from \"@m4ike1/ion-agent-core\";\nimport { type Static, Type } from \"typebox\";\nimport type { ExtensionContext, ToolDefinition } from \"../extensions/types.ts\";\nimport type { WebDocument } from \"../webfetch/backend.ts\";\nimport { isFallbackEligible } from \"../webfetch/backend-firecrawl.ts\";\nimport { WEBFETCH_DEFAULT_WINDOW_CHARS, WebDocumentStore } from \"../webfetch/doc-store.ts\";\nimport { WebfetchError } from \"../webfetch/guardrails.ts\";\nimport { webextractRenderers } from \"./renderers/webfetch.ts\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.ts\";\nimport { resolveFetchBackends, type WebfetchToolOptions } from \"./webfetch.ts\";\n\nconst webextractSchema = Type.Object({\n\tref: Type.Optional(\n\t\tType.String({ description: \"web:// reference from a webfetch result or earlier webextract call\" }),\n\t),\n\turl: Type.Optional(Type.String({ description: \"Page URL to fetch directly (exactly one of ref or url)\" })),\n\toffsetChars: Type.Optional(Type.Number({ description: \"Character offset into the document (default 0)\" })),\n\tlengthChars: Type.Optional(\n\t\tType.Number({ description: `Characters to return (default ${WEBFETCH_DEFAULT_WINDOW_CHARS})` }),\n\t),\n});\n\nexport const webextractToolSystemPromptContribution = {\n\tsnippet: \"Read full page text from a web:// ref or URL (windowed)\",\n\tguidelines: [\n\t\t\"Use webextract to read webfetch results: it pages large documents with offsetChars/lengthChars instead of dumping whole pages\",\n\t],\n} as const;\n\nexport type WebextractToolInput = Static<typeof webextractSchema>;\n\nexport interface WebextractToolDetails {\n\tref: string;\n\ttotalChars: number;\n\toffsetChars: number;\n\tlengthChars: number;\n\tsource: \"firecrawl\" | \"undici\";\n\tfallbackNotice?: string;\n}\n\nexport interface WebextractToolOptions extends WebfetchToolOptions {}\n\nfunction validatedWindow(\n\toffsetChars: number | undefined,\n\tlengthChars: number | undefined,\n): {\n\toffsetChars: number;\n\tlengthChars: number;\n} {\n\tif (offsetChars !== undefined && (!Number.isFinite(offsetChars) || offsetChars < 0)) {\n\t\tthrow new Error(\"offsetChars must be a non-negative number\");\n\t}\n\tif (lengthChars !== undefined && (!Number.isFinite(lengthChars) || lengthChars <= 0)) {\n\t\tthrow new Error(\"lengthChars must be a positive number\");\n\t}\n\treturn { offsetChars: offsetChars ?? 0, lengthChars: lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS };\n}\n\nexport function formatExtractWindow(\n\ttitle: string,\n\turl: string,\n\twindow: { ref: string; totalChars: number; offsetChars: number; lengthChars: number; text: string },\n): string {\n\tconst end = window.offsetChars + window.lengthChars;\n\tconst lines = [\n\t\ttitle,\n\t\t`${url} (${window.ref})`,\n\t\t`[chars ${window.offsetChars}-${end} of ${window.totalChars}]`,\n\t\t\"\",\n\t\twindow.text,\n\t];\n\tif (end < window.totalChars) {\n\t\tlines.push(\"\", `[Use offsetChars=${end} to continue.]`);\n\t}\n\treturn lines.join(\"\\n\");\n}\n\nexport function createWebextractToolDefinition(\n\t_cwd: string,\n\toptions?: WebextractToolOptions,\n): ToolDefinition<typeof webextractSchema, WebextractToolDetails> {\n\tconst { primary, fallback } = resolveFetchBackends(options);\n\tconst store = options?.store ?? new WebDocumentStore();\n\n\tasync function fetchDocument(\n\t\turl: string,\n\t\tsignal?: AbortSignal,\n\t): Promise<{\n\t\tdoc: WebDocument;\n\t\tsource: \"firecrawl\" | \"undici\";\n\t\tfallbackNotice?: string;\n\t}> {\n\t\ttry {\n\t\t\tconst doc = await primary.fetch(url, { signal });\n\t\t\treturn { doc, source: primary.id };\n\t\t} catch (error) {\n\t\t\tif (!isFallbackEligible(error)) throw error;\n\t\t\t// The keyless-unavailable message is a multi-line remediation hint; keep the\n\t\t\t// per-page notice short and point at the full guidance instead.\n\t\t\tconst reason =\n\t\t\t\terror instanceof WebfetchError && error.code === \"unavailable\"\n\t\t\t\t\t? \"keyless tier refused this network; set firecrawlApiKey or FIRECRAWL_API_KEY\"\n\t\t\t\t\t: error instanceof Error\n\t\t\t\t\t\t? error.message\n\t\t\t\t\t\t: String(error);\n\t\t\tconst doc = await fallback.fetch(url, { signal });\n\t\t\treturn { doc, source: fallback.id, fallbackNotice: `Firecrawl unavailable (${reason}); used direct fetch.` };\n\t\t}\n\t}\n\n\treturn {\n\t\tname: \"webextract\",\n\t\tlabel: \"webextract\",\n\t\tdescription: `Read the full text of a fetched web page, windowed by character offset so large pages never flood context. Provide exactly one of ref (a web:// reference from webfetch) or url (fetched on demand through Firecrawl with direct-fetch fallback). First read of a search-result ref fetches the page; later reads page from the stored document.`,\n\t\tpromptSnippet: webextractToolSystemPromptContribution.snippet,\n\t\tpromptGuidelines: [...webextractToolSystemPromptContribution.guidelines],\n\t\tparameters: webextractSchema,\n\t\tasync execute(\n\t\t\t_toolCallId,\n\t\t\t{\n\t\t\t\tref,\n\t\t\t\turl,\n\t\t\t\toffsetChars,\n\t\t\t\tlengthChars,\n\t\t\t}: { ref?: string; url?: string; offsetChars?: number; lengthChars?: number },\n\t\t\tsignal?: AbortSignal,\n\t\t\t_onUpdate?,\n\t\t\t_ctx?: ExtensionContext,\n\t\t) {\n\t\t\tif ((ref === undefined) === (url === undefined)) {\n\t\t\t\tthrow new Error(\"Provide exactly one of ref or url\");\n\t\t\t}\n\t\t\tconst window = validatedWindow(offsetChars, lengthChars);\n\t\t\tlet effectiveRef = ref;\n\t\t\tlet source: \"firecrawl\" | \"undici\" | undefined;\n\t\t\tlet fallbackNotice: string | undefined;\n\t\t\tif (url !== undefined) {\n\t\t\t\tif (!url.trim()) {\n\t\t\t\t\tthrow new Error(\"URL must not be empty\");\n\t\t\t\t}\n\t\t\t\tconst fetched = await fetchDocument(url, signal);\n\t\t\t\tsource = fetched.source;\n\t\t\t\tfallbackNotice = fetched.fallbackNotice;\n\t\t\t\teffectiveRef = store.store(fetched.doc, fetched.source);\n\t\t\t} else {\n\t\t\t\tconst stored = store.get(effectiveRef as string);\n\t\t\t\tif (!stored) {\n\t\t\t\t\tthrow new Error(`Unknown web reference: ${effectiveRef}`);\n\t\t\t\t}\n\t\t\t\tsource = stored.source;\n\t\t\t\tif (!stored.complete) {\n\t\t\t\t\tconst fetched = await fetchDocument(stored.doc.url, signal);\n\t\t\t\t\tstore.update(stored.ref, fetched.doc, fetched.source);\n\t\t\t\t\tsource = fetched.source;\n\t\t\t\t\tfallbackNotice = fetched.fallbackNotice;\n\t\t\t\t}\n\t\t\t}\n\t\t\tconst stored = store.get(effectiveRef as string);\n\t\t\tif (!stored) {\n\t\t\t\tthrow new Error(`Unknown web reference: ${effectiveRef}`);\n\t\t\t}\n\t\t\tif (window.offsetChars > stored.doc.markdown.length) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Offset ${window.offsetChars} is beyond end of document (${stored.doc.markdown.length} chars total)`,\n\t\t\t\t);\n\t\t\t}\n\t\t\tconst extracted = store.extract(stored.ref, window);\n\t\t\tconst details: WebextractToolDetails = {\n\t\t\t\tref: stored.ref,\n\t\t\t\ttotalChars: extracted.totalChars,\n\t\t\t\toffsetChars: extracted.offsetChars,\n\t\t\t\tlengthChars: extracted.lengthChars,\n\t\t\t\tsource,\n\t\t\t};\n\t\t\tif (fallbackNotice !== undefined) details.fallbackNotice = fallbackNotice;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\", text: formatExtractWindow(stored.doc.title, stored.doc.url, extracted) }],\n\t\t\t\tdetails,\n\t\t\t};\n\t\t},\n\t\t...webextractRenderers,\n\t};\n}\n\nexport function createWebextractTool(\n\t_cwd: string,\n\toptions?: WebextractToolOptions,\n): AgentTool<typeof webextractSchema> {\n\treturn wrapToolDefinition(createWebextractToolDefinition(_cwd, options));\n}\n"]}

package/dist/core/tools/webextract.js

@@ -0,0 +1,130 @@
+import { Type } from "typebox";
+import { isFallbackEligible } from "../webfetch/backend-firecrawl.js";
+import { WEBFETCH_DEFAULT_WINDOW_CHARS, WebDocumentStore } from "../webfetch/doc-store.js";
+import { WebfetchError } from "../webfetch/guardrails.js";
+import { webextractRenderers } from "./renderers/webfetch.js";
+import { wrapToolDefinition } from "./tool-definition-wrapper.js";
+import { resolveFetchBackends } from "./webfetch.js";
+const webextractSchema = Type.Object({
+    ref: Type.Optional(Type.String({ description: "web:// reference from a webfetch result or earlier webextract call" })),
+    url: Type.Optional(Type.String({ description: "Page URL to fetch directly (exactly one of ref or url)" })),
+    offsetChars: Type.Optional(Type.Number({ description: "Character offset into the document (default 0)" })),
+    lengthChars: Type.Optional(Type.Number({ description: `Characters to return (default ${WEBFETCH_DEFAULT_WINDOW_CHARS})` })),
+});
+export const webextractToolSystemPromptContribution = {
+    snippet: "Read full page text from a web:// ref or URL (windowed)",
+    guidelines: [
+        "Use webextract to read webfetch results: it pages large documents with offsetChars/lengthChars instead of dumping whole pages",
+    ],
+};
+function validatedWindow(offsetChars, lengthChars) {
+    if (offsetChars !== undefined && (!Number.isFinite(offsetChars) || offsetChars < 0)) {
+        throw new Error("offsetChars must be a non-negative number");
+    }
+    if (lengthChars !== undefined && (!Number.isFinite(lengthChars) || lengthChars <= 0)) {
+        throw new Error("lengthChars must be a positive number");
+    }
+    return { offsetChars: offsetChars ?? 0, lengthChars: lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS };
+}
+export function formatExtractWindow(title, url, window) {
+    const end = window.offsetChars + window.lengthChars;
+    const lines = [
+        title,
+        `${url} (${window.ref})`,
+        `[chars ${window.offsetChars}-${end} of ${window.totalChars}]`,
+        "",
+        window.text,
+    ];
+    if (end < window.totalChars) {
+        lines.push("", `[Use offsetChars=${end} to continue.]`);
+    }
+    return lines.join("\n");
+}
+export function createWebextractToolDefinition(_cwd, options) {
+    const { primary, fallback } = resolveFetchBackends(options);
+    const store = options?.store ?? new WebDocumentStore();
+    async function fetchDocument(url, signal) {
+        try {
+            const doc = await primary.fetch(url, { signal });
+            return { doc, source: primary.id };
+        }
+        catch (error) {
+            if (!isFallbackEligible(error))
+                throw error;
+            // The keyless-unavailable message is a multi-line remediation hint; keep the
+            // per-page notice short and point at the full guidance instead.
+            const reason = error instanceof WebfetchError && error.code === "unavailable"
+                ? "keyless tier refused this network; set firecrawlApiKey or FIRECRAWL_API_KEY"
+                : error instanceof Error
+                    ? error.message
+                    : String(error);
+            const doc = await fallback.fetch(url, { signal });
+            return { doc, source: fallback.id, fallbackNotice: `Firecrawl unavailable (${reason}); used direct fetch.` };
+        }
+    }
+    return {
+        name: "webextract",
+        label: "webextract",
+        description: `Read the full text of a fetched web page, windowed by character offset so large pages never flood context. Provide exactly one of ref (a web:// reference from webfetch) or url (fetched on demand through Firecrawl with direct-fetch fallback). First read of a search-result ref fetches the page; later reads page from the stored document.`,
+        promptSnippet: webextractToolSystemPromptContribution.snippet,
+        promptGuidelines: [...webextractToolSystemPromptContribution.guidelines],
+        parameters: webextractSchema,
+        async execute(_toolCallId, { ref, url, offsetChars, lengthChars, }, signal, _onUpdate, _ctx) {
+            if ((ref === undefined) === (url === undefined)) {
+                throw new Error("Provide exactly one of ref or url");
+            }
+            const window = validatedWindow(offsetChars, lengthChars);
+            let effectiveRef = ref;
+            let source;
+            let fallbackNotice;
+            if (url !== undefined) {
+                if (!url.trim()) {
+                    throw new Error("URL must not be empty");
+                }
+                const fetched = await fetchDocument(url, signal);
+                source = fetched.source;
+                fallbackNotice = fetched.fallbackNotice;
+                effectiveRef = store.store(fetched.doc, fetched.source);
+            }
+            else {
+                const stored = store.get(effectiveRef);
+                if (!stored) {
+                    throw new Error(`Unknown web reference: ${effectiveRef}`);
+                }
+                source = stored.source;
+                if (!stored.complete) {
+                    const fetched = await fetchDocument(stored.doc.url, signal);
+                    store.update(stored.ref, fetched.doc, fetched.source);
+                    source = fetched.source;
+                    fallbackNotice = fetched.fallbackNotice;
+                }
+            }
+            const stored = store.get(effectiveRef);
+            if (!stored) {
+                throw new Error(`Unknown web reference: ${effectiveRef}`);
+            }
+            if (window.offsetChars > stored.doc.markdown.length) {
+                throw new Error(`Offset ${window.offsetChars} is beyond end of document (${stored.doc.markdown.length} chars total)`);
+            }
+            const extracted = store.extract(stored.ref, window);
+            const details = {
+                ref: stored.ref,
+                totalChars: extracted.totalChars,
+                offsetChars: extracted.offsetChars,
+                lengthChars: extracted.lengthChars,
+                source,
+            };
+            if (fallbackNotice !== undefined)
+                details.fallbackNotice = fallbackNotice;
+            return {
+                content: [{ type: "text", text: formatExtractWindow(stored.doc.title, stored.doc.url, extracted) }],
+                details,
+            };
+        },
+        ...webextractRenderers,
+    };
+}
+export function createWebextractTool(_cwd, options) {
+    return wrapToolDefinition(createWebextractToolDefinition(_cwd, options));
+}
+//# sourceMappingURL=webextract.js.map

package/dist/core/tools/webextract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webextract.js","sourceRoot":"","sources":["../../../src/core/tools/webextract.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,6BAA6B,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAA4B,MAAM,eAAe,CAAC;AAE/E,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC;IACpC,GAAG,EAAE,IAAI,CAAC,QAAQ,CACjB,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,oEAAoE,EAAE,CAAC,CAClG;IACD,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,wDAAwD,EAAE,CAAC,CAAC;IAC1G,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,gDAAgD,EAAE,CAAC,CAAC;IAC1G,WAAW,EAAE,IAAI,CAAC,QAAQ,CACzB,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,iCAAiC,6BAA6B,GAAG,EAAE,CAAC,CAC/F;CACD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sCAAsC,GAAG;IACrD,OAAO,EAAE,yDAAyD;IAClE,UAAU,EAAE;QACX,+HAA+H;KAC/H;CACQ,CAAC;AAeX,SAAS,eAAe,CACvB,WAA+B,EAC/B,WAA+B,EAI9B;IACD,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,WAAW,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,WAAW,IAAI,CAAC,EAAE,WAAW,EAAE,WAAW,IAAI,6BAA6B,EAAE,CAAC;AAAA,CACpG;AAED,MAAM,UAAU,mBAAmB,CAClC,KAAa,EACb,GAAW,EACX,MAAmG,EAC1F;IACT,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACpD,MAAM,KAAK,GAAG;QACb,KAAK;QACL,GAAG,GAAG,KAAK,MAAM,CAAC,GAAG,GAAG;QACxB,UAAU,MAAM,CAAC,WAAW,IAAI,GAAG,OAAO,MAAM,CAAC,UAAU,GAAG;QAC9D,EAAE;QACF,MAAM,CAAC,IAAI;KACX,CAAC;IACF,IAAI,GAAG,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,GAAG,gBAAgB,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACxB;AAED,MAAM,UAAU,8BAA8B,CAC7C,IAAY,EACZ,OAA+B,EACkC;IACjE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,gBAAgB,EAAE,CAAC;IAEvD,KAAK,UAAU,aAAa,CAC3B,GAAW,EACX,MAAoB,EAKlB;QACF,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YACjD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;YAC5C,6EAA6E;YAC7E,gEAAgE;YAChE,MAAM,MAAM,GACX,KAAK,YAAY,aAAa,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;gBAC7D,CAAC,CAAC,6EAA6E;gBAC/E,CAAC,CAAC,KAAK,YAAY,KAAK;oBACvB,CAAC,CAAC,KAAK,CAAC,OAAO;oBACf,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAClD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,EAAE,cAAc,EAAE,0BAA0B,MAAM,uBAAuB,EAAE,CAAC;QAC9G,CAAC;IAAA,CACD;IAED,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,YAAY;QACnB,WAAW,EAAE,kVAAkV;QAC/V,aAAa,EAAE,sCAAsC,CAAC,OAAO;QAC7D,gBAAgB,EAAE,CAAC,GAAG,sCAAsC,CAAC,UAAU,CAAC;QACxE,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,OAAO,CACZ,WAAW,EACX,EACC,GAAG,EACH,GAAG,EACH,WAAW,EACX,WAAW,GACiE,EAC7E,MAAoB,EACpB,SAAU,EACV,IAAuB,EACtB;YACD,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,KAAK,CAAC,GAAG,KAAK,SAAS,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACtD,CAAC;YACD,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACzD,IAAI,YAAY,GAAG,GAAG,CAAC;YACvB,IAAI,MAA0C,CAAC;YAC/C,IAAI,cAAkC,CAAC;YACvC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBAC1C,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBACjD,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;gBACxB,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;gBACxC,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACP,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,YAAsB,CAAC,CAAC;gBACjD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAC;gBAC3D,CAAC;gBACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACtB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;oBAC5D,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;oBACtD,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBACxB,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;gBACzC,CAAC;YACF,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,YAAsB,CAAC,CAAC;YACjD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,IAAI,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CACd,UAAU,MAAM,CAAC,WAAW,+BAA+B,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,eAAe,CACpG,CAAC;YACH,CAAC;YACD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACpD,MAAM,OAAO,GAA0B;gBACtC,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,MAAM;aACN,CAAC;YACF,IAAI,cAAc,KAAK,SAAS;gBAAE,OAAO,CAAC,cAAc,GAAG,cAAc,CAAC;YAC1E,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC;gBACnG,OAAO;aACP,CAAC;QAAA,CACF;QACD,GAAG,mBAAmB;KACtB,CAAC;AAAA,CACF;AAED,MAAM,UAAU,oBAAoB,CACnC,IAAY,EACZ,OAA+B,EACM;IACrC,OAAO,kBAAkB,CAAC,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACzE","sourcesContent":["import type { AgentTool } from \"@m4ike1/ion-agent-core\";\nimport { type Static, Type } from \"typebox\";\nimport type { ExtensionContext, ToolDefinition } from \"../extensions/types.ts\";\nimport type { WebDocument } from \"../webfetch/backend.ts\";\nimport { isFallbackEligible } from \"../webfetch/backend-firecrawl.ts\";\nimport { WEBFETCH_DEFAULT_WINDOW_CHARS, WebDocumentStore } from \"../webfetch/doc-store.ts\";\nimport { WebfetchError } from \"../webfetch/guardrails.ts\";\nimport { webextractRenderers } from \"./renderers/webfetch.ts\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.ts\";\nimport { resolveFetchBackends, type WebfetchToolOptions } from \"./webfetch.ts\";\n\nconst webextractSchema = Type.Object({\n\tref: Type.Optional(\n\t\tType.String({ description: \"web:// reference from a webfetch result or earlier webextract call\" }),\n\t),\n\turl: Type.Optional(Type.String({ description: \"Page URL to fetch directly (exactly one of ref or url)\" })),\n\toffsetChars: Type.Optional(Type.Number({ description: \"Character offset into the document (default 0)\" })),\n\tlengthChars: Type.Optional(\n\t\tType.Number({ description: `Characters to return (default ${WEBFETCH_DEFAULT_WINDOW_CHARS})` }),\n\t),\n});\n\nexport const webextractToolSystemPromptContribution = {\n\tsnippet: \"Read full page text from a web:// ref or URL (windowed)\",\n\tguidelines: [\n\t\t\"Use webextract to read webfetch results: it pages large documents with offsetChars/lengthChars instead of dumping whole pages\",\n\t],\n} as const;\n\nexport type WebextractToolInput = Static<typeof webextractSchema>;\n\nexport interface WebextractToolDetails {\n\tref: string;\n\ttotalChars: number;\n\toffsetChars: number;\n\tlengthChars: number;\n\tsource: \"firecrawl\" | \"undici\";\n\tfallbackNotice?: string;\n}\n\nexport interface WebextractToolOptions extends WebfetchToolOptions {}\n\nfunction validatedWindow(\n\toffsetChars: number | undefined,\n\tlengthChars: number | undefined,\n): {\n\toffsetChars: number;\n\tlengthChars: number;\n} {\n\tif (offsetChars !== undefined && (!Number.isFinite(offsetChars) || offsetChars < 0)) {\n\t\tthrow new Error(\"offsetChars must be a non-negative number\");\n\t}\n\tif (lengthChars !== undefined && (!Number.isFinite(lengthChars) || lengthChars <= 0)) {\n\t\tthrow new Error(\"lengthChars must be a positive number\");\n\t}\n\treturn { offsetChars: offsetChars ?? 0, lengthChars: lengthChars ?? WEBFETCH_DEFAULT_WINDOW_CHARS };\n}\n\nexport function formatExtractWindow(\n\ttitle: string,\n\turl: string,\n\twindow: { ref: string; totalChars: number; offsetChars: number; lengthChars: number; text: string },\n): string {\n\tconst end = window.offsetChars + window.lengthChars;\n\tconst lines = [\n\t\ttitle,\n\t\t`${url} (${window.ref})`,\n\t\t`[chars ${window.offsetChars}-${end} of ${window.totalChars}]`,\n\t\t\"\",\n\t\twindow.text,\n\t];\n\tif (end < window.totalChars) {\n\t\tlines.push(\"\", `[Use offsetChars=${end} to continue.]`);\n\t}\n\treturn lines.join(\"\\n\");\n}\n\nexport function createWebextractToolDefinition(\n\t_cwd: string,\n\toptions?: WebextractToolOptions,\n): ToolDefinition<typeof webextractSchema, WebextractToolDetails> {\n\tconst { primary, fallback } = resolveFetchBackends(options);\n\tconst store = options?.store ?? new WebDocumentStore();\n\n\tasync function fetchDocument(\n\t\turl: string,\n\t\tsignal?: AbortSignal,\n\t): Promise<{\n\t\tdoc: WebDocument;\n\t\tsource: \"firecrawl\" | \"undici\";\n\t\tfallbackNotice?: string;\n\t}> {\n\t\ttry {\n\t\t\tconst doc = await primary.fetch(url, { signal });\n\t\t\treturn { doc, source: primary.id };\n\t\t} catch (error) {\n\t\t\tif (!isFallbackEligible(error)) throw error;\n\t\t\t// The keyless-unavailable message is a multi-line remediation hint; keep the\n\t\t\t// per-page notice short and point at the full guidance instead.\n\t\t\tconst reason =\n\t\t\t\terror instanceof WebfetchError && error.code === \"unavailable\"\n\t\t\t\t\t? \"keyless tier refused this network; set firecrawlApiKey or FIRECRAWL_API_KEY\"\n\t\t\t\t\t: error instanceof Error\n\t\t\t\t\t\t? error.message\n\t\t\t\t\t\t: String(error);\n\t\t\tconst doc = await fallback.fetch(url, { signal });\n\t\t\treturn { doc, source: fallback.id, fallbackNotice: `Firecrawl unavailable (${reason}); used direct fetch.` };\n\t\t}\n\t}\n\n\treturn {\n\t\tname: \"webextract\",\n\t\tlabel: \"webextract\",\n\t\tdescription: `Read the full text of a fetched web page, windowed by character offset so large pages never flood context. Provide exactly one of ref (a web:// reference from webfetch) or url (fetched on demand through Firecrawl with direct-fetch fallback). First read of a search-result ref fetches the page; later reads page from the stored document.`,\n\t\tpromptSnippet: webextractToolSystemPromptContribution.snippet,\n\t\tpromptGuidelines: [...webextractToolSystemPromptContribution.guidelines],\n\t\tparameters: webextractSchema,\n\t\tasync execute(\n\t\t\t_toolCallId,\n\t\t\t{\n\t\t\t\tref,\n\t\t\t\turl,\n\t\t\t\toffsetChars,\n\t\t\t\tlengthChars,\n\t\t\t}: { ref?: string; url?: string; offsetChars?: number; lengthChars?: number },\n\t\t\tsignal?: AbortSignal,\n\t\t\t_onUpdate?,\n\t\t\t_ctx?: ExtensionContext,\n\t\t) {\n\t\t\tif ((ref === undefined) === (url === undefined)) {\n\t\t\t\tthrow new Error(\"Provide exactly one of ref or url\");\n\t\t\t}\n\t\t\tconst window = validatedWindow(offsetChars, lengthChars);\n\t\t\tlet effectiveRef = ref;\n\t\t\tlet source: \"firecrawl\" | \"undici\" | undefined;\n\t\t\tlet fallbackNotice: string | undefined;\n\t\t\tif (url !== undefined) {\n\t\t\t\tif (!url.trim()) {\n\t\t\t\t\tthrow new Error(\"URL must not be empty\");\n\t\t\t\t}\n\t\t\t\tconst fetched = await fetchDocument(url, signal);\n\t\t\t\tsource = fetched.source;\n\t\t\t\tfallbackNotice = fetched.fallbackNotice;\n\t\t\t\teffectiveRef = store.store(fetched.doc, fetched.source);\n\t\t\t} else {\n\t\t\t\tconst stored = store.get(effectiveRef as string);\n\t\t\t\tif (!stored) {\n\t\t\t\t\tthrow new Error(`Unknown web reference: ${effectiveRef}`);\n\t\t\t\t}\n\t\t\t\tsource = stored.source;\n\t\t\t\tif (!stored.complete) {\n\t\t\t\t\tconst fetched = await fetchDocument(stored.doc.url, signal);\n\t\t\t\t\tstore.update(stored.ref, fetched.doc, fetched.source);\n\t\t\t\t\tsource = fetched.source;\n\t\t\t\t\tfallbackNotice = fetched.fallbackNotice;\n\t\t\t\t}\n\t\t\t}\n\t\t\tconst stored = store.get(effectiveRef as string);\n\t\t\tif (!stored) {\n\t\t\t\tthrow new Error(`Unknown web reference: ${effectiveRef}`);\n\t\t\t}\n\t\t\tif (window.offsetChars > stored.doc.markdown.length) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t`Offset ${window.offsetChars} is beyond end of document (${stored.doc.markdown.length} chars total)`,\n\t\t\t\t);\n\t\t\t}\n\t\t\tconst extracted = store.extract(stored.ref, window);\n\t\t\tconst details: WebextractToolDetails = {\n\t\t\t\tref: stored.ref,\n\t\t\t\ttotalChars: extracted.totalChars,\n\t\t\t\toffsetChars: extracted.offsetChars,\n\t\t\t\tlengthChars: extracted.lengthChars,\n\t\t\t\tsource,\n\t\t\t};\n\t\t\tif (fallbackNotice !== undefined) details.fallbackNotice = fallbackNotice;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\", text: formatExtractWindow(stored.doc.title, stored.doc.url, extracted) }],\n\t\t\t\tdetails,\n\t\t\t};\n\t\t},\n\t\t...webextractRenderers,\n\t};\n}\n\nexport function createWebextractTool(\n\t_cwd: string,\n\toptions?: WebextractToolOptions,\n): AgentTool<typeof webextractSchema> {\n\treturn wrapToolDefinition(createWebextractToolDefinition(_cwd, options));\n}\n"]}

package/dist/core/tools/webfetch.d.ts

@@ -0,0 +1,104 @@
+import type { AgentTool } from "@m4ike1/ion-agent-core";
+import { type Static, Type } from "typebox";
+import type { ToolDefinition } from "../extensions/types.ts";
+import type { WebFetchBackend, WebSearchBackend, WebSearchResult } from "../webfetch/backend.ts";
+import { WebDocumentStore } from "../webfetch/doc-store.ts";
+import { type ResolvedSearchIntent } from "../webfetch/search-intent.ts";
+export type { WebDocument, WebFetchBackend, WebSearchBackend, WebSearchResult } from "../webfetch/backend.ts";
+export { type ExtractWindow, type StoredWebDocument, WEBFETCH_DEFAULT_WINDOW_CHARS, type WebDocumentSource, WebDocumentStore, } from "../webfetch/doc-store.ts";
+export { parseRecency, type ResolvedSearchIntent, resolveSearchIntent, SEARCH_DEFAULT_WEB_RESULTS, SEARCH_MAX_PER_SOURCE, type SearchFilters, type SearchIntent, SearchIntentError, type SearchResultCounts, totalRequestedResults, } from "../webfetch/search-intent.ts";
+export { type ParsedSearchQuery, parseSearchQueryTags } from "../webfetch/search-tags.ts";
+declare const webfetchSchema: Type.TObject<{
+    query: Type.TString;
+    results: Type.TOptional<Type.TObject<{
+        web: Type.TOptional<Type.TNumber>;
+        images: Type.TOptional<Type.TNumber>;
+        news: Type.TOptional<Type.TNumber>;
+    }>>;
+    filters: Type.TOptional<Type.TObject<{
+        domains: Type.TOptional<Type.TArray<Type.TString>>;
+        recency: Type.TOptional<Type.TString>;
+        language: Type.TOptional<Type.TString>;
+    }>>;
+    limit: Type.TOptional<Type.TNumber>;
+}>;
+export declare const webfetchToolSystemPromptContribution: {
+    readonly snippet: "Search the web (results stored as web:// refs, full text via webextract)";
+    readonly guidelines: readonly ["Use webfetch instead of curl for web research: results are stored as web:// refs, then read full text on demand with webextract instead of pasting whole pages", "Pass results/filters for structured control (per-source counts, domains, recency). Tag sugar like [images=3] [news=5] [recent=7d] works only when those args are omitted.", "Naming a source replaces the default web-only request: results={images:3} returns images only.", "Image results and recency/domain filters need Firecrawl; if it is unavailable, ask for plain web results without filters."];
+};
+export type WebfetchToolInput = Static<typeof webfetchSchema>;
+export interface WebfetchToolDetails {
+    refs: string[];
+    backend: "firecrawl" | "exa" | "undici";
+    warning?: string;
+    creditsUsed?: number;
+    /** Set when an earlier backend in the chain failed and a later one answered. */
+    backendNotice?: string;
+    /** Per-source counts actually requested after tag sugar and clamping. */
+    requested?: {
+        web?: number;
+        images?: number;
+        news?: number;
+    };
+    /** Tag sugar that was recognized, e.g. `[images=3]`. */
+    appliedTags?: string[];
+    /** Tag sugar that looked like a directive but could not be applied. */
+    tagWarnings?: string[];
+}
+/**
+ * Pluggable backends for the web tools.
+ * Override these to test without network or to front a different search provider.
+ */
+export interface WebOperations {
+    /** Search backend. Default: the ordered chain Firecrawl -> Exa. */
+    searchBackend?: WebSearchBackend;
+    /** Extra search backends tried after `searchBackend` fails. Default: Exa. */
+    fallbackSearchBackends?: WebSearchBackend[];
+    /** Primary fetch backend. Default: Firecrawl. */
+    fetchBackend?: WebFetchBackend;
+    /** Direct-fetch fallback used on Firecrawl transport/auth failures. Default: undici. */
+    fallbackBackend?: WebFetchBackend;
+    /** Shared session document store. Default: a fresh store per tool instance. */
+    store?: WebDocumentStore;
+}
+export interface WebfetchToolOptions extends WebOperations {
+    /** Firecrawl API key. Default: `FIRECRAWL_API_KEY` env. Omitted entirely when unset (keyless tier). */
+    apiKey?: string;
+    /** Exa API key. Default: `EXA_API_KEY` env. Unset uses the keyless hosted MCP search tier. */
+    exaApiKey?: string;
+}
+export declare function resolveWebApiKey(explicitApiKey?: string): string | undefined;
+/**
+ * Build the search intent from the structured arguments, falling back to tag
+ * sugar in the query when `results`/`filters` are absent. An explicit structured
+ * call always wins; sugar is only a convenience for terse queries.
+ */
+export declare function buildSearchIntent(input: {
+    query: string;
+    results?: {
+        web?: number;
+        images?: number;
+        news?: number;
+    };
+    filters?: {
+        domains?: string[];
+        recency?: string;
+        language?: string;
+    };
+    limit?: number;
+}): {
+    intent: ResolvedSearchIntent;
+    appliedTags: string[];
+    warnings: string[];
+};
+export declare function formatSearchResults(query: string, hits: Array<{
+    result: WebSearchResult;
+    ref: string;
+}>): string;
+export declare function createWebfetchToolDefinition(_cwd: string, options?: WebfetchToolOptions): ToolDefinition<typeof webfetchSchema, WebfetchToolDetails>;
+export declare function resolveFetchBackends(options?: WebfetchToolOptions): {
+    primary: WebFetchBackend;
+    fallback: WebFetchBackend;
+};
+export declare function createWebfetchTool(_cwd: string, options?: WebfetchToolOptions): AgentTool<typeof webfetchSchema>;
+//# sourceMappingURL=webfetch.d.ts.map

package/dist/core/tools/webfetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,KAAK,EAAoB,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC/E,OAAO,KAAK,EAAe,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAS9G,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAE5D,OAAO,EAAE,KAAK,oBAAoB,EAA8C,MAAM,8BAA8B,CAAC;AAKrH,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9G,OAAO,EACN,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,6BAA6B,EAC7B,KAAK,iBAAiB,EACtB,gBAAgB,GAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACN,YAAY,EACZ,KAAK,oBAAoB,EACzB,mBAAmB,EACnB,0BAA0B,EAC1B,qBAAqB,EACrB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,iBAAiB,EACjB,KAAK,kBAAkB,EACvB,qBAAqB,GACrB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,KAAK,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAgB1F,QAAA,MAAM,cAAc;;;;;;;;;;;;;EAQlB,CAAC;AAEH,eAAO,MAAM,oCAAoC;;;CAQvC,CAAC;AAEX,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AAK9D,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,OAAO,EAAE,WAAW,GAAG,KAAK,GAAG,QAAQ,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yEAAyE;IACzE,SAAS,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7D,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC7B,mEAAmE;IACnE,aAAa,CAAC,EAAE,gBAAgB,CAAC;IACjC,6EAA6E;IAC7E,sBAAsB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC5C,iDAAiD;IACjD,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,wFAAwF;IACxF,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,+EAA+E;IAC/E,KAAK,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED,MAAM,WAAW,mBAAoB,SAAQ,aAAa;IACzD,uGAAuG;IACvG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8FAA8F;IAC9F,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,gBAAgB,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE5E;AAsDD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtE,KAAK,CAAC,EAAE,MAAM,CAAC;CACf,GAAG;IAAE,MAAM,EAAE,oBAAoB,CAAC;IAAC,WAAW,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAsC9E;AAWD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAWhH;AAED,wBAAgB,4BAA4B,CAC3C,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,CAAC,CAyE5D;AAMD,wBAAgB,oBAAoB,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG;IACpE,OAAO,EAAE,eAAe,CAAC;IACzB,QAAQ,EAAE,eAAe,CAAC;CAC1B,CAKA;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAEhH","sourcesContent":["import type { AgentTool } from \"@m4ike1/ion-agent-core\";\nimport { type Static, Type } from \"typebox\";\nimport type { ExtensionContext, ToolDefinition } from \"../extensions/types.ts\";\nimport type { WebDocument, WebFetchBackend, WebSearchBackend, WebSearchResult } from \"../webfetch/backend.ts\";\nimport { createExaBackend, isExaFallbackEligible, resolveExaApiKey } from \"../webfetch/backend-exa.ts\";\nimport {\n\tcreateFirecrawlBackend,\n\ttype FirecrawlBackend,\n\tisFallbackEligible,\n\tresolveFirecrawlApiKey,\n} from \"../webfetch/backend-firecrawl.ts\";\nimport { createUndiciBackend } from \"../webfetch/backend-undici.ts\";\nimport { WebDocumentStore } from \"../webfetch/doc-store.ts\";\nimport { WebfetchError } from \"../webfetch/guardrails.ts\";\nimport { type ResolvedSearchIntent, resolveSearchIntent, totalRequestedResults } from \"../webfetch/search-intent.ts\";\nimport { parseSearchQueryTags } from \"../webfetch/search-tags.ts\";\nimport { webfetchRenderers } from \"./renderers/webfetch.ts\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.ts\";\n\nexport type { WebDocument, WebFetchBackend, WebSearchBackend, WebSearchResult } from \"../webfetch/backend.ts\";\nexport {\n\ttype ExtractWindow,\n\ttype StoredWebDocument,\n\tWEBFETCH_DEFAULT_WINDOW_CHARS,\n\ttype WebDocumentSource,\n\tWebDocumentStore,\n} from \"../webfetch/doc-store.ts\";\nexport {\n\tparseRecency,\n\ttype ResolvedSearchIntent,\n\tresolveSearchIntent,\n\tSEARCH_DEFAULT_WEB_RESULTS,\n\tSEARCH_MAX_PER_SOURCE,\n\ttype SearchFilters,\n\ttype SearchIntent,\n\tSearchIntentError,\n\ttype SearchResultCounts,\n\ttotalRequestedResults,\n} from \"../webfetch/search-intent.ts\";\nexport { type ParsedSearchQuery, parseSearchQueryTags } from \"../webfetch/search-tags.ts\";\n\nconst resultCountsSchema = Type.Object({\n\tweb: Type.Optional(Type.Number({ description: \"Web page results (1-100, default 5 when results is omitted)\" })),\n\timages: Type.Optional(Type.Number({ description: \"Image results (1-100); Firecrawl only\" })),\n\tnews: Type.Optional(Type.Number({ description: \"News results (1-100)\" })),\n});\n\nconst searchFiltersSchema = Type.Object({\n\tdomains: Type.Optional(\n\t\tType.Array(Type.String(), { description: \"Restrict results to these domains or domain prefixes\" }),\n\t),\n\trecency: Type.Optional(Type.String({ description: \"Maximum result age, e.g. 24h, 7d, 2w, 1m\" })),\n\tlanguage: Type.Optional(Type.String({ description: \"Two-letter language/country hint\" })),\n});\n\nconst webfetchSchema = Type.Object({\n\tquery: Type.String({\n\t\tdescription:\n\t\t\t\"Web search query. Tag sugar is accepted when the structured args below are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] your query\",\n\t}),\n\tresults: Type.Optional(resultCountsSchema),\n\tfilters: Type.Optional(searchFiltersSchema),\n\tlimit: Type.Optional(Type.Number({ description: \"Cap on total results across all sources (1-100)\" })),\n});\n\nexport const webfetchToolSystemPromptContribution = {\n\tsnippet: \"Search the web (results stored as web:// refs, full text via webextract)\",\n\tguidelines: [\n\t\t\"Use webfetch instead of curl for web research: results are stored as web:// refs, then read full text on demand with webextract instead of pasting whole pages\",\n\t\t\"Pass results/filters for structured control (per-source counts, domains, recency). Tag sugar like [images=3] [news=5] [recent=7d] works only when those args are omitted.\",\n\t\t\"Naming a source replaces the default web-only request: results={images:3} returns images only.\",\n\t\t\"Image results and recency/domain filters need Firecrawl; if it is unavailable, ask for plain web results without filters.\",\n\t],\n} as const;\n\nexport type WebfetchToolInput = Static<typeof webfetchSchema>;\n\nconst DEFAULT_LIMIT = 5;\nconst MAX_LIMIT = 100;\n\nexport interface WebfetchToolDetails {\n\trefs: string[];\n\tbackend: \"firecrawl\" | \"exa\" | \"undici\";\n\twarning?: string;\n\tcreditsUsed?: number;\n\t/** Set when an earlier backend in the chain failed and a later one answered. */\n\tbackendNotice?: string;\n\t/** Per-source counts actually requested after tag sugar and clamping. */\n\trequested?: { web?: number; images?: number; news?: number };\n\t/** Tag sugar that was recognized, e.g. `[images=3]`. */\n\tappliedTags?: string[];\n\t/** Tag sugar that looked like a directive but could not be applied. */\n\ttagWarnings?: string[];\n}\n\n/**\n * Pluggable backends for the web tools.\n * Override these to test without network or to front a different search provider.\n */\nexport interface WebOperations {\n\t/** Search backend. Default: the ordered chain Firecrawl -> Exa. */\n\tsearchBackend?: WebSearchBackend;\n\t/** Extra search backends tried after `searchBackend` fails. Default: Exa. */\n\tfallbackSearchBackends?: WebSearchBackend[];\n\t/** Primary fetch backend. Default: Firecrawl. */\n\tfetchBackend?: WebFetchBackend;\n\t/** Direct-fetch fallback used on Firecrawl transport/auth failures. Default: undici. */\n\tfallbackBackend?: WebFetchBackend;\n\t/** Shared session document store. Default: a fresh store per tool instance. */\n\tstore?: WebDocumentStore;\n}\n\nexport interface WebfetchToolOptions extends WebOperations {\n\t/** Firecrawl API key. Default: `FIRECRAWL_API_KEY` env. Omitted entirely when unset (keyless tier). */\n\tapiKey?: string;\n\t/** Exa API key. Default: `EXA_API_KEY` env. Unset uses the keyless hosted MCP search tier. */\n\texaApiKey?: string;\n}\n\nexport function resolveWebApiKey(explicitApiKey?: string): string | undefined {\n\treturn resolveFirecrawlApiKey(explicitApiKey);\n}\n\nfunction clampToolLimit(limit: number | undefined): number {\n\tif (limit === undefined || !Number.isFinite(limit)) return DEFAULT_LIMIT;\n\treturn Math.min(MAX_LIMIT, Math.max(1, Math.floor(limit)));\n}\n\nfunction resolveSearchBackends(options?: WebfetchToolOptions): WebSearchBackend[] {\n\tif (options?.searchBackend) {\n\t\treturn [options.searchBackend, ...(options.fallbackSearchBackends ?? [])];\n\t}\n\treturn [\n\t\tcreateFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),\n\t\tcreateExaBackend({ apiKey: resolveExaApiKey(options?.exaApiKey) }),\n\t];\n}\n\n/**\n * A search backend is skipped for auth/quota/transport/timeout/empty failures\n * and keyless-tier refusals. The `unavailable` code is matched structurally so a\n * backend compiled in a different module graph still counts.\n */\nfunction isSearchFallbackEligible(error: unknown): boolean {\n\tif (isFallbackEligible(error) || isExaFallbackEligible(error)) return true;\n\treturn error instanceof Error && \"code\" in error && (error as { code?: unknown }).code === \"unavailable\";\n}\n\n/**\n * Try each search backend in order. A backend is only skipped when it reports a\n * fallback-eligible failure — a bad request still surfaces, and results are\n * never merged across backends.\n */\nasync function searchWithFallback(\n\tbackends: WebSearchBackend[],\n\tintent: ResolvedSearchIntent,\n\tsearchOptions: { limit: number; signal?: AbortSignal },\n): Promise<{ backend: WebSearchBackend; results: WebSearchResult[]; skipped: string[] }> {\n\tconst skipped: string[] = [];\n\tlet lastError: unknown;\n\tfor (const backend of backends) {\n\t\ttry {\n\t\t\tconst results = await backend.search(intent, searchOptions);\n\t\t\treturn { backend, results, skipped };\n\t\t} catch (error) {\n\t\t\tif (!isSearchFallbackEligible(error)) throw error;\n\t\t\tlastError = error;\n\t\t\tskipped.push(`${backend.id}: ${error instanceof Error ? error.message : String(error)}`);\n\t\t}\n\t}\n\tthrow lastError instanceof Error\n\t\t? lastError\n\t\t: new WebfetchError(\"No web search backend is available\", \"unavailable\");\n}\n\n/**\n * Build the search intent from the structured arguments, falling back to tag\n * sugar in the query when `results`/`filters` are absent. An explicit structured\n * call always wins; sugar is only a convenience for terse queries.\n */\nexport function buildSearchIntent(input: {\n\tquery: string;\n\tresults?: { web?: number; images?: number; news?: number };\n\tfilters?: { domains?: string[]; recency?: string; language?: string };\n\tlimit?: number;\n}): { intent: ResolvedSearchIntent; appliedTags: string[]; warnings: string[] } {\n\tconst structured = input.results !== undefined || input.filters !== undefined;\n\tlet query = input.query;\n\tlet results = input.results;\n\tlet filters = input.filters;\n\tlet appliedTags: string[] = [];\n\tlet warnings: string[] = [];\n\n\tif (!structured) {\n\t\tconst parsed = parseSearchQueryTags(input.query);\n\t\tquery = parsed.intent.query;\n\t\tresults = parsed.intent.results;\n\t\tfilters = parsed.intent.filters;\n\t\tappliedTags = parsed.appliedTags;\n\t\twarnings = parsed.warnings;\n\t}\n\n\tconst resolved = resolveSearchIntent({ query, results, filters });\n\t// `limit` is a total across sources: scale the requested counts down together\n\t// rather than truncating one modality after the backend answered.\n\tconst requestedTotal = totalRequestedResults(resolved);\n\tif (input.limit !== undefined && Number.isFinite(input.limit) && input.limit < requestedTotal) {\n\t\tconst cap = Math.max(1, Math.floor(input.limit));\n\t\tconst scale = cap / requestedTotal;\n\t\tconst scaled: Record<string, number> = {};\n\t\tlet assigned = 0;\n\t\tconst sources = Object.keys(resolved.results) as Array<\"web\" | \"images\" | \"news\">;\n\t\tfor (const [index, source] of sources.entries()) {\n\t\t\tconst count = resolved.results[source] ?? 0;\n\t\t\tconst share =\n\t\t\t\tindex === sources.length - 1 ? Math.max(0, cap - assigned) : Math.max(1, Math.floor(count * scale));\n\t\t\tscaled[source] = Math.min(count, share);\n\t\t\tassigned += scaled[source];\n\t\t}\n\t\tresolved.results = { ...resolved.results, ...scaled };\n\t\twarnings = [...warnings, `limit ${cap} reduced the requested ${requestedTotal} results per source`];\n\t}\n\treturn { intent: resolved, appliedTags, warnings };\n}\n\nfunction readFirecrawlMeta(backend: WebSearchBackend): { warning?: string; creditsUsed?: number } {\n\tif (!(\"lastMeta\" in backend)) return {};\n\tconst meta = (backend as FirecrawlBackend).lastMeta;\n\tconst result: { warning?: string; creditsUsed?: number } = {};\n\tif (meta?.warning !== undefined) result.warning = meta.warning;\n\tif (meta?.creditsUsed !== undefined) result.creditsUsed = meta.creditsUsed;\n\treturn result;\n}\n\nexport function formatSearchResults(query: string, hits: Array<{ result: WebSearchResult; ref: string }>): string {\n\tconst lines = hits.map(\n\t\t({ result, ref }, index) =>\n\t\t\t`${index + 1}. ${result.title}\\n   ${result.url} (${ref})${result.snippet ? `\\n   ${result.snippet}` : \"\"}`,\n\t);\n\treturn [\n\t\t`Search results for \"${query}\":`,\n\t\t...lines,\n\t\t\"\",\n\t\t\"Full text is not included. Use webextract with a web:// ref to read a result.\",\n\t].join(\"\\n\");\n}\n\nexport function createWebfetchToolDefinition(\n\t_cwd: string,\n\toptions?: WebfetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebfetchToolDetails> {\n\tconst backends = resolveSearchBackends(options);\n\tconst store = options?.store ?? new WebDocumentStore();\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription: `Search the web for a query. Returns titles, URLs, and snippets; each web result is stored as a web:// ref for on-demand full text via webextract. Control modality and cardinality with results: {web, images, news} (counts are per source), and filters: {domains, recency, language}. Tag sugar works when those are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] query. Search tries Firecrawl first (keyless tier without a key, and the only source of image results), then Exa (REST with an API key, or the keyless hosted tier) when Firecrawl is unavailable.`,\n\t\tpromptSnippet: webfetchToolSystemPromptContribution.snippet,\n\t\tpromptGuidelines: [...webfetchToolSystemPromptContribution.guidelines],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(\n\t\t\t_toolCallId,\n\t\t\t{ query, results, filters, limit }: WebfetchToolInput,\n\t\t\tsignal?: AbortSignal,\n\t\t\t_onUpdate?,\n\t\t\t_ctx?: ExtensionContext,\n\t\t) {\n\t\t\tif (!query?.trim()) {\n\t\t\t\tthrow new Error(\"Search query must not be empty\");\n\t\t\t}\n\t\t\tconst built = buildSearchIntent({ query, results, filters, limit });\n\t\t\tconst intent = built.intent;\n\t\t\tconst effectiveLimit = clampToolLimit(limit);\n\t\t\tconst {\n\t\t\t\tbackend,\n\t\t\t\tresults: hits,\n\t\t\t\tskipped,\n\t\t\t} = await searchWithFallback(backends, intent, {\n\t\t\t\tlimit: effectiveLimit,\n\t\t\t\tsignal,\n\t\t\t});\n\t\t\tconst meta = readFirecrawlMeta(backend);\n\t\t\tconst backendNotice = skipped.length > 0 ? `Search backend fallback: ${skipped.join(\"; \")}` : undefined;\n\t\t\tconst requested = {\n\t\t\t\t...(intent.results.images !== undefined ? { images: intent.results.images } : {}),\n\t\t\t\t...(intent.results.news !== undefined ? { news: intent.results.news } : {}),\n\t\t\t\t...(intent.results.web !== undefined ? { web: intent.results.web } : {}),\n\t\t\t};\n\t\t\tconst extraDetails = {\n\t\t\t\trequested,\n\t\t\t\t...(built.appliedTags.length > 0 ? { appliedTags: built.appliedTags } : {}),\n\t\t\t\t...(built.warnings.length > 0 ? { tagWarnings: built.warnings } : {}),\n\t\t\t};\n\t\t\tif (hits.length === 0) {\n\t\t\t\tconst details: WebfetchToolDetails = {\n\t\t\t\t\trefs: [],\n\t\t\t\t\tbackend: backend.id,\n\t\t\t\t\t...meta,\n\t\t\t\t\tbackendNotice,\n\t\t\t\t\t...extraDetails,\n\t\t\t\t};\n\t\t\t\t// Zero results after a backend was skipped is not the same as \"nothing\n\t\t\t\t// matched\": say which backend could not serve the request.\n\t\t\t\tconst text = backendNotice\n\t\t\t\t\t? `No results found for \"${intent.query}\". ${backendNotice}`\n\t\t\t\t\t: `No results found for \"${intent.query}\"`;\n\t\t\t\treturn { content: [{ type: \"text\", text }], details };\n\t\t\t}\n\t\t\tconst stored = hits.slice(0, effectiveLimit).map((result) => ({\n\t\t\t\tresult,\n\t\t\t\tref: store.store(resultToDocument(result), backend.id === \"undici\" ? \"undici\" : \"firecrawl\", false),\n\t\t\t}));\n\t\t\tconst details: WebfetchToolDetails = {\n\t\t\t\trefs: stored.map((hit) => hit.ref),\n\t\t\t\tbackend: backend.id,\n\t\t\t\t...meta,\n\t\t\t\tbackendNotice,\n\t\t\t\t...extraDetails,\n\t\t\t};\n\t\t\treturn { content: [{ type: \"text\", text: formatSearchResults(intent.query, stored) }], details };\n\t\t},\n\t\t...webfetchRenderers,\n\t};\n}\n\nfunction resultToDocument(result: WebSearchResult): WebDocument {\n\treturn { url: result.url, title: result.title, markdown: result.snippet, fetchedAt: new Date().toISOString() };\n}\n\nexport function resolveFetchBackends(options?: WebfetchToolOptions): {\n\tprimary: WebFetchBackend;\n\tfallback: WebFetchBackend;\n} {\n\treturn {\n\t\tprimary: options?.fetchBackend ?? createFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),\n\t\tfallback: options?.fallbackBackend ?? createUndiciBackend(),\n\t};\n}\n\nexport function createWebfetchTool(_cwd: string, options?: WebfetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebfetchToolDefinition(_cwd, options));\n}\n"]}

package/dist/core/tools/webfetch.js

@@ -0,0 +1,231 @@
+import { Type } from "typebox";
+import { createExaBackend, isExaFallbackEligible, resolveExaApiKey } from "../webfetch/backend-exa.js";
+import { createFirecrawlBackend, isFallbackEligible, resolveFirecrawlApiKey, } from "../webfetch/backend-firecrawl.js";
+import { createUndiciBackend } from "../webfetch/backend-undici.js";
+import { WebDocumentStore } from "../webfetch/doc-store.js";
+import { WebfetchError } from "../webfetch/guardrails.js";
+import { resolveSearchIntent, totalRequestedResults } from "../webfetch/search-intent.js";
+import { parseSearchQueryTags } from "../webfetch/search-tags.js";
+import { webfetchRenderers } from "./renderers/webfetch.js";
+import { wrapToolDefinition } from "./tool-definition-wrapper.js";
+export { WEBFETCH_DEFAULT_WINDOW_CHARS, WebDocumentStore, } from "../webfetch/doc-store.js";
+export { parseRecency, resolveSearchIntent, SEARCH_DEFAULT_WEB_RESULTS, SEARCH_MAX_PER_SOURCE, SearchIntentError, totalRequestedResults, } from "../webfetch/search-intent.js";
+export { parseSearchQueryTags } from "../webfetch/search-tags.js";
+const resultCountsSchema = Type.Object({
+    web: Type.Optional(Type.Number({ description: "Web page results (1-100, default 5 when results is omitted)" })),
+    images: Type.Optional(Type.Number({ description: "Image results (1-100); Firecrawl only" })),
+    news: Type.Optional(Type.Number({ description: "News results (1-100)" })),
+});
+const searchFiltersSchema = Type.Object({
+    domains: Type.Optional(Type.Array(Type.String(), { description: "Restrict results to these domains or domain prefixes" })),
+    recency: Type.Optional(Type.String({ description: "Maximum result age, e.g. 24h, 7d, 2w, 1m" })),
+    language: Type.Optional(Type.String({ description: "Two-letter language/country hint" })),
+});
+const webfetchSchema = Type.Object({
+    query: Type.String({
+        description: "Web search query. Tag sugar is accepted when the structured args below are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] your query",
+    }),
+    results: Type.Optional(resultCountsSchema),
+    filters: Type.Optional(searchFiltersSchema),
+    limit: Type.Optional(Type.Number({ description: "Cap on total results across all sources (1-100)" })),
+});
+export const webfetchToolSystemPromptContribution = {
+    snippet: "Search the web (results stored as web:// refs, full text via webextract)",
+    guidelines: [
+        "Use webfetch instead of curl for web research: results are stored as web:// refs, then read full text on demand with webextract instead of pasting whole pages",
+        "Pass results/filters for structured control (per-source counts, domains, recency). Tag sugar like [images=3] [news=5] [recent=7d] works only when those args are omitted.",
+        "Naming a source replaces the default web-only request: results={images:3} returns images only.",
+        "Image results and recency/domain filters need Firecrawl; if it is unavailable, ask for plain web results without filters.",
+    ],
+};
+const DEFAULT_LIMIT = 5;
+const MAX_LIMIT = 100;
+export function resolveWebApiKey(explicitApiKey) {
+    return resolveFirecrawlApiKey(explicitApiKey);
+}
+function clampToolLimit(limit) {
+    if (limit === undefined || !Number.isFinite(limit))
+        return DEFAULT_LIMIT;
+    return Math.min(MAX_LIMIT, Math.max(1, Math.floor(limit)));
+}
+function resolveSearchBackends(options) {
+    if (options?.searchBackend) {
+        return [options.searchBackend, ...(options.fallbackSearchBackends ?? [])];
+    }
+    return [
+        createFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),
+        createExaBackend({ apiKey: resolveExaApiKey(options?.exaApiKey) }),
+    ];
+}
+/**
+ * A search backend is skipped for auth/quota/transport/timeout/empty failures
+ * and keyless-tier refusals. The `unavailable` code is matched structurally so a
+ * backend compiled in a different module graph still counts.
+ */
+function isSearchFallbackEligible(error) {
+    if (isFallbackEligible(error) || isExaFallbackEligible(error))
+        return true;
+    return error instanceof Error && "code" in error && error.code === "unavailable";
+}
+/**
+ * Try each search backend in order. A backend is only skipped when it reports a
+ * fallback-eligible failure — a bad request still surfaces, and results are
+ * never merged across backends.
+ */
+async function searchWithFallback(backends, intent, searchOptions) {
+    const skipped = [];
+    let lastError;
+    for (const backend of backends) {
+        try {
+            const results = await backend.search(intent, searchOptions);
+            return { backend, results, skipped };
+        }
+        catch (error) {
+            if (!isSearchFallbackEligible(error))
+                throw error;
+            lastError = error;
+            skipped.push(`${backend.id}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    throw lastError instanceof Error
+        ? lastError
+        : new WebfetchError("No web search backend is available", "unavailable");
+}
+/**
+ * Build the search intent from the structured arguments, falling back to tag
+ * sugar in the query when `results`/`filters` are absent. An explicit structured
+ * call always wins; sugar is only a convenience for terse queries.
+ */
+export function buildSearchIntent(input) {
+    const structured = input.results !== undefined || input.filters !== undefined;
+    let query = input.query;
+    let results = input.results;
+    let filters = input.filters;
+    let appliedTags = [];
+    let warnings = [];
+    if (!structured) {
+        const parsed = parseSearchQueryTags(input.query);
+        query = parsed.intent.query;
+        results = parsed.intent.results;
+        filters = parsed.intent.filters;
+        appliedTags = parsed.appliedTags;
+        warnings = parsed.warnings;
+    }
+    const resolved = resolveSearchIntent({ query, results, filters });
+    // `limit` is a total across sources: scale the requested counts down together
+    // rather than truncating one modality after the backend answered.
+    const requestedTotal = totalRequestedResults(resolved);
+    if (input.limit !== undefined && Number.isFinite(input.limit) && input.limit < requestedTotal) {
+        const cap = Math.max(1, Math.floor(input.limit));
+        const scale = cap / requestedTotal;
+        const scaled = {};
+        let assigned = 0;
+        const sources = Object.keys(resolved.results);
+        for (const [index, source] of sources.entries()) {
+            const count = resolved.results[source] ?? 0;
+            const share = index === sources.length - 1 ? Math.max(0, cap - assigned) : Math.max(1, Math.floor(count * scale));
+            scaled[source] = Math.min(count, share);
+            assigned += scaled[source];
+        }
+        resolved.results = { ...resolved.results, ...scaled };
+        warnings = [...warnings, `limit ${cap} reduced the requested ${requestedTotal} results per source`];
+    }
+    return { intent: resolved, appliedTags, warnings };
+}
+function readFirecrawlMeta(backend) {
+    if (!("lastMeta" in backend))
+        return {};
+    const meta = backend.lastMeta;
+    const result = {};
+    if (meta?.warning !== undefined)
+        result.warning = meta.warning;
+    if (meta?.creditsUsed !== undefined)
+        result.creditsUsed = meta.creditsUsed;
+    return result;
+}
+export function formatSearchResults(query, hits) {
+    const lines = hits.map(({ result, ref }, index) => `${index + 1}. ${result.title}\n   ${result.url} (${ref})${result.snippet ? `\n   ${result.snippet}` : ""}`);
+    return [
+        `Search results for "${query}":`,
+        ...lines,
+        "",
+        "Full text is not included. Use webextract with a web:// ref to read a result.",
+    ].join("\n");
+}
+export function createWebfetchToolDefinition(_cwd, options) {
+    const backends = resolveSearchBackends(options);
+    const store = options?.store ?? new WebDocumentStore();
+    return {
+        name: "webfetch",
+        label: "webfetch",
+        description: `Search the web for a query. Returns titles, URLs, and snippets; each web result is stored as a web:// ref for on-demand full text via webextract. Control modality and cardinality with results: {web, images, news} (counts are per source), and filters: {domains, recency, language}. Tag sugar works when those are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] query. Search tries Firecrawl first (keyless tier without a key, and the only source of image results), then Exa (REST with an API key, or the keyless hosted tier) when Firecrawl is unavailable.`,
+        promptSnippet: webfetchToolSystemPromptContribution.snippet,
+        promptGuidelines: [...webfetchToolSystemPromptContribution.guidelines],
+        parameters: webfetchSchema,
+        async execute(_toolCallId, { query, results, filters, limit }, signal, _onUpdate, _ctx) {
+            if (!query?.trim()) {
+                throw new Error("Search query must not be empty");
+            }
+            const built = buildSearchIntent({ query, results, filters, limit });
+            const intent = built.intent;
+            const effectiveLimit = clampToolLimit(limit);
+            const { backend, results: hits, skipped, } = await searchWithFallback(backends, intent, {
+                limit: effectiveLimit,
+                signal,
+            });
+            const meta = readFirecrawlMeta(backend);
+            const backendNotice = skipped.length > 0 ? `Search backend fallback: ${skipped.join("; ")}` : undefined;
+            const requested = {
+                ...(intent.results.images !== undefined ? { images: intent.results.images } : {}),
+                ...(intent.results.news !== undefined ? { news: intent.results.news } : {}),
+                ...(intent.results.web !== undefined ? { web: intent.results.web } : {}),
+            };
+            const extraDetails = {
+                requested,
+                ...(built.appliedTags.length > 0 ? { appliedTags: built.appliedTags } : {}),
+                ...(built.warnings.length > 0 ? { tagWarnings: built.warnings } : {}),
+            };
+            if (hits.length === 0) {
+                const details = {
+                    refs: [],
+                    backend: backend.id,
+                    ...meta,
+                    backendNotice,
+                    ...extraDetails,
+                };
+                // Zero results after a backend was skipped is not the same as "nothing
+                // matched": say which backend could not serve the request.
+                const text = backendNotice
+                    ? `No results found for "${intent.query}". ${backendNotice}`
+                    : `No results found for "${intent.query}"`;
+                return { content: [{ type: "text", text }], details };
+            }
+            const stored = hits.slice(0, effectiveLimit).map((result) => ({
+                result,
+                ref: store.store(resultToDocument(result), backend.id === "undici" ? "undici" : "firecrawl", false),
+            }));
+            const details = {
+                refs: stored.map((hit) => hit.ref),
+                backend: backend.id,
+                ...meta,
+                backendNotice,
+                ...extraDetails,
+            };
+            return { content: [{ type: "text", text: formatSearchResults(intent.query, stored) }], details };
+        },
+        ...webfetchRenderers,
+    };
+}
+function resultToDocument(result) {
+    return { url: result.url, title: result.title, markdown: result.snippet, fetchedAt: new Date().toISOString() };
+}
+export function resolveFetchBackends(options) {
+    return {
+        primary: options?.fetchBackend ?? createFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),
+        fallback: options?.fallbackBackend ?? createUndiciBackend(),
+    };
+}
+export function createWebfetchTool(_cwd, options) {
+    return wrapToolDefinition(createWebfetchToolDefinition(_cwd, options));
+}
+//# sourceMappingURL=webfetch.js.map

package/dist/core/tools/webfetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACvG,OAAO,EACN,sBAAsB,EAEtB,kBAAkB,EAClB,sBAAsB,GACtB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAA6B,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrH,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAGlE,OAAO,EAGN,6BAA6B,EAE7B,gBAAgB,GAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACN,YAAY,EAEZ,mBAAmB,EACnB,0BAA0B,EAC1B,qBAAqB,EAGrB,iBAAiB,EAEjB,qBAAqB,GACrB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAA0B,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAE1F,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,6DAA6D,EAAE,CAAC,CAAC;IAC/G,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,uCAAuC,EAAE,CAAC,CAAC;IAC5F,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,sBAAsB,EAAE,CAAC,CAAC;CACzE,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,IAAI,CAAC,QAAQ,CACrB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,sDAAsD,EAAE,CAAC,CAClG;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,0CAA0C,EAAE,CAAC,CAAC;IAChG,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC,CAAC;CACzF,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;QAClB,WAAW,EACV,2JAA2J;KAC5J,CAAC;IACF,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC1C,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAC3C,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,iDAAiD,EAAE,CAAC,CAAC;CACrG,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oCAAoC,GAAG;IACnD,OAAO,EAAE,0EAA0E;IACnF,UAAU,EAAE;QACX,gKAAgK;QAChK,2KAA2K;QAC3K,gGAAgG;QAChG,2HAA2H;KAC3H;CACQ,CAAC;AAIX,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,SAAS,GAAG,GAAG,CAAC;AAyCtB,MAAM,UAAU,gBAAgB,CAAC,cAAuB,EAAsB;IAC7E,OAAO,sBAAsB,CAAC,cAAc,CAAC,CAAC;AAAA,CAC9C;AAED,SAAS,cAAc,CAAC,KAAyB,EAAU;IAC1D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC;IACzE,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAAA,CAC3D;AAED,SAAS,qBAAqB,CAAC,OAA6B,EAAsB;IACjF,IAAI,OAAO,EAAE,aAAa,EAAE,CAAC;QAC5B,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,OAAO,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO;QACN,sBAAsB,CAAC,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;QACrE,gBAAgB,CAAC,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;KAClE,CAAC;AAAA,CACF;AAED;;;;GAIG;AACH,SAAS,wBAAwB,CAAC,KAAc,EAAW;IAC1D,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,qBAAqB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3E,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA4B,CAAC,IAAI,KAAK,aAAa,CAAC;AAAA,CACzG;AAED;;;;GAIG;AACH,KAAK,UAAU,kBAAkB,CAChC,QAA4B,EAC5B,MAA4B,EAC5B,aAAsD,EACkC;IACxF,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,SAAkB,CAAC;IACvB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YAC5D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;YAClD,SAAS,GAAG,KAAK,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,EAAE,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1F,CAAC;IACF,CAAC;IACD,MAAM,SAAS,YAAY,KAAK;QAC/B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,aAAa,CAAC,oCAAoC,EAAE,aAAa,CAAC,CAAC;AAAA,CAC1E;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAKjC,EAA+E;IAC/E,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC;IAC9E,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IACxB,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC5B,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC5B,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,QAAQ,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjD,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QAC5B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAChC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAChC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACjC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;IAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,kEAAkE;IAClE,MAAM,cAAc,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IACvD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,cAAc,EAAE,CAAC;QAC/F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,GAAG,GAAG,cAAc,CAAC;QACnC,MAAM,MAAM,GAA2B,EAAE,CAAC;QAC1C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAqC,CAAC;QAClF,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,KAAK,GACV,KAAK,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC;YACrG,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACxC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;QACtD,QAAQ,GAAG,CAAC,GAAG,QAAQ,EAAE,SAAS,GAAG,0BAA0B,cAAc,qBAAqB,CAAC,CAAC;IACrG,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAAA,CACnD;AAED,SAAS,iBAAiB,CAAC,OAAyB,EAA8C;IACjG,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,IAAI,GAAI,OAA4B,CAAC,QAAQ,CAAC;IACpD,MAAM,MAAM,GAA+C,EAAE,CAAC;IAC9D,IAAI,IAAI,EAAE,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC/D,IAAI,IAAI,EAAE,WAAW,KAAK,SAAS;QAAE,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IAC3E,OAAO,MAAM,CAAC;AAAA,CACd;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,IAAqD,EAAU;IACjH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACrB,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAC1B,GAAG,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,QAAQ,MAAM,CAAC,GAAG,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5G,CAAC;IACF,OAAO;QACN,uBAAuB,KAAK,IAAI;QAChC,GAAG,KAAK;QACR,EAAE;QACF,+EAA+E;KAC/E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACb;AAED,MAAM,UAAU,4BAA4B,CAC3C,IAAY,EACZ,OAA6B,EACgC;IAC7D,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,gBAAgB,EAAE,CAAC;IACvD,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EAAE,gkBAAgkB;QAC7kB,aAAa,EAAE,oCAAoC,CAAC,OAAO;QAC3D,gBAAgB,EAAE,CAAC,GAAG,oCAAoC,CAAC,UAAU,CAAC;QACtE,UAAU,EAAE,cAAc;QAC1B,KAAK,CAAC,OAAO,CACZ,WAAW,EACX,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAqB,EACrD,MAAoB,EACpB,SAAU,EACV,IAAuB,EACtB;YACD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACnD,CAAC;YACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACpE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC5B,MAAM,cAAc,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,EACL,OAAO,EACP,OAAO,EAAE,IAAI,EACb,OAAO,GACP,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE;gBAC9C,KAAK,EAAE,cAAc;gBACrB,MAAM;aACN,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,4BAA4B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACxG,MAAM,SAAS,GAAG;gBACjB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjF,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC;YACF,MAAM,YAAY,GAAG;gBACpB,SAAS;gBACT,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAwB;oBACpC,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,OAAO,CAAC,EAAE;oBACnB,GAAG,IAAI;oBACP,aAAa;oBACb,GAAG,YAAY;iBACf,CAAC;gBACF,uEAAuE;gBACvE,2DAA2D;gBAC3D,MAAM,IAAI,GAAG,aAAa;oBACzB,CAAC,CAAC,yBAAyB,MAAM,CAAC,KAAK,MAAM,aAAa,EAAE;oBAC5D,CAAC,CAAC,yBAAyB,MAAM,CAAC,KAAK,GAAG,CAAC;gBAC5C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;YACvD,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAC7D,MAAM;gBACN,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC;aACnG,CAAC,CAAC,CAAC;YACJ,MAAM,OAAO,GAAwB;gBACpC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;gBAClC,OAAO,EAAE,OAAO,CAAC,EAAE;gBACnB,GAAG,IAAI;gBACP,aAAa;gBACb,GAAG,YAAY;aACf,CAAC;YACF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;QAAA,CACjG;QACD,GAAG,iBAAiB;KACpB,CAAC;AAAA,CACF;AAED,SAAS,gBAAgB,CAAC,MAAuB,EAAe;IAC/D,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;AAAA,CAC/G;AAED,MAAM,UAAU,oBAAoB,CAAC,OAA6B,EAGhE;IACD,OAAO;QACN,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,sBAAsB,CAAC,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC;QACvG,QAAQ,EAAE,OAAO,EAAE,eAAe,IAAI,mBAAmB,EAAE;KAC3D,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,OAA6B,EAAoC;IACjH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACvE","sourcesContent":["import type { AgentTool } from \"@m4ike1/ion-agent-core\";\nimport { type Static, Type } from \"typebox\";\nimport type { ExtensionContext, ToolDefinition } from \"../extensions/types.ts\";\nimport type { WebDocument, WebFetchBackend, WebSearchBackend, WebSearchResult } from \"../webfetch/backend.ts\";\nimport { createExaBackend, isExaFallbackEligible, resolveExaApiKey } from \"../webfetch/backend-exa.ts\";\nimport {\n\tcreateFirecrawlBackend,\n\ttype FirecrawlBackend,\n\tisFallbackEligible,\n\tresolveFirecrawlApiKey,\n} from \"../webfetch/backend-firecrawl.ts\";\nimport { createUndiciBackend } from \"../webfetch/backend-undici.ts\";\nimport { WebDocumentStore } from \"../webfetch/doc-store.ts\";\nimport { WebfetchError } from \"../webfetch/guardrails.ts\";\nimport { type ResolvedSearchIntent, resolveSearchIntent, totalRequestedResults } from \"../webfetch/search-intent.ts\";\nimport { parseSearchQueryTags } from \"../webfetch/search-tags.ts\";\nimport { webfetchRenderers } from \"./renderers/webfetch.ts\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.ts\";\n\nexport type { WebDocument, WebFetchBackend, WebSearchBackend, WebSearchResult } from \"../webfetch/backend.ts\";\nexport {\n\ttype ExtractWindow,\n\ttype StoredWebDocument,\n\tWEBFETCH_DEFAULT_WINDOW_CHARS,\n\ttype WebDocumentSource,\n\tWebDocumentStore,\n} from \"../webfetch/doc-store.ts\";\nexport {\n\tparseRecency,\n\ttype ResolvedSearchIntent,\n\tresolveSearchIntent,\n\tSEARCH_DEFAULT_WEB_RESULTS,\n\tSEARCH_MAX_PER_SOURCE,\n\ttype SearchFilters,\n\ttype SearchIntent,\n\tSearchIntentError,\n\ttype SearchResultCounts,\n\ttotalRequestedResults,\n} from \"../webfetch/search-intent.ts\";\nexport { type ParsedSearchQuery, parseSearchQueryTags } from \"../webfetch/search-tags.ts\";\n\nconst resultCountsSchema = Type.Object({\n\tweb: Type.Optional(Type.Number({ description: \"Web page results (1-100, default 5 when results is omitted)\" })),\n\timages: Type.Optional(Type.Number({ description: \"Image results (1-100); Firecrawl only\" })),\n\tnews: Type.Optional(Type.Number({ description: \"News results (1-100)\" })),\n});\n\nconst searchFiltersSchema = Type.Object({\n\tdomains: Type.Optional(\n\t\tType.Array(Type.String(), { description: \"Restrict results to these domains or domain prefixes\" }),\n\t),\n\trecency: Type.Optional(Type.String({ description: \"Maximum result age, e.g. 24h, 7d, 2w, 1m\" })),\n\tlanguage: Type.Optional(Type.String({ description: \"Two-letter language/country hint\" })),\n});\n\nconst webfetchSchema = Type.Object({\n\tquery: Type.String({\n\t\tdescription:\n\t\t\t\"Web search query. Tag sugar is accepted when the structured args below are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] your query\",\n\t}),\n\tresults: Type.Optional(resultCountsSchema),\n\tfilters: Type.Optional(searchFiltersSchema),\n\tlimit: Type.Optional(Type.Number({ description: \"Cap on total results across all sources (1-100)\" })),\n});\n\nexport const webfetchToolSystemPromptContribution = {\n\tsnippet: \"Search the web (results stored as web:// refs, full text via webextract)\",\n\tguidelines: [\n\t\t\"Use webfetch instead of curl for web research: results are stored as web:// refs, then read full text on demand with webextract instead of pasting whole pages\",\n\t\t\"Pass results/filters for structured control (per-source counts, domains, recency). Tag sugar like [images=3] [news=5] [recent=7d] works only when those args are omitted.\",\n\t\t\"Naming a source replaces the default web-only request: results={images:3} returns images only.\",\n\t\t\"Image results and recency/domain filters need Firecrawl; if it is unavailable, ask for plain web results without filters.\",\n\t],\n} as const;\n\nexport type WebfetchToolInput = Static<typeof webfetchSchema>;\n\nconst DEFAULT_LIMIT = 5;\nconst MAX_LIMIT = 100;\n\nexport interface WebfetchToolDetails {\n\trefs: string[];\n\tbackend: \"firecrawl\" | \"exa\" | \"undici\";\n\twarning?: string;\n\tcreditsUsed?: number;\n\t/** Set when an earlier backend in the chain failed and a later one answered. */\n\tbackendNotice?: string;\n\t/** Per-source counts actually requested after tag sugar and clamping. */\n\trequested?: { web?: number; images?: number; news?: number };\n\t/** Tag sugar that was recognized, e.g. `[images=3]`. */\n\tappliedTags?: string[];\n\t/** Tag sugar that looked like a directive but could not be applied. */\n\ttagWarnings?: string[];\n}\n\n/**\n * Pluggable backends for the web tools.\n * Override these to test without network or to front a different search provider.\n */\nexport interface WebOperations {\n\t/** Search backend. Default: the ordered chain Firecrawl -> Exa. */\n\tsearchBackend?: WebSearchBackend;\n\t/** Extra search backends tried after `searchBackend` fails. Default: Exa. */\n\tfallbackSearchBackends?: WebSearchBackend[];\n\t/** Primary fetch backend. Default: Firecrawl. */\n\tfetchBackend?: WebFetchBackend;\n\t/** Direct-fetch fallback used on Firecrawl transport/auth failures. Default: undici. */\n\tfallbackBackend?: WebFetchBackend;\n\t/** Shared session document store. Default: a fresh store per tool instance. */\n\tstore?: WebDocumentStore;\n}\n\nexport interface WebfetchToolOptions extends WebOperations {\n\t/** Firecrawl API key. Default: `FIRECRAWL_API_KEY` env. Omitted entirely when unset (keyless tier). */\n\tapiKey?: string;\n\t/** Exa API key. Default: `EXA_API_KEY` env. Unset uses the keyless hosted MCP search tier. */\n\texaApiKey?: string;\n}\n\nexport function resolveWebApiKey(explicitApiKey?: string): string | undefined {\n\treturn resolveFirecrawlApiKey(explicitApiKey);\n}\n\nfunction clampToolLimit(limit: number | undefined): number {\n\tif (limit === undefined || !Number.isFinite(limit)) return DEFAULT_LIMIT;\n\treturn Math.min(MAX_LIMIT, Math.max(1, Math.floor(limit)));\n}\n\nfunction resolveSearchBackends(options?: WebfetchToolOptions): WebSearchBackend[] {\n\tif (options?.searchBackend) {\n\t\treturn [options.searchBackend, ...(options.fallbackSearchBackends ?? [])];\n\t}\n\treturn [\n\t\tcreateFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),\n\t\tcreateExaBackend({ apiKey: resolveExaApiKey(options?.exaApiKey) }),\n\t];\n}\n\n/**\n * A search backend is skipped for auth/quota/transport/timeout/empty failures\n * and keyless-tier refusals. The `unavailable` code is matched structurally so a\n * backend compiled in a different module graph still counts.\n */\nfunction isSearchFallbackEligible(error: unknown): boolean {\n\tif (isFallbackEligible(error) || isExaFallbackEligible(error)) return true;\n\treturn error instanceof Error && \"code\" in error && (error as { code?: unknown }).code === \"unavailable\";\n}\n\n/**\n * Try each search backend in order. A backend is only skipped when it reports a\n * fallback-eligible failure — a bad request still surfaces, and results are\n * never merged across backends.\n */\nasync function searchWithFallback(\n\tbackends: WebSearchBackend[],\n\tintent: ResolvedSearchIntent,\n\tsearchOptions: { limit: number; signal?: AbortSignal },\n): Promise<{ backend: WebSearchBackend; results: WebSearchResult[]; skipped: string[] }> {\n\tconst skipped: string[] = [];\n\tlet lastError: unknown;\n\tfor (const backend of backends) {\n\t\ttry {\n\t\t\tconst results = await backend.search(intent, searchOptions);\n\t\t\treturn { backend, results, skipped };\n\t\t} catch (error) {\n\t\t\tif (!isSearchFallbackEligible(error)) throw error;\n\t\t\tlastError = error;\n\t\t\tskipped.push(`${backend.id}: ${error instanceof Error ? error.message : String(error)}`);\n\t\t}\n\t}\n\tthrow lastError instanceof Error\n\t\t? lastError\n\t\t: new WebfetchError(\"No web search backend is available\", \"unavailable\");\n}\n\n/**\n * Build the search intent from the structured arguments, falling back to tag\n * sugar in the query when `results`/`filters` are absent. An explicit structured\n * call always wins; sugar is only a convenience for terse queries.\n */\nexport function buildSearchIntent(input: {\n\tquery: string;\n\tresults?: { web?: number; images?: number; news?: number };\n\tfilters?: { domains?: string[]; recency?: string; language?: string };\n\tlimit?: number;\n}): { intent: ResolvedSearchIntent; appliedTags: string[]; warnings: string[] } {\n\tconst structured = input.results !== undefined || input.filters !== undefined;\n\tlet query = input.query;\n\tlet results = input.results;\n\tlet filters = input.filters;\n\tlet appliedTags: string[] = [];\n\tlet warnings: string[] = [];\n\n\tif (!structured) {\n\t\tconst parsed = parseSearchQueryTags(input.query);\n\t\tquery = parsed.intent.query;\n\t\tresults = parsed.intent.results;\n\t\tfilters = parsed.intent.filters;\n\t\tappliedTags = parsed.appliedTags;\n\t\twarnings = parsed.warnings;\n\t}\n\n\tconst resolved = resolveSearchIntent({ query, results, filters });\n\t// `limit` is a total across sources: scale the requested counts down together\n\t// rather than truncating one modality after the backend answered.\n\tconst requestedTotal = totalRequestedResults(resolved);\n\tif (input.limit !== undefined && Number.isFinite(input.limit) && input.limit < requestedTotal) {\n\t\tconst cap = Math.max(1, Math.floor(input.limit));\n\t\tconst scale = cap / requestedTotal;\n\t\tconst scaled: Record<string, number> = {};\n\t\tlet assigned = 0;\n\t\tconst sources = Object.keys(resolved.results) as Array<\"web\" | \"images\" | \"news\">;\n\t\tfor (const [index, source] of sources.entries()) {\n\t\t\tconst count = resolved.results[source] ?? 0;\n\t\t\tconst share =\n\t\t\t\tindex === sources.length - 1 ? Math.max(0, cap - assigned) : Math.max(1, Math.floor(count * scale));\n\t\t\tscaled[source] = Math.min(count, share);\n\t\t\tassigned += scaled[source];\n\t\t}\n\t\tresolved.results = { ...resolved.results, ...scaled };\n\t\twarnings = [...warnings, `limit ${cap} reduced the requested ${requestedTotal} results per source`];\n\t}\n\treturn { intent: resolved, appliedTags, warnings };\n}\n\nfunction readFirecrawlMeta(backend: WebSearchBackend): { warning?: string; creditsUsed?: number } {\n\tif (!(\"lastMeta\" in backend)) return {};\n\tconst meta = (backend as FirecrawlBackend).lastMeta;\n\tconst result: { warning?: string; creditsUsed?: number } = {};\n\tif (meta?.warning !== undefined) result.warning = meta.warning;\n\tif (meta?.creditsUsed !== undefined) result.creditsUsed = meta.creditsUsed;\n\treturn result;\n}\n\nexport function formatSearchResults(query: string, hits: Array<{ result: WebSearchResult; ref: string }>): string {\n\tconst lines = hits.map(\n\t\t({ result, ref }, index) =>\n\t\t\t`${index + 1}. ${result.title}\\n   ${result.url} (${ref})${result.snippet ? `\\n   ${result.snippet}` : \"\"}`,\n\t);\n\treturn [\n\t\t`Search results for \"${query}\":`,\n\t\t...lines,\n\t\t\"\",\n\t\t\"Full text is not included. Use webextract with a web:// ref to read a result.\",\n\t].join(\"\\n\");\n}\n\nexport function createWebfetchToolDefinition(\n\t_cwd: string,\n\toptions?: WebfetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebfetchToolDetails> {\n\tconst backends = resolveSearchBackends(options);\n\tconst store = options?.store ?? new WebDocumentStore();\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription: `Search the web for a query. Returns titles, URLs, and snippets; each web result is stored as a web:// ref for on-demand full text via webextract. Control modality and cardinality with results: {web, images, news} (counts are per source), and filters: {domains, recency, language}. Tag sugar works when those are omitted: [images=3] [news=5] [recent=7d] [domains=openai.com] [img] query. Search tries Firecrawl first (keyless tier without a key, and the only source of image results), then Exa (REST with an API key, or the keyless hosted tier) when Firecrawl is unavailable.`,\n\t\tpromptSnippet: webfetchToolSystemPromptContribution.snippet,\n\t\tpromptGuidelines: [...webfetchToolSystemPromptContribution.guidelines],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(\n\t\t\t_toolCallId,\n\t\t\t{ query, results, filters, limit }: WebfetchToolInput,\n\t\t\tsignal?: AbortSignal,\n\t\t\t_onUpdate?,\n\t\t\t_ctx?: ExtensionContext,\n\t\t) {\n\t\t\tif (!query?.trim()) {\n\t\t\t\tthrow new Error(\"Search query must not be empty\");\n\t\t\t}\n\t\t\tconst built = buildSearchIntent({ query, results, filters, limit });\n\t\t\tconst intent = built.intent;\n\t\t\tconst effectiveLimit = clampToolLimit(limit);\n\t\t\tconst {\n\t\t\t\tbackend,\n\t\t\t\tresults: hits,\n\t\t\t\tskipped,\n\t\t\t} = await searchWithFallback(backends, intent, {\n\t\t\t\tlimit: effectiveLimit,\n\t\t\t\tsignal,\n\t\t\t});\n\t\t\tconst meta = readFirecrawlMeta(backend);\n\t\t\tconst backendNotice = skipped.length > 0 ? `Search backend fallback: ${skipped.join(\"; \")}` : undefined;\n\t\t\tconst requested = {\n\t\t\t\t...(intent.results.images !== undefined ? { images: intent.results.images } : {}),\n\t\t\t\t...(intent.results.news !== undefined ? { news: intent.results.news } : {}),\n\t\t\t\t...(intent.results.web !== undefined ? { web: intent.results.web } : {}),\n\t\t\t};\n\t\t\tconst extraDetails = {\n\t\t\t\trequested,\n\t\t\t\t...(built.appliedTags.length > 0 ? { appliedTags: built.appliedTags } : {}),\n\t\t\t\t...(built.warnings.length > 0 ? { tagWarnings: built.warnings } : {}),\n\t\t\t};\n\t\t\tif (hits.length === 0) {\n\t\t\t\tconst details: WebfetchToolDetails = {\n\t\t\t\t\trefs: [],\n\t\t\t\t\tbackend: backend.id,\n\t\t\t\t\t...meta,\n\t\t\t\t\tbackendNotice,\n\t\t\t\t\t...extraDetails,\n\t\t\t\t};\n\t\t\t\t// Zero results after a backend was skipped is not the same as \"nothing\n\t\t\t\t// matched\": say which backend could not serve the request.\n\t\t\t\tconst text = backendNotice\n\t\t\t\t\t? `No results found for \"${intent.query}\". ${backendNotice}`\n\t\t\t\t\t: `No results found for \"${intent.query}\"`;\n\t\t\t\treturn { content: [{ type: \"text\", text }], details };\n\t\t\t}\n\t\t\tconst stored = hits.slice(0, effectiveLimit).map((result) => ({\n\t\t\t\tresult,\n\t\t\t\tref: store.store(resultToDocument(result), backend.id === \"undici\" ? \"undici\" : \"firecrawl\", false),\n\t\t\t}));\n\t\t\tconst details: WebfetchToolDetails = {\n\t\t\t\trefs: stored.map((hit) => hit.ref),\n\t\t\t\tbackend: backend.id,\n\t\t\t\t...meta,\n\t\t\t\tbackendNotice,\n\t\t\t\t...extraDetails,\n\t\t\t};\n\t\t\treturn { content: [{ type: \"text\", text: formatSearchResults(intent.query, stored) }], details };\n\t\t},\n\t\t...webfetchRenderers,\n\t};\n}\n\nfunction resultToDocument(result: WebSearchResult): WebDocument {\n\treturn { url: result.url, title: result.title, markdown: result.snippet, fetchedAt: new Date().toISOString() };\n}\n\nexport function resolveFetchBackends(options?: WebfetchToolOptions): {\n\tprimary: WebFetchBackend;\n\tfallback: WebFetchBackend;\n} {\n\treturn {\n\t\tprimary: options?.fetchBackend ?? createFirecrawlBackend({ apiKey: resolveWebApiKey(options?.apiKey) }),\n\t\tfallback: options?.fallbackBackend ?? createUndiciBackend(),\n\t};\n}\n\nexport function createWebfetchTool(_cwd: string, options?: WebfetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebfetchToolDefinition(_cwd, options));\n}\n"]}