daemora 1.0.0-alpha.0

package/dist/tools/core/webFetch.js

@@ -0,0 +1,265 @@
+/**
+ * web_fetch — fetch a URL and return *readable* text.
+ *
+ * Where `fetch_url` is a raw HTTP client (for calling APIs and getting
+ * status codes back), `web_fetch` is the "read me this page" tool:
+ *
+ *   1. SSRF defense — block private / loopback hosts.
+ *   2. GitHub `blob` links auto-convert to `raw.githubusercontent.com`.
+ *   3. Primary: fetch the page, strip script/style/nav/footer, remove
+ *      hidden elements and prompt-injection-y invisible unicode, convert
+ *      to plain text.
+ *   4. Fallback: Firecrawl (if FIRECRAWL_API_KEY is in the vault) — for
+ *      JS-rendered pages or anything our regex pipeline can't crack.
+ *   5. 15-minute LRU cache (100 URLs max) so repeated reads don't burn
+ *      bandwidth / quota.
+ */
+import { z } from "zod";
+import { ProviderError, TimeoutError, ValidationError } from "../../util/errors.js";
+const CACHE_TTL_MS = 15 * 60_000;
+const CACHE_MAX = 100;
+const inputSchema = z.object({
+    url: z.string().url(),
+    maxChars: z.number().int().positive().max(1_000_000).default(100_000)
+        .describe("Hard cap on returned text. Truncates with an ellipsis marker."),
+    timeoutMs: z.number().int().positive().max(60_000).default(20_000),
+    forceFirecrawl: z.boolean().default(false)
+        .describe("Skip the local extractor and go straight to Firecrawl (if configured)."),
+});
+const cache = new Map();
+export function clearWebFetchCache() {
+    const n = cache.size;
+    cache.clear();
+    return n;
+}
+export function makeWebFetchTool(cfg) {
+    return {
+        name: "web_fetch",
+        description: "Fetch a web page and return clean readable text. Strips scripts/nav/hidden elements. Falls back to Firecrawl for JS-rendered pages when FIRECRAWL_API_KEY is set.",
+        category: "network",
+        source: { kind: "core" },
+        alwaysOn: true,
+        tags: ["fetch", "web", "content", "readability", "scrape"],
+        inputSchema,
+        async execute({ url, maxChars, timeoutMs, forceFirecrawl }, { abortSignal, logger }) {
+            const started = Date.now();
+            // SSRF check
+            assertPublicUrl(url);
+            // Normalise GitHub blob → raw
+            const normalUrl = convertGitHubBlobUrl(url);
+            // Cache hit
+            const hit = cache.get(normalUrl);
+            if (hit && Date.now() < hit.expiresAt) {
+                logger.info("web_fetch cache hit", { url: normalUrl });
+                return { ...hit.result, cached: true };
+            }
+            // Firecrawl first if explicitly requested.
+            if (forceFirecrawl) {
+                const firecrawlText = await firecrawlExtract(cfg, normalUrl, timeoutMs);
+                if (firecrawlText === null) {
+                    throw new ProviderError("Firecrawl requested but FIRECRAWL_API_KEY not set or call failed", "firecrawl");
+                }
+                return cacheAndReturn(normalUrl, {
+                    url: normalUrl,
+                    status: 200,
+                    extractor: "firecrawl",
+                    ...finaliseText(firecrawlText, maxChars),
+                    cached: false,
+                    durationMs: Date.now() - started,
+                });
+            }
+            // Local pipeline.
+            const timer = AbortSignal.timeout(timeoutMs);
+            const composite = composeSignals([abortSignal, timer]);
+            let res;
+            try {
+                res = await fetch(normalUrl, {
+                    method: "GET",
+                    signal: composite,
+                    headers: {
+                        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                    },
+                });
+            }
+            catch (e) {
+                if (e.name === "TimeoutError")
+                    throw new TimeoutError(`web_fetch ${normalUrl}`, timeoutMs);
+                if (abortSignal.aborted)
+                    throw new ValidationError("Fetch cancelled");
+                throw new ProviderError(`Fetch failed: ${e.message}`, "web_fetch");
+            }
+            // For non-HTML (JSON / text / PDF / image) we return as-is, respecting maxChars.
+            const ct = res.headers.get("content-type") ?? "";
+            if (!ct.includes("html") && !ct.includes("xml")) {
+                const text = (await res.text()).slice(0, maxChars + 1);
+                return cacheAndReturn(normalUrl, {
+                    url: normalUrl,
+                    status: res.status,
+                    extractor: "local",
+                    ...finaliseText(text, maxChars),
+                    cached: false,
+                    durationMs: Date.now() - started,
+                });
+            }
+            let html;
+            try {
+                html = await res.text();
+            }
+            catch (e) {
+                throw new ProviderError(`Failed reading body: ${e.message}`, "web_fetch");
+            }
+            const localText = extractReadableText(html);
+            // Short result → try Firecrawl if available before giving up.
+            if (localText.length < 200) {
+                const firecrawlText = await firecrawlExtract(cfg, normalUrl, timeoutMs);
+                if (firecrawlText && firecrawlText.length >= localText.length) {
+                    logger.info("web_fetch used firecrawl fallback", { url: normalUrl });
+                    return cacheAndReturn(normalUrl, {
+                        url: normalUrl,
+                        status: res.status,
+                        extractor: "firecrawl",
+                        ...finaliseText(firecrawlText, maxChars),
+                        cached: false,
+                        durationMs: Date.now() - started,
+                    });
+                }
+            }
+            return cacheAndReturn(normalUrl, {
+                url: normalUrl,
+                status: res.status,
+                extractor: "local",
+                ...finaliseText(localText, maxChars),
+                cached: false,
+                durationMs: Date.now() - started,
+            });
+        },
+    };
+}
+// ── Helpers ─────────────────────────────────────────────────────
+function cacheAndReturn(url, result) {
+    if (cache.size >= CACHE_MAX) {
+        const first = cache.keys().next().value;
+        if (first)
+            cache.delete(first);
+    }
+    cache.set(url, { result, expiresAt: Date.now() + CACHE_TTL_MS });
+    return result;
+}
+function finaliseText(text, maxChars) {
+    const stripped = stripInvisibleUnicode(text).replace(/\r\n?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+    if (stripped.length <= maxChars)
+        return { text: stripped, truncated: false };
+    return { text: `${stripped.slice(0, maxChars)}\n\n…[truncated — exceeded ${maxChars} chars]`, truncated: true };
+}
+const INVISIBLE_UNICODE_RE = /[\u200B-\u200F\u202A-\u202E\u2060-\u2064\u206A-\u206F\uFEFF]/gu;
+function stripInvisibleUnicode(s) {
+    return s.replace(INVISIBLE_UNICODE_RE, "");
+}
+const STRIP_BLOCK_RE = /<(script|style|template|svg|canvas|iframe|object|embed|noscript|nav|footer|aside|form)\b[\s\S]*?<\/\1\s*>/gi;
+const COMMENT_RE = /<!--[\s\S]*?-->/g;
+const HIDDEN_CLASS_RE = /<[^>]+\sclass\s*=\s*["'][^"']*\b(sr-only|visually-hidden|d-none|hidden|invisible|offscreen|screen-reader-only)\b[^"']*["'][^>]*>[\s\S]*?<\/[^>]+>/gi;
+const HIDDEN_STYLE_RE = /<[^>]+\sstyle\s*=\s*["'][^"']*(display\s*:\s*none|visibility\s*:\s*hidden)[^"']*["'][^>]*>[\s\S]*?<\/[^>]+>/gi;
+const HIDDEN_ATTR_RE = /<[^>]+\s(?:aria-hidden\s*=\s*["']true["']|hidden(?:\s|=|>))[^>]*>[\s\S]*?<\/[^>]+>/gi;
+const TAG_RE = /<[^>]+>/g;
+const HTML_ENTITIES = {
+    "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"',
+    "&#39;": "'", "&apos;": "'", "&#x27;": "'", "&#x2F;": "/",
+    "&nbsp;": " ",
+};
+const ENTITY_RE = /&(?:#x[0-9a-f]+|#\d+|[a-z]+);/gi;
+function extractReadableText(html) {
+    let out = html;
+    out = out.replace(COMMENT_RE, "");
+    out = out.replace(STRIP_BLOCK_RE, "");
+    out = out.replace(HIDDEN_CLASS_RE, "");
+    out = out.replace(HIDDEN_STYLE_RE, "");
+    out = out.replace(HIDDEN_ATTR_RE, "");
+    // Prefer <article> / <main> if present.
+    const main = out.match(/<(?:article|main)[^>]*>([\s\S]*?)<\/(?:article|main)>/i);
+    if (main?.[1])
+        out = main[1];
+    // Linebreak-aware block tag close
+    out = out.replace(/<\/(p|div|li|h[1-6]|br|tr)>/gi, "\n");
+    out = out.replace(/<li[^>]*>/gi, "- ");
+    out = out.replace(TAG_RE, "");
+    out = out.replace(ENTITY_RE, (m) => {
+        if (HTML_ENTITIES[m])
+            return HTML_ENTITIES[m];
+        if (m.startsWith("&#x"))
+            return String.fromCharCode(parseInt(m.slice(3, -1), 16));
+        if (m.startsWith("&#"))
+            return String.fromCharCode(parseInt(m.slice(2, -1), 10));
+        return m;
+    });
+    return out.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+async function firecrawlExtract(cfg, url, timeoutMs) {
+    const key = cfg.vault.get("FIRECRAWL_API_KEY")?.reveal();
+    if (!key)
+        return null;
+    try {
+        const res = await fetch("https://api.firecrawl.dev/v1/scrape", {
+            method: "POST",
+            headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
+            body: JSON.stringify({ url, formats: ["markdown"], onlyMainContent: true, timeout: Math.min(timeoutMs, 15_000) }),
+            signal: AbortSignal.timeout(Math.min(timeoutMs + 5_000, 30_000)),
+        });
+        if (!res.ok)
+            return null;
+        const data = (await res.json());
+        return data.data?.markdown ?? data.data?.content ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+const GITHUB_BLOB_RE = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/blob\/(.+)$/;
+function convertGitHubBlobUrl(url) {
+    const m = url.match(GITHUB_BLOB_RE);
+    return m ? `https://raw.githubusercontent.com/${m[1]}/${m[2]}/${m[3]}` : url;
+}
+// SSRF — block loopback, RFC1918, link-local, cloud metadata, localhost.
+const PRIVATE_HOST_REGEX = [
+    /^127\./,
+    /^10\./,
+    /^172\.(1[6-9]|2[0-9]|3[01])\./,
+    /^192\.168\./,
+    /^169\.254\./,
+    /^0\./,
+    /^::1$/,
+    /^fc00:/i,
+    /^fe80:/i,
+    /^localhost$/i,
+];
+function assertPublicUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new ValidationError(`Invalid URL: ${url}`);
+    }
+    if (!/^https?:$/i.test(parsed.protocol)) {
+        throw new ValidationError(`Blocked protocol: ${parsed.protocol}`);
+    }
+    const host = parsed.hostname.toLowerCase();
+    if (PRIVATE_HOST_REGEX.some((r) => r.test(host))) {
+        throw new ValidationError(`Blocked private / loopback host: ${host}`);
+    }
+}
+function composeSignals(signals) {
+    const native = AbortSignal.any;
+    if (typeof native === "function")
+        return native(Array.from(signals));
+    const ctrl = new AbortController();
+    for (const s of signals) {
+        if (s.aborted) {
+            ctrl.abort(s.reason);
+            break;
+        }
+        s.addEventListener("abort", () => ctrl.abort(s.reason), { once: true });
+    }
+    return ctrl.signal;
+}
+//# sourceMappingURL=webFetch.js.map

package/dist/tools/core/webFetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webFetch.js","sourceRoot":"","sources":["../../../src/tools/core/webFetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGpF,MAAM,YAAY,GAAG,EAAE,GAAG,MAAM,CAAC;AACjC,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;SAClE,QAAQ,CAAC,+DAA+D,CAAC;IAC5E,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;IAClE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;SACvC,QAAQ,CAAC,wEAAwE,CAAC;CACtF,CAAC,CAAC;AAYH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyD,CAAC;AAE/E,MAAM,UAAU,kBAAkB;IAChC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC;IACrB,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkB;IACjD,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,WAAW,EACT,mKAAmK;QACrK,QAAQ,EAAE,SAAS;QACnB,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QACxB,QAAQ,EAAE,IAAI;QACd,IAAI,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,QAAQ,CAAC;QAC1D,WAAW;QACX,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE;YACjF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE3B,aAAa;YACb,eAAe,CAAC,GAAG,CAAC,CAAC;YAErB,8BAA8B;YAC9B,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;YAE5C,YAAY;YACZ,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;gBACvD,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACzC,CAAC;YAED,2CAA2C;YAC3C,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;gBACxE,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;oBAC3B,MAAM,IAAI,aAAa,CAAC,kEAAkE,EAAE,WAAW,CAAC,CAAC;gBAC3G,CAAC;gBACD,OAAO,cAAc,CAAC,SAAS,EAAE;oBAC/B,GAAG,EAAE,SAAS;oBACd,MAAM,EAAE,GAAG;oBACX,SAAS,EAAE,WAAW;oBACtB,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC;oBACxC,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;iBACjC,CAAC,CAAC;YACL,CAAC;YAED,kBAAkB;YAClB,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;YACvD,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBAC3B,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,SAAS;oBACjB,OAAO,EAAE;wBACP,YAAY,EACV,uHAAuH;wBACzH,MAAM,EAAE,iEAAiE;qBAC1E;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAK,CAAW,CAAC,IAAI,KAAK,cAAc;oBAAE,MAAM,IAAI,YAAY,CAAC,aAAa,SAAS,EAAE,EAAE,SAAS,CAAC,CAAC;gBACtG,IAAI,WAAW,CAAC,OAAO;oBAAE,MAAM,IAAI,eAAe,CAAC,iBAAiB,CAAC,CAAC;gBACtE,MAAM,IAAI,aAAa,CAAC,iBAAkB,CAAW,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;YAChF,CAAC;YAED,iFAAiF;YACjF,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO,cAAc,CAAC,SAAS,EAAE;oBAC/B,GAAG,EAAE,SAAS;oBACd,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,SAAS,EAAE,OAAO;oBAClB,GAAG,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC;oBAC/B,MAAM,EAAE,KAAK;oBACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;iBACjC,CAAC,CAAC;YACL,CAAC;YAED,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC1B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,aAAa,CAAC,wBAAyB,CAAW,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;YACvF,CAAC;YAED,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;YAC5C,8DAA8D;YAC9D,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC3B,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;gBACxE,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBAC9D,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;oBACrE,OAAO,cAAc,CAAC,SAAS,EAAE;wBAC/B,GAAG,EAAE,SAAS;wBACd,MAAM,EAAE,GAAG,CAAC,MAAM;wBAClB,SAAS,EAAE,WAAW;wBACtB,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC;wBACxC,MAAM,EAAE,KAAK;wBACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;qBACjC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,cAAc,CAAC,SAAS,EAAE;gBAC/B,GAAG,EAAE,SAAS;gBACd,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,OAAO;gBAClB,GAAG,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC;gBACpC,MAAM,EAAE,KAAK;gBACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aACjC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,mEAAmE;AAEnE,SAAS,cAAc,CAAC,GAAW,EAAE,MAAsB;IACzD,IAAI,KAAK,CAAC,IAAI,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACxC,IAAI,KAAK;YAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IACjE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,QAAgB;IAClD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACvG,IAAI,QAAQ,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,8BAA8B,QAAQ,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAClH,CAAC;AAED,MAAM,oBAAoB,GAAG,gEAAgE,CAAC;AAC9F,SAAS,qBAAqB,CAAC,CAAS;IACtC,OAAO,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,cAAc,GAAG,6GAA6G,CAAC;AACrI,MAAM,UAAU,GAAG,kBAAkB,CAAC;AACtC,MAAM,eAAe,GAAG,qJAAqJ,CAAC;AAC9K,MAAM,eAAe,GAAG,+GAA+G,CAAC;AACxI,MAAM,cAAc,GAAG,sFAAsF,CAAC;AAC9G,MAAM,MAAM,GAAG,UAAU,CAAC;AAE1B,MAAM,aAAa,GAA2B;IAC5C,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG;IACrD,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG;IACzD,QAAQ,EAAE,GAAG;CACd,CAAC;AACF,MAAM,SAAS,GAAG,iCAAiC,CAAC;AAEpD,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAClC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACtC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACvC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACvC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACtC,wCAAwC;IACxC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;IACjF,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QAAE,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7B,kCAAkC;IAClC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,+BAA+B,EAAE,IAAI,CAAC,CAAC;IACzD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IACvC,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC9B,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE;QACjC,IAAI,aAAa,CAAC,CAAC,CAAC;YAAE,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClF,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAkB,EAAE,GAAW,EAAE,SAAiB;IAChF,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IACzD,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,qCAAqC,EAAE;YAC7D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/E,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;YACjH,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,KAAK,EAAE,MAAM,CAAC,CAAC;SACjE,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuD,CAAC;QACtF,OAAO,IAAI,CAAC,IAAI,EAAE,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,IAAI,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,uDAAuD,CAAC;AAC/E,SAAS,oBAAoB,CAAC,GAAW;IACvC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,OAAO,CAAC,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;AAC/E,CAAC;AAED,yEAAyE;AACzE,MAAM,kBAAkB,GAAG;IACzB,QAAQ;IACR,OAAO;IACP,+BAA+B;IAC/B,aAAa;IACb,aAAa;IACb,MAAM;IACN,OAAO;IACP,SAAS;IACT,SAAS;IACT,cAAc;CACf,CAAC;AAEF,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,eAAe,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,eAAe,CAAC,qBAAqB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,eAAe,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAA+B;IACrD,MAAM,MAAM,GAAI,WAAsE,CAAC,GAAG,CAAC;IAC3F,IAAI,OAAO,MAAM,KAAK,UAAU;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAAC,MAAM;QAAC,CAAC;QAC/C,CAAC,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC"}

package/dist/tools/core/webSearch.d.ts

@@ -0,0 +1,55 @@
+/**
+ * web_search — multi-provider internet search with automatic fallback.
+ *
+ * Tries providers in priority order and falls through to the next on
+ * any failure (missing key, rate limit, HTTP error, timeout):
+ *
+ *   Tavily → Perplexity → Brave → SearXNG → DuckDuckGo
+ *
+ * DuckDuckGo needs no API key, so there's always *something* working
+ * as long as the process has outbound internet. The earlier providers
+ * are better quality when a key is present — Tavily/Perplexity even
+ * return a synthesised AI answer alongside the raw hits.
+ *
+ * Results cached in-memory for 5 minutes (LRU, max 50 entries) so
+ * back-to-back identical queries don't burn quota.
+ */
+import { z } from "zod";
+import type { ConfigManager } from "../../config/ConfigManager.js";
+import { TimeoutError } from "../../util/errors.js";
+import type { ToolDef } from "../types.js";
+declare const inputSchema: z.ZodObject<{
+    query: z.ZodString;
+    count: z.ZodDefault<z.ZodNumber>;
+    freshness: z.ZodOptional<z.ZodEnum<["day", "week", "month", "year"]>>;
+    language: z.ZodOptional<z.ZodString>;
+    provider: z.ZodOptional<z.ZodEnum<["firecrawl", "tavily", "perplexity", "brave", "searxng", "ddg"]>>;
+}, "strip", z.ZodTypeAny, {
+    count: number;
+    query: string;
+    provider?: "perplexity" | "brave" | "firecrawl" | "tavily" | "searxng" | "ddg" | undefined;
+    language?: string | undefined;
+    freshness?: "year" | "month" | "day" | "week" | undefined;
+}, {
+    query: string;
+    provider?: "perplexity" | "brave" | "firecrawl" | "tavily" | "searxng" | "ddg" | undefined;
+    count?: number | undefined;
+    language?: string | undefined;
+    freshness?: "year" | "month" | "day" | "week" | undefined;
+}>;
+export interface WebSearchHit {
+    readonly title: string;
+    readonly url: string;
+    readonly description: string;
+}
+export interface WebSearchResult {
+    readonly query: string;
+    readonly provider: string;
+    readonly aiAnswer: string | null;
+    readonly totalReturned: number;
+    readonly results: readonly WebSearchHit[];
+}
+export declare function clearWebSearchCache(): number;
+export declare function makeWebSearchTool(cfg: ConfigManager): ToolDef<typeof inputSchema, WebSearchResult>;
+export { TimeoutError };
+//# sourceMappingURL=webSearch.d.ts.map

package/dist/tools/core/webSearch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webSearch.d.ts","sourceRoot":"","sources":["../../../src/tools/core/webSearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAGL,YAAY,EAEb,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAc3C,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;EAOf,CAAC;AAEH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;CAC3C;AA+BD,wBAAgB,mBAAmB,IAAI,MAAM,CAI5C;AAmQD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,WAAW,EAAE,eAAe,CAAC,CAkElG;AAqCD,OAAO,EAAE,YAAY,EAAE,CAAC"}