@botbotgo/kit-builtin 0.0.92

package/dist/src/http/duckduckgoSearch.js

@@ -0,0 +1,190 @@
+import { parse } from "node-html-parser";
+import { getExtensionContext } from "../../index.js";
+import { DEFAULT_TIMEOUT_MS, DEFAULT_MAX_HTTP_BYTES, DEFAULT_HTTP_USER_AGENT, DEFAULT_BLOCKED_CIDRS, } from "./defaults.js";
+import { createTaggedError } from "@botbotgo/kit/sdk";
+import { validateUrl } from "../platform/securityCompat.js";
+import { overrideWithConfig } from "../util/overrideWithConfig.js";
+const DUCKDUCKGO_API = "https://api.duckduckgo.com/";
+const DUCKDUCKGO_HTML = "https://html.duckduckgo.com/html/";
+function getDuckConfig() {
+    const defaults = {
+        timeoutMs: DEFAULT_TIMEOUT_MS,
+        maxBytes: DEFAULT_MAX_HTTP_BYTES,
+        userAgent: DEFAULT_HTTP_USER_AGENT,
+        blockedCidrs: DEFAULT_BLOCKED_CIDRS,
+        allowedHosts: [],
+        blockedHosts: [],
+    };
+    const cfg = overrideWithConfig(defaults, getExtensionContext().config);
+    return {
+        timeoutMs: cfg.timeoutMs ?? defaults.timeoutMs,
+        maxBytes: cfg.maxBytes ?? defaults.maxBytes,
+        userAgent: cfg.userAgent ?? defaults.userAgent,
+        blockedCidrs: Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs,
+        allowedHosts: Array.isArray(cfg.allowedHosts) ? cfg.allowedHosts : defaults.allowedHosts,
+        blockedHosts: Array.isArray(cfg.blockedHosts) ? cfg.blockedHosts : defaults.blockedHosts,
+    };
+}
+function requireQuery(args) {
+    const query = (args.query ?? args.q)?.trim();
+    if (!query)
+        throw createTaggedError("DUCKDUCKGO_INVALID", "query is required (pass 'query' or 'q')", {});
+    return query;
+}
+async function fetchTextWithTimeout(args) {
+    await validateUrl(args.url, {
+        allowedHosts: args.allowedHosts,
+        blockedHosts: args.blockedHosts,
+        blockedCidrs: args.blockedCidrs,
+    });
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), args.timeoutMs);
+    let response;
+    try {
+        response = await fetch(args.url, {
+            method: "GET",
+            headers: { "User-Agent": args.userAgent },
+            signal: controller.signal,
+        });
+    }
+    catch (err) {
+        clearTimeout(timer);
+        if (err instanceof Error && err.name === "AbortError") {
+            throw createTaggedError("HTTP_TIMEOUT", `DuckDuckGo search timed out after ${args.timeoutMs}ms`, {
+                timeoutMs: args.timeoutMs,
+            });
+        }
+        throw createTaggedError("UPSTREAM_ERROR", `DuckDuckGo search failed: ${err instanceof Error ? err.message : String(err)}`, {});
+    }
+    clearTimeout(timer);
+    const text = await response.text();
+    const bytes = Buffer.byteLength(text, "utf-8");
+    if (bytes > args.maxBytes) {
+        throw createTaggedError("HTTP_TOO_LARGE", `DuckDuckGo response ${bytes} bytes exceeds limit of ${args.maxBytes} bytes`, {
+            bytes,
+            limit: args.maxBytes,
+        });
+    }
+    return text;
+}
+function parseDuckApi(text, query) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw createTaggedError("UPSTREAM_ERROR", "DuckDuckGo returned invalid JSON", {
+            query,
+            textPreview: text.slice(0, 200),
+        });
+    }
+}
+function collectInstantApiResults(raw, maxResults) {
+    const results = [];
+    if (Array.isArray(raw.Results)) {
+        for (const item of raw.Results.slice(0, maxResults)) {
+            if (!item.FirstURL)
+                continue;
+            results.push({
+                url: item.FirstURL,
+                title: item.Text ?? item.FirstURL,
+                snippet: item.Text,
+            });
+        }
+    }
+    const relatedTopics = [];
+    if (Array.isArray(raw.RelatedTopics)) {
+        for (const topic of raw.RelatedTopics.slice(0, maxResults)) {
+            const text = topic.Text ?? topic.Result;
+            if (text)
+                relatedTopics.push({ text, url: topic.FirstURL });
+        }
+    }
+    return { results, relatedTopics };
+}
+async function fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, userAgent, allowedHosts, blockedHosts, blockedCidrs) {
+    const htmlUrl = `${DUCKDUCKGO_HTML}?q=${encodeURIComponent(query)}`;
+    try {
+        await validateUrl(htmlUrl, { allowedHosts, blockedHosts, blockedCidrs });
+    }
+    catch {
+        return [];
+    }
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    let response;
+    try {
+        response = await fetch(htmlUrl, {
+            method: "GET",
+            headers: { "User-Agent": userAgent },
+            signal: controller.signal,
+        });
+    }
+    catch {
+        clearTimeout(timer);
+        return [];
+    }
+    clearTimeout(timer);
+    if (!response.ok)
+        return [];
+    const text = await response.text();
+    if (Buffer.byteLength(text, "utf-8") > maxBytes)
+        return [];
+    const root = parse(text, { comment: false });
+    const out = [];
+    const resultBlocks = root.querySelectorAll(".result");
+    for (let i = 0; i < Math.min(resultBlocks.length, maxResults); i += 1) {
+        const block = resultBlocks[i];
+        const linkEl = block?.querySelector(".result__a");
+        const href = linkEl?.getAttribute("href");
+        const title = (linkEl?.textContent ?? "").trim().replace(/\s+/g, " ");
+        const snippet = (block?.querySelector(".result__snippet")?.textContent ?? "")
+            .trim()
+            .replace(/\s+/g, " ")
+            .slice(0, 500);
+        if (href && title)
+            out.push({ url: href, title, snippet: snippet || undefined });
+    }
+    return out;
+}
+export async function duckduckgoSearch(args) {
+    const cfg = getDuckConfig();
+    const query = requireQuery(args);
+    const maxResults = args.maxResults ?? 10;
+    const timeoutMs = args.timeoutMs ?? cfg.timeoutMs;
+    const maxBytes = args.maxBytes ?? cfg.maxBytes;
+    const url = `${DUCKDUCKGO_API}?q=${encodeURIComponent(query)}&format=json`;
+    const text = await fetchTextWithTimeout({
+        url,
+        timeoutMs,
+        userAgent: cfg.userAgent,
+        maxBytes,
+        allowedHosts: cfg.allowedHosts,
+        blockedHosts: cfg.blockedHosts,
+        blockedCidrs: cfg.blockedCidrs,
+    });
+    const raw = parseDuckApi(text, query);
+    const { results, relatedTopics } = collectInstantApiResults(raw, maxResults);
+    if (results.length === 0 && relatedTopics.length === 0) {
+        const htmlResults = await fetchDuckDuckGoHtmlSearch(query, maxResults, timeoutMs, maxBytes, cfg.userAgent, cfg.allowedHosts, cfg.blockedHosts, cfg.blockedCidrs);
+        if (htmlResults.length > 0)
+            results.push(...htmlResults);
+    }
+    return {
+        result: {
+            query,
+            abstract: raw.Abstract ?? raw.AbstractText ?? undefined,
+            abstractUrl: raw.AbstractURL ?? undefined,
+            abstractSource: raw.AbstractSource ?? undefined,
+            heading: raw.Heading ?? undefined,
+            results,
+            relatedTopics,
+        },
+        evidence: [{
+                type: "url",
+                ref: url,
+                summary: `DuckDuckGo search: "${query}" → ${results.length} results`,
+                createdAt: new Date().toISOString(),
+            }],
+    };
+}
+//# sourceMappingURL=duckduckgoSearch.js.map

package/dist/src/http/duckduckgoSearch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckduckgoSearch.js","sourceRoot":"","sources":["../../../src/http/duckduckgoSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG,6BAA6B,CAAC;AACrD,MAAM,eAAe,GAAG,mCAAmC,CAAC;AAgC5D,SAAS,aAAa;IACpB,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC;IACvE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC3C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;KACzF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,IAAmC;IACvD,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC7C,IAAI,CAAC,KAAK;QAAE,MAAM,iBAAiB,CAAC,oBAAoB,EAAE,yCAAyC,EAAE,EAAE,CAAC,CAAC;IACzG,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,IAQnC;IACC,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE;QAC1B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;KAChC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CAAC,cAAc,EAAE,qCAAqC,IAAI,CAAC,SAAS,IAAI,EAAE;gBAC/F,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,6BAA6B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjI,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC1B,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,uBAAuB,KAAK,2BAA2B,IAAI,CAAC,QAAQ,QAAQ,EAAE;YACtH,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,QAAQ;SACrB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;IAC/C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAuB,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,kCAAkC,EAAE;YAC5E,KAAK;YACL,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,GAAuB,EAAE,UAAkB;IAC3E,MAAM,OAAO,GAA4D,EAAE,CAAC;IAC5E,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,QAAQ;gBAAE,SAAS;YAC7B,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ;gBACjC,OAAO,EAAE,IAAI,CAAC,IAAI;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAA0C,EAAE,CAAC;IAChE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;YACxC,IAAI,IAAI;gBAAE,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,KAAa,EACb,UAAkB,EAClB,SAAiB,EACjB,QAAgB,EAChB,SAAiB,EACjB,YAAsB,EACtB,YAAsB,EACtB,YAAsB;IAEtB,MAAM,OAAO,GAAG,GAAG,eAAe,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;IACpE,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,OAAO,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC9B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,YAAY,CAAC,KAAK,CAAC,CAAC;IAEpB,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,QAAQ;QAAE,OAAO,EAAE,CAAC;IAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,GAAG,GAA4D,EAAE,CAAC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACtE,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,kBAAkB,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;aAC1E,IAAI,EAAE;aACN,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACjB,IAAI,IAAI,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAMtC;IACC,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAC/C,MAAM,GAAG,GAAG,GAAG,cAAc,MAAM,kBAAkB,CAAC,KAAK,CAAC,cAAc,CAAC;IAE3E,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC;QACtC,GAAG;QACH,SAAS;QACT,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,QAAQ;QACR,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,wBAAwB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC7E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,MAAM,yBAAyB,CACjD,KAAK,EACL,UAAU,EACV,SAAS,EACT,QAAQ,EACR,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,YAAY,CACjB,CAAC;QACF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,KAAK;YACL,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,IAAI,SAAS;YACvD,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;YACzC,cAAc,EAAE,GAAG,CAAC,cAAc,IAAI,SAAS;YAC/C,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,SAAS;YACjC,OAAO;YACP,aAAa;SACd;QACD,QAAQ,EAAE,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,uBAAuB,KAAK,OAAO,OAAO,CAAC,MAAM,UAAU;gBACpE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/src/http/fetchJson.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Fetch JSON from HTTP endpoint with SSRF/safety checks and size limits.
+ * @tool
+ * @effect none
+ */
+export declare function fetchJson(args: {
+    url?: string;
+    uri?: string;
+    method?: "GET" | "POST";
+    headers?: Record<string, string>;
+    body?: string;
+    timeoutMs?: number;
+    maxBytes?: number;
+}): Promise<{
+    result: {
+        url: string;
+        status: number;
+        json: unknown;
+        bytes: number;
+    };
+    evidence: {
+        type: string;
+        ref: string;
+        summary: string;
+        createdAt: string;
+    }[];
+}>;
+//# sourceMappingURL=fetchJson.d.ts.map

package/dist/src/http/fetchJson.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchJson.d.ts","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"AA4HA;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;;;;;;GAuCA"}

package/dist/src/http/fetchJson.js

@@ -0,0 +1,139 @@
+import { getExtensionContext } from "../../index.js";
+import { DEFAULT_TIMEOUT_MS, DEFAULT_MAX_HTTP_BYTES, DEFAULT_HTTP_USER_AGENT, DEFAULT_BLOCKED_CIDRS, } from "./defaults.js";
+import { createTaggedError } from "@botbotgo/kit/sdk";
+import { validateUrl } from "../platform/securityCompat.js";
+import { overrideWithConfig } from "../util/overrideWithConfig.js";
+function getFetchJsonConfig() {
+    const defaults = {
+        timeoutMs: DEFAULT_TIMEOUT_MS,
+        maxBytes: DEFAULT_MAX_HTTP_BYTES,
+        userAgent: DEFAULT_HTTP_USER_AGENT,
+        blockedCidrs: DEFAULT_BLOCKED_CIDRS,
+        allowedHosts: [],
+        blockedHosts: [],
+    };
+    const cfg = overrideWithConfig(defaults, getExtensionContext().config);
+    return {
+        timeoutMs: cfg.timeoutMs ?? defaults.timeoutMs,
+        maxBytes: cfg.maxBytes ?? defaults.maxBytes,
+        userAgent: cfg.userAgent ?? defaults.userAgent,
+        blockedCidrs: Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs,
+        allowedHosts: Array.isArray(cfg.allowedHosts) ? cfg.allowedHosts : defaults.allowedHosts,
+        blockedHosts: Array.isArray(cfg.blockedHosts) ? cfg.blockedHosts : defaults.blockedHosts,
+    };
+}
+function parseFetchJsonInput(args) {
+    const url = (args.url ?? args.uri)?.trim();
+    if (!url)
+        throw createTaggedError("HTTP_INVALID", "url is required (pass 'url' or 'uri')", {});
+    return {
+        url,
+        method: args.method ?? "GET",
+        headers: { ...(args.headers ?? {}) },
+        body: args.body ?? undefined,
+        timeoutMs: args.timeoutMs,
+        maxBytes: args.maxBytes,
+    };
+}
+async function fetchJsonResponse(args) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), args.timeoutMs);
+    try {
+        return await fetch(args.url, {
+            method: args.method,
+            headers: args.headers,
+            body: args.body,
+            signal: controller.signal,
+        });
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === "AbortError") {
+            throw createTaggedError("HTTP_TIMEOUT", `Request to ${args.url} timed out after ${args.timeoutMs}ms`, {
+                url: args.url,
+                timeoutMs: args.timeoutMs,
+            });
+        }
+        throw createTaggedError("UPSTREAM_ERROR", `Fetch failed for ${args.url}: ${err instanceof Error ? err.message : String(err)}`, {
+            url: args.url,
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function checkBodySizeLimit(url, response, text, maxBytes) {
+    const contentLength = response.headers.get("content-length");
+    if (contentLength && parseInt(contentLength, 10) > maxBytes) {
+        throw createTaggedError("HTTP_TOO_LARGE", `Response Content-Length ${contentLength} exceeds limit of ${maxBytes} bytes`, {
+            url,
+            contentLength: parseInt(contentLength, 10),
+            limit: maxBytes,
+        });
+    }
+    const bytes = Buffer.byteLength(text, "utf-8");
+    if (bytes > maxBytes) {
+        throw createTaggedError("HTTP_TOO_LARGE", `Response body ${bytes} bytes exceeds limit of ${maxBytes} bytes`, {
+            url,
+            bytes,
+            limit: maxBytes,
+        });
+    }
+    return bytes;
+}
+function parseJsonOrThrow(url, response, text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw createTaggedError("UPSTREAM_ERROR", `Failed to parse JSON response from ${url}: ${text.slice(0, 200)}`, {
+            url,
+            status: response.status,
+            textPreview: text.slice(0, 500),
+        });
+    }
+}
+/**
+ * Fetch JSON from HTTP endpoint with SSRF/safety checks and size limits.
+ * @tool
+ * @effect none
+ */
+export async function fetchJson(args) {
+    const cfg = getFetchJsonConfig();
+    const input = parseFetchJsonInput(args);
+    const timeoutMs = input.timeoutMs ?? cfg.timeoutMs;
+    const maxBytes = input.maxBytes ?? cfg.maxBytes;
+    await validateUrl(input.url, {
+        allowedHosts: cfg.allowedHosts,
+        blockedHosts: cfg.blockedHosts,
+        blockedCidrs: cfg.blockedCidrs,
+    });
+    if (!input.headers.Accept && !input.headers.accept)
+        input.headers.Accept = "application/json";
+    if (!input.headers["User-Agent"] && !input.headers["user-agent"])
+        input.headers["User-Agent"] = cfg.userAgent;
+    const response = await fetchJsonResponse({
+        url: input.url,
+        method: input.method,
+        headers: input.headers,
+        body: input.body,
+        timeoutMs,
+    });
+    const text = await response.text();
+    const bytes = checkBodySizeLimit(input.url, response, text, maxBytes);
+    const json = parseJsonOrThrow(input.url, response, text);
+    return {
+        result: {
+            url: input.url,
+            status: response.status,
+            json,
+            bytes,
+        },
+        evidence: [{
+                type: "url",
+                ref: input.url,
+                summary: `${input.method} ${input.url} → ${response.status} JSON (${bytes} bytes)`,
+                createdAt: new Date().toISOString(),
+            }],
+    };
+}
+//# sourceMappingURL=fetchJson.js.map

package/dist/src/http/fetchJson.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchJson.js","sourceRoot":"","sources":["../../../src/http/fetchJson.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAWnE,SAAS,kBAAkB;IACzB,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC;IACvE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC3C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;KACzF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,IAQ5B;IACC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC3C,IAAI,CAAC,GAAG;QAAE,MAAM,iBAAiB,CAAC,cAAc,EAAE,uCAAuC,EAAE,EAAE,CAAC,CAAC;IAC/F,OAAO;QACL,GAAG;QACH,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;QAC5B,OAAO,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE;QACpC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,SAAS;QAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;KACxB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAMhC;IACC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CAAC,cAAc,EAAE,cAAc,IAAI,CAAC,GAAG,oBAAoB,IAAI,CAAC,SAAS,IAAI,EAAE;gBACpG,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,oBAAoB,IAAI,CAAC,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;YAC7H,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAY,EAAE,QAAgB;IACzF,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC;QAC5D,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,2BAA2B,aAAa,qBAAqB,QAAQ,QAAQ,EAAE;YACvH,GAAG;YACH,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;YAC1C,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACrB,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,iBAAiB,KAAK,2BAA2B,QAAQ,QAAQ,EAAE;YAC3G,GAAG;YACH,KAAK;YACL,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAY;IACrE,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,sCAAsC,GAAG,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;YAC5G,GAAG;YACH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAQ/B;IACC,MAAM,GAAG,GAAG,kBAAkB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IACnD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAEhD,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE;QAC3B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM;QAAE,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC;IAC9F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC;IAE9G,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC;QACvC,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS;KACV,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAEzD,OAAO;QACL,MAAM,EAAE;YACN,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI;YACJ,KAAK;SACN;QACD,QAAQ,EAAE,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,OAAO,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,QAAQ,CAAC,MAAM,UAAU,KAAK,SAAS;gBAClF,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/src/http/fetchPageMainContent.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Fetch page and extract main text content heuristically.
+ * @tool
+ * @effect none
+ */
+export declare function fetchPageMainContent(args: {
+    url?: string;
+    uri?: string;
+    timeoutMs?: number;
+    maxBytes?: number;
+}): Promise<{
+    result: {
+        url: string;
+        status: number;
+        title: string | undefined;
+        mainContent: string;
+        bytes: number;
+        isHtml: boolean;
+    };
+    evidence: {
+        type: string;
+        ref: string;
+        summary: string;
+        createdAt: string;
+    }[];
+}>;
+//# sourceMappingURL=fetchPageMainContent.d.ts.map

package/dist/src/http/fetchPageMainContent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchPageMainContent.d.ts","sourceRoot":"","sources":["../../../src/http/fetchPageMainContent.ts"],"names":[],"mappings":"AAoIA;;;;GAIG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;;;;;;;;GAgDA"}

package/dist/src/http/fetchPageMainContent.js

@@ -0,0 +1,164 @@
+import { parse } from "node-html-parser";
+import { getExtensionContext } from "../../index.js";
+import { DEFAULT_TIMEOUT_MS, DEFAULT_MAX_HTTP_BYTES, DEFAULT_HTTP_USER_AGENT, DEFAULT_BLOCKED_CIDRS, } from "./defaults.js";
+import { createTaggedError } from "@botbotgo/kit/sdk";
+import { validateUrl } from "../platform/securityCompat.js";
+import { overrideWithConfig } from "../util/overrideWithConfig.js";
+const MAIN_SELECTORS = [
+    "main",
+    "article",
+    "[role='main']",
+    "#content",
+    "#main",
+    ".content",
+    ".main-content",
+    ".article-body",
+    ".post-content",
+    ".entry-content",
+];
+function getMainContentConfig() {
+    const defaults = {
+        timeoutMs: DEFAULT_TIMEOUT_MS,
+        maxBytes: DEFAULT_MAX_HTTP_BYTES,
+        userAgent: DEFAULT_HTTP_USER_AGENT,
+        blockedCidrs: DEFAULT_BLOCKED_CIDRS,
+        allowedHosts: [],
+        blockedHosts: [],
+    };
+    const cfg = overrideWithConfig(defaults, getExtensionContext().config);
+    return {
+        timeoutMs: cfg.timeoutMs ?? defaults.timeoutMs,
+        maxBytes: cfg.maxBytes ?? defaults.maxBytes,
+        userAgent: cfg.userAgent ?? defaults.userAgent,
+        blockedCidrs: Array.isArray(cfg.blockedCidrs) ? cfg.blockedCidrs : defaults.blockedCidrs,
+        allowedHosts: Array.isArray(cfg.allowedHosts) ? cfg.allowedHosts : defaults.allowedHosts,
+        blockedHosts: Array.isArray(cfg.blockedHosts) ? cfg.blockedHosts : defaults.blockedHosts,
+    };
+}
+function extractMainContent(html) {
+    const root = parse(html, { comment: false });
+    const title = (root.querySelector("title")?.textContent ?? "").trim().replace(/\s+/g, " ");
+    let mainEl = null;
+    for (const selector of MAIN_SELECTORS) {
+        mainEl = root.querySelector(selector);
+        if (mainEl)
+            break;
+    }
+    if (!mainEl) {
+        const body = root.querySelector("body");
+        if (body) {
+            for (const tag of ["script", "style", "nav", "header", "footer", "aside", "noscript", "iframe"]) {
+                body.querySelectorAll(tag).forEach((el) => el.remove());
+            }
+            mainEl = body;
+        }
+        else {
+            mainEl = root;
+        }
+    }
+    return {
+        title,
+        mainContent: (mainEl.textContent ?? "").trim().replace(/\s+/g, " "),
+    };
+}
+function parseUrl(args) {
+    const url = (args.url ?? args.uri)?.trim();
+    if (!url)
+        throw createTaggedError("HTTP_INVALID", "url is required (pass 'url' or 'uri')", {});
+    return url;
+}
+async function fetchPageText(args) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), args.timeoutMs);
+    try {
+        return await fetch(args.url, {
+            method: "GET",
+            headers: { "User-Agent": args.userAgent },
+            signal: controller.signal,
+        });
+    }
+    catch (err) {
+        if (err instanceof Error && err.name === "AbortError") {
+            throw createTaggedError("HTTP_TIMEOUT", `Request to ${args.url} timed out after ${args.timeoutMs}ms`, {
+                url: args.url,
+                timeoutMs: args.timeoutMs,
+            });
+        }
+        throw createTaggedError("UPSTREAM_ERROR", `Fetch failed for ${args.url}: ${err instanceof Error ? err.message : String(err)}`, {
+            url: args.url,
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function enforceSizeLimit(url, response, body, maxBytes) {
+    const contentLength = response.headers.get("content-length");
+    if (contentLength && parseInt(contentLength, 10) > maxBytes) {
+        throw createTaggedError("HTTP_TOO_LARGE", `Response Content-Length ${contentLength} exceeds limit of ${maxBytes} bytes`, {
+            url,
+            contentLength: parseInt(contentLength, 10),
+            limit: maxBytes,
+        });
+    }
+    const rawBytes = Buffer.byteLength(body, "utf-8");
+    if (rawBytes > maxBytes) {
+        throw createTaggedError("HTTP_TOO_LARGE", `Response body ${rawBytes} bytes exceeds limit of ${maxBytes} bytes`, {
+            url,
+            bytes: rawBytes,
+            limit: maxBytes,
+        });
+    }
+    return rawBytes;
+}
+/**
+ * Fetch page and extract main text content heuristically.
+ * @tool
+ * @effect none
+ */
+export async function fetchPageMainContent(args) {
+    const cfg = getMainContentConfig();
+    const url = parseUrl(args);
+    const timeoutMs = args.timeoutMs ?? cfg.timeoutMs;
+    const maxBytes = args.maxBytes ?? cfg.maxBytes;
+    await validateUrl(url, {
+        allowedHosts: cfg.allowedHosts,
+        blockedHosts: cfg.blockedHosts,
+        blockedCidrs: cfg.blockedCidrs,
+    });
+    const response = await fetchPageText({ url, timeoutMs, userAgent: cfg.userAgent });
+    const rawText = await response.text();
+    enforceSizeLimit(url, response, rawText, maxBytes);
+    const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
+    const isHtml = contentType.includes("text/html");
+    let title;
+    let mainContent = rawText;
+    if (isHtml && rawText.trim().length > 0) {
+        try {
+            const extracted = extractMainContent(rawText);
+            title = extracted.title || undefined;
+            mainContent = extracted.mainContent;
+        }
+        catch {
+            mainContent = rawText;
+        }
+    }
+    const bytes = Buffer.byteLength(mainContent, "utf-8");
+    return {
+        result: {
+            url,
+            status: response.status,
+            title,
+            mainContent,
+            bytes,
+            isHtml,
+        },
+        evidence: [{
+                type: "url",
+                ref: url,
+                summary: `${url} → ${response.status} main content (${bytes} bytes)`,
+                createdAt: new Date().toISOString(),
+            }],
+    };
+}
+//# sourceMappingURL=fetchPageMainContent.js.map

package/dist/src/http/fetchPageMainContent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchPageMainContent.js","sourceRoot":"","sources":["../../../src/http/fetchPageMainContent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAEnE,MAAM,cAAc,GAAG;IACrB,MAAM;IACN,SAAS;IACT,eAAe;IACf,UAAU;IACV,OAAO;IACP,UAAU;IACV,eAAe;IACf,eAAe;IACf,eAAe;IACf,gBAAgB;CACjB,CAAC;AAWF,SAAS,oBAAoB;IAC3B,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,kBAAkB;QAC7B,QAAQ,EAAE,sBAAsB;QAChC,SAAS,EAAE,uBAAuB;QAClC,YAAY,EAAE,qBAAqB;QACnC,YAAY,EAAE,EAAc;QAC5B,YAAY,EAAE,EAAc;KAC7B,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC,MAAM,CAAC,CAAC;IACvE,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC3C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS;QAC9C,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;QACxF,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY;KACzF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAE3F,IAAI,MAAM,GAA0C,IAAI,CAAC;IACzD,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACtC,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,MAAM;YAAE,MAAM;IACpB,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,IAAI,EAAE,CAAC;YACT,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAChG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpE,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAoC;IACpD,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IAC3C,IAAI,CAAC,GAAG;QAAE,MAAM,iBAAiB,CAAC,cAAc,EAAE,uCAAuC,EAAE,EAAE,CAAC,CAAC;IAC/F,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAA2D;IACtF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;YACzC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACtD,MAAM,iBAAiB,CAAC,cAAc,EAAE,cAAc,IAAI,CAAC,GAAG,oBAAoB,IAAI,CAAC,SAAS,IAAI,EAAE;gBACpG,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,oBAAoB,IAAI,CAAC,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;YAC7H,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,QAAkB,EAAE,IAAY,EAAE,QAAgB;IACvF,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC;QAC5D,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,2BAA2B,aAAa,qBAAqB,QAAQ,QAAQ,EAAE;YACvH,GAAG;YACH,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC;YAC1C,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAClD,IAAI,QAAQ,GAAG,QAAQ,EAAE,CAAC;QACxB,MAAM,iBAAiB,CAAC,gBAAgB,EAAE,iBAAiB,QAAQ,2BAA2B,QAAQ,QAAQ,EAAE;YAC9G,GAAG;YACH,KAAK,EAAE,QAAQ;YACf,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,GAAG,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAE/C,MAAM,WAAW,CAAC,GAAG,EAAE;QACrB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,YAAY,EAAE,GAAG,CAAC,YAAY;KAC/B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IAEnD,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/E,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACjD,IAAI,KAAyB,CAAC;IAC9B,IAAI,WAAW,GAAG,OAAO,CAAC;IAE1B,IAAI,MAAM,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC9C,KAAK,GAAG,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC;YACrC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,OAAO,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACtD,OAAO;QACL,MAAM,EAAE;YACN,GAAG;YACH,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK;YACL,WAAW;YACX,KAAK;YACL,MAAM;SACP;QACD,QAAQ,EAAE,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,GAAG,EAAE,GAAG;gBACR,OAAO,EAAE,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,kBAAkB,KAAK,SAAS;gBACpE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/src/http/fetchText.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Fetch text body from HTTP endpoint with SSRF/safety checks and size limits.
+ * @tool
+ * @effect none
+ */
+export declare function fetchText(args: {
+    url?: string;
+    uri?: string;
+    method?: "GET" | "POST";
+    headers?: Record<string, string>;
+    body?: string;
+    timeoutMs?: number;
+    maxBytes?: number;
+}): Promise<{
+    result: {
+        url: string;
+        status: number;
+        headers: Record<string, string>;
+        text: string;
+        bytes: number;
+    };
+    evidence: {
+        type: string;
+        ref: string;
+        summary: string;
+        createdAt: string;
+    }[];
+}>;
+//# sourceMappingURL=fetchText.d.ts.map

package/dist/src/http/fetchText.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchText.d.ts","sourceRoot":"","sources":["../../../src/http/fetchText.ts"],"names":[],"mappings":"AAqIA;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;;;;;;;;;;;;;;GA4CA"}