@poolzin/pool-bot 2026.2.21 → 2026.2.23

package/dist/agents/tools/web-fetch.js

@@ -1,14 +1,22 @@
 import { Type } from "@sinclair/typebox";
-import { closeDispatcher, createPinnedDispatcher, resolvePinnedHostname, SsrFBlockedError, } from "../../infra/net/ssrf.js";
+import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js";
+import { SsrFBlockedError } from "../../infra/net/ssrf.js";
+import { logDebug } from "../../logger.js";
+import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js";
+import { normalizeSecretInput } from "../../utils/normalize-secret-input.js";
 import { stringEnum } from "../schema/typebox.js";
 import { jsonResult, readNumberParam, readStringParam } from "./common.js";
-import { DEFAULT_CACHE_TTL_MINUTES, DEFAULT_TIMEOUT_SECONDS, normalizeCacheKey, readCache, readResponseText, resolveCacheTtlMs, resolveTimeoutSeconds, withTimeout, writeCache, } from "./web-shared.js";
 import { extractReadableContent, htmlToMarkdown, markdownToText, truncateText, } from "./web-fetch-utils.js";
+import { DEFAULT_CACHE_TTL_MINUTES, DEFAULT_TIMEOUT_SECONDS, normalizeCacheKey, readCache, readResponseText, resolveCacheTtlMs, resolveTimeoutSeconds, withTimeout, writeCache, } from "./web-shared.js";
 export { extractReadableContent } from "./web-fetch-utils.js";
 const EXTRACT_MODES = ["markdown", "text"];
 const DEFAULT_FETCH_MAX_CHARS = 50_000;
+const DEFAULT_FETCH_MAX_RESPONSE_BYTES = 2_000_000;
+const FETCH_MAX_RESPONSE_BYTES_MIN = 32_000;
+const FETCH_MAX_RESPONSE_BYTES_MAX = 10_000_000;
 const DEFAULT_FETCH_MAX_REDIRECTS = 3;
 const DEFAULT_ERROR_MAX_CHARS = 4_000;
+const DEFAULT_ERROR_MAX_BYTES = 64_000;
 const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev";
 const DEFAULT_FIRECRAWL_MAX_AGE_MS = 172_800_000;
 const DEFAULT_FETCH_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
@@ -26,38 +34,63 @@ const WebFetchSchema = Type.Object({
 });
 function resolveFetchConfig(cfg) {
     const fetch = cfg?.tools?.web?.fetch;
-    if (!fetch || typeof fetch !== "object")
+    if (!fetch || typeof fetch !== "object") {
         return undefined;
+    }
     return fetch;
 }
 function resolveFetchEnabled(params) {
-    if (typeof params.fetch?.enabled === "boolean")
+    if (typeof params.fetch?.enabled === "boolean") {
         return params.fetch.enabled;
+    }
     return true;
 }
 function resolveFetchReadabilityEnabled(fetch) {
-    if (typeof fetch?.readability === "boolean")
+    if (typeof fetch?.readability === "boolean") {
         return fetch.readability;
+    }
     return true;
 }
+function resolveFetchMaxCharsCap(fetch) {
+    const raw = fetch && "maxCharsCap" in fetch && typeof fetch.maxCharsCap === "number"
+        ? fetch.maxCharsCap
+        : undefined;
+    if (typeof raw !== "number" || !Number.isFinite(raw)) {
+        return DEFAULT_FETCH_MAX_CHARS;
+    }
+    return Math.max(100, Math.floor(raw));
+}
+function resolveFetchMaxResponseBytes(fetch) {
+    const raw = fetch && "maxResponseBytes" in fetch && typeof fetch.maxResponseBytes === "number"
+        ? fetch.maxResponseBytes
+        : undefined;
+    if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) {
+        return DEFAULT_FETCH_MAX_RESPONSE_BYTES;
+    }
+    const value = Math.floor(raw);
+    return Math.min(FETCH_MAX_RESPONSE_BYTES_MAX, Math.max(FETCH_MAX_RESPONSE_BYTES_MIN, value));
+}
 function resolveFirecrawlConfig(fetch) {
-    if (!fetch || typeof fetch !== "object")
+    if (!fetch || typeof fetch !== "object") {
         return undefined;
+    }
     const firecrawl = "firecrawl" in fetch ? fetch.firecrawl : undefined;
-    if (!firecrawl || typeof firecrawl !== "object")
+    if (!firecrawl || typeof firecrawl !== "object") {
         return undefined;
+    }
     return firecrawl;
 }
 function resolveFirecrawlApiKey(firecrawl) {
     const fromConfig = firecrawl && "apiKey" in firecrawl && typeof firecrawl.apiKey === "string"
-        ? firecrawl.apiKey.trim()
+        ? normalizeSecretInput(firecrawl.apiKey)
         : "";
-    const fromEnv = (process.env.FIRECRAWL_API_KEY ?? "").trim();
+    const fromEnv = normalizeSecretInput(process.env.FIRECRAWL_API_KEY);
     return fromConfig || fromEnv || undefined;
 }
 function resolveFirecrawlEnabled(params) {
-    if (typeof params.firecrawl?.enabled === "boolean")
+    if (typeof params.firecrawl?.enabled === "boolean") {
         return params.firecrawl.enabled;
+    }
     return Boolean(params.apiKey);
 }
 function resolveFirecrawlBaseUrl(firecrawl) {
@@ -67,28 +100,32 @@ function resolveFirecrawlBaseUrl(firecrawl) {
     return raw || DEFAULT_FIRECRAWL_BASE_URL;
 }
 function resolveFirecrawlOnlyMainContent(firecrawl) {
-    if (typeof firecrawl?.onlyMainContent === "boolean")
+    if (typeof firecrawl?.onlyMainContent === "boolean") {
         return firecrawl.onlyMainContent;
+    }
     return true;
 }
 function resolveFirecrawlMaxAgeMs(firecrawl) {
     const raw = firecrawl && "maxAgeMs" in firecrawl && typeof firecrawl.maxAgeMs === "number"
         ? firecrawl.maxAgeMs
         : undefined;
-    if (typeof raw !== "number" || !Number.isFinite(raw))
+    if (typeof raw !== "number" || !Number.isFinite(raw)) {
         return undefined;
+    }
     const parsed = Math.max(0, Math.floor(raw));
     return parsed > 0 ? parsed : undefined;
 }
 function resolveFirecrawlMaxAgeMsOrDefault(firecrawl) {
     const resolved = resolveFirecrawlMaxAgeMs(firecrawl);
-    if (typeof resolved === "number")
+    if (typeof resolved === "number") {
         return resolved;
+    }
     return DEFAULT_FIRECRAWL_MAX_AGE_MS;
 }
-function resolveMaxChars(value, fallback) {
+function resolveMaxChars(value, fallback, cap) {
     const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
-    return Math.max(100, Math.floor(parsed));
+    const clamped = Math.max(100, Math.floor(parsed));
+    return Math.min(clamped, cap);
 }
 function resolveMaxRedirects(value, fallback) {
     const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
@@ -96,79 +133,17 @@ function resolveMaxRedirects(value, fallback) {
 }
 function looksLikeHtml(value) {
     const trimmed = value.trimStart();
-    if (!trimmed)
+    if (!trimmed) {
         return false;
+    }
     const head = trimmed.slice(0, 256).toLowerCase();
     return head.startsWith("<!doctype html") || head.startsWith("<html");
 }
-function isRedirectStatus(status) {
-    return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
-}
-async function fetchWithRedirects(params) {
-    const signal = withTimeout(undefined, params.timeoutSeconds * 1000);
-    const visited = new Set();
-    let currentUrl = params.url;
-    let redirectCount = 0;
-    while (true) {
-        let parsedUrl;
-        try {
-            parsedUrl = new URL(currentUrl);
-        }
-        catch {
-            throw new Error("Invalid URL: must be http or https");
-        }
-        if (!["http:", "https:"].includes(parsedUrl.protocol)) {
-            throw new Error("Invalid URL: must be http or https");
-        }
-        const pinned = await resolvePinnedHostname(parsedUrl.hostname);
-        const dispatcher = createPinnedDispatcher(pinned);
-        let res;
-        try {
-            res = await fetch(parsedUrl.toString(), {
-                method: "GET",
-                headers: {
-                    Accept: "*/*",
-                    "User-Agent": params.userAgent,
-                    "Accept-Language": "en-US,en;q=0.9",
-                },
-                signal,
-                redirect: "manual",
-                dispatcher,
-            });
-        }
-        catch (err) {
-            await closeDispatcher(dispatcher);
-            throw err;
-        }
-        if (isRedirectStatus(res.status)) {
-            const location = res.headers.get("location");
-            if (!location) {
-                await closeDispatcher(dispatcher);
-                throw new Error(`Redirect missing location header (${res.status})`);
-            }
-            redirectCount += 1;
-            if (redirectCount > params.maxRedirects) {
-                await closeDispatcher(dispatcher);
-                throw new Error(`Too many redirects (limit: ${params.maxRedirects})`);
-            }
-            const nextUrl = new URL(location, parsedUrl).toString();
-            if (visited.has(nextUrl)) {
-                await closeDispatcher(dispatcher);
-                throw new Error("Redirect loop detected");
-            }
-            visited.add(nextUrl);
-            void res.body?.cancel();
-            await closeDispatcher(dispatcher);
-            currentUrl = nextUrl;
-            continue;
-        }
-        return { response: res, finalUrl: currentUrl, dispatcher };
-    }
-}
 function formatWebFetchErrorDetail(params) {
     const { detail, contentType, maxChars } = params;
-    if (!detail)
+    if (!detail) {
         return "";
+    }
     let text = detail;
     const contentTypeLower = contentType?.toLowerCase();
     if (contentTypeLower?.includes("text/html") || looksLikeHtml(detail)) {
@@ -179,6 +154,102 @@ function formatWebFetchErrorDetail(params) {
     const truncated = truncateText(text.trim(), maxChars);
     return truncated.text;
 }
+function redactUrlForDebugLog(rawUrl) {
+    try {
+        const parsed = new URL(rawUrl);
+        return parsed.pathname && parsed.pathname !== "/" ? `${parsed.origin}/...` : parsed.origin;
+    }
+    catch {
+        return "[invalid-url]";
+    }
+}
+const WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD = wrapWebContent("", "web_fetch").length;
+const WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD = wrapExternalContent("", {
+    source: "web_fetch",
+    includeWarning: false,
+}).length;
+function wrapWebFetchContent(value, maxChars) {
+    if (maxChars <= 0) {
+        return { text: "", truncated: true, rawLength: 0, wrappedLength: 0 };
+    }
+    const includeWarning = maxChars >= WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD;
+    const wrapperOverhead = includeWarning
+        ? WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD
+        : WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD;
+    if (wrapperOverhead > maxChars) {
+        const minimal = includeWarning
+            ? wrapWebContent("", "web_fetch")
+            : wrapExternalContent("", { source: "web_fetch", includeWarning: false });
+        const truncatedWrapper = truncateText(minimal, maxChars);
+        return {
+            text: truncatedWrapper.text,
+            truncated: true,
+            rawLength: 0,
+            wrappedLength: truncatedWrapper.text.length,
+        };
+    }
+    const maxInner = Math.max(0, maxChars - wrapperOverhead);
+    let truncated = truncateText(value, maxInner);
+    let wrappedText = includeWarning
+        ? wrapWebContent(truncated.text, "web_fetch")
+        : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+    if (wrappedText.length > maxChars) {
+        const excess = wrappedText.length - maxChars;
+        const adjustedMaxInner = Math.max(0, maxInner - excess);
+        truncated = truncateText(value, adjustedMaxInner);
+        wrappedText = includeWarning
+            ? wrapWebContent(truncated.text, "web_fetch")
+            : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+    }
+    return {
+        text: wrappedText,
+        truncated: truncated.truncated,
+        rawLength: truncated.text.length,
+        wrappedLength: wrappedText.length,
+    };
+}
+function wrapWebFetchField(value) {
+    if (!value) {
+        return value;
+    }
+    return wrapExternalContent(value, { source: "web_fetch", includeWarning: false });
+}
+function buildFirecrawlWebFetchPayload(params) {
+    const wrapped = wrapWebFetchContent(params.firecrawl.text, params.maxChars);
+    const wrappedTitle = params.firecrawl.title
+        ? wrapWebFetchField(params.firecrawl.title)
+        : undefined;
+    return {
+        url: params.rawUrl, // Keep raw for tool chaining
+        finalUrl: params.firecrawl.finalUrl || params.finalUrlFallback, // Keep raw
+        status: params.firecrawl.status ?? params.statusFallback,
+        contentType: "text/markdown", // Protocol metadata, don't wrap
+        title: wrappedTitle,
+        extractMode: params.extractMode,
+        extractor: "firecrawl",
+        externalContent: {
+            untrusted: true,
+            source: "web_fetch",
+            wrapped: true,
+        },
+        truncated: wrapped.truncated,
+        length: wrapped.wrappedLength,
+        rawLength: wrapped.rawLength, // Actual content length, not wrapped
+        wrappedLength: wrapped.wrappedLength,
+        fetchedAt: new Date().toISOString(),
+        tookMs: params.tookMs,
+        text: wrapped.text,
+        warning: wrapWebFetchField(params.firecrawl.warning),
+    };
+}
+function normalizeContentType(value) {
+    if (!value) {
+        return undefined;
+    }
+    const [raw] = value.split(";");
+    const trimmed = raw?.trim();
+    return trimmed || undefined;
+}
 export async function fetchFirecrawlContent(params) {
     const endpoint = resolveFirecrawlEndpoint(params.baseUrl);
     const body = {
@@ -201,8 +272,8 @@ export async function fetchFirecrawlContent(params) {
     });
     const payload = (await res.json());
     if (!res.ok || payload?.success === false) {
-        const detail = payload?.error || res.statusText;
-        throw new Error(`Firecrawl fetch failed (${res.status}): ${detail}`.trim());
+        const detail = payload?.error ?? "";
+        throw new Error(`Firecrawl fetch failed (${res.status}): ${wrapWebContent(detail || res.statusText, "web_fetch")}`.trim());
     }
     const data = payload?.data ?? {};
     const rawText = typeof data.markdown === "string"
@@ -219,11 +290,50 @@ export async function fetchFirecrawlContent(params) {
         warning: payload?.warning,
     };
 }
+function toFirecrawlContentParams(params) {
+    if (!params.firecrawlEnabled || !params.firecrawlApiKey) {
+        return null;
+    }
+    return {
+        url: params.url,
+        extractMode: params.extractMode,
+        apiKey: params.firecrawlApiKey,
+        baseUrl: params.firecrawlBaseUrl,
+        onlyMainContent: params.firecrawlOnlyMainContent,
+        maxAgeMs: params.firecrawlMaxAgeMs,
+        proxy: params.firecrawlProxy,
+        storeInCache: params.firecrawlStoreInCache,
+        timeoutSeconds: params.firecrawlTimeoutSeconds,
+    };
+}
+async function maybeFetchFirecrawlWebFetchPayload(params) {
+    const firecrawlParams = toFirecrawlContentParams({
+        ...params,
+        url: params.urlToFetch,
+        extractMode: params.extractMode,
+    });
+    if (!firecrawlParams) {
+        return null;
+    }
+    const firecrawl = await fetchFirecrawlContent(firecrawlParams);
+    const payload = buildFirecrawlWebFetchPayload({
+        firecrawl,
+        rawUrl: params.url,
+        finalUrlFallback: params.finalUrlFallback,
+        statusFallback: params.statusFallback,
+        extractMode: params.extractMode,
+        maxChars: params.maxChars,
+        tookMs: params.tookMs,
+    });
+    writeCache(FETCH_CACHE, params.cacheKey, payload, params.cacheTtlMs);
+    return payload;
+}
 async function runWebFetch(params) {
     const cacheKey = normalizeCacheKey(`fetch:${params.url}:${params.extractMode}:${params.maxChars}`);
     const cached = readCache(FETCH_CACHE, cacheKey);
-    if (cached)
+    if (cached) {
         return { ...cached.value, cached: true };
+    }
     let parsedUrl;
     try {
         parsedUrl = new URL(params.url);
@@ -236,103 +346,88 @@ async function runWebFetch(params) {
     }
     const start = Date.now();
     let res;
-    let dispatcher = null;
+    let release = null;
     let finalUrl = params.url;
     try {
-        const result = await fetchWithRedirects({
+        const result = await fetchWithSsrFGuard({
             url: params.url,
             maxRedirects: params.maxRedirects,
-            timeoutSeconds: params.timeoutSeconds,
-            userAgent: params.userAgent,
+            timeoutMs: params.timeoutSeconds * 1000,
+            init: {
+                headers: {
+                    Accept: "text/markdown, text/html;q=0.9, */*;q=0.1",
+                    "User-Agent": params.userAgent,
+                    "Accept-Language": "en-US,en;q=0.9",
+                },
+            },
         });
         res = result.response;
         finalUrl = result.finalUrl;
-        dispatcher = result.dispatcher;
+        release = result.release;
+        // Cloudflare Markdown for Agents — log token budget hint when present
+        const markdownTokens = res.headers.get("x-markdown-tokens");
+        if (markdownTokens) {
+            logDebug(`[web-fetch] x-markdown-tokens: ${markdownTokens} (${redactUrlForDebugLog(finalUrl)})`);
+        }
     }
     catch (error) {
         if (error instanceof SsrFBlockedError) {
             throw error;
         }
-        if (params.firecrawlEnabled && params.firecrawlApiKey) {
-            const firecrawl = await fetchFirecrawlContent({
-                url: finalUrl,
-                extractMode: params.extractMode,
-                apiKey: params.firecrawlApiKey,
-                baseUrl: params.firecrawlBaseUrl,
-                onlyMainContent: params.firecrawlOnlyMainContent,
-                maxAgeMs: params.firecrawlMaxAgeMs,
-                proxy: params.firecrawlProxy,
-                storeInCache: params.firecrawlStoreInCache,
-                timeoutSeconds: params.firecrawlTimeoutSeconds,
-            });
-            const truncated = truncateText(firecrawl.text, params.maxChars);
-            const payload = {
-                url: params.url,
-                finalUrl: firecrawl.finalUrl || finalUrl,
-                status: firecrawl.status ?? 200,
-                contentType: "text/markdown",
-                title: firecrawl.title,
-                extractMode: params.extractMode,
-                extractor: "firecrawl",
-                truncated: truncated.truncated,
-                length: truncated.text.length,
-                fetchedAt: new Date().toISOString(),
-                tookMs: Date.now() - start,
-                text: truncated.text,
-                warning: firecrawl.warning,
-            };
-            writeCache(FETCH_CACHE, cacheKey, payload, params.cacheTtlMs);
+        const payload = await maybeFetchFirecrawlWebFetchPayload({
+            ...params,
+            urlToFetch: finalUrl,
+            finalUrlFallback: finalUrl,
+            statusFallback: 200,
+            cacheKey,
+            tookMs: Date.now() - start,
+        });
+        if (payload) {
             return payload;
         }
         throw error;
     }
     try {
         if (!res.ok) {
-            if (params.firecrawlEnabled && params.firecrawlApiKey) {
-                const firecrawl = await fetchFirecrawlContent({
-                    url: params.url,
-                    extractMode: params.extractMode,
-                    apiKey: params.firecrawlApiKey,
-                    baseUrl: params.firecrawlBaseUrl,
-                    onlyMainContent: params.firecrawlOnlyMainContent,
-                    maxAgeMs: params.firecrawlMaxAgeMs,
-                    proxy: params.firecrawlProxy,
-                    storeInCache: params.firecrawlStoreInCache,
-                    timeoutSeconds: params.firecrawlTimeoutSeconds,
-                });
-                const truncated = truncateText(firecrawl.text, params.maxChars);
-                const payload = {
-                    url: params.url,
-                    finalUrl: firecrawl.finalUrl || finalUrl,
-                    status: firecrawl.status ?? res.status,
-                    contentType: "text/markdown",
-                    title: firecrawl.title,
-                    extractMode: params.extractMode,
-                    extractor: "firecrawl",
-                    truncated: truncated.truncated,
-                    length: truncated.text.length,
-                    fetchedAt: new Date().toISOString(),
-                    tookMs: Date.now() - start,
-                    text: truncated.text,
-                    warning: firecrawl.warning,
-                };
-                writeCache(FETCH_CACHE, cacheKey, payload, params.cacheTtlMs);
+            const payload = await maybeFetchFirecrawlWebFetchPayload({
+                ...params,
+                urlToFetch: params.url,
+                finalUrlFallback: finalUrl,
+                statusFallback: res.status,
+                cacheKey,
+                tookMs: Date.now() - start,
+            });
+            if (payload) {
                 return payload;
             }
-            const rawDetail = await readResponseText(res);
+            const rawDetailResult = await readResponseText(res, { maxBytes: DEFAULT_ERROR_MAX_BYTES });
+            const rawDetail = rawDetailResult.text;
             const detail = formatWebFetchErrorDetail({
                 detail: rawDetail,
                 contentType: res.headers.get("content-type"),
                 maxChars: DEFAULT_ERROR_MAX_CHARS,
             });
-            throw new Error(`Web fetch failed (${res.status}): ${detail || res.statusText}`);
+            const wrappedDetail = wrapWebFetchContent(detail || res.statusText, DEFAULT_ERROR_MAX_CHARS);
+            throw new Error(`Web fetch failed (${res.status}): ${wrappedDetail.text}`);
         }
         const contentType = res.headers.get("content-type") ?? "application/octet-stream";
-        const body = await readResponseText(res);
+        const normalizedContentType = normalizeContentType(contentType) ?? "application/octet-stream";
+        const bodyResult = await readResponseText(res, { maxBytes: params.maxResponseBytes });
+        const body = bodyResult.text;
+        const responseTruncatedWarning = bodyResult.truncated
+            ? `Response body truncated after ${params.maxResponseBytes} bytes.`
+            : undefined;
         let title;
         let extractor = "raw";
         let text = body;
-        if (contentType.includes("text/html")) {
+        if (contentType.includes("text/markdown")) {
+            // Cloudflare Markdown for Agents: server returned pre-rendered markdown
+            extractor = "cf-markdown";
+            if (params.extractMode === "text") {
+                text = markdownToText(body);
+            }
+        }
+        else if (contentType.includes("text/html")) {
             if (params.readabilityEnabled) {
                 const readable = await extractReadableContent({
                     html: body,
@@ -370,43 +465,47 @@ async function runWebFetch(params) {
                 extractor = "raw";
             }
         }
-        const truncated = truncateText(text, params.maxChars);
+        const wrapped = wrapWebFetchContent(text, params.maxChars);
+        const wrappedTitle = title ? wrapWebFetchField(title) : undefined;
+        const wrappedWarning = wrapWebFetchField(responseTruncatedWarning);
         const payload = {
-            url: params.url,
-            finalUrl,
+            url: params.url, // Keep raw for tool chaining
+            finalUrl, // Keep raw
             status: res.status,
-            contentType,
-            title,
+            contentType: normalizedContentType, // Protocol metadata, don't wrap
+            title: wrappedTitle,
             extractMode: params.extractMode,
             extractor,
-            truncated: truncated.truncated,
-            length: truncated.text.length,
+            externalContent: {
+                untrusted: true,
+                source: "web_fetch",
+                wrapped: true,
+            },
+            truncated: wrapped.truncated,
+            length: wrapped.wrappedLength,
+            rawLength: wrapped.rawLength, // Actual content length, not wrapped
+            wrappedLength: wrapped.wrappedLength,
             fetchedAt: new Date().toISOString(),
             tookMs: Date.now() - start,
-            text: truncated.text,
+            text: wrapped.text,
+            warning: wrappedWarning,
         };
         writeCache(FETCH_CACHE, cacheKey, payload, params.cacheTtlMs);
         return payload;
     }
     finally {
-        await closeDispatcher(dispatcher);
+        if (release) {
+            await release();
+        }
     }
 }
 async function tryFirecrawlFallback(params) {
-    if (!params.firecrawlEnabled || !params.firecrawlApiKey)
+    const firecrawlParams = toFirecrawlContentParams(params);
+    if (!firecrawlParams) {
         return null;
+    }
     try {
-        const firecrawl = await fetchFirecrawlContent({
-            url: params.url,
-            extractMode: params.extractMode,
-            apiKey: params.firecrawlApiKey,
-            baseUrl: params.firecrawlBaseUrl,
-            onlyMainContent: params.firecrawlOnlyMainContent,
-            maxAgeMs: params.firecrawlMaxAgeMs,
-            proxy: params.firecrawlProxy,
-            storeInCache: params.firecrawlStoreInCache,
-            timeoutSeconds: params.firecrawlTimeoutSeconds,
-        });
+        const firecrawl = await fetchFirecrawlContent(firecrawlParams);
         return { text: firecrawl.text, title: firecrawl.title };
     }
     catch {
@@ -415,8 +514,9 @@ async function tryFirecrawlFallback(params) {
 }
 function resolveFirecrawlEndpoint(baseUrl) {
     const trimmed = baseUrl.trim();
-    if (!trimmed)
+    if (!trimmed) {
         return `${DEFAULT_FIRECRAWL_BASE_URL}/v2/scrape`;
+    }
     try {
         const url = new URL(trimmed);
         if (url.pathname && url.pathname !== "/") {
@@ -431,8 +531,9 @@ function resolveFirecrawlEndpoint(baseUrl) {
 }
 export function createWebFetchTool(options) {
     const fetch = resolveFetchConfig(options?.config);
-    if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed }))
+    if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed })) {
         return null;
+    }
     const readabilityEnabled = resolveFetchReadabilityEnabled(fetch);
     const firecrawl = resolveFirecrawlConfig(fetch);
     const firecrawlApiKey = resolveFirecrawlApiKey(firecrawl);
@@ -443,6 +544,7 @@ export function createWebFetchTool(options) {
     const firecrawlTimeoutSeconds = resolveTimeoutSeconds(firecrawl?.timeoutSeconds ?? fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS);
     const userAgent = (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) ||
         DEFAULT_FETCH_USER_AGENT;
+    const maxResponseBytes = resolveFetchMaxResponseBytes(fetch);
     return {
         label: "Web Fetch",
         name: "web_fetch",
@@ -453,10 +555,12 @@ export function createWebFetchTool(options) {
             const url = readStringParam(params, "url", { required: true });
             const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown";
             const maxChars = readNumberParam(params, "maxChars", { integer: true });
+            const maxCharsCap = resolveFetchMaxCharsCap(fetch);
             const result = await runWebFetch({
                 url,
                 extractMode,
-                maxChars: resolveMaxChars(maxChars ?? fetch?.maxChars, DEFAULT_FETCH_MAX_CHARS),
+                maxChars: resolveMaxChars(maxChars ?? fetch?.maxChars, DEFAULT_FETCH_MAX_CHARS, maxCharsCap),
+                maxResponseBytes,
                 maxRedirects: resolveMaxRedirects(fetch?.maxRedirects, DEFAULT_FETCH_MAX_REDIRECTS),
                 timeoutSeconds: resolveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS),
                 cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),