@mono-agent/agent-runtime 0.20.11 → 0.21.0

package/src/agent/tools/web-fetch.js

@@ -1,15 +1,14 @@
+import { withWebDeadline, coordinatedWebRequest, webRequestFailure } from "./web-request.js";
 // @ts-check
 
-import { Readability } from "@mozilla/readability";
-import { Defuddle as parseDefuddle } from "defuddle/node";
-import { DOMParser, parseHTML } from "linkedom";
-import { extractText as extractPdfText, getDocumentProxy } from "unpdf";
 import { passthroughSandbox } from "../sandbox-seam.js";
 import { DEFAULT_MAX_TOOL_OUTPUT_CHARS } from "./shared/constants.js";
 import { capChars } from "./shared/output-truncation.js";
 import { readToolRuntime } from "./shared/runtime-context.js";
 import { resolveSandboxPolicy } from "./shared/tool-context.js";
 import { renderWithAgentBrowser } from "./web-browser-render.js";
+import { contentKind, decodeWebBytes, extractWebDocument, markdownToText, shouldAutoRender } from "./web-document-extractor.js";
+import { assertNoWebAccessInterstitial } from "./web-access-interstitial.js";
 
 const FETCH_TIMEOUT_MS = 15_000;
 const MAX_REDIRECTS = 5;
@@ -28,22 +27,23 @@ class WebFetchError extends Error {
   /**
    * @param {string} code
    * @param {string} message
-   * @param {{retryable?: boolean, statusCode?: number}} [options]
+   * @param {{retryable?: boolean, statusCode?: number, retryAfterMs?: number}} [options]
    */
-  constructor(code, message, { retryable = false, statusCode } = {}) {
+  constructor(code, message, { retryable = false, statusCode, retryAfterMs } = {}) {
     super(message);
     this.name = "WebFetchError";
     this.code = code;
     this.retryable = retryable;
     this.statusCode = statusCode;
+    this.retryAfterMs = retryAfterMs;
   }
 }
 
 /**
  * Compatibility wrapper for direct callers.
  *
- * @param {{url: string, headers?: Record<string, string>, max_output_chars?: number, format?: string, render?: string}} params
- * @param {{sandboxPolicy?: any, sandboxEngine?: any, ctx?: any, signal?: AbortSignal, retryDelaysMs?: number[], fetchConfig?: any, fetchImpl?: typeof fetch, browserRenderer?: typeof renderWithAgentBrowser, namespace?: string, registerCleanup?: (cleanup: () => Promise<void>) => () => void}} [options]
+ * @param {{url: string, headers?: Record<string, string>, max_output_chars?: number, format?: string, render?: string, start_line?: number, max_lines?: number}} params
+ * @param {{documentOnly?: boolean, coordinator?: any, sandboxPolicy?: any, sandboxEngine?: any, ctx?: any, signal?: AbortSignal, retryDelaysMs?: number[], fetchConfig?: any, fetchImpl?: typeof fetch, browserRenderer?: typeof renderWithAgentBrowser, namespace?: string, registerCleanup?: (cleanup: () => Promise<void>) => () => void}} [options]
  */
 export async function webFetchToolImpl(params, options = {}) {
   return (await performWebFetch(params, options)).text;
@@ -52,18 +52,41 @@ export async function webFetchToolImpl(params, options = {}) {
 /**
  * Fetch and locally extract one public URL.
  *
- * @param {{url: string, headers?: Record<string, string>, max_output_chars?: number, format?: string, render?: string}} params
- * @param {{sandboxPolicy?: any, sandboxEngine?: any, ctx?: any, signal?: AbortSignal, retryDelaysMs?: number[], fetchConfig?: any, fetchImpl?: typeof fetch, browserRenderer?: typeof renderWithAgentBrowser, namespace?: string, registerCleanup?: (cleanup: () => Promise<void>) => () => void}} [options]
+ * @param {{url: string, headers?: Record<string, string>, max_output_chars?: number, format?: string, render?: string, start_line?: number, max_lines?: number}} params
+ * @param {{documentOnly?: boolean, coordinator?: any, sandboxPolicy?: any, sandboxEngine?: any, ctx?: any, signal?: AbortSignal, retryDelaysMs?: number[], fetchConfig?: any, fetchImpl?: typeof fetch, browserRenderer?: typeof renderWithAgentBrowser, namespace?: string, registerCleanup?: (cleanup: () => Promise<void>) => () => void}} [options]
  */
-export async function performWebFetch(
+export async function performWebFetch(params, options = {}) {
+  const started = Date.now();
+  try {
+    return await withWebDeadline(options.signal, 45_000, async (signal) => {
+      const result = await performFetch(params, { ...options, signal });
+      if (signal.aborted && !result.error) return failure("Error: WebFetch was aborted or exceeded its deadline.", signal.reason?.code === "deadline_exceeded" ? "deadline_exceeded" : "aborted", started);
+      return result;
+    });
+  } catch (error) {
+    const normalized = webRequestFailure(error, "http", options.signal);
+    return failure(`Error: ${normalized.message}`, normalized.code, started, { retryAfterMs: normalized.retryAfterMs });
+  }
+}
+
+/**
+ * Fetch and locally extract one public URL.
+ *
+ * @param {{url: string, headers?: Record<string, string>, max_output_chars?: number, format?: string, render?: string, start_line?: number, max_lines?: number}} params
+ * @param {{documentOnly?: boolean, coordinator?: any, sandboxPolicy?: any, sandboxEngine?: any, ctx?: any, signal?: AbortSignal, retryDelaysMs?: number[], fetchConfig?: any, fetchImpl?: typeof fetch, browserRenderer?: typeof renderWithAgentBrowser, namespace?: string, registerCleanup?: (cleanup: () => Promise<void>) => () => void}} [options]
+ */
+async function performFetch(
   {
     url,
     headers = {},
     max_output_chars,
     format = "markdown",
     render,
+    start_line, max_lines,
   },
   {
+    coordinator,
+    documentOnly = false,
     sandboxPolicy,
     sandboxEngine,
     ctx,
@@ -77,6 +100,10 @@ export async function performWebFetch(
   } = {},
 ) {
   const startedAt = Date.now();
+  if ((start_line !== undefined && (!Number.isSafeInteger(start_line) || start_line < 1))
+    || (max_lines !== undefined && (!Number.isSafeInteger(max_lines) || max_lines < 1 || max_lines > 10_000))) {
+    return failure("Error: start_line must be positive; max_lines must be between 1 and 10000.", "invalid_range", startedAt);
+  }
   let parsed;
   try { parsed = new URL(url); } catch {
     return failure("Error: Invalid URL", "invalid_url", startedAt);
@@ -115,6 +142,52 @@ export async function performWebFetch(
   const resolvedCtx = ctx ?? readToolRuntime();
   const sandbox = resolvedCtx.sandbox ?? passthroughSandbox;
   const policy = resolveSandboxPolicy(resolvedCtx, sandboxPolicy);
+  if (requestedRender === "always") {
+    if (!sandbox.networkAllowsUrl(policy, parsed.href)) {
+      return failure("Error: Network access denied by sandbox policy.", "network_denied", startedAt, {
+        backend: "agent-browser", browserRecommended: true, renderReason: "explicit",
+      });
+    }
+    try {
+      const renderedResult = await coordinatedWebRequest(coordinator, "fetch", parsed.origin, signal, async () => ({
+        ok: true,
+        rendered: await browserRenderer(parsed.href, {
+          browserCommand: fetchSettings.browserCommand,
+          namespace,
+          sandboxPolicy: policy,
+          sandboxEngine,
+          ctx: resolvedCtx,
+          signal,
+          registerCleanup,
+        }),
+      }));
+      const rendered = normalizeBrowserResult(renderedResult.rendered, parsed.href);
+      assertNoWebAccessInterstitial({ url: rendered.finalUrl, text: rendered.text });
+      const renderedBody = outputFormat === "text" ? markdownToText(rendered.text) : rendered.text;
+      const document = {
+        body: renderedBody,
+        finalUrl: rendered.finalUrl,
+        outcome: {
+          status: "ok", code: "ok", retryable: false, attempts: 1,
+          backend: "agent-browser", cacheHit: false, durationMs: Date.now() - startedAt,
+          bytes: Buffer.byteLength(rendered.text, "utf8"), queueWaitMs: renderedResult.coordinationWaitMs,
+          backendDurationMs: renderedResult.backendDurationMs, truncated: renderedBody.length > maxChars,
+          redirectCount: 0, rendered: true, renderFailed: false, browserRecommended: false,
+          renderReason: "explicit", contentKind: "html", extractionStage: "browser", parserFailureCount: 0,
+          parserFailures: [],
+        },
+      };
+      return documentOnly ? { text: "", error: false, outcome: document.outcome, document }
+        : formatWebFetchDocument(document, { start_line, max_lines, max_output_chars: maxChars }, resolvedCtx);
+    } catch (error) {
+      const code = ["access_challenge", "authentication_required", "network_denied"].includes(error?.code)
+        ? error.code : "browser_render_failed";
+      return failure(`Error rendering URL: ${error?.message || String(error)}`, code, startedAt, {
+        attempts: 1, backend: "agent-browser", rendered: false, renderFailed: true,
+        browserRecommended: code === "browser_render_failed", renderReason: "explicit",
+      });
+    }
+  }
   const delays = Array.isArray(retryDelaysMs)
     ? retryDelaysMs.slice(0, 2).map((value) => Math.max(0, Number(value) || 0))
     : [];
@@ -123,18 +196,23 @@ export async function performWebFetch(
   let finalUrl = parsed.href;
   let redirectCount = 0;
   let responseBytes = 0;
+  let queueWaitMs = 0;
+  let backendDurationMs = 0;
   let bytes;
 
   for (let attempt = 0; attempt <= delays.length; attempt += 1) {
     attempts += 1;
     try {
       const fetched = await fetchFollowingRedirects(parsed, {
+        coordinator,
         headers: requestHeaders.headers,
         sandbox,
         policy,
         signal,
         fetchImpl,
       });
+      queueWaitMs += fetched.queueWaitMs;
+      backendDurationMs += fetched.backendDurationMs;
       response = fetched.response;
       finalUrl = fetched.url;
       redirectCount = fetched.redirects;
@@ -144,12 +222,12 @@ export async function performWebFetch(
         await waitForRetry(delay, signal);
         continue;
       }
-      bytes = await readResponseBytes(response);
+      bytes = fetched.bytes;
       responseBytes = bytes.byteLength;
       break;
     } catch (error) {
       if (signal?.aborted) {
-        return failure("Error fetching URL: request aborted", "aborted", startedAt, {
+        return failure("Error fetching URL: request aborted", signal.reason?.code === "deadline_exceeded" ? "deadline_exceeded" : "aborted", startedAt, {
           attempts,
           retryable: false,
         });
@@ -176,6 +254,9 @@ export async function performWebFetch(
         attempts,
         retryable: normalized.retryable,
         statusCode: normalized.statusCode ?? response?.status,
+        retryAfterMs: normalized.retryAfterMs, queueWaitMs, backendDurationMs,
+        browserRecommended: requestedRender === "auto"
+          && !["network_denied", "redirect_network_denied", "aborted", "deadline_exceeded"].includes(normalized.code),
       });
     }
   }
@@ -186,10 +267,46 @@ export async function performWebFetch(
 
   const contentType = response.headers.get("content-type") || "";
   const responseKind = contentKind(contentType, bytes);
+  let decodedForExtraction;
+  if (responseKind !== "binary") {
+    try {
+      decodedForExtraction = decodeWebBytes(bytes, contentType, responseKind);
+      assertNoWebAccessInterstitial({
+        url: finalUrl,
+        text: decodedForExtraction.text,
+        statusCode: response.status,
+      });
+    } catch (error) {
+      if (["access_challenge", "authentication_required"].includes(error?.code)) {
+        return failure(`Error fetching URL: ${error.message}`, error.code, startedAt, {
+          attempts,
+          statusCode: response.status,
+          bytes: responseBytes,
+          backend: "http",
+          redirectCount,
+        });
+      }
+      // Unsupported encodings are still handled by the extraction path below,
+      // where their decoding metadata and browser recommendation are retained.
+      decodedForExtraction = undefined;
+    }
+  } else if ([401, 407].includes(response.status)) {
+    try {
+      assertNoWebAccessInterstitial({ url: finalUrl, statusCode: response.status });
+    } catch (error) {
+      return failure(`Error fetching URL: ${error.message}`, error.code, startedAt, {
+        attempts,
+        statusCode: response.status,
+        bytes: responseBytes,
+        backend: "http",
+        redirectCount,
+      });
+    }
+  }
   if (!response.ok) {
     const preview = responseKind === "binary"
       ? "(binary response body omitted)"
-      : decodeBytes(bytes, contentType).slice(0, 500);
+      : safeDecodePreview(bytes, contentType, responseKind);
     const errorText = [
       `HTTP ${response.status}`,
       `[BEGIN UNTRUSTED WEB ERROR BODY source=${JSON.stringify(finalUrl)}]`,
@@ -203,6 +320,7 @@ export async function performWebFetch(
       bytes: responseBytes,
       backend: "http",
       redirectCount,
+      browserRecommended: requestedRender === "auto" && [406, 415].includes(response.status),
     });
   }
   if (responseKind === "binary") {
@@ -216,14 +334,40 @@ export async function performWebFetch(
   }
 
   let extracted;
+  let decoding;
   try {
-    extracted = await extractResponse(bytes, {
+    decoding = decodedForExtraction ?? decodeWebBytes(bytes, contentType, responseKind);
+    extracted = await extractWebDocument(bytes, {
       contentType,
       format: outputFormat,
       url: finalUrl,
     });
   } catch (error) {
-    return failure(`Error extracting URL: ${error?.message || String(error)}`, "extraction_failed", startedAt, {
+    return failure(`Error extracting URL: ${error?.message || String(error)}`, error?.code || "extraction_failed", startedAt, {
+      attempts,
+      statusCode: response.status,
+      bytes: responseBytes,
+      backend: "http",
+      redirectCount,
+      browserRecommended: requestedRender === "auto" && responseKind === "html",
+      contentKind: responseKind,
+      ...(decoding === undefined ? {} : {
+        charset: decoding.charset,
+        charsetSource: decoding.charsetSource,
+        hadDecodingReplacement: decoding.hadDecodingReplacement,
+      }),
+      ...(Array.isArray(error?.parserFailures) ? { parserFailures: error.parserFailures.slice(0, 3) } : {}),
+    });
+  }
+
+  try {
+    assertNoWebAccessInterstitial({
+      url: finalUrl,
+      text: extracted.readableText,
+      statusCode: response.status,
+    });
+  } catch (error) {
+    return failure(`Error fetching URL: ${error.message}`, error.code, startedAt, {
       attempts,
       statusCode: response.status,
       bytes: responseBytes,
@@ -235,13 +379,13 @@ export async function performWebFetch(
   const shouldRender = responseKind === "html"
     && (
       requestedRender === "always"
-      || (requestedRender === "auto" && shouldAutoRender(extracted.readableText, decodeBytes(bytes, contentType)))
+      || (requestedRender === "auto" && shouldAutoRender(extracted.readableText, decodedText(bytes, contentType, responseKind)))
     );
   let backend = "http";
   let renderFailed = false;
   if (shouldRender) {
     try {
-      const rendered = await browserRenderer(finalUrl, {
+      const renderedResult = await coordinatedWebRequest(coordinator, "fetch", new URL(finalUrl).origin, signal, async () => ({ ok: true, rendered: await browserRenderer(finalUrl, {
         browserCommand: fetchSettings.browserCommand,
         namespace,
         sandboxPolicy,
@@ -249,16 +393,32 @@ export async function performWebFetch(
         ctx: resolvedCtx,
         signal,
         registerCleanup,
-      });
+      }) }));
+      queueWaitMs += renderedResult.coordinationWaitMs;
+      backendDurationMs += renderedResult.backendDurationMs;
+      const rendered = normalizeBrowserResult(renderedResult.rendered, finalUrl);
+      assertNoWebAccessInterstitial({ url: rendered.finalUrl, text: rendered.text });
+      signal?.throwIfAborted();
       extracted = {
-        body: outputFormat === "text" ? markdownToText(rendered) : rendered,
-        readableText: markdownToText(rendered),
+        body: outputFormat === "text" ? markdownToText(rendered.text) : rendered.text,
+        readableText: markdownToText(rendered.text),
         title: extracted.title,
+        charset: extracted.charset,
+        charsetSource: extracted.charsetSource,
+        hadDecodingReplacement: extracted.hadDecodingReplacement,
+        extractionStage: "browser",
+        parserFailureCount: extracted.parserFailureCount,
+        parserFailures: extracted.parserFailures,
       };
+      finalUrl = rendered.finalUrl;
       backend = "agent-browser";
     } catch (error) {
-      if (requestedRender === "always") {
-        return failure(`Error rendering URL: ${error?.message || String(error)}`, "browser_render_failed", startedAt, {
+      if (signal?.aborted) return failure("Error: WebFetch rendering was aborted.", "aborted", startedAt);
+      const terminalCode = ["access_challenge", "authentication_required", "network_denied"].includes(error?.code)
+        ? error.code : null;
+      if (terminalCode || requestedRender === "always" || error?.code === "coordination_unavailable") {
+        const code = terminalCode ?? "browser_render_failed";
+        return failure(`Error rendering URL: ${error?.message || String(error)}`, code, startedAt, {
           attempts,
           statusCode: response.status,
           bytes: responseBytes,
@@ -270,15 +430,11 @@ export async function performWebFetch(
     }
   }
 
+  if (responseKind === "html" && backend === "http" && shouldAutoRender(extracted.readableText, decodedText(bytes, contentType, responseKind))) {
+    return failure("Error: Page contains an unusable loading shell; no readable evidence was retrieved.", "unusable_content", startedAt, { backend, rendered: false, renderFailed, browserRecommended: true });
+  }
   const body = extracted.body || "(no readable content)";
-  const capped = capChars(body, { label: "WebFetch", maxChars, ctx: resolvedCtx });
-  const text = [
-    `[BEGIN UNTRUSTED WEB CONTENT source=${JSON.stringify(finalUrl)}]`,
-    capped,
-    "[END UNTRUSTED WEB CONTENT]",
-  ].join("\n");
-  return {
-    text,
+  const document = { body, finalUrl,
     outcome: {
       status: "ok",
       code: renderFailed ? "ok_static_render_failed" : "ok",
@@ -288,19 +444,37 @@ export async function performWebFetch(
       cacheHit: false,
       durationMs: Date.now() - startedAt,
       bytes: responseBytes,
+      queueWaitMs, backendDurationMs,
       truncated: body.length > maxChars,
       statusCode: response.status,
       redirectCount,
       rendered: backend === "agent-browser",
       renderFailed,
+      browserRecommended: renderFailed,
+      ...(backend === "agent-browser" ? { renderReason: "sparse_html" } : {}),
       contentKind: responseKind,
+      charset: extracted.charset,
+      charsetSource: extracted.charsetSource,
+      hadDecodingReplacement: extracted.hadDecodingReplacement,
+      extractionStage: extracted.extractionStage,
+      parserFailureCount: extracted.parserFailureCount ?? 0,
+      parserFailures: extracted.parserFailures ?? [],
     },
-    error: false,
-  };
+    };
+  return documentOnly ? { text: "", error: false, outcome: document.outcome, document }
+    : formatWebFetchDocument(document, { start_line, max_lines, max_output_chars: maxChars }, resolvedCtx);
+}
+
+function normalizeBrowserResult(value, requestedUrl) {
+  if (typeof value === "string") return { text: value, finalUrl: requestedUrl };
+  if (value && typeof value.text === "string" && typeof value.finalUrl === "string") return value;
+  throw Object.assign(new Error("Browser renderer returned an invalid result."), { code: "browser_render_failed" });
 }
 
 async function fetchFollowingRedirects(initialUrl, options) {
   let current = new URL(initialUrl.href);
+  let queueWaitMs = 0;
+  let backendDurationMs = 0;
   for (let hop = 0; hop <= MAX_REDIRECTS; hop += 1) {
     if (!options.sandbox.networkAllowsUrl(options.policy, current.href)) {
       throw new WebFetchError(
@@ -310,14 +484,25 @@ async function fetchFollowingRedirects(initialUrl, options) {
           : "Network access denied by sandbox policy (redirect).",
       );
     }
-    const response = await options.fetchImpl(current, {
-      headers: options.headers,
-      redirect: "manual",
-      signal: requestSignal(options.signal),
-    });
+    const fetched = await coordinatedWebRequest(options.coordinator, "fetch", current.origin, options.signal, async () => {
+      const response = await options.fetchImpl(current, {
+        headers: options.headers, redirect: "manual", signal: requestSignal(options.signal),
+      });
+      const redirect = response.status >= 300 && response.status < 400 && response.headers.has("location");
+      let bytes;
+      if (redirect) { await response.body?.cancel(); bytes = new Uint8Array(); }
+      else bytes = await readResponseBytes(response);
+      return { response, bytes };
+    }, ({ response }) => ({
+      status: response.status === 429 ? "rate_limited" : response.status >= 500 ? "unavailable" : "ok",
+      ...(response.status === 429 ? { retryAfterMs: retryAfterMilliseconds(response) } : {}),
+    }));
+    queueWaitMs += fetched.coordinationWaitMs;
+    backendDurationMs += fetched.backendDurationMs;
+    const { response, bytes } = fetched;
     const location = response.headers.get("location");
     if (response.status < 300 || response.status >= 400 || !location) {
-      return { response, url: current.href, redirects: hop };
+      return { response, bytes, url: current.href, redirects: hop, queueWaitMs, backendDurationMs };
     }
     if (hop === MAX_REDIRECTS) {
       try { await response.body?.cancel(); } catch { /* best effort */ }
@@ -368,166 +553,13 @@ async function readResponseBytes(response) {
   return new Uint8Array(Buffer.concat(chunks));
 }
 
-async function extractResponse(bytes, { contentType, format, url }) {
-  const kind = contentKind(contentType, bytes);
-  const raw = decodeBytes(bytes, contentType);
-  if (format === "raw") return { body: raw, readableText: raw, title: "" };
-  if (kind === "pdf") {
-    const pdf = await getDocumentProxy(bytes);
-    try {
-      const extracted = await extractPdfText(pdf, { mergePages: true });
-      const body = String(extracted.text || "").trim();
-      return {
-        body,
-        readableText: body,
-        title: "",
-      };
-    } finally {
-      try { await /** @type {any} */ (pdf).destroy?.(); } catch { /* best effort */ }
-    }
-  }
-  if (kind === "json") {
-    let parsed;
-    try { parsed = JSON.parse(raw); } catch {
-      return { body: raw, readableText: raw, title: "" };
-    }
-    const pretty = JSON.stringify(parsed, null, 2);
-    return {
-      body: format === "markdown" ? `\`\`\`json\n${pretty}\n\`\`\`` : pretty,
-      readableText: pretty,
-      title: "",
-    };
-  }
-  if (kind === "xml") {
-    const body = extractXml(raw, format);
-    return { body, readableText: markdownToText(body), title: "" };
-  }
-  if (kind === "html") {
-    return extractHtml(raw, url, format);
-  }
-  return { body: raw.trim(), readableText: raw.trim(), title: "" };
-}
-
-async function extractHtml(html, url, format) {
-  let title = "";
-  let markdown = "";
-  try {
-    const { document } = parseHTML(html);
-    const parsed = await parseDefuddle(/** @type {any} */ (document), url, {
-      markdown: true,
-      separateMarkdown: true,
-      useAsync: false,
-    });
-    title = String(parsed.title || "").trim();
-    markdown = String(parsed.contentMarkdown || parsed.content || "").trim();
-  } catch { /* Readability fallback below */ }
-
-  if (meaningfulCharacters(markdown) < 1) {
-    try {
-      const { document } = parseHTML(html);
-      const article = new Readability(/** @type {any} */ (document)).parse();
-      if (article) {
-        title ||= String(article.title || "").trim();
-        markdown = htmlToText(article.content || article.textContent || "");
-      }
-    } catch { /* final body-text fallback below */ }
-  }
-
-  if (meaningfulCharacters(markdown) < 1) {
-    const { document } = parseHTML(html);
-    title ||= collapseWhitespace(document.querySelector("title")?.textContent);
-    markdown = collapseDocumentText(document.body?.textContent || "");
-  }
-  const readableText = markdownToText(markdown);
-  if (format === "text") return { body: readableText, readableText, title };
-  const body = title && !markdown.trimStart().startsWith(`# ${title}`)
-    ? `# ${title}\n\n${markdown}`
-    : markdown;
-  return { body, readableText, title };
-}
-
-function extractXml(xml, format) {
-  const document = new DOMParser().parseFromString(String(xml || ""), "text/xml");
-  const entries = [...document.querySelectorAll("item, entry")].slice(0, 50);
-  if (entries.length === 0) {
-    return collapseDocumentText(document.documentElement?.textContent || xml);
-  }
-  const blocks = entries.map((entry) => {
-    const title = collapseWhitespace(entry.querySelector("title")?.textContent) || "Untitled";
-    const linkElement = entry.querySelector("link");
-    const link = linkElement?.getAttribute("href") || collapseWhitespace(linkElement?.textContent);
-    const description = collapseWhitespace(
-      entry.querySelector("description, summary, content")?.textContent,
-    );
-    if (format === "text") return [title, link, description].filter(Boolean).join("\n");
-    return [
-      `## ${title}`,
-      link ? `[${link}](${link})` : "",
-      description,
-    ].filter(Boolean).join("\n\n");
-  });
-  return blocks.join("\n\n");
-}
-
-function contentKind(contentType, bytes) {
-  const mime = String(contentType || "").split(";", 1)[0].trim().toLowerCase();
-  if (mime === "application/pdf" || startsWithPdf(bytes)) return "pdf";
-  if (mime.includes("json") || mime.endsWith("+json")) return "json";
-  if (
-    mime.includes("xml")
-    || mime.includes("rss")
-    || mime.includes("atom")
-    || mime.endsWith("+xml")
-  ) return "xml";
-  if (mime.includes("html") || looksLikeHtml(bytes)) return "html";
-  if (
-    mime.startsWith("text/")
-    || [
-      "application/ecmascript",
-      "application/graphql",
-      "application/javascript",
-      "application/rtf",
-      "application/sql",
-      "application/x-httpd-php",
-      "application/x-yaml",
-      "application/yaml",
-    ].includes(mime)
-  ) return "text";
-  if (!mime && !looksBinary(bytes)) return "text";
-  return "binary";
-}
-
-function startsWithPdf(bytes) {
-  return Buffer.from(bytes.subarray(0, 5)).toString("ascii") === "%PDF-";
-}
-
-function looksLikeHtml(bytes) {
-  return /^\s*(?:<!doctype html|<html|<head|<body)/iu.test(
-    Buffer.from(bytes.subarray(0, 512)).toString("utf8"),
-  );
-}
-
-function looksBinary(bytes) {
-  const sample = bytes.subarray(0, Math.min(bytes.byteLength, 1_024));
-  if (sample.byteLength === 0) return false;
-  let controls = 0;
-  for (const byte of sample) {
-    if (byte === 0) return true;
-    if (byte < 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0c && byte !== 0x0d) {
-      controls += 1;
-    }
-  }
-  return controls / sample.byteLength > 0.1;
+function decodedText(bytes, contentType, kind) {
+  return decodeWebBytes(bytes, contentType, kind).text;
 }
 
-function decodeBytes(bytes, contentType) {
-  const match = String(contentType || "").match(/charset\s*=\s*["']?([^;"'\s]+)/iu);
-  const charset = match?.[1] || "utf-8";
-  try {
-    return new TextDecoder(charset, { fatal: false }).decode(bytes);
-  } catch {
-    return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
-  }
+function safeDecodePreview(bytes, contentType, kind) {
+  try { return decodedText(bytes, contentType, kind).slice(0, 500); }
+  catch { return new TextDecoder("utf-8", { fatal: false }).decode(bytes).slice(0, 500); }
 }
 
 function normalizeRequestHeaders(headers) {
@@ -568,51 +600,6 @@ function normalizeFetchConfig(input) {
   return { render, browserCommand: browserCommand.trim() };
 }
 
-function shouldAutoRender(readableText, html) {
-  if (meaningfulCharacters(readableText) >= 200) return false;
-  const scriptCount = (html.match(/<script\b/giu) || []).length;
-  const hasAppRoot = /<(?:div|main)[^>]+(?:id|class)=["'][^"']*(?:app|root|__next|nuxt|svelte)[^"']*["']/iu.test(html);
-  const hasSpaAssets = /\b(?:webpack|__NEXT_DATA__|vite|hydration|data-reactroot)\b/iu.test(html);
-  return scriptCount >= 2 && (hasAppRoot || hasSpaAssets);
-}
-
-function meaningfulCharacters(value) {
-  return markdownToText(value).replace(/\s/gu, "").length;
-}
-
-function htmlToText(value) {
-  try {
-    const { document } = parseHTML(String(value || ""));
-    return collapseDocumentText(document.body?.textContent || document.documentElement?.textContent || "");
-  } catch {
-    return collapseDocumentText(String(value || "").replace(/<[^>]+>/gu, " "));
-  }
-}
-
-function markdownToText(value) {
-  return collapseDocumentText(
-    String(value || "")
-      .replace(/```[\s\S]*?```/gu, (block) => block.replace(/^```[^\n]*\n?|```$/gu, ""))
-      .replace(/!\[([^\]]*)\]\([^)]*\)/gu, "$1")
-      .replace(/\[([^\]]+)\]\([^)]*\)/gu, "$1")
-      .replace(/^[#>*+-]+\s*/gmu, "")
-      .replace(/[*_`~]/gu, ""),
-  );
-}
-
-function collapseDocumentText(value) {
-  return String(value || "")
-    .replace(/\r/gu, "")
-    .replace(/[ \t]+\n/gu, "\n")
-    .replace(/\n{3,}/gu, "\n\n")
-    .replace(/[ \t]{2,}/gu, " ")
-    .trim();
-}
-
-function collapseWhitespace(value) {
-  return String(value || "").replace(/\s+/gu, " ").trim();
-}
-
 function requestSignal(signal) {
   const timeout = AbortSignal.timeout(FETCH_TIMEOUT_MS);
   return signal ? AbortSignal.any([signal, timeout]) : timeout;
@@ -662,7 +649,7 @@ function normalizeFetchError(error) {
   return new WebFetchError(
     timedOut ? "timeout" : (aborted ? "aborted" : (code || "request_failed")),
     error?.message || String(error),
-    { retryable: transient },
+    { retryable: transient, retryAfterMs: error?.retryAfterMs },
   );
 }
 
@@ -689,3 +676,44 @@ function positiveInteger(value, fallback) {
   const number = Number(value);
   return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback;
 }
+
+function retryAfterMilliseconds(response) {
+  const raw = response.headers.get("retry-after");
+  if (!raw) return undefined;
+  const seconds = Number(raw);
+  const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(raw) - Date.now();
+  return Number.isFinite(delay) ? Math.max(0, delay) : undefined;
+}
+
+export function formatWebFetchDocument(document, params, ctx) {
+  if ((params.start_line !== undefined && (!Number.isSafeInteger(params.start_line) || params.start_line < 1))
+    || (params.max_lines !== undefined && (!Number.isSafeInteger(params.max_lines) || params.max_lines < 1 || params.max_lines > 10000))) {
+    return failure("Error: Invalid WebFetch line range.", "invalid_range", Date.now());
+  }
+  const { body, finalUrl } = document;
+  const ranged = params.start_line !== undefined || params.max_lines !== undefined;
+  const lines = body.split("\n");
+  const start = params.start_line ?? 1;
+  const count = params.max_lines ?? 200;
+  const selected = ranged ? lines.slice(start - 1, start - 1 + count).join("\n") : body;
+  const maxChars = positiveInteger(params.max_output_chars, DEFAULT_MAX_TOOL_OUTPUT_CHARS);
+  const capped = capChars(selected, { label: "WebFetch", maxChars, ctx });
+  const shownLines = capped === selected ? (selected ? selected.split("\n").length : 0)
+    : Math.max(0, capped.slice(0, capped.lastIndexOf("[truncated WebFetch output:")).split("\n").length - 1);
+  const end = Math.min(lines.length, start - 1 + shownLines);
+  const continuation = end < lines.length ? Math.max(start, end + 1) : null;
+  const continuationHint = continuation === start && capped !== selected
+    ? `The next line exceeds the output budget. Increase max_output_chars or read the saved output artifact; repeating this range with the same budget cannot advance.`
+    : `Continue with WebFetch url=${JSON.stringify(finalUrl)} start_line=${continuation} max_lines=${count}.`;
+  return {
+    text: [`[BEGIN UNTRUSTED WEB CONTENT source=${JSON.stringify(finalUrl)}]`,
+      ...(ranged || continuation ? [`[Lines ${start}-${end} of ${lines.length}.]`] : []),
+      capped,
+      ...(continuation ? [`[${continuationHint}]`] : []),
+      "[END UNTRUSTED WEB CONTENT]"].join("\n"),
+    outcome: { ...document.outcome, truncated: selected.length > maxChars || end < lines.length,
+      startLine: start, endLine: end, totalLines: lines.length, nextLine: continuation },
+    error: false,
+    document,
+  };
+}