chainlesschain 0.166.32 → 0.166.36

package/src/lib/web-fetch.js

@@ -10,8 +10,11 @@ import https from "node:https";
 import { lookup as dnsLookup } from "node:dns";
 import { URL } from "node:url";
 import { createHash } from "node:crypto";
+import { StringDecoder } from "node:string_decoder";
+import { webFetchSnapshots } from "./web-fetch-snapshots.js";
 
-const DEFAULT_MAX_BYTES = 2_000_000;
+const DEFAULT_MAX_BYTES = 10_000_000;
+const DEFAULT_MAX_CHARS = 20_000;
 const DEFAULT_TIMEOUT_MS = 15_000;
 
 // RFC 1918 + loopback + link-local (IPv4 and IPv6). Blocked by default unless
@@ -99,13 +102,23 @@ export function checkAllowed(urlStr, config = {}) {
   return { allowed: true, url: parsed };
 }
 
-export function htmlToMarkdown(html) {
-  if (!html) return "";
+function stripHtmlNoise(html) {
   let text = String(html);
   // Strip scripts/styles entirely
   text = text.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "");
   text = text.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "");
   text = text.replace(/<!--[\s\S]*?-->/g, "");
+  // A bounded download may end inside a script, stylesheet, comment or tag.
+  // Closed blocks were removed above; discard any unfinished non-content tail.
+  return text
+    .replace(/<(?:script|style)\b[^>]*>[\s\S]*$/gi, "")
+    .replace(/<!--[\s\S]*$/g, "")
+    .replace(/<[^>]*$/g, "");
+}
+
+export function htmlToMarkdown(html) {
+  if (!html) return "";
+  let text = stripHtmlNoise(html);
   // Convert headings
   text = text.replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, n, inner) => {
     return `\n\n${"#".repeat(Number(n))} ${inner.trim()}\n\n`;
@@ -141,7 +154,10 @@ function stripTags(html) {
   return String(html).replace(/<[^>]+>/g, "");
 }
 
-function webFetchCacheKey(parsed, { format, maxBytes, maxRedirects, headers }) {
+function webFetchCacheKey(
+  parsed,
+  { format, maxBytes, maxChars, offset, onOverflow, maxRedirects, headers },
+) {
   // A caller-provided header can carry credentials or change representation.
   // Keep those requests out of the shared host cache entirely.
   if (
@@ -155,6 +171,9 @@ function webFetchCacheKey(parsed, { format, maxBytes, maxRedirects, headers }) {
     url: parsed.toString(),
     format,
     maxBytes,
+    maxChars,
+    offset,
+    onOverflow,
     maxRedirects,
   });
   // Do not retain a raw URL/query as a long-lived Map key in the host cache.
@@ -211,12 +230,15 @@ export function makeSafeLookup(allowPrivateHosts, deps = _deps) {
 
 async function _doRequest(
   parsed,
-  { maxBytes, timeout, headers, allowPrivateHosts },
+  { maxBytes, timeout, headers, allowPrivateHosts, truncateOnOverflow },
 ) {
   const lib = parsed.protocol === "https:" ? https : http;
   return new Promise((resolve, reject) => {
     let timer;
+    let settled = false;
     const settle = (error, result) => {
+      if (settled) return;
+      settled = true;
       clearTimeout(timer);
       if (error) reject(error);
       else resolve(result);
@@ -250,13 +272,38 @@ async function _doRequest(
         }
         const chunks = [];
         let size = 0;
+        const complete = (downloadTruncated = false) => {
+          const buffer = Buffer.concat(chunks);
+          settle(null, {
+            statusCode: res.statusCode,
+            headers: res.headers,
+            // With a partial response, discard an incomplete final UTF-8
+            // sequence instead of inventing a replacement character.
+            body: downloadTruncated
+              ? new StringDecoder("utf8").write(buffer)
+              : buffer.toString("utf8"),
+            bytes: buffer.length,
+            downloadTruncated,
+          });
+        };
         res.on("data", (chunk) => {
+          if (settled) return;
+          const available = maxBytes - size;
           size += chunk.length;
           if (size > maxBytes) {
+            if (truncateOnOverflow) {
+              if (available > 0) chunks.push(chunk.subarray(0, available));
+              complete(true);
+              res.destroy();
+              req.destroy();
+              return;
+            }
             const error = Object.assign(
               new Error(`response exceeds maxBytes (${maxBytes})`),
               {
                 code: "ERR_RESPONSE_TOO_LARGE",
+                maxBytes,
+                receivedBytes: size,
               },
             );
             settle(error);
@@ -267,11 +314,7 @@ async function _doRequest(
           chunks.push(chunk);
         });
         res.on("end", () => {
-          settle(null, {
-            statusCode: res.statusCode,
-            headers: res.headers,
-            body: Buffer.concat(chunks).toString("utf8"),
-          });
+          if (!settled) complete();
         });
         res.on("error", (error) => settle(error));
         res.on("aborted", () =>
@@ -303,6 +346,14 @@ export async function webFetch(url, options = {}) {
   const {
     format = "markdown",
     maxBytes = DEFAULT_MAX_BYTES,
+    maxChars = DEFAULT_MAX_CHARS,
+    offset = 0,
+    snapshotId = null,
+    query = null,
+    maxMatches = 20,
+    contextChars = 150,
+    caseSensitive = false,
+    onOverflow = "truncate",
     timeout = DEFAULT_TIMEOUT_MS,
     config = {},
     headers = {},
@@ -321,29 +372,115 @@ export async function webFetch(url, options = {}) {
   if (
     !Number.isSafeInteger(maxBytes) ||
     maxBytes <= 0 ||
+    !Number.isSafeInteger(maxChars) ||
+    maxChars <= 0 ||
+    !Number.isSafeInteger(offset) ||
+    offset < 0 ||
+    (snapshotId !== null && (typeof snapshotId !== "string" || !snapshotId)) ||
     !Number.isSafeInteger(timeout) ||
     timeout <= 0 ||
     timeout > 2_147_483_647 ||
     !Number.isSafeInteger(maxRedirects) ||
-    maxRedirects < 0
+    maxRedirects < 0 ||
+    !["truncate", "error"].includes(onOverflow)
   ) {
     return {
-      error: "invalid web_fetch size, timeout or redirect limit",
+      error: "invalid web_fetch size, output length, timeout or redirect limit",
       code: "ERR_FETCH_OPTIONS",
       retryable: false,
     };
   }
 
+  if (query !== null && (!snapshotId || typeof query !== "string" || !query)) {
+    return {
+      error:
+        "Download the URL first, then pass snapshotId and a non-empty query to search its saved text",
+      code: "ERR_FETCH_OPTIONS",
+      retryable: false,
+    };
+  }
+  if (format === "json" && (snapshotId || offset !== 0)) {
+    return {
+      error:
+        "JSON values cannot be paged. Use format=text to save and read a JSON document in chunks.",
+      code: "ERR_FETCH_OPTIONS",
+      retryable: false,
+    };
+  }
+  const snapshotIdentity = createHash("sha256")
+    .update(
+      JSON.stringify({
+        url: check.url.toString(),
+        format,
+        headers,
+        allowPrivateHosts: config.allowPrivateHosts === true,
+        allowedDomains: config.allowedDomains || ["*"],
+      }),
+    )
+    .digest("hex");
+  const readSnapshot = (id) =>
+    webFetchSnapshots.read(
+      id,
+      snapshotIdentity,
+      hostResourceBudget,
+      offset,
+      maxChars,
+    );
+  if (snapshotId) {
+    try {
+      const result = readSnapshot(snapshotId);
+      if (result.downloadTruncated && onOverflow === "error") {
+        return {
+          error:
+            "The saved download is incomplete. Fetch again without snapshotId and with a larger maxBytes.",
+          code: "ERR_RESPONSE_TOO_LARGE",
+          retryable: false,
+        };
+      }
+      if (query !== null && !result.error) {
+        const search = await webFetchSnapshots.search(
+          snapshotId,
+          snapshotIdentity,
+          hostResourceBudget,
+          {
+            pattern: query,
+            offset,
+            maxMatches,
+            contextChars,
+            caseSensitive,
+            url: check.url.toString(),
+          },
+        );
+        return search.error ? search : { ...search, cached: true };
+      }
+      return result.error ? result : { ...result, cached: true };
+    } catch {
+      return {
+        error:
+          "Unable to read the local webpage snapshot. Fetch the URL again without snapshotId.",
+        code: "ERR_FETCH_SNAPSHOT_READ",
+        retryable: false,
+      };
+    }
+  }
+
   const cacheKey = webFetchCacheKey(check.url, {
     format,
     maxBytes,
+    maxChars,
+    offset,
+    onOverflow,
     maxRedirects,
     headers,
   });
   if (cacheKey && hostResourceBudget?.getWebFetch) {
     try {
       const cached = hostResourceBudget.getWebFetch(cacheKey);
-      if (cached) return { ...cached, cached: true };
+      if (cached) {
+        if (!cached.snapshotId) return { ...cached, cached: true };
+        const page = readSnapshot(cached.snapshotId);
+        if (!page.error) return { ...page, cached: true };
+      }
     } catch (error) {
       return hostBudgetUnavailable("web_fetch", error);
     }
@@ -380,6 +517,8 @@ export async function webFetch(url, options = {}) {
         timeout: remaining,
         headers,
         allowPrivateHosts: config.allowPrivateHosts,
+        // An incomplete JSON document cannot be returned as a valid value.
+        truncateOnOverflow: onOverflow === "truncate" && format !== "json",
       });
       if (!response.redirect) break;
       if (++redirects > maxRedirects) {
@@ -436,7 +575,9 @@ export async function webFetch(url, options = {}) {
     if (format === "markdown") {
       output = /html/i.test(contentType) ? htmlToMarkdown(body) : body;
     } else if (format === "text") {
-      output = /html/i.test(contentType) ? stripTags(body) : body;
+      output = /html/i.test(contentType)
+        ? stripTags(stripHtmlNoise(body))
+        : body;
     } else if (format === "json") {
       try {
         output = JSON.parse(body);
@@ -451,15 +592,37 @@ export async function webFetch(url, options = {}) {
       }
     }
 
-    const result = {
+    // Download limits apply to the raw page, which can contain much more
+    // markup than the extracted text. Bound model context AFTER extraction;
+    // a short requested answer must not abort an otherwise readable page.
+    const totalChars =
+      format !== "json" && typeof output === "string" ? output.length : null;
+    const downloadTruncated = response.downloadTruncated === true;
+    let result = {
       url: parsed.toString(),
       statusCode,
       contentType,
       format: outputFormat,
-      bytes: Buffer.byteLength(body, "utf8"),
+      bytes: response.bytes,
       content: output,
+      ...(downloadTruncated ? { downloadTruncated: true, maxBytes } : {}),
+      ...(downloadTruncated
+        ? {
+            hint: `The raw response exceeded maxBytes=${maxBytes}. Only the downloaded prefix is included; totalChars measures that prefix after extraction, not the complete page. Increase maxBytes to fetch more, or request a focused page/API endpoint. Use maxChars to control returned text. Do not treat this as the complete page.`,
+          }
+        : {}),
     };
-    if (cacheKey && hostResourceBudget?.putWebFetch) {
+    if (totalChars !== null) {
+      // Save the extracted document once, then read only the requested range
+      // from disk. Subsequent snapshot reads never contact the origin server.
+      const id = webFetchSnapshots.save(
+        result,
+        snapshotIdentity,
+        hostResourceBudget,
+      );
+      result = readSnapshot(id);
+    }
+    if (!downloadTruncated && cacheKey && hostResourceBudget?.putWebFetch) {
       try {
         hostResourceBudget.putWebFetch(cacheKey, result);
       } catch {
@@ -472,6 +635,15 @@ export async function webFetch(url, options = {}) {
     return {
       error: `web_fetch failed: ${error.message}`,
       code,
+      ...(code === "ERR_RESPONSE_TOO_LARGE"
+        ? {
+            maxBytes,
+            receivedBytes: error.receivedBytes,
+            ...(maxBytes < DEFAULT_MAX_BYTES
+              ? { suggestedMaxBytes: DEFAULT_MAX_BYTES }
+              : {}),
+          }
+        : {}),
       retryable: [
         "ETIMEDOUT",
         "ECONNRESET",
@@ -480,7 +652,9 @@ export async function webFetch(url, options = {}) {
       ].includes(code),
       hint:
         code === "ERR_RESPONSE_TOO_LARGE"
-          ? "Request a smaller resource or a focused log excerpt. Do not repeatedly download the same full log."
+          ? maxBytes < DEFAULT_MAX_BYTES
+            ? `The raw response exceeded maxBytes=${maxBytes}, before text extraction. To limit returned text, use maxChars and omit maxBytes (default ${DEFAULT_MAX_BYTES}). Retry once with corrected limits, not the same small download limit.`
+            : "The raw response exceeded the download limit. Request a smaller resource or focused log excerpt, or explicitly raise maxBytes within the permitted resource budget. maxChars only limits the returned text."
           : "Use the error code to check connectivity, DNS or access. Retry a transient failure at most once, then change approach or report the blocker.",
     };
   } finally {

package/src/lib/web-search.js

@@ -15,7 +15,7 @@ import http from "http";
 import https from "https";
 import { URL } from "url";
 
-const DEFAULT_MAX_BYTES = 2_000_000;
+const DEFAULT_MAX_BYTES = 10_000_000;
 const DEFAULT_TIMEOUT_MS = 15_000;
 const DEFAULT_MAX_RESULTS = 8;
 export const MAX_WEB_SEARCH_RESULTS = 20;
@@ -98,6 +98,15 @@ function _request(
   const parsed = new URL(urlStr);
   const lib = parsed.protocol === "https:" ? _deps.https : _deps.http;
   return new Promise((resolve, reject) => {
+    let timer;
+    let settled = false;
+    const settle = (error, result) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (error) reject(error);
+      else resolve(result);
+    };
     const req = lib.request(
       {
         method,
@@ -113,28 +122,66 @@ function _request(
         timeout,
       },
       (res) => {
+        if (res.statusCode >= 400) {
+          settle(
+            Object.assign(new Error(`HTTP ${res.statusCode}`), {
+              code: "ERR_HTTP_STATUS",
+              statusCode: res.statusCode,
+              retryAfter: res.headers?.["retry-after"],
+            }),
+          );
+          res.destroy?.();
+          req.destroy();
+          return;
+        }
         const chunks = [];
         let size = 0;
         res.on("data", (chunk) => {
+          if (settled) return;
           size += chunk.length;
           if (size > maxBytes) {
-            req.destroy(new Error(`response exceeds maxBytes (${maxBytes})`));
+            settle(
+              Object.assign(
+                new Error(`response exceeds maxBytes (${maxBytes})`),
+                {
+                  code: "ERR_RESPONSE_TOO_LARGE",
+                },
+              ),
+            );
+            res.destroy?.();
+            req.destroy();
             return;
           }
           chunks.push(chunk);
         });
         res.on("end", () =>
-          resolve({
+          settle(null, {
             statusCode: res.statusCode,
             headers: res.headers,
             body: Buffer.concat(chunks).toString("utf8"),
           }),
         );
-        res.on("error", reject);
+        res.on("error", (error) => settle(error));
+        res.on("aborted", () =>
+          settle(
+            Object.assign(new Error("response aborted"), {
+              code: "ECONNRESET",
+            }),
+          ),
+        );
       },
     );
-    req.on("error", reject);
-    req.on("timeout", () => req.destroy(new Error("request timeout")));
+    const abortTimeout = () => {
+      settle(
+        Object.assign(new Error("request timeout"), { code: "ETIMEDOUT" }),
+      );
+      req.destroy();
+    };
+    req.on("error", (error) => settle(error));
+    req.on("timeout", abortTimeout);
+    // An overall deadline also bounds DNS and continuously streaming results.
+    timer = setTimeout(abortTimeout, timeout);
+    timer.unref?.();
     if (body != null) req.write(body);
     req.end();
   });
@@ -285,6 +332,19 @@ async function _searchDuckDuckGo(query, { maxResults, timeout, maxBytes }) {
     return { error: `duckduckgo HTTP ${res.statusCode}` };
   }
   const html = res.body;
+  if (
+    res.statusCode === 202 ||
+    (res.statusCode >= 300 && res.statusCode < 400) ||
+    /anomaly-modal|challenge-form|anomaly\.js|captcha/i.test(html)
+  ) {
+    return {
+      error:
+        "duckduckgo: verification challenge; search results are unavailable",
+      code: "ERR_SEARCH_CHALLENGE",
+      retryable: false,
+      hint: "Do not report this as no results or repeat the same request. Configure a keyed search provider (Tavily, Brave, Bocha or Qianfan), or use your configured SearXNG instance.",
+    };
+  }
   const results = [];
   const anchorRe =
     /<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
@@ -453,8 +513,26 @@ export async function webSearch(query, options = {}) {
   const config = options.config || {};
   const provider = resolveProvider(options, config);
   let maxResults = normalizeMaxResults(options.maxResults ?? config.maxResults);
-  const timeout = Number(options.timeout) || DEFAULT_TIMEOUT_MS;
-  const maxBytes = Number(options.maxBytes) || DEFAULT_MAX_BYTES;
+  const timeout = options.timeout ?? config.timeout ?? DEFAULT_TIMEOUT_MS;
+  const maxBytes = options.maxBytes ?? config.maxBytes ?? DEFAULT_MAX_BYTES;
+  const maxSnippetChars =
+    options.maxSnippetChars ?? config.maxSnippetChars ?? 2000;
+  if (
+    !Number.isSafeInteger(timeout) ||
+    timeout <= 0 ||
+    timeout > 2_147_483_647 ||
+    !Number.isSafeInteger(maxBytes) ||
+    maxBytes <= 0 ||
+    !Number.isSafeInteger(maxSnippetChars) ||
+    maxSnippetChars <= 0 ||
+    maxSnippetChars > 10000
+  ) {
+    return {
+      error: "web_search: invalid download, snippet or timeout limit",
+      code: "ERR_SEARCH_OPTIONS",
+      retryable: false,
+    };
+  }
 
   if (!SUPPORTED_PROVIDERS.includes(provider)) {
     return {
@@ -533,17 +611,47 @@ export async function webSearch(query, options = {}) {
       return {
         error: `web_search (${provider}) failed: ${err.message}`,
         provider,
+        code: err.code || "ERR_WEB_SEARCH",
+        ...(err.statusCode ? { statusCode: err.statusCode } : {}),
+        ...(err.retryAfter ? { retryAfter: String(err.retryAfter) } : {}),
+        retryable:
+          err.statusCode === 429 ||
+          err.statusCode >= 500 ||
+          ["ETIMEDOUT", "ECONNRESET", "EAI_AGAIN"].includes(err.code),
+        hint:
+          err.code === "ERR_RESPONSE_TOO_LARGE"
+            ? "Search response exceeds maxBytes. Raise maxBytes for a larger download; use maxResults and maxSnippetChars to limit returned results, not maxBytes."
+            : err.statusCode === 429
+              ? "Honor Retry-After before a bounded retry. Do not repeat the same search immediately."
+              : "Check the search provider's configuration, API key and connectivity. Do not treat a provider failure as no results or invent source URLs.",
       };
     }
 
     if (out && out.error) return { ...out, provider };
-    const results = (out && out.results) || [];
+    let truncated = false;
+    const bound = (value, limit) => {
+      const text = String(value || "");
+      if (text.length <= limit) return text;
+      truncated = true;
+      return text.slice(
+        0,
+        /[\uD800-\uDBFF]/.test(text[limit - 1]) ? limit - 1 : limit,
+      );
+    };
+    const results = ((out && out.results) || []).map((result) => ({
+      ...result,
+      title: bound(result.title, 512),
+      snippet: bound(result.snippet, maxSnippetChars),
+    }));
+    const answer = bound(out?.answer, 8000);
     return {
       query: q,
       provider,
       count: results.length,
       results,
-      answer: (out && out.answer) || "",
+      answer,
+      ...(truncated ? { truncated: true, maxSnippetChars } : {}),
+      hint: "These are search results, not full webpages. Cite the returned source URLs. Use web_fetch only when a page needs closer reading; download it once, then continue with snapshotId and nextOffset as offset.",
     };
   } finally {
     toolLease?.release();

package/src/runtime/agent-core.js

@@ -1097,6 +1097,8 @@ Key behaviors:
 - For long-running commands (builds, full test suites, dev servers) set run_shell { run_in_background: true } to get a task_id back immediately, then poll output and completion with check_shell { task_id }. Kill a backgrounded server with check_shell { task_id, kill: true } when finished
 - When asked about git status, diff, log, or other repository operations, use the git tool instead of run_shell
 - When asked about files or code, use search_files to locate relevant sections, then read_file with offset/limit. Follow nextRead for large files. Reuse unchanged content already in context instead of repeatedly reading the same page; re-read when the file changes or the earlier content is no longer available.
+- For long text files, use search_files with path and pattern to find matching lines/columns and nearby text; read the returned nextRead location instead of paging from the top.
+- For online discovery without a known URL, use web_search with keywords. Return relevant source links and snippets; use web_fetch only when the source needs closer reading. For a known URL, fetch it directly. Download large pages once, then use snapshotId to search or read local chunks. Never invent URLs or claim a blocked search returned no results.
 - Large task documents include a sampled outline with line numbers. Use it to locate relevant unfinished work and inspect those sections instead of scanning the entire document unless the task requires full coverage. File coverage and excerpts survive compaction; reading to EOF is not completing the user's task.
 - For task counts or completion status, search the named document for the latest status/summary table first, then read that section. Distinguish historical entries from current status and repository work from external acceptance. Once the relevant entries are verified, answer; do not read every implementation detail just to count tasks.
 - Before renaming or changing a symbol, use code_intelligence (action: references/definition) to find every real usage instead of guessing with text search. It degrades to "unavailable" when no language server is installed — fall back to search_files then.
@@ -1704,6 +1706,8 @@ function agentFileToolPathRequests(name, args = {}, workspaceRoots = []) {
         },
       ];
     case "search_files":
+      if (typeof args.path === "string" && args.path.length > 0)
+        return one("path", "read");
       return typeof args.directory === "string" && args.directory.length > 0
         ? [
             {
@@ -7076,6 +7080,14 @@ async function executeToolInner(
         const result = await webFetch(args.url, {
           format: args.format,
           maxBytes: args.maxBytes,
+          maxChars: args.maxChars,
+          onOverflow: args.onOverflow,
+          snapshotId: args.snapshotId,
+          query: args.query,
+          maxMatches: args.maxMatches,
+          contextChars: args.contextChars,
+          caseSensitive: args.caseSensitive,
+          offset: args.offset,
           timeout: args.timeout,
           config: webFetchConfig,
           hostResourceBudget,
@@ -7104,6 +7116,8 @@ async function executeToolInner(
         const result = await webSearch(args.query, {
           provider: args.provider,
           maxResults: args.maxResults,
+          maxBytes: args.maxBytes,
+          maxSnippetChars: args.maxSnippetChars,
           timeout: args.timeout,
           config: webSearchConfig,
           hostResourceBudget,
@@ -7660,6 +7674,36 @@ async function executeToolInner(
     }
 
     case "search_files": {
+      if (typeof args.path === "string" && args.path.length > 0) {
+        const file = path.resolve(cwd, args.path);
+        const { credentialFileReason } =
+          await import("../lib/credential-guard.js");
+        const reason =
+          credentialFileReason(file) ||
+          (fs.existsSync(file)
+            ? credentialFileReason(fs.realpathSync(file))
+            : null);
+        if (reason)
+          return attachDescriptor({
+            error:
+              "Credential file content is protected; use read_file with confirmation",
+            code: "ERR_CREDENTIAL_FILE",
+          });
+        const { searchTextFile } = await import("../lib/text-file-search.js");
+        const result = await searchTextFile(file, { ...args, hostResourceBudget });
+        return attachDescriptor({
+          ...result,
+          path: file,
+          ...(result.matches
+            ? {
+                matches: result.matches.map((match) => ({
+                  ...match,
+                  nextRead: { path: args.path, ...match.nextRead },
+                })),
+              }
+            : {}),
+        });
+      }
       // An explicit directory scopes the search to one root; otherwise span
       // cwd plus any --add-dir roots so cross-package searches find matches.
       const extraRoots = Array.isArray(additionalDirectories)