@kolisachint/hoocode-agent 0.5.43 → 0.5.45

package/CHANGELOG.md

@@ -1,5 +1,64 @@
 # Changelog
 
+## [0.5.45] - 2026-08-31
+
+## [0.5.44] - 2026-08-31
+
+### Added
+
+- `webfetch` takes an `offset`, so a long page can be read to the end.
+
+  With a `webtools` binary that reports paging offsets, a cut page comes back
+  with the window it covers and the offset that continues it, and passing that
+  offset back reads the next window. Windows tile the document exactly, so
+  nothing is skipped or repeated, and a whole document costs one budget per
+  window instead of one copy of the page per attempt.
+
+  Against an older binary the fields are absent and the note falls back to
+  advising a larger `maxTokens`, the only honest advice when there is no offset
+  to resume at. `--offset` is sent only when non-zero, so a read from the start
+  never passes a flag an older binary would reject.
+
+- `webfetch` says when a page was cut off, and how to get the rest.
+
+  A fetch that overran its token budget came back with the binary's bare
+  `…[truncated]` marker: enough to tell the model something was missing,
+  nothing it could act on, so a long document was a dead end rather than a
+  first page. The result now carries the budget the cut was made at and the
+  ways past it, `details.truncated` records it, and the TUI marks the token
+  line `(truncated at N)` instead of showing a prefix and a complete page
+  identically.
+
+  The note names the *clamped* budget, not the requested one — advice to raise
+  `maxTokens` past the 25000 cap would point somewhere that changes nothing.
+
+- `websearch` reports what its results cost, as `webfetch` already did.
+
+  Search is the one web tool with no token budget of its own — snippet length
+  is whatever the backend returns — so the estimate the binary already sends
+  back is the only thing that makes an expensive query visible before it is
+  already in context, and it is the number `maxResults` is tuned against. It
+  was being parsed into the result details and then dropped at render.
+
+### Fixed
+
+- The agent-selection gold set covers `code-review` and `security-review`.
+
+  0.5.42 added both agents without cases for them, and the eval's coverage
+  assertion — every agent in the roster is expected by at least one case — has
+  failed on `main` ever since, so every branch cut from it inherits a red
+  `bun-test (coding-agent)`. The assertion earns its keep: an agent no case asks
+  for scores as a permanent miss, so a roster can grow agents nobody selects
+  while the eval reads as though the descriptions got worse.
+
+  Three cases each, plus one more `expect: null` — review phrasing over a diff
+  already pasted into the conversation, where a subagent that cannot see it is
+  strictly worse. All authored from how the ask is actually phrased rather than
+  from the agent descriptions, per the fixture's own circularity note. The two
+  review agents ship with identical tools, isolation, background flag and cost,
+  so like `explore`/`plan` they are separated on the ask itself: is this
+  correct, versus can this be attacked.
+
 ## [0.5.43] - 2026-08-31
 
 ### Added

package/dist/core/tools/webfetch.d.ts

@@ -8,11 +8,20 @@ declare const webfetchSchema: Type.TObject<{
     url: Type.TString;
     maxTokens: Type.TOptional<Type.TNumber>;
     output: Type.TOptional<Type.TUnion<[Type.TLiteral<"text">, Type.TLiteral<"markdown">]>>;
+    offset: Type.TOptional<Type.TNumber>;
 }>;
 export interface WebFetchToolDetails {
     finalUrl?: string;
     title?: string;
     tokenEstimate?: number;
+    /** The page continued past the token budget: what came back is a prefix. */
+    truncated?: boolean;
+    /** The budget the cut was made at, so the TUI can say what to raise. */
+    maxTokens?: number;
+    /** Estimated tokens of the whole page, when the binary reports it. */
+    totalTokenEstimate?: number;
+    /** Where to resume reading, when the binary reports paging offsets. */
+    nextOffset?: number;
     contentType?: string;
     media?: string;
     /** Non-"ok" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */

package/dist/core/tools/webfetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAMN,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;EAalB,CAAC;AAIH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,MAAM,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC7D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACtC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAiCD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAgExE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tconst body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAON,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;;EAmBlB,CAAC;AAIH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,MAAM,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC7D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACtC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAyDD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAuFxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}`;\n\t}\n\treturn text;\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Only sent when non-zero: an older binary rejects the unknown flag,\n\t\t\t// and a page read from the start never needs it.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/webfetch.js

@@ -4,7 +4,7 @@ import { keyHint } from "../../modes/interactive/components/keybinding-hints.js"
 import { theme as appTheme } from "../../modes/interactive/theme/theme.js";
 import { getTextOutput, invalidArgText, str } from "./render-utils.js";
 import { wrapToolDefinition } from "./tool-definition-wrapper.js";
-import { blockedHostForUrl, fetchStatusNote, resolveWebtoolsTimeoutSecs, resolveWebtoolsTLSConfig, runWebtools, WebToolsCache, } from "./webtools-shared.js";
+import { blockedHostForUrl, fetchStatusNote, isTruncatedContent, resolveWebtoolsTimeoutSecs, resolveWebtoolsTLSConfig, runWebtools, WebToolsCache, } from "./webtools-shared.js";
 const DEFAULT_MAX_TOKENS = 4000;
 // Hard ceiling so a single fetch can never flood the context window, regardless
 // of what the model requests. The binary still applies its own soft cap.
@@ -23,6 +23,9 @@ const webfetchSchema = Type.Object({
     output: Type.Optional(Type.Union([Type.Literal("text"), Type.Literal("markdown")], {
         description: "Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).",
     })),
+    offset: Type.Optional(Type.Number({
+        description: "Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.",
+    })),
 });
 function formatWebfetchCall(args) {
     const url = str(args?.url);
@@ -45,10 +48,32 @@ function formatWebfetchResult(result, options, showImages) {
     }
     const tokenEstimate = result.details?.tokenEstimate;
     if (tokenEstimate !== undefined) {
-        text += `\n${appTheme.fg("muted", `~${tokenEstimate} tokens`)}`;
+        // A cut page and a complete one cost the same at the budget, so the number
+        // alone reads as "this is the page". Mark the ones that are a prefix.
+        const cut = result.details?.truncated
+            ? appTheme.fg("warning", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)
+            : "";
+        text += `\n${appTheme.fg("muted", `~${tokenEstimate} tokens`)}${cut}`;
     }
     return text;
 }
+/**
+ * What to tell the model when a page did not fit.
+ *
+ * With paging offsets it is a position to resume at, which is the whole point:
+ * the rest of the document is one call away and costs another window, not
+ * another copy of the page. Without them (an older binary) the only truthful
+ * advice is a larger budget.
+ */
+function continuationNote(result, offset, maxTokens) {
+    const next = result.next_offset;
+    if (next === undefined) {
+        return `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;
+    }
+    const total = result.total_token_estimate;
+    const progress = total !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;
+    return `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? "?"} (${progress}); continue with offset=${next}`;
+}
 export function createWebFetchToolDefinition(cwd, options) {
     const cache = options?.cache ?? new WebToolsCache();
     // Resolve CA/insecure plumbing and the request timeout once (settings
@@ -62,9 +87,10 @@ export function createWebFetchToolDefinition(cwd, options) {
         promptSnippet: "Fetch a URL and return clean, token-efficient page content",
         promptGuidelines: [
             "Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.",
+            "A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.",
         ],
         parameters: webfetchSchema,
-        async execute(_toolCallId, { url, maxTokens, output }, signal) {
+        async execute(_toolCallId, { url, maxTokens, output, offset }, signal) {
             if (signal?.aborted)
                 throw new Error("Operation aborted");
             // Policy gate (.webtoolsignore). SSRF/private-address blocking lives in
@@ -75,21 +101,42 @@ export function createWebFetchToolDefinition(cwd, options) {
             }
             const effectiveMaxTokens = clampMaxTokens(maxTokens);
             const format = output ?? "text";
-            const cacheKey = `${format}:${effectiveMaxTokens}:${url}`;
+            const effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;
+            const cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;
             const args = ["--url", url, "--max-tokens", String(effectiveMaxTokens), "--output", format];
+            // Only sent when non-zero: an older binary rejects the unknown flag,
+            // and a page read from the start never needs it.
+            if (effectiveOffset > 0)
+                args.push("--offset", String(effectiveOffset));
             const result = await cache.getOrCompute(cacheKey, signal, (sig) => runWebtools("fetch", args, cwd, sig, timeoutSecs, tlsConfig));
             const header = result.title ? `${result.title}\n${result.final_url}\n\n` : `${result.final_url}\n\n`;
             // An empty body and a JavaScript-rendered shell look identical in the
             // content alone. Say which it was, so the page is not read as "nothing
             // to say" when it simply needs a browser.
             const note = fetchStatusNote(result.status);
-            const body = note ? `${result.content}\n\n[webtools: ${note}]`.trimStart() : result.content;
+            let body = note ? `${result.content}\n\n[webtools: ${note}]`.trimStart() : result.content;
+            // A cut page used to end in a bare elision marker: the model could see
+            // that something was missing but had no way to act on it, so a long
+            // document was a dead end rather than a first page. Say where the
+            // window sits and how to continue past it.
+            //
+            // The binary's own flag is authoritative; the marker is the fallback
+            // for binaries older than the paging fields, where the only honest
+            // advice is a larger budget because there is no offset to resume at.
+            const truncated = result.truncated ?? isTruncatedContent(result.content);
+            if (truncated) {
+                body += `\n\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;
+            }
             return {
                 content: [{ type: "text", text: header + body }],
                 details: {
                     finalUrl: result.final_url,
                     title: result.title,
                     tokenEstimate: result.token_estimate,
+                    truncated,
+                    maxTokens: effectiveMaxTokens,
+                    totalTokenEstimate: result.total_token_estimate,
+                    nextOffset: result.next_offset,
                     contentType: result.content_type,
                     media: result.media,
                     status: result.status,

package/dist/core/tools/webfetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,wFAAwF;AACxF,MAAM,UAAU,cAAc,CAAC,SAAkB,EAAU;IAC1D,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAAA,CAC3C;AAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,UAAU,cAAc,GAAG;KAC5H,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;CACD,CAAC,CAAC;AAqBH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;SAC3I;QACD,UAAU,EAAE,cAAc;QAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAqB,EAAE,MAAoB,EAAE;YAC/F,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,GAAG,EAAE,CAAC;YAE1D,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC5F,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,sEAAsE;YACtE,uEAAuE;YACvE,0CAA0C;YAC1C,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,kBAAkB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5F,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC;gBACzD,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACrB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tconst body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,wFAAwF;AACxF,MAAM,UAAU,cAAc,CAAC,SAAkB,EAAU;IAC1D,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAAA,CAC3C;AAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,UAAU,cAAc,GAAG;KAC5H,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EACV,wMAAwM;KACzM,CAAC,CACF;CACD,CAAC,CAAC;AA6BH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,EAAE,SAAS;YACpC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,kBAAkB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC;YACxF,CAAC,CAAC,EAAE,CAAC;QACN,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,GAAG,GAAG,EAAE,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,MAAsB,EAAE,MAAc,EAAE,SAAiB,EAAU;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,yBAAyB,SAAS,8FAA8F,cAAc,sDAAsD,CAAC;IAC7M,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAC1C,MAAM,QAAQ,GACb,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,cAAc,SAAS,CAAC;IAC3G,OAAO,iBAAiB,MAAM,CAAC,MAAM,IAAI,MAAM,IAAI,IAAI,OAAO,MAAM,CAAC,WAAW,IAAI,GAAG,KAAK,QAAQ,2BAA2B,IAAI,EAAE,CAAC;AAAA,CACtI;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;YAC3I,8WAA4W;SAC5W;QACD,UAAU,EAAE,cAAc;QAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAqB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9G,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,eAAe,IAAI,GAAG,EAAE,CAAC;YAE7E,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC5F,qEAAqE;YACrE,iDAAiD;YACjD,IAAI,eAAe,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,sEAAsE;YACtE,uEAAuE;YACvE,0CAA0C;YAC1C,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,kBAAkB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAE1F,uEAAuE;YACvE,oEAAoE;YACpE,kEAAkE;YAClE,2CAA2C;YAC3C,EAAE;YACF,qEAAqE;YACrE,mEAAmE;YACnE,qEAAqE;YACrE,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzE,IAAI,SAAS,EAAE,CAAC;gBACf,IAAI,IAAI,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,kBAAkB,CAAC,GAAG,CAAC;YAC5F,CAAC;YAED,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC;gBACzD,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,SAAS;oBACT,SAAS,EAAE,kBAAkB;oBAC7B,kBAAkB,EAAE,MAAM,CAAC,oBAAoB;oBAC/C,UAAU,EAAE,MAAM,CAAC,WAAW;oBAC9B,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACrB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}`;\n\t}\n\treturn text;\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Only sent when non-zero: an older binary rejects the unknown flag,\n\t\t\t// and a page read from the start never needs it.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/websearch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"websearch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAON,KAAK,eAAe,EAEpB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAK9B,QAAA,MAAM,eAAe;;;;EAYnB,CAAC;AAIH,MAAM,WAAW,oBAAoB;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,oBAAqB,SAAQ,iBAAiB;IAC9D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IACvC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAwDD,wBAAgB,6BAA6B,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,oBAAoB,GAC5B,cAAc,CAAC,OAAO,eAAe,EAAE,oBAAoB,GAAG,SAAS,CAAC,CA4D1E;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC,OAAO,eAAe,CAAC,CAElH","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\ntype WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n\t/** Which backend answered — a fallback from a keyed provider is otherwise silent. */\n\tprovider?: string;\n}\n\nexport interface WebSearchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web and return ranked results as titles + snippets with reference-style [N] links. Defaults to keyless DuckDuckGo; Brave, Tavily and SearXNG are used instead when credentials are configured for the webtools binary. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t\tprovider: output.provider,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"websearch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAON,KAAK,eAAe,EAEpB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAK9B,QAAA,MAAM,eAAe;;;;EAYnB,CAAC;AAIH,MAAM,WAAW,oBAAoB;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uFAAqF;IACrF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,oBAAqB,SAAQ,iBAAiB;IAC9D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;IACvC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAgED,wBAAgB,6BAA6B,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,oBAAoB,GAC5B,cAAc,CAAC,OAAO,eAAe,EAAE,oBAAoB,GAAG,SAAS,CAAC,CA4D1E;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC,OAAO,eAAe,CAAC,CAElH","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\ntype WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n\t/** Which backend answered — a fallback from a keyed provider is otherwise silent. */\n\tprovider?: string;\n}\n\nexport interface WebSearchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\t// What the results cost in context, as webfetch reports for a page. Search is\n\t// the web tool with no token budget of its own — snippet length is whatever\n\t// the backend returns — so the number is the only thing that makes an\n\t// expensive query visible, and it is what `maxResults` is tuned against.\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web and return ranked results as titles + snippets with reference-style [N] links. Defaults to keyless DuckDuckGo; Brave, Tavily and SearXNG are used instead when credentials are configured for the webtools binary. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t\tprovider: output.provider,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/websearch.js

@@ -61,6 +61,14 @@ function formatWebsearchResult(result, options, showImages) {
             text += `${appTheme.fg("muted", `\n... (${remaining} more lines,`)} ${keyHint("app.tools.expand", "to expand")})`;
         }
     }
+    // What the results cost in context, as webfetch reports for a page. Search is
+    // the web tool with no token budget of its own — snippet length is whatever
+    // the backend returns — so the number is the only thing that makes an
+    // expensive query visible, and it is what `maxResults` is tuned against.
+    const tokenEstimate = result.details?.tokenEstimate;
+    if (tokenEstimate !== undefined) {
+        text += `\n${appTheme.fg("muted", `~${tokenEstimate} tokens`)}`;
+    }
     return text;
 }
 export function createWebSearchToolDefinition(cwd, options) {

package/dist/core/tools/websearch.js.map

@@ -1 +1 @@
-{"version":3,"file":"websearch.js","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,iDAAiD,mBAAmB,UAAU,eAAe,GAAG;KAC7G,CAAC,CACF;IACD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;QACrD,WAAW,EAAE,8DAA8D;KAC3E,CAAC,CACF;CACD,CAAC,CAAC;AAmBH;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAA2B,EAAE,WAAmB,EAAU;IAClG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,qCAAqC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,mBAAmB,KAAK,IAAI,MAAM,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAAA,CAChC,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACvE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,IAAI,QAAQ,WAAW,UAAU,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oCAAoC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,mBAAmB,CAAC,IAAyD,EAAU;IAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,YAAY,GACjB,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACrG,MAAM,KAAK,GAAG,IAAI,EAAE,UAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,qBAAqB,CAC7B,MAA2F,EAC3F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,6BAA6B,CAC5C,GAAW,EACX,OAA8B,EAC6C;IAC3E,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAmB,CAAC;IACrE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,WAAW,EACV,iVAAiV;QAClV,aAAa,EAAE,qDAAqD;QACpE,gBAAgB,EAAE;YACjB,6HAA6H;SAC7H;QACD,UAAU,EAAE,eAAe;QAC3B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAsB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,IAAI,mBAAmB,CAAC,CAAC,CAAC;YAC/F,MAAM,QAAQ,GAAG,GAAG,YAAY,IAAI,UAAU,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;YAEvE,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACvE,IAAI,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAkB,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC9E,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO;gBACnB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAA,CACnD,CAAC;gBACH,CAAC,CAAC,UAAU,CAAC;YACd,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAEpD,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;gBACtF,OAAO,EAAE;oBACR,WAAW,EAAE,IAAI,CAAC,MAAM;oBACxB,WAAW;oBACX,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBACzB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,OAA8B,EAAqC;IACnH,OAAO,kBAAkB,CAAC,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACvE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\ntype WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n\t/** Which backend answered — a fallback from a keyed provider is otherwise silent. */\n\tprovider?: string;\n}\n\nexport interface WebSearchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web and return ranked results as titles + snippets with reference-style [N] links. Defaults to keyless DuckDuckGo; Brave, Tavily and SearXNG are used instead when credentials are configured for the webtools binary. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t\tprovider: output.provider,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"websearch.js","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,iDAAiD,mBAAmB,UAAU,eAAe,GAAG;KAC7G,CAAC,CACF;IACD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;QACrD,WAAW,EAAE,8DAA8D;KAC3E,CAAC,CACF;CACD,CAAC,CAAC;AAmBH;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAA2B,EAAE,WAAmB,EAAU;IAClG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,qCAAqC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,mBAAmB,KAAK,IAAI,MAAM,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAAA,CAChC,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACvE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,IAAI,QAAQ,WAAW,UAAU,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oCAAoC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,mBAAmB,CAAC,IAAyD,EAAU;IAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,YAAY,GACjB,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACrG,MAAM,KAAK,GAAG,IAAI,EAAE,UAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,qBAAqB,CAC7B,MAA2F,EAC3F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,8EAA8E;IAC9E,8EAA4E;IAC5E,wEAAsE;IACtE,yEAAyE;IACzE,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,6BAA6B,CAC5C,GAAW,EACX,OAA8B,EAC6C;IAC3E,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAmB,CAAC;IACrE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,WAAW,EACV,iVAAiV;QAClV,aAAa,EAAE,qDAAqD;QACpE,gBAAgB,EAAE;YACjB,6HAA6H;SAC7H;QACD,UAAU,EAAE,eAAe;QAC3B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAsB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,IAAI,mBAAmB,CAAC,CAAC,CAAC;YAC/F,MAAM,QAAQ,GAAG,GAAG,YAAY,IAAI,UAAU,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;YAEvE,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACvE,IAAI,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAkB,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC9E,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO;gBACnB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAA,CACnD,CAAC;gBACH,CAAC,CAAC,UAAU,CAAC;YACd,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAEpD,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;gBACtF,OAAO,EAAE;oBACR,WAAW,EAAE,IAAI,CAAC,MAAM;oBACxB,WAAW;oBACX,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBACzB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,OAA8B,EAAqC;IACnH,OAAO,kBAAkB,CAAC,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACvE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\ntype WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n\t/** Which backend answered — a fallback from a keyed provider is otherwise silent. */\n\tprovider?: string;\n}\n\nexport interface WebSearchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\t// What the results cost in context, as webfetch reports for a page. Search is\n\t// the web tool with no token budget of its own — snippet length is whatever\n\t// the backend returns — so the number is the only thing that makes an\n\t// expensive query visible, and it is what `maxResults` is tuned against.\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web and return ranked results as titles + snippets with reference-style [N] links. Defaults to keyless DuckDuckGo; Brave, Tavily and SearXNG are used instead when credentials are configured for the webtools binary. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t\tprovider: output.provider,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/webtools-shared.d.ts

@@ -24,6 +24,21 @@ type IgnoreMatcher = ReturnType<typeof ignore>;
 export type WebFetchContentStatus = "ok" | "empty" | "needs_js" | "too_complex";
 /** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */
 export type WebSearchStatus = "ok" | "empty" | "blocked";
+/**
+ * The elision marker the binary appends when `--max-tokens` cuts a body
+ * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text
+ * and markdown — route their budgeting through `truncate_to_tokens`, so its
+ * presence is what tells us a page continued past what we were handed. The
+ * binary reports no structured flag today; when it grows one, prefer that and
+ * keep this as the fallback for older binaries.
+ */
+export declare const WEBTOOLS_TRUNCATION_MARKER = "\u2026[truncated]";
+/**
+ * Whether a fetch came back cut off. Substring rather than suffix: the marker
+ * lands at the end of the *body*, and the reference block is assembled after
+ * it.
+ */
+export declare function isTruncatedContent(content: string | undefined): boolean;
 /** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */
 export declare function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined;
 interface WebFetchReference {
@@ -41,6 +56,21 @@ interface WebFetchMetadata {
 export interface WebFetchResult {
     title?: string;
     final_url: string;
+    /**
+     * Where this window sits in the extracted document. Absent on binaries
+     * older than the paging fields, which is why every consumer falls back to
+     * the elision marker (see {@link isTruncatedContent}) rather than treating
+     * a missing `next_offset` as "the page ended here".
+     */
+    offset?: number;
+    /** Byte offset to resume at, absent when the document ended in this window. */
+    next_offset?: number;
+    /** Size of the whole extracted body, the space offsets index into. */
+    total_bytes?: number;
+    /** Estimated tokens of the whole extracted body, before budget and window. */
+    total_token_estimate?: number;
+    /** The binary's own truncation flag; authoritative when present. */
+    truncated?: boolean;
     content: string;
     content_type: string;
     media: string;

package/dist/core/tools/webtools-shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAgB/C;;;;;;;GAOG;AACH,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,OAAO,GAAG,UAAU,GAAG,aAAa,CAAC;AAEhF,6GAA6G;AAC7G,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD,yFAAyF;AACzF,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW7F;AAED,UAAU,iBAAiB;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,gBAAgB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AASD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,oFAAoF;IACpF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAQD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,CAAC,EAAE,iBAAiB,GAAG,iBAAiB,CAKzF;AAOD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpE;AAMD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7B,MAAM,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9B,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAClD,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,yBAAyB;IACzC,qFAAqF;IACrF,UAAU,EAAE,OAAO,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,0EAAwE;IACxE,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC;IAC5B,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAcD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,yBAAyB,CAkBtG;AAiFD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,EACnD,SAAS,CAAC,EAAE,iBAAiB,GAC3B,OAAO,CAAC,CAAC,CAAC,CAkCZ;AAsBD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;IAED;;;;;;;;OAQG;IACG,YAAY,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1C,OAAO,CAAC,CAAC,CAAC,CA6CZ;CACD;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CA2BzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
+{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAgB/C;;;;;;;GAOG;AACH,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,OAAO,GAAG,UAAU,GAAG,aAAa,CAAC;AAEhF,6GAA6G;AAC7G,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,eAAO,MAAM,0BAA0B,sBAAiB,CAAC;AAEzD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEvE;AAED,yFAAyF;AACzF,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW7F;AAED,UAAU,iBAAiB;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,gBAAgB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AASD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,oFAAoF;IACpF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAQD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,CAAC,EAAE,iBAAiB,GAAG,iBAAiB,CAKzF;AAOD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpE;AAMD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7B,MAAM,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9B,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAClD,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,yBAAyB;IACzC,qFAAqF;IACrF,UAAU,EAAE,OAAO,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,0EAAwE;IACxE,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC;IAC5B,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAcD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,yBAAyB,CAkBtG;AAiFD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,EACnD,SAAS,CAAC,EAAE,iBAAiB,GAC3B,OAAO,CAAC,CAAC,CAAC,CAkCZ;AAsBD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;IAED;;;;;;;;OAQG;IACG,YAAY,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1C,OAAO,CAAC,CAAC,CAAC,CA6CZ;CACD;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CA2BzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}

package/dist/core/tools/webtools-shared.js

@@ -26,6 +26,23 @@ const WEBTOOLS_MIN_TIMEOUT_SECS = 1;
 const WEBTOOLS_MAX_TIMEOUT_SECS = 120;
 /** How long a successful result stays cached, mirroring the documented 15-min TTL. */
 const CACHE_TTL_MS = 15 * 60 * 1000;
+/**
+ * The elision marker the binary appends when `--max-tokens` cuts a body
+ * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text
+ * and markdown — route their budgeting through `truncate_to_tokens`, so its
+ * presence is what tells us a page continued past what we were handed. The
+ * binary reports no structured flag today; when it grows one, prefer that and
+ * keep this as the fallback for older binaries.
+ */
+export const WEBTOOLS_TRUNCATION_MARKER = "…[truncated]";
+/**
+ * Whether a fetch came back cut off. Substring rather than suffix: the marker
+ * lands at the end of the *body*, and the reference block is assembled after
+ * it.
+ */
+export function isTruncatedContent(content) {
+    return typeof content === "string" && content.includes(WEBTOOLS_TRUNCATION_MARKER);
+}
 /** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */
 export function fetchStatusNote(status) {
     switch (status) {

package/dist/core/tools/webtools-shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,qEAAqE;AACrE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmBpC,yFAAyF;AACzF,MAAM,UAAU,eAAe,CAAC,MAAyC,EAAsB;IAC9F,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,OAAO;YACX,OAAO,sCAAsC,CAAC;QAC/C,KAAK,UAAU;YACd,OAAO,6FAA6F,CAAC;QACtG,KAAK,aAAa;YACjB,OAAO,mEAAmE,CAAC;QAC5E;YACC,OAAO,SAAS,CAAC;IACnB,CAAC;AAAA,CACD;AAuDD,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAcnJ,SAAS,WAAW,CAAC,KAAyB,EAAW;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,OAAO,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,CAAC;AAAA,CAC3E;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAA6B,EAAqB;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,SAAS,EAAE,QAAQ,IAAI,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC3F,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,iFAAiF;AACjF,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAA,CAClG;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAiB,EAAU;IACrE,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC/C,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,6BAA6B,CAAC;AAAA,CACrC;AAwCD,8EAA8E;AAC9E,MAAM,qBAAqB,GAA8E;IACxG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,wBAAwB,EAAE,eAAe,CAAC,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,EAAE;IAC3E,4EAA4E;IAC5E,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,sBAAsB,CAAC,EAAE;CACvD,CAAC;AAEF,SAAS,OAAO,CAAC,KAAyB,EAAW;IACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAAA,CAC5D;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAA+B,EAA6B;IACvG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,MAAM,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChG,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC7B,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,qBAAqB,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACtD,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC3G,IAAI,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7G,IAAI,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAEhH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAAA,CAC7B;AAED,sEAAsE;AACtE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;AACjD,SAAS,QAAQ,CAAC,OAAe,EAAQ;IACxC,IAAI,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO;IAChD,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,IAAY,EAAW;IAC9C,IAAI,CAAC;QACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9B,QAAQ,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,QAAQ,CAAC,6CAA6C,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC;QAC1E,OAAO,KAAK,CAAC;IACd,CAAC;AAAA,CACD;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,MAAqC,EAAY;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,IAAI,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,QAAQ,CACP,yFAAyF;YACxF,qFAAqF,CACtF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;GAKG;AACH,MAAM,4BAA4B,GAAuC;IACxE,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;CACT,CAAC;AAEF,mGAAmG;AACnG,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,UAA8B,EAAE,MAAc,EAAE,MAAc,EAAE,IAAY,EAAU;IAChH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAwB,CAAC;QACzD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CACN,4GAA0G;gBAC1G,uDAAuD,CACvD,CAAC;QACH,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,iDAAiD;IAClD,CAAC;IACD,OAAO,YAAY,UAAU,qBAAqB,IAAI,EAAE,CAAC;AAAA,CACzD;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACnD,SAA6B,EAChB;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,YAAY,GAAG,WAAW,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,CAAC,YAAY,GAAG,2BAA2B,CAAC,GAAG,IAAI,CAAC;IAC3E,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACxC,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,WAAW,CAC/B,UAAU,EACV,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,CAAC,EAC7E,GAAG,EACH;QACC,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CACD,CAAC;IAEF,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,YAAY,GAAG,CAAC,CAAC;IAC9F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAsBD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEhE,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,MAA+B,EAC/B,OAA4C,EAC/B;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAE1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YAAA,CACD,CAAC,EAAE,CAAC;YACL,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,MAAM,CAAC,OAAO,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAAA,CACrD,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,MAAM,CAAC,OAAO;gBACd,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAA,CAC/F,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IAAA,CACD;CACD;AAkBD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAe,EAAU;IACjD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,GAAG,IAAI,SAAS,CAAC;QACzB,CAAC;IAAA,CACD,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IAEF,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
+{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,qEAAqE;AACrE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmBpC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,gBAAc,CAAC;AAEzD;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA2B,EAAW;IACxE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;AAAA,CACnF;AAED,yFAAyF;AACzF,MAAM,UAAU,eAAe,CAAC,MAAyC,EAAsB;IAC9F,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,OAAO;YACX,OAAO,sCAAsC,CAAC;QAC/C,KAAK,UAAU;YACd,OAAO,6FAA6F,CAAC;QACtG,KAAK,aAAa;YACjB,OAAO,mEAAmE,CAAC;QAC5E;YACC,OAAO,SAAS,CAAC;IACnB,CAAC;AAAA,CACD;AAsED,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAcnJ,SAAS,WAAW,CAAC,KAAyB,EAAW;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,OAAO,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,CAAC;AAAA,CAC3E;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAA6B,EAAqB;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,SAAS,EAAE,QAAQ,IAAI,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC3F,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,iFAAiF;AACjF,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAA,CAClG;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAiB,EAAU;IACrE,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC/C,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,6BAA6B,CAAC;AAAA,CACrC;AAwCD,8EAA8E;AAC9E,MAAM,qBAAqB,GAA8E;IACxG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,wBAAwB,EAAE,eAAe,CAAC,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,EAAE;IAC3E,4EAA4E;IAC5E,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,sBAAsB,CAAC,EAAE;CACvD,CAAC;AAEF,SAAS,OAAO,CAAC,KAAyB,EAAW;IACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAAA,CAC5D;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAA+B,EAA6B;IACvG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,MAAM,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChG,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC7B,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,qBAAqB,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACtD,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC3G,IAAI,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7G,IAAI,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAEhH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAAA,CAC7B;AAED,sEAAsE;AACtE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;AACjD,SAAS,QAAQ,CAAC,OAAe,EAAQ;IACxC,IAAI,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO;IAChD,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,IAAY,EAAW;IAC9C,IAAI,CAAC;QACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9B,QAAQ,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,QAAQ,CAAC,6CAA6C,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC;QAC1E,OAAO,KAAK,CAAC;IACd,CAAC;AAAA,CACD;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,MAAqC,EAAY;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,IAAI,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,QAAQ,CACP,yFAAyF;YACxF,qFAAqF,CACtF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;GAKG;AACH,MAAM,4BAA4B,GAAuC;IACxE,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;CACT,CAAC;AAEF,mGAAmG;AACnG,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,UAA8B,EAAE,MAAc,EAAE,MAAc,EAAE,IAAY,EAAU;IAChH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAwB,CAAC;QACzD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CACN,4GAA0G;gBAC1G,uDAAuD,CACvD,CAAC;QACH,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,iDAAiD;IAClD,CAAC;IACD,OAAO,YAAY,UAAU,qBAAqB,IAAI,EAAE,CAAC;AAAA,CACzD;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACnD,SAA6B,EAChB;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,YAAY,GAAG,WAAW,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,CAAC,YAAY,GAAG,2BAA2B,CAAC,GAAG,IAAI,CAAC;IAC3E,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACxC,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,WAAW,CAC/B,UAAU,EACV,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,CAAC,EAC7E,GAAG,EACH;QACC,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CACD,CAAC;IAEF,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,YAAY,GAAG,CAAC,CAAC;IAC9F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAsBD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEhE,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,MAA+B,EAC/B,OAA4C,EAC/B;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAE1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YAAA,CACD,CAAC,EAAE,CAAC;YACL,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,MAAM,CAAC,OAAO,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAAA,CACrD,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,MAAM,CAAC,OAAO;gBACd,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAA,CAC/F,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IAAA,CACD;CACD;AAkBD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAe,EAAU;IACjD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,GAAG,IAAI,SAAS,CAAC;QACzB,CAAC;IAAA,CACD,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IAEF,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}

package/docs/settings.md

@@ -352,6 +352,23 @@ Environment variables:
 On Android/Termux the published Linux builds do not run; install with
 `pkg install <name>` instead.
 
+#### Reading a long page
+
+`webfetch` budgets its output in tokens (`maxTokens`, default 4000, hard cap
+25000). When a page runs past that, the result says so — the budget it stopped
+at, and the ways past it — so a long document reads as a first page rather than
+a dead end. The TUI marks the same fetch `~4000 tokens (truncated at 4000)`.
+
+When the `webtools` binary reports paging offsets, the note names the offset to
+continue at; pass it back as `offset` to read the next window. Windows tile the
+document exactly, so a long page costs one budget per window rather than one
+copy of the page per attempt. Against an older binary the note falls back to
+advising a larger `maxTokens`.
+
+Prefer a more specific URL or `#anchor` over paging a whole document: extraction
+has already dropped nav and boilerplate, so the first few thousand tokens are
+usually the article itself.
+
 #### Web search providers
 
 `websearch` needs no configuration: it defaults to keyless DuckDuckGo Lite.

package/examples/extensions/custom-provider-anthropic/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-anthropic",
 	"private": true,
-	"version": "0.3.43",
+	"version": "0.3.45",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/custom-provider-gitlab-duo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-gitlab-duo",
 	"private": true,
-	"version": "0.3.43",
+	"version": "0.3.45",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/sandbox/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-sandbox",
 	"private": true,
-	"version": "0.3.43",
+	"version": "0.3.45",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/with-deps/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-with-deps",
 	"private": true,
-	"version": "0.3.43",
+	"version": "0.3.45",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kolisachint/hoocode-agent",
-	"version": "0.5.43",
+	"version": "0.5.45",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"hoocodeConfig": {
@@ -50,9 +50,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@kolisachint/hoocode-agent-core": "^0.5.43",
-		"@kolisachint/hoocode-ai": "^0.5.43",
-		"@kolisachint/hoocode-tui": "^0.5.43",
+		"@kolisachint/hoocode-agent-core": "^0.5.45",
+		"@kolisachint/hoocode-ai": "^0.5.45",
+		"@kolisachint/hoocode-tui": "^0.5.45",
 		"@silvia-odwyer/photon-node": "^0.3.4",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",