@kolisachint/hoocode-agent 0.5.45 → 0.5.46

package/CHANGELOG.md

@@ -1,11 +1,30 @@
 # Changelog
 
+## [0.5.46] - 2026-08-31
+
 ## [0.5.45] - 2026-08-31
 
 ## [0.5.44] - 2026-08-31
 
 ### Added
 
+- `webfetch` takes `outline: true`, mapping a long page instead of reading it.
+
+  Paging made a long document readable front to back, which is the wrong shape
+  when the answer is in one section and the rest is overhead. An outline returns
+  the headings with the offset that reads each section and what it costs, so a
+  page of any length maps for a few dozen tokens and only the section worth
+  reading is fetched. Outline offsets are the offsets paging already uses, so a
+  row feeds straight back into `offset`.
+
+  The TUI marks such a result `outline, N sections`, since an outline's token
+  cost is its own and not the page's. Needs `webtools` v0.5.0 or newer; an older
+  binary rejects the flag, and the error now names the binary and how to update
+  it rather than passing an argument-parser message through — the fix for an
+  unknown flag is never the call site.
+
+### Added
+
 - `webfetch` takes an `offset`, so a long page can be read to the end.
 
   With a `webtools` binary that reports paging offsets, a cut page comes back

package/dist/core/tools/webfetch.d.ts

@@ -9,6 +9,7 @@ declare const webfetchSchema: Type.TObject<{
     maxTokens: Type.TOptional<Type.TNumber>;
     output: Type.TOptional<Type.TUnion<[Type.TLiteral<"text">, Type.TLiteral<"markdown">]>>;
     offset: Type.TOptional<Type.TNumber>;
+    outline: Type.TOptional<Type.TBoolean>;
 }>;
 export interface WebFetchToolDetails {
     finalUrl?: string;
@@ -22,6 +23,8 @@ export interface WebFetchToolDetails {
     totalTokenEstimate?: number;
     /** Where to resume reading, when the binary reports paging offsets. */
     nextOffset?: number;
+    /** How many sections an outline listed, for the TUI to show at a glance. */
+    sectionCount?: number;
     contentType?: string;
     media?: string;
     /** Non-"ok" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */

package/dist/core/tools/webfetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAON,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;;EAmBlB,CAAC;AAIH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,MAAM,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC7D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACtC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAyDD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAuFxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}`;\n\t}\n\treturn text;\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Only sent when non-zero: an older binary rejects the unknown flag,\n\t\t\t// and a page read from the start never needs it.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAON,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;;;EAyBlB,CAAC;AAIH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,MAAM,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC7D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACtC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAgFD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAkGxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n\toutline: Type.Optional(\n\t\tType.Boolean({\n\t\t\tdescription:\n\t\t\t\t\"Return the page's headings, each with the offset that reads its section and what that section costs, instead of the page body. Map a long page with this first, then fetch the one section you need at its offset.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\t/** How many sections an outline listed, for the TUI to show at a glance. */\n\tsectionCount?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\t// An outline's cost is its own, not the page's, so say which was read.\n\t\tconst sections = result.details?.sectionCount;\n\t\tconst map = sections !== undefined ? appTheme.fg(\"muted\", ` · outline, ${sections} sections`) : \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}${map}`;\n\t}\n\treturn text;\n}\n\n/**\n * Turn an older binary's argument-parsing error into advice.\n *\n * `--offset` and `--outline` postdate binaries already in the wild, which\n * reject an unknown flag with a parser message naming it. That message says\n * nothing about what to do, and the fix is never the call site — it is the\n * binary — so name it here. Returns undefined for anything else, leaving the\n * original error to speak for itself.\n */\nfunction unsupportedFlagError(error: unknown, wantsOutline: boolean, offset: number): Error | undefined {\n\tconst message = error instanceof Error ? error.message : String(error);\n\tif (!/unexpected argument|unrecognized|unknown (option|argument)/i.test(message)) return undefined;\n\n\tconst flag = wantsOutline && /outline/.test(message) ? \"--outline\" : offset > 0 ? \"--offset\" : undefined;\n\tif (!flag) return undefined;\n\treturn new Error(\n\t\t`the installed webtools binary does not support ${flag}; update it (or delete it from the hoocode bin directory to re-download). Original error: ${message}`,\n\t);\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t\t\"For a long page whose relevant part is unknown, fetch it once with `outline: true`: that costs a few dozen tokens and returns the headings with the offset and cost of each section. Then fetch the one section at its offset instead of paging the whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset, outline }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst wantsOutline = outline === true;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${wantsOutline}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Both flags are sent only when asked for: an older binary rejects an\n\t\t\t// unknown argument, and neither is needed to read a page from the start.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tif (wantsOutline) args.push(\"--outline\");\n\t\t\tconst result = await cache\n\t\t\t\t.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t\t)\n\t\t\t\t.catch((error: unknown) => {\n\t\t\t\t\t// A binary predating these flags rejects them as unknown arguments,\n\t\t\t\t\t// which surfaces as an argument-parsing error naming the flag. Say\n\t\t\t\t\t// what to do about it rather than passing the raw parser message on.\n\t\t\t\t\tthrow unsupportedFlagError(error, wantsOutline, effectiveOffset) ?? error;\n\t\t\t\t});\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\tsectionCount: result.outline?.length,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/webfetch.js

@@ -26,6 +26,9 @@ const webfetchSchema = Type.Object({
     offset: Type.Optional(Type.Number({
         description: "Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.",
     })),
+    outline: Type.Optional(Type.Boolean({
+        description: "Return the page's headings, each with the offset that reads its section and what that section costs, instead of the page body. Map a long page with this first, then fetch the one section you need at its offset.",
+    })),
 });
 function formatWebfetchCall(args) {
     const url = str(args?.url);
@@ -53,10 +56,31 @@ function formatWebfetchResult(result, options, showImages) {
         const cut = result.details?.truncated
             ? appTheme.fg("warning", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)
             : "";
-        text += `\n${appTheme.fg("muted", `~${tokenEstimate} tokens`)}${cut}`;
+        // An outline's cost is its own, not the page's, so say which was read.
+        const sections = result.details?.sectionCount;
+        const map = sections !== undefined ? appTheme.fg("muted", ` · outline, ${sections} sections`) : "";
+        text += `\n${appTheme.fg("muted", `~${tokenEstimate} tokens`)}${cut}${map}`;
     }
     return text;
 }
+/**
+ * Turn an older binary's argument-parsing error into advice.
+ *
+ * `--offset` and `--outline` postdate binaries already in the wild, which
+ * reject an unknown flag with a parser message naming it. That message says
+ * nothing about what to do, and the fix is never the call site — it is the
+ * binary — so name it here. Returns undefined for anything else, leaving the
+ * original error to speak for itself.
+ */
+function unsupportedFlagError(error, wantsOutline, offset) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (!/unexpected argument|unrecognized|unknown (option|argument)/i.test(message))
+        return undefined;
+    const flag = wantsOutline && /outline/.test(message) ? "--outline" : offset > 0 ? "--offset" : undefined;
+    if (!flag)
+        return undefined;
+    return new Error(`the installed webtools binary does not support ${flag}; update it (or delete it from the hoocode bin directory to re-download). Original error: ${message}`);
+}
 /**
  * What to tell the model when a page did not fit.
  *
@@ -88,9 +112,10 @@ export function createWebFetchToolDefinition(cwd, options) {
         promptGuidelines: [
             "Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.",
             "A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.",
+            "For a long page whose relevant part is unknown, fetch it once with `outline: true`: that costs a few dozen tokens and returns the headings with the offset and cost of each section. Then fetch the one section at its offset instead of paging the whole document.",
         ],
         parameters: webfetchSchema,
-        async execute(_toolCallId, { url, maxTokens, output, offset }, signal) {
+        async execute(_toolCallId, { url, maxTokens, output, offset, outline }, signal) {
             if (signal?.aborted)
                 throw new Error("Operation aborted");
             // Policy gate (.webtoolsignore). SSRF/private-address blocking lives in
@@ -102,13 +127,23 @@ export function createWebFetchToolDefinition(cwd, options) {
             const effectiveMaxTokens = clampMaxTokens(maxTokens);
             const format = output ?? "text";
             const effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;
-            const cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;
+            const wantsOutline = outline === true;
+            const cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${wantsOutline}:${url}`;
             const args = ["--url", url, "--max-tokens", String(effectiveMaxTokens), "--output", format];
-            // Only sent when non-zero: an older binary rejects the unknown flag,
-            // and a page read from the start never needs it.
+            // Both flags are sent only when asked for: an older binary rejects an
+            // unknown argument, and neither is needed to read a page from the start.
             if (effectiveOffset > 0)
                 args.push("--offset", String(effectiveOffset));
-            const result = await cache.getOrCompute(cacheKey, signal, (sig) => runWebtools("fetch", args, cwd, sig, timeoutSecs, tlsConfig));
+            if (wantsOutline)
+                args.push("--outline");
+            const result = await cache
+                .getOrCompute(cacheKey, signal, (sig) => runWebtools("fetch", args, cwd, sig, timeoutSecs, tlsConfig))
+                .catch((error) => {
+                // A binary predating these flags rejects them as unknown arguments,
+                // which surfaces as an argument-parsing error naming the flag. Say
+                // what to do about it rather than passing the raw parser message on.
+                throw unsupportedFlagError(error, wantsOutline, effectiveOffset) ?? error;
+            });
             const header = result.title ? `${result.title}\n${result.final_url}\n\n` : `${result.final_url}\n\n`;
             // An empty body and a JavaScript-rendered shell look identical in the
             // content alone. Say which it was, so the page is not read as "nothing
@@ -135,6 +170,7 @@ export function createWebFetchToolDefinition(cwd, options) {
                     tokenEstimate: result.token_estimate,
                     truncated,
                     maxTokens: effectiveMaxTokens,
+                    sectionCount: result.outline?.length,
                     totalTokenEstimate: result.total_token_estimate,
                     nextOffset: result.next_offset,
                     contentType: result.content_type,

package/dist/core/tools/webfetch.js.map

@@ -1 +1 @@
-{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,wFAAwF;AACxF,MAAM,UAAU,cAAc,CAAC,SAAkB,EAAU;IAC1D,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAAA,CAC3C;AAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,UAAU,cAAc,GAAG;KAC5H,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EACV,wMAAwM;KACzM,CAAC,CACF;CACD,CAAC,CAAC;AA6BH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,EAAE,SAAS;YACpC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,kBAAkB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC;YACxF,CAAC,CAAC,EAAE,CAAC;QACN,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,GAAG,GAAG,EAAE,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,MAAsB,EAAE,MAAc,EAAE,SAAiB,EAAU;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,yBAAyB,SAAS,8FAA8F,cAAc,sDAAsD,CAAC;IAC7M,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAC1C,MAAM,QAAQ,GACb,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,cAAc,SAAS,CAAC;IAC3G,OAAO,iBAAiB,MAAM,CAAC,MAAM,IAAI,MAAM,IAAI,IAAI,OAAO,MAAM,CAAC,WAAW,IAAI,GAAG,KAAK,QAAQ,2BAA2B,IAAI,EAAE,CAAC;AAAA,CACtI;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;YAC3I,8WAA4W;SAC5W;QACD,UAAU,EAAE,cAAc;QAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAqB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9G,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,eAAe,IAAI,GAAG,EAAE,CAAC;YAE7E,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC5F,qEAAqE;YACrE,iDAAiD;YACjD,IAAI,eAAe,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC5E,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,sEAAsE;YACtE,uEAAuE;YACvE,0CAA0C;YAC1C,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,kBAAkB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAE1F,uEAAuE;YACvE,oEAAoE;YACpE,kEAAkE;YAClE,2CAA2C;YAC3C,EAAE;YACF,qEAAqE;YACrE,mEAAmE;YACnE,qEAAqE;YACrE,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzE,IAAI,SAAS,EAAE,CAAC;gBACf,IAAI,IAAI,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,kBAAkB,CAAC,GAAG,CAAC;YAC5F,CAAC;YAED,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC;gBACzD,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,SAAS;oBACT,SAAS,EAAE,kBAAkB;oBAC7B,kBAAkB,EAAE,MAAM,CAAC,oBAAoB;oBAC/C,UAAU,EAAE,MAAM,CAAC,WAAW;oBAC9B,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACrB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}`;\n\t}\n\treturn text;\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Only sent when non-zero: an older binary rejects the unknown flag,\n\t\t\t// and a page read from the start never needs it.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
+{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,0BAA0B,EAC1B,wBAAwB,EACxB,WAAW,EAGX,aAAa,GAEb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,wFAAwF;AACxF,MAAM,UAAU,cAAc,CAAC,SAAkB,EAAU;IAC1D,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAAA,CAC3C;AAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,UAAU,cAAc,GAAG;KAC5H,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EACV,wMAAwM;KACzM,CAAC,CACF;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CACrB,IAAI,CAAC,OAAO,CAAC;QACZ,WAAW,EACV,oNAAoN;KACrN,CAAC,CACF;CACD,CAAC,CAAC;AA+BH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,2EAA2E;QAC3E,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,EAAE,SAAS;YACpC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,kBAAkB,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,aAAa,GAAG,CAAC;YACxF,CAAC,CAAC,EAAE,CAAC;QACN,uEAAuE;QACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC;QAC9C,MAAM,GAAG,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAe,QAAQ,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,KAAc,EAAE,YAAqB,EAAE,MAAc,EAAqB;IACvG,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,CAAC,6DAA6D,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAEnG,MAAM,IAAI,GAAG,YAAY,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IACzG,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,IAAI,KAAK,CACf,kDAAkD,IAAI,6FAA6F,OAAO,EAAE,CAC5J,CAAC;AAAA,CACF;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,MAAsB,EAAE,MAAc,EAAE,SAAiB,EAAU;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,yBAAyB,SAAS,8FAA8F,cAAc,sDAAsD,CAAC;IAC7M,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAC1C,MAAM,QAAQ,GACb,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,cAAc,SAAS,CAAC;IAC3G,OAAO,iBAAiB,MAAM,CAAC,MAAM,IAAI,MAAM,IAAI,IAAI,OAAO,MAAM,CAAC,WAAW,IAAI,GAAG,KAAK,QAAQ,2BAA2B,IAAI,EAAE,CAAC;AAAA,CACtI;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,SAAS,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,0BAA0B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;YAC3I,8WAA4W;YAC5W,qQAAqQ;SACrQ;QACD,UAAU,EAAE,cAAc;QAC1B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAqB,EAAE,MAAoB,EAAE;YAChH,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9G,MAAM,YAAY,GAAG,OAAO,KAAK,IAAI,CAAC;YACtC,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,eAAe,IAAI,YAAY,IAAI,GAAG,EAAE,CAAC;YAE7F,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC5F,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,eAAe,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;YACxE,IAAI,YAAY;gBAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,KAAK;iBACxB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACvC,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAC5E;iBACA,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC;gBAC1B,oEAAoE;gBACpE,mEAAmE;gBACnE,qEAAqE;gBACrE,MAAM,oBAAoB,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,IAAI,KAAK,CAAC;YAAA,CAC1E,CAAC,CAAC;YAEJ,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,sEAAsE;YACtE,uEAAuE;YACvE,0CAA0C;YAC1C,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,kBAAkB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAE1F,uEAAuE;YACvE,oEAAoE;YACpE,kEAAkE;YAClE,2CAA2C;YAC3C,EAAE;YACF,qEAAqE;YACrE,mEAAmE;YACnE,qEAAqE;YACrE,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzE,IAAI,SAAS,EAAE,CAAC;gBACf,IAAI,IAAI,kBAAkB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,kBAAkB,CAAC,GAAG,CAAC;YAC5F,CAAC;YAED,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,CAAC;gBACzD,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,SAAS;oBACT,SAAS,EAAE,kBAAkB;oBAC7B,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM;oBACpC,kBAAkB,EAAE,MAAM,CAAC,oBAAoB;oBAC/C,UAAU,EAAE,MAAM,CAAC,WAAW;oBAC9B,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACrB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n\toutline: Type.Optional(\n\t\tType.Boolean({\n\t\t\tdescription:\n\t\t\t\t\"Return the page's headings, each with the offset that reads its section and what that section costs, instead of the page body. Map a long page with this first, then fetch the one section you need at its offset.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\t/** How many sections an outline listed, for the TUI to show at a glance. */\n\tsectionCount?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\t// An outline's cost is its own, not the page's, so say which was read.\n\t\tconst sections = result.details?.sectionCount;\n\t\tconst map = sections !== undefined ? appTheme.fg(\"muted\", ` · outline, ${sections} sections`) : \"\";\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}${map}`;\n\t}\n\treturn text;\n}\n\n/**\n * Turn an older binary's argument-parsing error into advice.\n *\n * `--offset` and `--outline` postdate binaries already in the wild, which\n * reject an unknown flag with a parser message naming it. That message says\n * nothing about what to do, and the fix is never the call site — it is the\n * binary — so name it here. Returns undefined for anything else, leaving the\n * original error to speak for itself.\n */\nfunction unsupportedFlagError(error: unknown, wantsOutline: boolean, offset: number): Error | undefined {\n\tconst message = error instanceof Error ? error.message : String(error);\n\tif (!/unexpected argument|unrecognized|unknown (option|argument)/i.test(message)) return undefined;\n\n\tconst flag = wantsOutline && /outline/.test(message) ? \"--outline\" : offset > 0 ? \"--offset\" : undefined;\n\tif (!flag) return undefined;\n\treturn new Error(\n\t\t`the installed webtools binary does not support ${flag}; update it (or delete it from the hoocode bin directory to re-download). Original error: ${message}`,\n\t);\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t\t\"For a long page whose relevant part is unknown, fetch it once with `outline: true`: that costs a few dozen tokens and returns the headings with the offset and cost of each section. Then fetch the one section at its offset instead of paging the whole document.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(_toolCallId, { url, maxTokens, output, offset, outline }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst wantsOutline = outline === true;\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${wantsOutline}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Both flags are sent only when asked for: an older binary rejects an\n\t\t\t// unknown argument, and neither is needed to read a page from the start.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tif (wantsOutline) args.push(\"--outline\");\n\t\t\tconst result = await cache\n\t\t\t\t.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t\t)\n\t\t\t\t.catch((error: unknown) => {\n\t\t\t\t\t// A binary predating these flags rejects them as unknown arguments,\n\t\t\t\t\t// which surfaces as an argument-parsing error naming the flag. Say\n\t\t\t\t\t// what to do about it rather than passing the raw parser message on.\n\t\t\t\t\tthrow unsupportedFlagError(error, wantsOutline, effectiveOffset) ?? error;\n\t\t\t\t});\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\tsectionCount: result.outline?.length,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}

package/dist/core/tools/webtools-shared.d.ts

@@ -53,6 +53,15 @@ interface WebFetchMetadata {
     lang?: string;
     site_name?: string;
 }
+/** One heading in a page's outline, and the span of text it opens. */
+export interface WebFetchOutlineSection {
+    level: number;
+    title: string;
+    /** Byte offset in the extracted text — the same space `offset` addresses. */
+    offset: number;
+    bytes: number;
+    token_estimate: number;
+}
 export interface WebFetchResult {
     title?: string;
     final_url: string;
@@ -71,6 +80,8 @@ export interface WebFetchResult {
     total_token_estimate?: number;
     /** The binary's own truncation flag; authoritative when present. */
     truncated?: boolean;
+    /** The page's headings, when an outline was requested. */
+    outline?: WebFetchOutlineSection[];
     content: string;
     content_type: string;
     media: string;

package/dist/core/tools/webtools-shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAgB/C;;;;;;;GAOG;AACH,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,OAAO,GAAG,UAAU,GAAG,aAAa,CAAC;AAEhF,6GAA6G;AAC7G,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,eAAO,MAAM,0BAA0B,sBAAiB,CAAC;AAEzD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEvE;AAED,yFAAyF;AACzF,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW7F;AAED,UAAU,iBAAiB;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,gBAAgB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AASD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,oFAAoF;IACpF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAQD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,CAAC,EAAE,iBAAiB,GAAG,iBAAiB,CAKzF;AAOD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpE;AAMD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7B,MAAM,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9B,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAClD,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,yBAAyB;IACzC,qFAAqF;IACrF,UAAU,EAAE,OAAO,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,0EAAwE;IACxE,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC;IAC5B,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAcD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,yBAAyB,CAkBtG;AAiFD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,EACnD,SAAS,CAAC,EAAE,iBAAiB,GAC3B,OAAO,CAAC,CAAC,CAAC,CAkCZ;AAsBD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;IAED;;;;;;;;OAQG;IACG,YAAY,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1C,OAAO,CAAC,CAAC,CAAC,CA6CZ;CACD;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CA2BzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
+{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAgB/C;;;;;;;GAOG;AACH,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,OAAO,GAAG,UAAU,GAAG,aAAa,CAAC;AAEhF,6GAA6G;AAC7G,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,eAAO,MAAM,0BAA0B,sBAAiB,CAAC;AAEzD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEvE;AAED,yFAAyF;AACzF,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAW7F;AAED,UAAU,iBAAiB;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,gBAAgB;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,sEAAsE;AACtE,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,+EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AASD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IACjC,oFAAoF;IACpF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAQD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,CAAC,EAAE,iBAAiB,GAAG,iBAAiB,CAKzF;AAOD;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAYpE;AAMD;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACtC,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE;QACX,KAAK,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7B,MAAM,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC9B,OAAO,CAAC,EAAE;YAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAClD,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAEjE,MAAM,WAAW,yBAAyB;IACzC,qFAAqF;IACrF,UAAU,EAAE,OAAO,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B,0EAAwE;IACxE,MAAM,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC;IAC5B,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAcD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,yBAAyB,CAkBtG;AAiFD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,EACnD,SAAS,CAAC,EAAE,iBAAiB,GAC3B,OAAO,CAAC,CAAC,CAAC,CAkCZ;AAsBD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;IAED;;;;;;;;OAQG;IACG,YAAY,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1C,OAAO,CAAC,CAAC,CAAC,CA6CZ;CACD;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CA2BzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\n/** One heading in a page's outline, and the span of text it opens. */\nexport interface WebFetchOutlineSection {\n\tlevel: number;\n\ttitle: string;\n\t/** Byte offset in the extracted text — the same space `offset` addresses. */\n\toffset: number;\n\tbytes: number;\n\ttoken_estimate: number;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\t/** The page's headings, when an outline was requested. */\n\toutline?: WebFetchOutlineSection[];\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}

package/dist/core/tools/webtools-shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,qEAAqE;AACrE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmBpC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,gBAAc,CAAC;AAEzD;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA2B,EAAW;IACxE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;AAAA,CACnF;AAED,yFAAyF;AACzF,MAAM,UAAU,eAAe,CAAC,MAAyC,EAAsB;IAC9F,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,OAAO;YACX,OAAO,sCAAsC,CAAC;QAC/C,KAAK,UAAU;YACd,OAAO,6FAA6F,CAAC;QACtG,KAAK,aAAa;YACjB,OAAO,mEAAmE,CAAC;QAC5E;YACC,OAAO,SAAS,CAAC;IACnB,CAAC;AAAA,CACD;AAsED,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAcnJ,SAAS,WAAW,CAAC,KAAyB,EAAW;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,OAAO,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,CAAC;AAAA,CAC3E;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAA6B,EAAqB;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,SAAS,EAAE,QAAQ,IAAI,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC3F,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,iFAAiF;AACjF,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAA,CAClG;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAiB,EAAU;IACrE,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC/C,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,6BAA6B,CAAC;AAAA,CACrC;AAwCD,8EAA8E;AAC9E,MAAM,qBAAqB,GAA8E;IACxG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,wBAAwB,EAAE,eAAe,CAAC,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,EAAE;IAC3E,4EAA4E;IAC5E,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,sBAAsB,CAAC,EAAE;CACvD,CAAC;AAEF,SAAS,OAAO,CAAC,KAAyB,EAAW;IACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAAA,CAC5D;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAA+B,EAA6B;IACvG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,MAAM,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChG,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC7B,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,qBAAqB,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACtD,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC3G,IAAI,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7G,IAAI,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAEhH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAAA,CAC7B;AAED,sEAAsE;AACtE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;AACjD,SAAS,QAAQ,CAAC,OAAe,EAAQ;IACxC,IAAI,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO;IAChD,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,IAAY,EAAW;IAC9C,IAAI,CAAC;QACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9B,QAAQ,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,QAAQ,CAAC,6CAA6C,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC;QAC1E,OAAO,KAAK,CAAC;IACd,CAAC;AAAA,CACD;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,MAAqC,EAAY;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,IAAI,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,QAAQ,CACP,yFAAyF;YACxF,qFAAqF,CACtF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;GAKG;AACH,MAAM,4BAA4B,GAAuC;IACxE,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;CACT,CAAC;AAEF,mGAAmG;AACnG,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,UAA8B,EAAE,MAAc,EAAE,MAAc,EAAE,IAAY,EAAU;IAChH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAwB,CAAC;QACzD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CACN,4GAA0G;gBAC1G,uDAAuD,CACvD,CAAC;QACH,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,iDAAiD;IAClD,CAAC;IACD,OAAO,YAAY,UAAU,qBAAqB,IAAI,EAAE,CAAC;AAAA,CACzD;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACnD,SAA6B,EAChB;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,YAAY,GAAG,WAAW,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,CAAC,YAAY,GAAG,2BAA2B,CAAC,GAAG,IAAI,CAAC;IAC3E,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACxC,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,WAAW,CAC/B,UAAU,EACV,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,CAAC,EAC7E,GAAG,EACH;QACC,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CACD,CAAC;IAEF,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,YAAY,GAAG,CAAC,CAAC;IAC9F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAsBD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEhE,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,MAA+B,EAC/B,OAA4C,EAC/B;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAE1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YAAA,CACD,CAAC,EAAE,CAAC;YACL,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,MAAM,CAAC,OAAO,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAAA,CACrD,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,MAAM,CAAC,OAAO;gBACd,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAA,CAC/F,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IAAA,CACD;CACD;AAkBD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAe,EAAU;IACjD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,GAAG,IAAI,SAAS,CAAC;QACzB,CAAC;IAAA,CACD,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IAEF,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
+{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,qEAAqE;AACrE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAmBpC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,gBAAc,CAAC;AAEzD;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA2B,EAAW;IACxE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;AAAA,CACnF;AAED,yFAAyF;AACzF,MAAM,UAAU,eAAe,CAAC,MAAyC,EAAsB;IAC9F,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,OAAO;YACX,OAAO,sCAAsC,CAAC;QAC/C,KAAK,UAAU;YACd,OAAO,6FAA6F,CAAC;QACtG,KAAK,aAAa;YACjB,OAAO,mEAAmE,CAAC;QAC5E;YACC,OAAO,SAAS,CAAC;IACnB,CAAC;AAAA,CACD;AAkFD,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAcnJ,SAAS,WAAW,CAAC,KAAyB,EAAW;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC9C,OAAO,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,CAAC;AAAA,CAC3E;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAA6B,EAAqB;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxG,MAAM,QAAQ,GAAG,SAAS,EAAE,QAAQ,IAAI,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC3F,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAAA,CAChC;AAED,iFAAiF;AACjF,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAA,CAClG;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAAiB,EAAU;IACrE,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzD,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IAC5D,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAC/C,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IACD,OAAO,6BAA6B,CAAC;AAAA,CACrC;AAwCD,8EAA8E;AAC9E,MAAM,qBAAqB,GAA8E;IACxG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,wBAAwB,EAAE,eAAe,CAAC,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,yBAAyB,EAAE,gBAAgB,CAAC,EAAE;IAC3E,4EAA4E;IAC5E,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,sBAAsB,CAAC,EAAE;CACvD,CAAC;AAEF,SAAS,OAAO,CAAC,KAAyB,EAAW;IACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAAA,CAC5D;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAA+B,EAA6B;IACvG,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,MAAM,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAChG,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QAC7B,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IACrD,CAAC;IAED,KAAK,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,qBAAqB,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACtD,CAAC;IACF,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,EAAE,SAAS,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC3G,IAAI,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7G,IAAI,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAEhH,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;AAAA,CAC7B;AAED,sEAAsE;AACtE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;AACjD,SAAS,QAAQ,CAAC,OAAe,EAAQ;IACxC,IAAI,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO;IAChD,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,gGAAgG;AAChG,SAAS,cAAc,CAAC,IAAY,EAAW;IAC9C,IAAI,CAAC;QACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9B,QAAQ,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,KAAK,CAAC;QACd,CAAC;QACD,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,QAAQ,CAAC,6CAA6C,IAAI,KAAK,MAAM,GAAG,CAAC,CAAC;QAC1E,OAAO,KAAK,CAAC;IACd,CAAC;AAAA,CACD;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,MAAqC,EAAY;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,IAAI,MAAM,CAAC,UAAU,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrB,QAAQ,CACP,yFAAyF;YACxF,qFAAqF,CACtF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED;;;;;GAKG;AACH,MAAM,4BAA4B,GAAuC;IACxE,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;CACT,CAAC;AAEF,mGAAmG;AACnG,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAEtC;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,UAA8B,EAAE,MAAc,EAAE,MAAc,EAAE,IAAY,EAAU;IAChH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IAExC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAwB,CAAC;QACzD,IAAI,MAAM,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;YAClC,OAAO,CACN,4GAA0G;gBAC1G,uDAAuD,CACvD,CAAC;QACH,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,iDAAiD;IAClD,CAAC;IACD,OAAO,YAAY,UAAU,qBAAqB,IAAI,EAAE,CAAC;AAAA,CACzD;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACnD,SAA6B,EAChB;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,0EAA0E;IAC1E,yEAAyE;IACzE,MAAM,YAAY,GAAG,WAAW,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,CAAC,YAAY,GAAG,2BAA2B,CAAC,GAAG,IAAI,CAAC;IAC3E,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IACxC,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,MAAM,GAAG,MAAM,WAAW,CAC/B,UAAU,EACV,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,CAAC,EAC7E,GAAG,EACH;QACC,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CACD,CAAC;IAEF,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,YAAY,GAAG,CAAC,CAAC;IAC9F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAsBD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEhE,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,MAA+B,EAC/B,OAA4C,EAC/B;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAE1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YAAA,CACD,CAAC,EAAE,CAAC;YACL,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,MAAM,CAAC,OAAO,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAAA,CACrD,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,MAAM,CAAC,OAAO;gBACd,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAA,CAC/F,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IAAA,CACD;CACD;AAkBD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAe,EAAU;IACjD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,GAAG,IAAI,SAAS,CAAC;QACzB,CAAC;IAAA,CACD,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IAEF,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache,\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n *   hosts both before a fetch and when filtering search result links, and\n * - the read-only check for whether `websearch` has a keyed backend configured.\n */\n\nimport { accessSync, constants, existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nconst WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** Lower/upper bounds on the effective request timeout (seconds). */\nconst WEBTOOLS_MIN_TIMEOUT_SECS = 1;\nconst WEBTOOLS_MAX_TIMEOUT_SECS = 120;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\n/**\n * Whether the binary actually extracted content (`FetchResult.status`).\n *\n * Optional here because an older `webtools` on PATH predates the field; absent\n * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely\n * blank page are both \"empty content, exit 0\" and the model reads either as\n * \"this page has nothing to say\".\n */\nexport type WebFetchContentStatus = \"ok\" | \"empty\" | \"needs_js\" | \"too_complex\";\n\n/** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */\nexport type WebSearchStatus = \"ok\" | \"empty\" | \"blocked\";\n\n/**\n * The elision marker the binary appends when `--max-tokens` cuts a body\n * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text\n * and markdown — route their budgeting through `truncate_to_tokens`, so its\n * presence is what tells us a page continued past what we were handed. The\n * binary reports no structured flag today; when it grows one, prefer that and\n * keep this as the fallback for older binaries.\n */\nexport const WEBTOOLS_TRUNCATION_MARKER = \"…[truncated]\";\n\n/**\n * Whether a fetch came back cut off. Substring rather than suffix: the marker\n * lands at the end of the *body*, and the reference block is assembled after\n * it.\n */\nexport function isTruncatedContent(content: string | undefined): boolean {\n\treturn typeof content === \"string\" && content.includes(WEBTOOLS_TRUNCATION_MARKER);\n}\n\n/** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */\nexport function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined {\n\tswitch (status) {\n\t\tcase \"empty\":\n\t\t\treturn \"the page parsed but contains no text\";\n\t\tcase \"needs_js\":\n\t\t\treturn \"no text content: the page renders its body with JavaScript, which webtools does not execute\";\n\t\tcase \"too_complex\":\n\t\t\treturn \"the document is too deeply nested to parse safely and was refused\";\n\t\tdefault:\n\t\t\treturn undefined;\n\t}\n}\n\ninterface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\ninterface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\n/** One heading in a page's outline, and the span of text it opens. */\nexport interface WebFetchOutlineSection {\n\tlevel: number;\n\ttitle: string;\n\t/** Byte offset in the extracted text — the same space `offset` addresses. */\n\toffset: number;\n\tbytes: number;\n\ttoken_estimate: number;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\t/**\n\t * Where this window sits in the extracted document. Absent on binaries\n\t * older than the paging fields, which is why every consumer falls back to\n\t * the elision marker (see {@link isTruncatedContent}) rather than treating\n\t * a missing `next_offset` as \"the page ended here\".\n\t */\n\toffset?: number;\n\t/** Byte offset to resume at, absent when the document ended in this window. */\n\tnext_offset?: number;\n\t/** Size of the whole extracted body, the space offsets index into. */\n\ttotal_bytes?: number;\n\t/** Estimated tokens of the whole extracted body, before budget and window. */\n\ttotal_token_estimate?: number;\n\t/** The binary's own truncation flag; authoritative when present. */\n\ttruncated?: boolean;\n\t/** The page's headings, when an outline was requested. */\n\toutline?: WebFetchOutlineSection[];\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebFetchContentStatus;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\t/** The URL that was requested, before any redirect (`final_url` is post-redirect). */\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\ninterface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n\t/** Absent on binaries older than the status field; treated as \"ok\". */\n\tstatus?: WebSearchStatus;\n\t/** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */\n\tprovider?: string;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`.\n * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the\n * binary has its own TLS stack, so it needs the CA / insecure flag passed in.\n */\nexport interface WebtoolsTLSConfig {\n\t/** Path to a PEM CA bundle forwarded as `--ca-cert <path>` (validated readable). */\n\tcaCertPath?: string;\n\t/** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */\n\tinsecure?: boolean;\n}\n\nfunction isTruthyEnv(value: string | undefined): boolean {\n\tif (!value) return false;\n\tconst normalized = value.trim().toLowerCase();\n\treturn normalized === \"1\" || normalized === \"true\" || normalized === \"yes\";\n}\n\n/**\n * Resolve the webtools TLS config from explicit overrides (e.g. settings.json\n * passed down from the tool factories) falling back to the environment\n * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded.\n */\nexport function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig {\n\tconst envCaCert = process.env.HOOCODE_WEBTOOLS_CA_CERT?.trim();\n\tconst caCertPath = overrides?.caCertPath ?? (envCaCert && envCaCert.length > 0 ? envCaCert : undefined);\n\tconst insecure = overrides?.insecure ?? isTruthyEnv(process.env.HOOCODE_WEBTOOLS_INSECURE);\n\treturn { caCertPath, insecure };\n}\n\n/** Clamp a request timeout to the supported range, flooring to whole seconds. */\nfunction clampTimeoutSecs(secs: number): number {\n\treturn Math.min(WEBTOOLS_MAX_TIMEOUT_SECS, Math.max(WEBTOOLS_MIN_TIMEOUT_SECS, Math.floor(secs)));\n}\n\n/**\n * Resolve the effective webtools request timeout (seconds) from an explicit\n * override (e.g. settings.json passed down from the tool factories) falling back\n * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors\n * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A\n * malformed or out-of-range env value falls back to the default; every result is\n * clamped to [1, 120].\n */\nexport function resolveWebtoolsTimeoutSecs(override?: number): number {\n\tif (override !== undefined && Number.isFinite(override)) {\n\t\treturn clampTimeoutSecs(override);\n\t}\n\tconst envRaw = process.env.HOOCODE_WEBTOOLS_TIMEOUT?.trim();\n\tif (envRaw) {\n\t\tconst envValue = Number(envRaw);\n\t\tif (Number.isFinite(envValue) && envValue > 0) {\n\t\t\treturn clampTimeoutSecs(envValue);\n\t\t}\n\t}\n\treturn WEBTOOLS_DEFAULT_TIMEOUT_SECS;\n}\n\n// ============================================================================\n// Search provider credentials\n// ============================================================================\n\n/**\n * The `webtools.search` block of `~/.hoocode/settings.json`.\n *\n * hoocode and the binary share that file: the binary reads its own `webtools`\n * key (snake_case, per its own schema) and ignores everything else, so these\n * keys are mirrored verbatim rather than camelCased. hoocode never writes them\n * — it only reads them to tell whether `websearch` has a keyed backend.\n */\nexport interface WebtoolsSearchSettings {\n\t/** Primary backend: \"duckduckgo\" | \"brave\" | \"tavily\" | \"searxng\". */\n\tprovider?: string;\n\t/** Backend tried when the primary fails; \"none\" disables the fallback. */\n\tfallback?: string;\n\tproviders?: {\n\t\tbrave?: { api_key?: string };\n\t\ttavily?: { api_key?: string };\n\t\tsearxng?: { base_url?: string; api_key?: string };\n\t};\n}\n\n/** A search backend that answers over an API contract instead of scraped HTML. */\nexport type KeyedSearchProvider = \"brave\" | \"tavily\" | \"searxng\";\n\nexport interface WebSearchCredentialStatus {\n\t/** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */\n\tconfigured: boolean;\n\t/** Which backend the credential belongs to, when one is configured. */\n\tprovider?: KeyedSearchProvider;\n\t/** Where the credential came from — env wins over the settings file. */\n\tsource?: \"env\" | \"settings\";\n\t/** The user explicitly asked for the keyless backend, so nothing is missing. */\n\texplicitKeyless?: boolean;\n}\n\n/** Env var names per keyed provider, in the precedence the binary applies. */\nconst SEARCH_CREDENTIAL_ENV: ReadonlyArray<{ provider: KeyedSearchProvider; vars: readonly string[] }> = [\n\t{ provider: \"brave\", vars: [\"WEBTOOLS_BRAVE_API_KEY\", \"BRAVE_API_KEY\"] },\n\t{ provider: \"tavily\", vars: [\"WEBTOOLS_TAVILY_API_KEY\", \"TAVILY_API_KEY\"] },\n\t// SearXNG is self-hosted: the endpoint is the credential, its key optional.\n\t{ provider: \"searxng\", vars: [\"WEBTOOLS_SEARXNG_URL\"] },\n];\n\nfunction hasText(value: string | undefined): boolean {\n\treturn typeof value === \"string\" && value.trim().length > 0;\n}\n\n/**\n * Whether `websearch` has a keyed backend configured, and where it came from.\n *\n * Mirrors the binary's own resolution order (env over settings file) for the\n * three keyed backends. This is a read-only check used to decide whether to\n * tell the user that search is running on keyless DuckDuckGo — it never\n * returns the credential itself, so a key cannot leak into the UI or a log.\n *\n * A provider pinned to `duckduckgo` (env or settings) is reported as\n * `explicitKeyless`: the user chose the scraped backend, so nothing is missing.\n */\nexport function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus {\n\tconst pinned = (process.env.WEBTOOLS_SEARCH_PROVIDER ?? search?.provider)?.trim().toLowerCase();\n\tif (pinned === \"duckduckgo\") {\n\t\treturn { configured: false, explicitKeyless: true };\n\t}\n\n\tfor (const { provider, vars } of SEARCH_CREDENTIAL_ENV) {\n\t\tif (vars.some((name) => hasText(process.env[name]))) {\n\t\t\treturn { configured: true, provider, source: \"env\" };\n\t\t}\n\t}\n\n\tconst providers = search?.providers;\n\tif (hasText(providers?.brave?.api_key)) return { configured: true, provider: \"brave\", source: \"settings\" };\n\tif (hasText(providers?.tavily?.api_key)) return { configured: true, provider: \"tavily\", source: \"settings\" };\n\tif (hasText(providers?.searxng?.base_url)) return { configured: true, provider: \"searxng\", source: \"settings\" };\n\n\treturn { configured: false };\n}\n\n// Warn at most once per distinct message for the life of the process.\nconst warnedWebtoolsMessages = new Set<string>();\nfunction warnOnce(message: string): void {\n\tif (warnedWebtoolsMessages.has(message)) return;\n\twarnedWebtoolsMessages.add(message);\n\tconsole.warn(chalk.yellow(`[webtools] ${message}`));\n}\n\n/** True only when `path` is a readable regular file; warns once and returns false otherwise. */\nfunction isReadableFile(path: string): boolean {\n\ttry {\n\t\tif (!statSync(path).isFile()) {\n\t\t\twarnOnce(`--ca-cert path is not a regular file, ignoring: ${path}`);\n\t\t\treturn false;\n\t\t}\n\t\taccessSync(path, constants.R_OK);\n\t\treturn true;\n\t} catch (error) {\n\t\tconst reason = error instanceof Error ? error.message : String(error);\n\t\twarnOnce(`--ca-cert path is not readable, ignoring: ${path} (${reason})`);\n\t\treturn false;\n\t}\n}\n\n/** Build the TLS-related argv flags forwarded to the binary (argv array, no shell). */\nfunction buildTLSArgs(config: WebtoolsTLSConfig | undefined): string[] {\n\tconst flags: string[] = [];\n\tif (!config) return flags;\n\tif (config.caCertPath && isReadableFile(config.caCertPath)) {\n\t\tflags.push(\"--ca-cert\", config.caCertPath);\n\t}\n\tif (config.insecure) {\n\t\twarnOnce(\n\t\t\t\"webtools running with --insecure: TLS verification is DISABLED for webfetch/websearch. \" +\n\t\t\t\t\"Prefer HOOCODE_WEBTOOLS_CA_CERT to trust your proxy's CA with verification kept on.\",\n\t\t);\n\t\tflags.push(\"--insecure\");\n\t}\n\treturn flags;\n}\n\n/**\n * The binary bounds a whole fetch (redirects + retries) at this multiple of the\n * per-request `--timeout`, so the spawn must outlive that or we kill a fetch the\n * binary would have finished. Search has no such budget, but it may try a\n * fallback provider after the primary fails, so it gets two requests' worth.\n */\nconst WHOLE_RUN_TIMEOUT_MULTIPLIER: Record<\"fetch\" | \"search\", number> = {\n\tfetch: 3,\n\tsearch: 2,\n};\n\n/** Extra wall-clock headroom (seconds) so the binary reports its own timeout before we kill it. */\nconst SPAWN_TIMEOUT_HEADROOM_SECS = 5;\n\n/**\n * Turn a non-zero exit into the most specific message available. `search` exits\n * non-zero on a blocked provider but writes its JSON (carrying `status`) to\n * stdout with nothing on stderr, so the generic \"exited with code 1\" would throw\n * away the only useful detail.\n */\nfunction describeFailedRun(subcommand: \"fetch\" | \"search\", stdout: string, stderr: string, code: number): string {\n\tconst trimmedStderr = stderr.trim();\n\tif (trimmedStderr) return trimmedStderr;\n\n\ttry {\n\t\tconst parsed = JSON.parse(stdout) as { status?: string };\n\t\tif (parsed?.status === \"blocked\") {\n\t\t\treturn (\n\t\t\t\t\"web search was blocked by the provider (bot challenge or rate limit) rather than returning no results — \" +\n\t\t\t\t\"retry later, or configure a different search provider\"\n\t\t\t);\n\t\t}\n\t} catch {\n\t\t// Not JSON: fall through to the generic message.\n\t}\n\treturn `webtools ${subcommand} exited with code ${code}`;\n}\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the\n * status carried on stdout when stderr is empty), or unparseable output. Callers\n * convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttlsConfig?: WebtoolsTLSConfig,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn headroom over the binary's own worst-case runtime so the\n\t// binary reports the timeout itself rather than being killed mid-flight.\n\tconst wholeRunSecs = timeoutSecs * WHOLE_RUN_TIMEOUT_MULTIPLIER[subcommand];\n\tconst spawnTimeoutMs = (wholeRunSecs + SPAWN_TIMEOUT_HEADROOM_SECS) * 1000;\n\tconst tlsArgs = buildTLSArgs(tlsConfig);\n\t// `--timeout` must be forwarded: without it the binary falls back to its own\n\t// default and the resolved setting/env value would never reach the request.\n\tconst result = await execCommand(\n\t\tbinaryPath,\n\t\t[subcommand, ...args, \"--timeout\", String(timeoutSecs), ...tlsArgs, \"--json\"],\n\t\tcwd,\n\t\t{\n\t\t\tsignal,\n\t\t\ttimeout: spawnTimeoutMs,\n\t\t},\n\t);\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${wholeRunSecs}s`);\n\tif (result.code !== 0) {\n\t\tthrow new Error(describeFailedRun(subcommand, result.stdout, result.stderr, result.code));\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}

package/docs/settings.md

@@ -352,6 +352,25 @@ Environment variables:
 On Android/Termux the published Linux builds do not run; install with
 `pkg install <name>` instead.
 
+#### Mapping a long page
+
+`webfetch(url, outline: true)` returns the page's headings instead of its body —
+each with the offset that reads its section and what that section costs:
+
+```
+  Installation — offset 0, ~143 tokens
+  Configuration — offset 557, ~145 tokens
+Read a section by fetching it at the offset shown.
+```
+
+A page maps for a few dozen tokens whatever its length, and the one section
+worth reading is then fetched at its own offset. Outline offsets are the same
+offsets paging uses, so a row feeds straight back into `offset`.
+
+Needs a `webtools` binary that supports `--outline` (v0.5.0 or newer). An older
+one rejects the flag, and hoocode reports that the binary needs updating rather
+than passing the parser message through.
+
 #### Reading a long page
 
 `webfetch` budgets its output in tokens (`maxTokens`, default 4000, hard cap

package/examples/extensions/custom-provider-anthropic/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-anthropic",
 	"private": true,
-	"version": "0.3.45",
+	"version": "0.3.46",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/custom-provider-gitlab-duo/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-custom-provider-gitlab-duo",
 	"private": true,
-	"version": "0.3.45",
+	"version": "0.3.46",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/sandbox/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-sandbox",
 	"private": true,
-	"version": "0.3.45",
+	"version": "0.3.46",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/examples/extensions/with-deps/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kolisachint/hoocode-extension-with-deps",
 	"private": true,
-	"version": "0.3.45",
+	"version": "0.3.46",
 	"type": "module",
 	"engines": {
 		"bun": ">=1.0.0"

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kolisachint/hoocode-agent",
-	"version": "0.5.45",
+	"version": "0.5.46",
 	"description": "Coding agent CLI with read, bash, edit, write tools and session management",
 	"type": "module",
 	"hoocodeConfig": {
@@ -50,9 +50,9 @@
 		"prepublishOnly": "npm run clean && npm run build"
 	},
 	"dependencies": {
-		"@kolisachint/hoocode-agent-core": "^0.5.45",
-		"@kolisachint/hoocode-ai": "^0.5.45",
-		"@kolisachint/hoocode-tui": "^0.5.45",
+		"@kolisachint/hoocode-agent-core": "^0.5.46",
+		"@kolisachint/hoocode-ai": "^0.5.46",
+		"@kolisachint/hoocode-tui": "^0.5.46",
 		"@silvia-odwyer/photon-node": "^0.3.4",
 		"chalk": "^5.5.0",
 		"cli-highlight": "^2.1.11",