@kolisachint/hoocode-agent 0.4.83 → 0.4.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/core/tools/webfetch.d.ts +2 -0
- package/dist/core/tools/webfetch.d.ts.map +1 -1
- package/dist/core/tools/webfetch.js +13 -8
- package/dist/core/tools/webfetch.js.map +1 -1
- package/dist/core/tools/websearch.d.ts.map +1 -1
- package/dist/core/tools/websearch.js +4 -8
- package/dist/core/tools/websearch.js.map +1 -1
- package/dist/core/tools/webtools-shared.d.ts +11 -0
- package/dist/core/tools/webtools-shared.d.ts.map +1 -1
- package/dist/core/tools/webtools-shared.js +81 -5
- package/dist/core/tools/webtools-shared.js.map +1 -1
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package.json +1 -1
- package/package.json +4 -4
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,8 @@ import type { AgentTool } from "@kolisachint/hoocode-agent-core";
|
|
|
2
2
|
import { type Static, Type } from "typebox";
|
|
3
3
|
import type { ToolDefinition } from "../extensions/types.js";
|
|
4
4
|
import { type WebFetchResult, WebToolsCache } from "./webtools-shared.js";
|
|
5
|
+
/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */
|
|
6
|
+
export declare function clampMaxTokens(requested?: number): number;
|
|
5
7
|
declare const webfetchSchema: Type.TObject<{
|
|
6
8
|
url: Type.TString;
|
|
7
9
|
maxTokens: Type.TOptional<Type.TNumber>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAIN,KAAK,cAAc,EACnB,aAAa,EACb,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAIN,KAAK,cAAc,EACnB,aAAa,EACb,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;EAalB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AAE9D,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACtC;AAiCD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAyDxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebFetchResult,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n});\n\nexport type WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\tcontentType?: string;\n\tmedia?: string;\n}\n\nexport interface WebFetchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\t// External network call with variable latency: run non-blocking so the\n\t\t// agent keeps reasoning while the page is fetched.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { url, maxTokens, output }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + result.content }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
|
|
@@ -6,10 +6,19 @@ import { getTextOutput, invalidArgText, str } from "./render-utils.js";
|
|
|
6
6
|
import { wrapToolDefinition } from "./tool-definition-wrapper.js";
|
|
7
7
|
import { blockedHostForUrl, runWebtools, WEBTOOLS_DEFAULT_TIMEOUT_SECS, WebToolsCache, } from "./webtools-shared.js";
|
|
8
8
|
const DEFAULT_MAX_TOKENS = 4000;
|
|
9
|
+
// Hard ceiling so a single fetch can never flood the context window, regardless
|
|
10
|
+
// of what the model requests. The binary still applies its own soft cap.
|
|
11
|
+
const MAX_TOKENS_CAP = 25000;
|
|
12
|
+
/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */
|
|
13
|
+
export function clampMaxTokens(requested) {
|
|
14
|
+
if (!requested || requested <= 0)
|
|
15
|
+
return DEFAULT_MAX_TOKENS;
|
|
16
|
+
return Math.min(requested, MAX_TOKENS_CAP);
|
|
17
|
+
}
|
|
9
18
|
const webfetchSchema = Type.Object({
|
|
10
19
|
url: Type.String({ description: "The URL to fetch (http or https)" }),
|
|
11
20
|
maxTokens: Type.Optional(Type.Number({
|
|
12
|
-
description: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS})`,
|
|
21
|
+
description: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,
|
|
13
22
|
})),
|
|
14
23
|
output: Type.Optional(Type.Union([Type.Literal("text"), Type.Literal("markdown")], {
|
|
15
24
|
description: "Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).",
|
|
@@ -63,15 +72,11 @@ export function createWebFetchToolDefinition(cwd, options) {
|
|
|
63
72
|
if (blockedHost) {
|
|
64
73
|
throw new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);
|
|
65
74
|
}
|
|
66
|
-
const effectiveMaxTokens = maxTokens
|
|
75
|
+
const effectiveMaxTokens = clampMaxTokens(maxTokens);
|
|
67
76
|
const format = output ?? "text";
|
|
68
77
|
const cacheKey = `${format}:${effectiveMaxTokens}:${url}`;
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const args = ["--url", url, "--max-tokens", String(effectiveMaxTokens), "--output", format];
|
|
72
|
-
result = await runWebtools("fetch", args, cwd, signal, WEBTOOLS_DEFAULT_TIMEOUT_SECS);
|
|
73
|
-
cache.set(cacheKey, result);
|
|
74
|
-
}
|
|
78
|
+
const args = ["--url", url, "--max-tokens", String(effectiveMaxTokens), "--output", format];
|
|
79
|
+
const result = await cache.getOrCompute(cacheKey, signal, (sig) => runWebtools("fetch", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS));
|
|
75
80
|
const header = result.title ? `${result.title}\n${result.final_url}\n\n` : `${result.final_url}\n\n`;
|
|
76
81
|
return {
|
|
77
82
|
content: [{ type: "text", text: header + result.content }],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,WAAW,EACX,6BAA6B,EAE7B,aAAa,GACb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,GAAG;KACpG,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;CACD,CAAC,CAAC;AAiBH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;SAC3I;QACD,UAAU,EAAE,cAAc;QAC1B,uEAAuE;QACvE,mDAAmD;QACnD,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAqB,EAAE,MAAoB,EAAE;YAC/F,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,SAAS,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,kBAAkB,CAAC;YACvF,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,GAAG,EAAE,CAAC;YAE1D,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC5F,MAAM,GAAG,MAAM,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,6BAA6B,CAAC,CAAC;gBACtG,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7B,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnE,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;iBACnB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebFetchResult,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n});\n\nexport type WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\tcontentType?: string;\n\tmedia?: string;\n}\n\nexport interface WebFetchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\t// External network call with variable latency: run non-blocking so the\n\t\t// agent keeps reasoning while the page is fetched.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { url, maxTokens, output }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = maxTokens && maxTokens > 0 ? maxTokens : DEFAULT_MAX_TOKENS;\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${url}`;\n\n\t\t\tlet result = cache.get(cacheKey);\n\t\t\tif (!result) {\n\t\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t\tresult = await runWebtools<WebFetchResult>(\"fetch\", args, cwd, signal, WEBTOOLS_DEFAULT_TIMEOUT_SECS);\n\t\t\t\tcache.set(cacheKey, result);\n\t\t\t}\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + result.content }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
|
|
1
|
+
{"version":3,"file":"webfetch.js","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,iBAAiB,EACjB,WAAW,EACX,6BAA6B,EAE7B,aAAa,GACb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,cAAc,GAAG,KAAK,CAAC;AAE7B,wFAAwF;AACxF,MAAM,UAAU,cAAc,CAAC,SAAkB,EAAU;IAC1D,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,kBAAkB,CAAC;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAAA,CAC3C;AAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kCAAkC,EAAE,CAAC;IACrE,SAAS,EAAE,IAAI,CAAC,QAAQ,CACvB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,kEAAkE,kBAAkB,UAAU,cAAc,GAAG;KAC5H,CAAC,CACF;IACD,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE;QAC5D,WAAW,EACV,mIAAmI;KACpI,CAAC,CACF;CACD,CAAC,CAAC;AAiBH,SAAS,kBAAkB,CAAC,IAAmD,EAAU;IACxF,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAC1G,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtF,OAAO,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,GAAG,MAAM,CAAC;AAAA,CACzG;AAED,SAAS,oBAAoB,CAC5B,MAA0F,EAC1F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;IACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,aAAa,SAAS,CAAC,EAAE,CAAC;IACjE,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,4BAA4B,CAC3C,GAAW,EACX,OAA6B,EAC4C;IACzE,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAkB,CAAC;IACpE,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;QACjB,WAAW,EACV,wUAAwU;QACzU,aAAa,EAAE,4DAA4D;QAC3E,gBAAgB,EAAE;YACjB,2IAA2I;SAC3I;QACD,UAAU,EAAE,cAAc;QAC1B,uEAAuE;QACvE,mDAAmD;QACnD,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAqB,EAAE,MAAoB,EAAE;YAC/F,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sCAAsC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,IAAI,MAAM,CAAC;YAChC,MAAM,QAAQ,GAAG,GAAG,MAAM,IAAI,kBAAkB,IAAI,GAAG,EAAE,CAAC;YAE1D,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC5F,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAiB,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,6BAA6B,CAAC,CACnF,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,MAAM,CAAC;YACrG,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnE,OAAO,EAAE;oBACR,QAAQ,EAAE,MAAM,CAAC,SAAS;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,aAAa,EAAE,MAAM,CAAC,cAAc;oBACpC,WAAW,EAAE,MAAM,CAAC,YAAY;oBAChC,KAAK,EAAE,MAAM,CAAC,KAAK;iBACnB;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;YACvC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAA6B,EAAoC;IAChH,OAAO,kBAAkB,CAAC,4BAA4B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACtE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebFetchResult,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n});\n\nexport type WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\tcontentType?: string;\n\tmedia?: string;\n}\n\nexport interface WebFetchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}`;\n\t}\n\treturn text;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\t// External network call with variable latency: run non-blocking so the\n\t\t// agent keeps reasoning while the page is fetched.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { url, maxTokens, output }: WebFetchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\tconst result = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS),\n\t\t\t);\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + result.content }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"websearch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAMN,KAAK,eAAe,EAEpB,aAAa,EACb,MAAM,sBAAsB,CAAC;AAK9B,QAAA,MAAM,eAAe;;;;EAYnB,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,eAAe,CAAC,CAAC;AAEhE,MAAM,WAAW,oBAAoB;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACpC,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;CACvC;AAwDD,wBAAgB,6BAA6B,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,oBAAoB,GAC5B,cAAc,CAAC,OAAO,eAAe,EAAE,oBAAoB,GAAG,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"websearch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAE,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAGtF,OAAO,EAMN,KAAK,eAAe,EAEpB,aAAa,EACb,MAAM,sBAAsB,CAAC;AAK9B,QAAA,MAAM,eAAe;;;;EAYnB,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,eAAe,CAAC,CAAC;AAEhE,MAAM,WAAW,oBAAoB;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACpC,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;CACvC;AAwDD,wBAAgB,6BAA6B,CAC5C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,oBAAoB,GAC5B,cAAc,CAAC,OAAO,eAAe,EAAE,oBAAoB,GAAG,SAAS,CAAC,CAyD1E;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC,OAAO,eAAe,CAAC,CAElH","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\nexport type WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n}\n\nexport interface WebSearchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web (DuckDuckGo, no API key) and return ranked results as titles + snippets with reference-style [N] links. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\t// External network call with variable latency: run non-blocking.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}
|
|
@@ -81,14 +81,10 @@ export function createWebSearchToolDefinition(cwd, options) {
|
|
|
81
81
|
throw new Error("Operation aborted");
|
|
82
82
|
const effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));
|
|
83
83
|
const cacheKey = `${effectiveMax}:${safeSearch ?? "default"}:${query}`;
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
args.push("--safe-search", safeSearch);
|
|
89
|
-
output = await runWebtools("search", args, cwd, signal, WEBTOOLS_DEFAULT_TIMEOUT_SECS);
|
|
90
|
-
cache.set(cacheKey, output);
|
|
91
|
-
}
|
|
84
|
+
const args = ["--query", query, "--max-results", String(effectiveMax)];
|
|
85
|
+
if (safeSearch)
|
|
86
|
+
args.push("--safe-search", safeSearch);
|
|
87
|
+
const output = await cache.getOrCompute(cacheKey, signal, (sig) => runWebtools("search", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS));
|
|
92
88
|
// Filter result links through .webtoolsignore policy.
|
|
93
89
|
const matcher = loadWebtoolsIgnore(cwd);
|
|
94
90
|
const allResults = output.results ?? [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"websearch.js","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,6BAA6B,EAG7B,aAAa,GACb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,iDAAiD,mBAAmB,UAAU,eAAe,GAAG;KAC7G,CAAC,CACF;IACD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;QACrD,WAAW,EAAE,8DAA8D;KAC3E,CAAC,CACF;CACD,CAAC,CAAC;AAeH;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAA2B,EAAE,WAAmB,EAAU;IAClG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,qCAAqC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,mBAAmB,KAAK,IAAI,MAAM,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAAA,CAChC,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACvE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,IAAI,QAAQ,WAAW,UAAU,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oCAAoC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,mBAAmB,CAAC,IAAyD,EAAU;IAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,YAAY,GACjB,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACrG,MAAM,KAAK,GAAG,IAAI,EAAE,UAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,qBAAqB,CAC7B,MAA2F,EAC3F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,6BAA6B,CAC5C,GAAW,EACX,OAA8B,EAC6C;IAC3E,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAmB,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,WAAW,EACV,sOAAsO;QACvO,aAAa,EAAE,qDAAqD;QACpE,gBAAgB,EAAE;YACjB,6HAA6H;SAC7H;QACD,UAAU,EAAE,eAAe;QAC3B,iEAAiE;QACjE,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAsB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,IAAI,mBAAmB,CAAC,CAAC,CAAC;YAC/F,MAAM,QAAQ,GAAG,GAAG,YAAY,IAAI,UAAU,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;YAEvE,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;gBACvE,IAAI,UAAU;oBAAE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;gBACvD,MAAM,GAAG,MAAM,WAAW,CAAkB,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,6BAA6B,CAAC,CAAC;gBACxG,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC7B,CAAC;YAED,sDAAsD;YACtD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO;gBACnB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAA,CACnD,CAAC;gBACH,CAAC,CAAC,UAAU,CAAC;YACd,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAEpD,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;gBACtF,OAAO,EAAE;oBACR,WAAW,EAAE,IAAI,CAAC,MAAM;oBACxB,WAAW;oBACX,aAAa,EAAE,MAAM,CAAC,cAAc;iBACpC;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,OAA8B,EAAqC;IACnH,OAAO,kBAAkB,CAAC,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACvE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\nexport type WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n}\n\nexport interface WebSearchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web (DuckDuckGo, no API key) and return ranked results as titles + snippets with reference-style [N] links. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\t// External network call with variable latency: run non-blocking.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tlet output = cache.get(cacheKey);\n\t\t\tif (!output) {\n\t\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\t\toutput = await runWebtools<WebSearchOutput>(\"search\", args, cwd, signal, WEBTOOLS_DEFAULT_TIMEOUT_SECS);\n\t\t\t\tcache.set(cacheKey, output);\n\t\t\t}\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}
|
|
1
|
+
{"version":3,"file":"websearch.js","sourceRoot":"","sources":["../../../src/core/tools/websearch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAC;AAChD,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,wDAAwD,CAAC;AACjF,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,wCAAwC,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EACN,UAAU,EACV,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,6BAA6B,EAG7B,aAAa,GACb,MAAM,sBAAsB,CAAC;AAE9B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,kBAAkB,EAAE,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;QACX,WAAW,EAAE,iDAAiD,mBAAmB,UAAU,eAAe,GAAG;KAC7G,CAAC,CACF;IACD,UAAU,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;QACrD,WAAW,EAAE,8DAA8D;KAC3E,CAAC,CACF;CACD,CAAC,CAAC;AAeH;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAA2B,EAAE,WAAmB,EAAU;IAClG,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,qCAAqC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,OAAO,mBAAmB,KAAK,IAAI,MAAM,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAAA,CAChC,CAAC,CAAC;IACH,IAAI,IAAI,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACvE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,IAAI,QAAQ,WAAW,UAAU,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,oCAAoC,CAAC;IACvG,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,mBAAmB,CAAC,IAAyD,EAAU;IAC/F,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,MAAM,YAAY,GACjB,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACrG,MAAM,KAAK,GAAG,IAAI,EAAE,UAAU,CAAC;IAC/B,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvG,IAAI,KAAK,KAAK,SAAS;QAAE,IAAI,IAAI,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,KAAK,GAAG,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,qBAAqB,CAC7B,MAA2F,EAC3F,OAAgC,EAChC,UAAmB,EACV;IACT,MAAM,MAAM,GAAG,aAAa,CAAC,MAAa,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC;QAC1C,IAAI,IAAI,KAAK,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACtF,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,IAAI,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,SAAS,cAAc,CAAC,IAAI,OAAO,CAAC,kBAAkB,EAAE,WAAW,CAAC,GAAG,CAAC;QACnH,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,MAAM,UAAU,6BAA6B,CAC5C,GAAW,EACX,OAA8B,EAC6C;IAC3E,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,IAAI,aAAa,EAAmB,CAAC;IACrE,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK,EAAE,WAAW;QAClB,WAAW,EACV,sOAAsO;QACvO,aAAa,EAAE,qDAAqD;QACpE,gBAAgB,EAAE;YACjB,6HAA6H;SAC7H;QACD,UAAU,EAAE,eAAe;QAC3B,iEAAiE;QACjE,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAsB,EAAE,MAAoB,EAAE;YACvG,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAE1D,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,IAAI,mBAAmB,CAAC,CAAC,CAAC;YAC/F,MAAM,QAAQ,GAAG,GAAG,YAAY,IAAI,UAAU,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;YAEvE,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACvE,IAAI,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CACjE,WAAW,CAAkB,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,6BAA6B,CAAC,CACrF,CAAC;YAEF,sDAAsD;YACtD,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;YACxC,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,OAAO;gBACnB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC5B,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAClC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAA,CACnD,CAAC;gBACH,CAAC,CAAC,UAAU,CAAC;YACd,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAEpD,OAAO;gBACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;gBACtF,OAAO,EAAE;oBACR,WAAW,EAAE,IAAI,CAAC,MAAM;oBACxB,WAAW;oBACX,aAAa,EAAE,MAAM,CAAC,cAAc;iBACpC;aACD,CAAC;QAAA,CACF;QACD,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;YACjC,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;YACxC,OAAO,IAAI,CAAC;QAAA,CACZ;QACD,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;YAC9C,MAAM,IAAI,GAAI,OAAO,CAAC,aAAkC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,MAAa,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QAAA,CACZ;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,OAA8B,EAAqC;IACnH,OAAO,kBAAkB,CAAC,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AAAA,CACvE","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\thostnameOf,\n\tisHostBlocked,\n\tloadWebtoolsIgnore,\n\trunWebtools,\n\tWEBTOOLS_DEFAULT_TIMEOUT_SECS,\n\ttype WebSearchOutput,\n\ttype WebSearchResultItem,\n\tWebToolsCache,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_RESULTS = 5;\nconst MAX_RESULTS_CAP = 10;\n\nconst websearchSchema = Type.Object({\n\tquery: Type.String({ description: \"The search query\" }),\n\tmaxResults: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Maximum number of results to return (default: ${DEFAULT_MAX_RESULTS}, max: ${MAX_RESULTS_CAP})`,\n\t\t}),\n\t),\n\tsafeSearch: Type.Optional(\n\t\tType.Union([Type.Literal(\"on\"), Type.Literal(\"off\")], {\n\t\t\tdescription: \"Safe search filter. Omit to use the search engine's default.\",\n\t\t}),\n\t),\n});\n\nexport type WebSearchToolInput = Static<typeof websearchSchema>;\n\nexport interface WebSearchToolDetails {\n\tresultCount?: number;\n\thiddenCount?: number;\n\ttokenEstimate?: number;\n}\n\nexport interface WebSearchToolOptions {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebSearchOutput>;\n}\n\n/**\n * Render the kept results in reference style: title + snippet with a trailing\n * [N] marker, then a `References:` block. Indices are renumbered after filtering\n * so the visible list stays contiguous.\n */\nfunction renderSearchText(query: string, kept: WebSearchResultItem[], hiddenCount: number): string {\n\tif (kept.length === 0) {\n\t\tconst suffix = hiddenCount > 0 ? ` (${hiddenCount} blocked by .webtoolsignore policy)` : \"\";\n\t\treturn `No results for \"${query}\"${suffix}`;\n\t}\n\tconst blocks: string[] = [];\n\tconst refs: string[] = [];\n\tkept.forEach((item, i) => {\n\t\tconst n = i + 1;\n\t\tblocks.push(`${item.title} [${n}]\\n${item.snippet}`);\n\t\trefs.push(`[${n}] ${item.url}`);\n\t});\n\tlet text = `${blocks.join(\"\\n\\n\")}\\n\\nReferences:\\n${refs.join(\"\\n\")}`;\n\tif (hiddenCount > 0) {\n\t\ttext += `\\n\\n[${hiddenCount} result${hiddenCount === 1 ? \"\" : \"s\"} hidden by .webtoolsignore policy]`;\n\t}\n\treturn text;\n}\n\nfunction formatWebsearchCall(args: { query?: string; maxResults?: number } | undefined): string {\n\tconst query = str(args?.query);\n\tconst queryDisplay =\n\t\tquery === null ? invalidArgText(appTheme) : query ? `\"${query}\"` : appTheme.fg(\"toolOutput\", \"...\");\n\tconst limit = args?.maxResults;\n\tlet text = appTheme.fg(\"toolTitle\", appTheme.bold(\"websearch \")) + appTheme.fg(\"accent\", queryDisplay);\n\tif (limit !== undefined) text += appTheme.fg(\"muted\", ` (${limit})`);\n\treturn text;\n}\n\nfunction formatWebsearchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebSearchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : 15;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\treturn text;\n}\n\nexport function createWebSearchToolDefinition(\n\tcwd: string,\n\toptions?: WebSearchToolOptions,\n): ToolDefinition<typeof websearchSchema, WebSearchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebSearchOutput>();\n\treturn {\n\t\tname: \"websearch\",\n\t\tlabel: \"websearch\",\n\t\tdescription:\n\t\t\t\"Search the web (DuckDuckGo, no API key) and return ranked results as titles + snippets with reference-style [N] links. Use to discover URLs, then webfetch to read a result in full. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Search the web and return ranked results with links\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use websearch to discover URLs when you do not already have one, then webfetch the most relevant result to read it in full.\",\n\t\t],\n\t\tparameters: websearchSchema,\n\t\t// External network call with variable latency: run non-blocking.\n\t\tbackground: true,\n\t\tasync execute(_toolCallId, { query, maxResults, safeSearch }: WebSearchToolInput, signal?: AbortSignal) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\tconst effectiveMax = Math.min(MAX_RESULTS_CAP, Math.max(1, maxResults ?? DEFAULT_MAX_RESULTS));\n\t\t\tconst cacheKey = `${effectiveMax}:${safeSearch ?? \"default\"}:${query}`;\n\n\t\t\tconst args = [\"--query\", query, \"--max-results\", String(effectiveMax)];\n\t\t\tif (safeSearch) args.push(\"--safe-search\", safeSearch);\n\t\t\tconst output = await cache.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\trunWebtools<WebSearchOutput>(\"search\", args, cwd, sig, WEBTOOLS_DEFAULT_TIMEOUT_SECS),\n\t\t\t);\n\n\t\t\t// Filter result links through .webtoolsignore policy.\n\t\t\tconst matcher = loadWebtoolsIgnore(cwd);\n\t\t\tconst allResults = output.results ?? [];\n\t\t\tconst kept = matcher\n\t\t\t\t? allResults.filter((item) => {\n\t\t\t\t\t\tconst host = hostnameOf(item.url);\n\t\t\t\t\t\treturn host ? !isHostBlocked(matcher, host) : true;\n\t\t\t\t\t})\n\t\t\t\t: allResults;\n\t\t\tconst hiddenCount = allResults.length - kept.length;\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: renderSearchText(query, kept, hiddenCount) }],\n\t\t\t\tdetails: {\n\t\t\t\t\tresultCount: kept.length,\n\t\t\t\t\thiddenCount,\n\t\t\t\t\ttokenEstimate: output.token_estimate,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebsearchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebSearchTool(cwd: string, options?: WebSearchToolOptions): AgentTool<typeof websearchSchema> {\n\treturn wrapToolDefinition(createWebSearchToolDefinition(cwd, options));\n}\n"]}
|
|
@@ -63,8 +63,19 @@ export interface WebSearchOutput {
|
|
|
63
63
|
export declare function runWebtools<T>(subcommand: "fetch" | "search", args: string[], cwd: string, signal?: AbortSignal, timeoutSecs?: number): Promise<T>;
|
|
64
64
|
export declare class WebToolsCache<T> {
|
|
65
65
|
private readonly entries;
|
|
66
|
+
private readonly inflight;
|
|
66
67
|
get(key: string): T | undefined;
|
|
67
68
|
set(key: string, value: T): void;
|
|
69
|
+
/**
|
|
70
|
+
* Return a cached value, join an identical in-flight computation, or start a
|
|
71
|
+
* new one — collapsing concurrent duplicate fetch/search calls onto a single
|
|
72
|
+
* subprocess. Successful results are cached; failures are not.
|
|
73
|
+
*
|
|
74
|
+
* Cancellation is shared safely: a caller whose own `signal` aborts rejects
|
|
75
|
+
* promptly and releases its reference, but the underlying work keeps running
|
|
76
|
+
* for the remaining callers and is only cancelled once none are left.
|
|
77
|
+
*/
|
|
78
|
+
getOrCompute(key: string, signal: AbortSignal | undefined, compute: (signal: AbortSignal) => Promise<T>): Promise<T>;
|
|
68
79
|
}
|
|
69
80
|
/**
|
|
70
81
|
* Build an {@link Ignore} matcher from `.webtoolsignore` policy files.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAE/C,8DAA8D;AAC9D,eAAO,MAAM,6BAA6B,KAAK,CAAC;AAShD,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACrB;AASD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,GACjD,OAAO,CAAC,CAAC,CAAC,CA0BZ;AAWD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAE5D,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;CACD;AAMD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAkBzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache, and\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n * hosts both before a fetch and when filtering search result links.\n */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nexport const WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\nexport interface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\nexport interface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\nexport interface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr), or\n * unparseable output. Callers convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn a little headroom over the binary's own request timeout so\n\t// the binary reports the timeout itself rather than being killed mid-flight.\n\tconst spawnTimeoutMs = (timeoutSecs + 5) * 1000;\n\tconst result = await execCommand(binaryPath, [subcommand, ...args, \"--json\"], cwd, {\n\t\tsignal,\n\t\ttimeout: spawnTimeoutMs,\n\t});\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${timeoutSecs}s`);\n\tif (result.code !== 0) {\n\t\tconst stderr = result.stderr.trim();\n\t\tthrow new Error(stderr || `webtools ${subcommand} exited with code ${result.code}`);\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\treturn found ? ig : undefined;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"webtools-shared.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAK5B,KAAK,aAAa,GAAG,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC;AAE/C,8DAA8D;AAC9D,eAAO,MAAM,6BAA6B,KAAK,CAAC;AAShD,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,UAAU,EAAE,kBAAkB,EAAE,CAAC;IACjC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACrB;AASD;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,UAAU,EAAE,OAAO,GAAG,QAAQ,EAC9B,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,WAAW,GAAE,MAAsC,GACjD,OAAO,CAAC,CAAC,CAAC,CA0BZ;AAsBD,qBAAa,aAAa,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;IAEhE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAQ9B;IAED,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAE/B;IAED;;;;;;;;OAQG;IACG,YAAY,CACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1C,OAAO,CAAC,CAAC,CAAC,CA6CZ;CACD;AAiCD;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CA2BzE;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAO1D;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAG3E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM9E","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache, and\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n * hosts both before a fetch and when filtering search result links.\n */\n\nimport { existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nexport const WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\nexport interface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\nexport interface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\nexport interface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr), or\n * unparseable output. Callers convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn a little headroom over the binary's own request timeout so\n\t// the binary reports the timeout itself rather than being killed mid-flight.\n\tconst spawnTimeoutMs = (timeoutSecs + 5) * 1000;\n\tconst result = await execCommand(binaryPath, [subcommand, ...args, \"--json\"], cwd, {\n\t\tsignal,\n\t\ttimeout: spawnTimeoutMs,\n\t});\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${timeoutSecs}s`);\n\tif (result.code !== 0) {\n\t\tconst stderr = result.stderr.trim();\n\t\tthrow new Error(stderr || `webtools ${subcommand} exited with code ${result.code}`);\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* - the `.webtoolsignore` policy matcher (gitignore semantics) used to block
|
|
11
11
|
* hosts both before a fetch and when filtering search result links.
|
|
12
12
|
*/
|
|
13
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
13
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
14
14
|
import { homedir } from "node:os";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import ignore from "ignore";
|
|
@@ -61,6 +61,7 @@ export async function runWebtools(subcommand, args, cwd, signal, timeoutSecs = W
|
|
|
61
61
|
}
|
|
62
62
|
export class WebToolsCache {
|
|
63
63
|
entries = new Map();
|
|
64
|
+
inflight = new Map();
|
|
64
65
|
get(key) {
|
|
65
66
|
const entry = this.entries.get(key);
|
|
66
67
|
if (!entry)
|
|
@@ -74,10 +75,78 @@ export class WebToolsCache {
|
|
|
74
75
|
set(key, value) {
|
|
75
76
|
this.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });
|
|
76
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Return a cached value, join an identical in-flight computation, or start a
|
|
80
|
+
* new one — collapsing concurrent duplicate fetch/search calls onto a single
|
|
81
|
+
* subprocess. Successful results are cached; failures are not.
|
|
82
|
+
*
|
|
83
|
+
* Cancellation is shared safely: a caller whose own `signal` aborts rejects
|
|
84
|
+
* promptly and releases its reference, but the underlying work keeps running
|
|
85
|
+
* for the remaining callers and is only cancelled once none are left.
|
|
86
|
+
*/
|
|
87
|
+
async getOrCompute(key, signal, compute) {
|
|
88
|
+
const cached = this.get(key);
|
|
89
|
+
if (cached !== undefined)
|
|
90
|
+
return cached;
|
|
91
|
+
if (signal?.aborted)
|
|
92
|
+
throw new Error("Operation aborted");
|
|
93
|
+
let entry = this.inflight.get(key);
|
|
94
|
+
if (!entry) {
|
|
95
|
+
const controller = new AbortController();
|
|
96
|
+
const promise = (async () => {
|
|
97
|
+
try {
|
|
98
|
+
const value = await compute(controller.signal);
|
|
99
|
+
this.set(key, value);
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
this.inflight.delete(key);
|
|
104
|
+
}
|
|
105
|
+
})();
|
|
106
|
+
entry = { promise, controller, refCount: 0 };
|
|
107
|
+
this.inflight.set(key, entry);
|
|
108
|
+
}
|
|
109
|
+
const joined = entry;
|
|
110
|
+
// Every joined caller (signalled or not) holds a reference; the shared work
|
|
111
|
+
// is cancelled only when an abort drops the count back to zero.
|
|
112
|
+
joined.refCount++;
|
|
113
|
+
if (!signal) {
|
|
114
|
+
return joined.promise;
|
|
115
|
+
}
|
|
116
|
+
const onAbort = () => {
|
|
117
|
+
if (joined.refCount > 0)
|
|
118
|
+
joined.refCount--;
|
|
119
|
+
if (joined.refCount === 0)
|
|
120
|
+
joined.controller.abort();
|
|
121
|
+
};
|
|
122
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
123
|
+
try {
|
|
124
|
+
return await Promise.race([
|
|
125
|
+
joined.promise,
|
|
126
|
+
new Promise((_, reject) => {
|
|
127
|
+
signal.addEventListener("abort", () => reject(new Error("Operation aborted")), { once: true });
|
|
128
|
+
}),
|
|
129
|
+
]);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
signal.removeEventListener("abort", onAbort);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
const ignoreCacheByCwd = new Map();
|
|
137
|
+
function ignoreSignature(files) {
|
|
138
|
+
return files
|
|
139
|
+
.map((file) => {
|
|
140
|
+
try {
|
|
141
|
+
const st = statSync(file);
|
|
142
|
+
return `${file}:${st.mtimeMs}:${st.size}`;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
return `${file}:absent`;
|
|
146
|
+
}
|
|
147
|
+
})
|
|
148
|
+
.join("|");
|
|
77
149
|
}
|
|
78
|
-
// ============================================================================
|
|
79
|
-
// .webtoolsignore policy matcher
|
|
80
|
-
// ============================================================================
|
|
81
150
|
/**
|
|
82
151
|
* Build an {@link Ignore} matcher from `.webtoolsignore` policy files.
|
|
83
152
|
*
|
|
@@ -94,6 +163,11 @@ export function loadWebtoolsIgnore(cwd) {
|
|
|
94
163
|
join(homedir(), ".webtoolsignore"),
|
|
95
164
|
join(cwd, ".webtoolsignore"),
|
|
96
165
|
];
|
|
166
|
+
const signature = ignoreSignature(files);
|
|
167
|
+
const cached = ignoreCacheByCwd.get(cwd);
|
|
168
|
+
if (cached && cached.signature === signature) {
|
|
169
|
+
return cached.matcher;
|
|
170
|
+
}
|
|
97
171
|
let found = false;
|
|
98
172
|
const ig = ignore();
|
|
99
173
|
for (const file of files) {
|
|
@@ -107,7 +181,9 @@ export function loadWebtoolsIgnore(cwd) {
|
|
|
107
181
|
// Unreadable policy file: ignore it rather than failing the tool call.
|
|
108
182
|
}
|
|
109
183
|
}
|
|
110
|
-
|
|
184
|
+
const matcher = found ? ig : undefined;
|
|
185
|
+
ignoreCacheByCwd.set(cwd, { signature, matcher });
|
|
186
|
+
return matcher;
|
|
111
187
|
}
|
|
112
188
|
/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */
|
|
113
189
|
export function hostnameOf(url) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEhD,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAoDpC,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAEnJ;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACtC;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,4EAA4E;IAC5E,6EAA6E;IAC7E,MAAM,cAAc,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,QAAQ,CAAC,EAAE,GAAG,EAAE;QAClF,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,WAAW,GAAG,CAAC,CAAC;IAC7F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,YAAY,UAAU,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAWD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAE5D,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;CACD;AAED,+EAA+E;AAC/E,iCAAiC;AACjC,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IACF,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CAC9B;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache, and\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n * hosts both before a fetch and when filtering search result links.\n */\n\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nexport const WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\nexport interface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\nexport interface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\nexport interface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr), or\n * unparseable output. Callers convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn a little headroom over the binary's own request timeout so\n\t// the binary reports the timeout itself rather than being killed mid-flight.\n\tconst spawnTimeoutMs = (timeoutSecs + 5) * 1000;\n\tconst result = await execCommand(binaryPath, [subcommand, ...args, \"--json\"], cwd, {\n\t\tsignal,\n\t\ttimeout: spawnTimeoutMs,\n\t});\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${timeoutSecs}s`);\n\tif (result.code !== 0) {\n\t\tconst stderr = result.stderr.trim();\n\t\tthrow new Error(stderr || `webtools ${subcommand} exited with code ${result.code}`);\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\treturn found ? ig : undefined;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"webtools-shared.js","sourceRoot":"","sources":["../../../src/core/tools/webtools-shared.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,8BAA8B,CAAC;AAC1D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIzC,8DAA8D;AAC9D,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEhD,sFAAsF;AACtF,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAoDpC,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,sBAAsB,GAC3B,mJAAiJ,CAAC;AAEnJ;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAA8B,EAC9B,IAAc,EACd,GAAW,EACX,MAAoB,EACpB,WAAW,GAAW,6BAA6B,EACtC;IACb,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAEzD,4EAA4E;IAC5E,6EAA6E;IAC7E,MAAM,cAAc,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,GAAG,IAAI,EAAE,QAAQ,CAAC,EAAE,GAAG,EAAE;QAClF,MAAM;QACN,OAAO,EAAE,cAAc;KACvB,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,oBAAoB,WAAW,GAAG,CAAC,CAAC;IAC7F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,YAAY,UAAU,qBAAqB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAM,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,0BAA0B,CAAC,CAAC;IACnE,CAAC;AAAA,CACD;AAsBD,MAAM,OAAO,aAAa;IACR,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEhE,GAAG,CAAC,GAAW,EAAiB;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,OAAO,SAAS,CAAC;QAClB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IAAA,CACnB;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAQ;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC;IAAA,CACvE;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,MAA+B,EAC/B,OAA4C,EAC/B;QACb,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAE1D,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBACrB,OAAO,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACV,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;YAAA,CACD,CAAC,EAAE,CAAC;YACL,KAAK,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC;QACrB,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,CAAC,QAAQ,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,MAAM,CAAC,OAAO,CAAC;QACvB,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;YACrB,IAAI,MAAM,CAAC,QAAQ,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;gBAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAAA,CACrD,CAAC;QACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACJ,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;gBACzB,MAAM,CAAC,OAAO;gBACd,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;oBACjC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAA,CAC/F,CAAC;aACF,CAAC,CAAC;QACJ,CAAC;gBAAS,CAAC;YACV,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IAAA,CACD;CACD;AAkBD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAA4B,CAAC;AAE7D,SAAS,eAAe,CAAC,KAAe,EAAU;IACjD,OAAO,KAAK;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACd,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,GAAG,IAAI,SAAS,CAAC;QACzB,CAAC;IAAA,CACD,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAC;AAAA,CACZ;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAA6B;IAC1E,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,WAAW,EAAE,EAAE,gBAAgB,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,iBAAiB,CAAC;QAClC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC;KAC5B,CAAC;IAEF,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;IACvB,CAAC;IAED,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,IAAI,CAAC;YACJ,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACnC,KAAK,GAAG,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACR,uEAAuE;QACxE,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACvC,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC;AAAA,CACf;AAED,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,GAAW,EAAsB;IAC3D,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,IAAI,IAAI,SAAS,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AAAA,CACD;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,IAAY,EAAW;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AAAA,CAC7B;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,GAAW,EAAsB;IAC/E,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAAA,CACvD","sourcesContent":["/**\n * Shared plumbing for the `webfetch` and `websearch` tools.\n *\n * Both tools shell out to the `webtools` binary (fetch / search subcommands,\n * resolved/downloaded via {@link ensureTool}) and parse its `--json` output.\n * This module owns:\n * - the spawn-and-parse runner,\n * - the locked JSON result types,\n * - a short-lived in-process result cache, and\n * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block\n * hosts both before a fetch and when filtering search result links.\n */\n\nimport { existsSync, readFileSync, statSync } from \"node:fs\";\nimport { homedir } from \"node:os\";\nimport { join } from \"node:path\";\nimport ignore from \"ignore\";\nimport { getAgentDir } from \"../../config.js\";\nimport { ensureTool } from \"../../utils/tools-manager.js\";\nimport { execCommand } from \"../exec.js\";\n\ntype IgnoreMatcher = ReturnType<typeof ignore>;\n\n/** Default request timeout (seconds) passed to the binary. */\nexport const WEBTOOLS_DEFAULT_TIMEOUT_SECS = 15;\n\n/** How long a successful result stays cached, mirroring the documented 15-min TTL. */\nconst CACHE_TTL_MS = 15 * 60 * 1000;\n\n// ============================================================================\n// Result types (locked against `webtools <cmd> --json`)\n// ============================================================================\n\nexport interface WebFetchReference {\n\tindex: number;\n\turl: string;\n\ttext?: string;\n}\n\nexport interface WebFetchMetadata {\n\tdescription?: string;\n\tauthor?: string;\n\tpublished?: string;\n\tlang?: string;\n\tsite_name?: string;\n}\n\nexport interface WebFetchResult {\n\ttitle?: string;\n\tfinal_url: string;\n\tcontent: string;\n\tcontent_type: string;\n\tmedia: string;\n\ttoken_estimate: number;\n\treferences: WebFetchReference[];\n\tmetadata?: WebFetchMetadata;\n\tsource: string;\n}\n\nexport interface WebSearchResultItem {\n\ttitle: string;\n\tsnippet: string;\n\turl: string;\n\tref_index: number;\n}\n\nexport interface WebSearchReference {\n\tindex: number;\n\turl: string;\n}\n\nexport interface WebSearchOutput {\n\tquery: string;\n\tresults: WebSearchResultItem[];\n\treferences: WebSearchReference[];\n\ttoken_estimate: number;\n\tresult_count: number;\n}\n\n// ============================================================================\n// Binary runner\n// ============================================================================\n\nconst BINARY_MISSING_MESSAGE =\n\t\"webtools binary unavailable and could not be downloaded — web tools require the `webtools` CLI on PATH or a published release for this platform\";\n\n/**\n * Run a `webtools` subcommand with `--json` and return parsed stdout.\n *\n * Throws on missing binary, non-zero exit (surfacing the binary's stderr), or\n * unparseable output. Callers convert thrown errors into tool error results.\n */\nexport async function runWebtools<T>(\n\tsubcommand: \"fetch\" | \"search\",\n\targs: string[],\n\tcwd: string,\n\tsignal?: AbortSignal,\n\ttimeoutSecs: number = WEBTOOLS_DEFAULT_TIMEOUT_SECS,\n): Promise<T> {\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\tconst binaryPath = await ensureTool(\"webtools\", true);\n\tif (!binaryPath) throw new Error(BINARY_MISSING_MESSAGE);\n\n\t// Give the spawn a little headroom over the binary's own request timeout so\n\t// the binary reports the timeout itself rather than being killed mid-flight.\n\tconst spawnTimeoutMs = (timeoutSecs + 5) * 1000;\n\tconst result = await execCommand(binaryPath, [subcommand, ...args, \"--json\"], cwd, {\n\t\tsignal,\n\t\ttimeout: spawnTimeoutMs,\n\t});\n\n\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\tif (result.killed) throw new Error(`webtools ${subcommand} timed out after ${timeoutSecs}s`);\n\tif (result.code !== 0) {\n\t\tconst stderr = result.stderr.trim();\n\t\tthrow new Error(stderr || `webtools ${subcommand} exited with code ${result.code}`);\n\t}\n\n\ttry {\n\t\treturn JSON.parse(result.stdout) as T;\n\t} catch {\n\t\tthrow new Error(`webtools ${subcommand} returned malformed JSON`);\n\t}\n}\n\n// ============================================================================\n// Result cache (per-process, short TTL)\n// ============================================================================\n\ninterface CacheEntry<T> {\n\tvalue: T;\n\texpiresAt: number;\n}\n\n/**\n * A computation shared by every caller that requested the same key while it was\n * still running. The subprocess is only aborted once *all* joined callers have\n * aborted, tracked by {@link refCount} against a shared {@link controller}.\n */\ninterface InFlightEntry<T> {\n\tpromise: Promise<T>;\n\tcontroller: AbortController;\n\trefCount: number;\n}\n\nexport class WebToolsCache<T> {\n\tprivate readonly entries = new Map<string, CacheEntry<T>>();\n\tprivate readonly inflight = new Map<string, InFlightEntry<T>>();\n\n\tget(key: string): T | undefined {\n\t\tconst entry = this.entries.get(key);\n\t\tif (!entry) return undefined;\n\t\tif (Date.now() >= entry.expiresAt) {\n\t\t\tthis.entries.delete(key);\n\t\t\treturn undefined;\n\t\t}\n\t\treturn entry.value;\n\t}\n\n\tset(key: string, value: T): void {\n\t\tthis.entries.set(key, { value, expiresAt: Date.now() + CACHE_TTL_MS });\n\t}\n\n\t/**\n\t * Return a cached value, join an identical in-flight computation, or start a\n\t * new one — collapsing concurrent duplicate fetch/search calls onto a single\n\t * subprocess. Successful results are cached; failures are not.\n\t *\n\t * Cancellation is shared safely: a caller whose own `signal` aborts rejects\n\t * promptly and releases its reference, but the underlying work keeps running\n\t * for the remaining callers and is only cancelled once none are left.\n\t */\n\tasync getOrCompute(\n\t\tkey: string,\n\t\tsignal: AbortSignal | undefined,\n\t\tcompute: (signal: AbortSignal) => Promise<T>,\n\t): Promise<T> {\n\t\tconst cached = this.get(key);\n\t\tif (cached !== undefined) return cached;\n\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\tlet entry = this.inflight.get(key);\n\t\tif (!entry) {\n\t\t\tconst controller = new AbortController();\n\t\t\tconst promise = (async () => {\n\t\t\t\ttry {\n\t\t\t\t\tconst value = await compute(controller.signal);\n\t\t\t\t\tthis.set(key, value);\n\t\t\t\t\treturn value;\n\t\t\t\t} finally {\n\t\t\t\t\tthis.inflight.delete(key);\n\t\t\t\t}\n\t\t\t})();\n\t\t\tentry = { promise, controller, refCount: 0 };\n\t\t\tthis.inflight.set(key, entry);\n\t\t}\n\n\t\tconst joined = entry;\n\t\t// Every joined caller (signalled or not) holds a reference; the shared work\n\t\t// is cancelled only when an abort drops the count back to zero.\n\t\tjoined.refCount++;\n\n\t\tif (!signal) {\n\t\t\treturn joined.promise;\n\t\t}\n\n\t\tconst onAbort = () => {\n\t\t\tif (joined.refCount > 0) joined.refCount--;\n\t\t\tif (joined.refCount === 0) joined.controller.abort();\n\t\t};\n\t\tsignal.addEventListener(\"abort\", onAbort, { once: true });\n\t\ttry {\n\t\t\treturn await Promise.race([\n\t\t\t\tjoined.promise,\n\t\t\t\tnew Promise<never>((_, reject) => {\n\t\t\t\t\tsignal.addEventListener(\"abort\", () => reject(new Error(\"Operation aborted\")), { once: true });\n\t\t\t\t}),\n\t\t\t]);\n\t\t} finally {\n\t\t\tsignal.removeEventListener(\"abort\", onAbort);\n\t\t}\n\t}\n}\n\n// ============================================================================\n// .webtoolsignore policy matcher\n// ============================================================================\n\n/**\n * Memoize the parsed matcher per cwd. The policy is consulted on every webfetch\n * (twice: permission gate + tool execute) and every websearch, so re-reading and\n * re-parsing three files each time is wasted sync I/O on the hot path. The cache\n * is invalidated by a cheap stat signature (existence + mtime + size) so an\n * edited `.webtoolsignore` still takes effect immediately — correctness matters\n * here because this gate enforces host policy.\n */\ninterface IgnoreCacheEntry {\n\tsignature: string;\n\tmatcher: IgnoreMatcher | undefined;\n}\nconst ignoreCacheByCwd = new Map<string, IgnoreCacheEntry>();\n\nfunction ignoreSignature(files: string[]): string {\n\treturn files\n\t\t.map((file) => {\n\t\t\ttry {\n\t\t\t\tconst st = statSync(file);\n\t\t\t\treturn `${file}:${st.mtimeMs}:${st.size}`;\n\t\t\t} catch {\n\t\t\t\treturn `${file}:absent`;\n\t\t\t}\n\t\t})\n\t\t.join(\"|\");\n}\n\n/**\n * Build an {@link Ignore} matcher from `.webtoolsignore` policy files.\n *\n * Precedence is project-after-user so a project file can re-allow (`!host`)\n * something the user blocked, matching gitignore layering. Returns undefined\n * when no policy files exist (the common case: everything allowed).\n *\n * Hosts are matched as single path components, so subdomains need an explicit\n * wildcard (`*.example.com`), exactly like gitignore directory matching.\n */\nexport function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined {\n\tconst files = [\n\t\tjoin(getAgentDir(), \"webtoolsignore\"),\n\t\tjoin(homedir(), \".webtoolsignore\"),\n\t\tjoin(cwd, \".webtoolsignore\"),\n\t];\n\n\tconst signature = ignoreSignature(files);\n\tconst cached = ignoreCacheByCwd.get(cwd);\n\tif (cached && cached.signature === signature) {\n\t\treturn cached.matcher;\n\t}\n\n\tlet found = false;\n\tconst ig = ignore();\n\tfor (const file of files) {\n\t\tif (!existsSync(file)) continue;\n\t\ttry {\n\t\t\tig.add(readFileSync(file, \"utf8\"));\n\t\t\tfound = true;\n\t\t} catch {\n\t\t\t// Unreadable policy file: ignore it rather than failing the tool call.\n\t\t}\n\t}\n\tconst matcher = found ? ig : undefined;\n\tignoreCacheByCwd.set(cwd, { signature, matcher });\n\treturn matcher;\n}\n\n/** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */\nexport function hostnameOf(url: string): string | undefined {\n\ttry {\n\t\tconst host = new URL(url).hostname.toLowerCase();\n\t\treturn host || undefined;\n\t} catch {\n\t\treturn undefined;\n\t}\n}\n\n/**\n * Whether a host is blocked by policy. A matcher is required; with no policy\n * files present callers treat every host as allowed.\n */\nexport function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean {\n\tif (!host) return false;\n\treturn matcher.ignores(host);\n}\n\n/**\n * Convenience used by the permission gate: returns the blocked host for a URL,\n * or undefined when the URL is allowed (or there is no policy / unparseable URL).\n */\nexport function blockedHostForUrl(cwd: string, url: string): string | undefined {\n\tconst matcher = loadWebtoolsIgnore(cwd);\n\tif (!matcher) return undefined;\n\tconst host = hostnameOf(url);\n\tif (!host) return undefined;\n\treturn isHostBlocked(matcher, host) ? host : undefined;\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kolisachint/hoocode-agent",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.84",
|
|
4
4
|
"description": "Coding agent CLI with read, bash, edit, write tools and session management",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"hoocodeConfig": {
|
|
@@ -50,9 +50,9 @@
|
|
|
50
50
|
"prepublishOnly": "npm run clean && npm run build"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@kolisachint/hoocode-agent-core": "^0.4.
|
|
54
|
-
"@kolisachint/hoocode-ai": "^0.4.
|
|
55
|
-
"@kolisachint/hoocode-tui": "^0.4.
|
|
53
|
+
"@kolisachint/hoocode-agent-core": "^0.4.84",
|
|
54
|
+
"@kolisachint/hoocode-ai": "^0.4.84",
|
|
55
|
+
"@kolisachint/hoocode-tui": "^0.4.84",
|
|
56
56
|
"@silvia-odwyer/photon-node": "^0.3.4",
|
|
57
57
|
"chalk": "^5.5.0",
|
|
58
58
|
"cli-highlight": "^2.1.11",
|