@warlock.js/ai-tools 4.15.0 → 4.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/cjs/index.cjs +44 -14
- package/cjs/index.cjs.map +1 -1
- package/esm/contracts/http.type.d.mts +13 -1
- package/esm/contracts/http.type.d.mts.map +1 -1
- package/esm/contracts/web.type.d.mts +13 -1
- package/esm/contracts/web.type.d.mts.map +1 -1
- package/esm/errors.d.mts +10 -4
- package/esm/errors.d.mts.map +1 -1
- package/esm/errors.mjs.map +1 -1
- package/esm/http/http-request.d.mts +9 -1
- package/esm/http/http-request.d.mts.map +1 -1
- package/esm/http/http-request.mjs +25 -13
- package/esm/http/http-request.mjs.map +1 -1
- package/esm/web/fetch-url.d.mts +9 -1
- package/esm/web/fetch-url.d.mts.map +1 -1
- package/esm/web/fetch-url.mjs +21 -3
- package/esm/web/fetch-url.mjs.map +1 -1
- package/llms-full.txt +6 -2
- package/llms.txt +1 -1
- package/package.json +2 -2
- package/skills/use-web-and-http-tools/SKILL.md +6 -2
|
@@ -35,9 +35,21 @@ interface HttpRequestOptions {
|
|
|
35
35
|
allowMethods?: HttpMethod[];
|
|
36
36
|
/**
|
|
37
37
|
* Host allowlist, enforced before the request (an SSRF guardrail).
|
|
38
|
-
* When set, any other host is rejected.
|
|
38
|
+
* When set, any other host is rejected. Redirect targets are held to
|
|
39
|
+
* the same allowlist.
|
|
39
40
|
*/
|
|
40
41
|
allowHosts?: string[];
|
|
42
|
+
/**
|
|
43
|
+
* Permit requests to private / loopback / link-local / cloud-metadata
|
|
44
|
+
* addresses. Off by default: the tool refuses internal targets (and
|
|
45
|
+
* hostnames that resolve to them) even when `allowHosts` is unset, so
|
|
46
|
+
* a prompt-injected URL cannot reach `169.254.169.254` or
|
|
47
|
+
* `localhost`. Enable only for a tool that is meant to call an
|
|
48
|
+
* internal service.
|
|
49
|
+
*
|
|
50
|
+
* @default false
|
|
51
|
+
*/
|
|
52
|
+
allowPrivateNetwork?: boolean;
|
|
41
53
|
/** Static headers merged into every call (e.g. an API key). */
|
|
42
54
|
headers?: Record<string, string>;
|
|
43
55
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.type.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/contracts/http.type.ts"],"mappings":";;AAUA;;;;AAAsB;AAOtB;;;KAPY,UAAA;;;;;;UAOK,kBAAA;
|
|
1
|
+
{"version":3,"file":"http.type.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/contracts/http.type.ts"],"mappings":";;AAUA;;;;AAAsB;AAOtB;;;KAPY,UAAA;;;;;;UAOK,kBAAA;EAqCf;;;;;EA/BA,IAAA;EAiDe;;;;EA5Cf,OAAA;EAiDS;;;;;;EA1CT,YAAA,GAAe,UAAA;EA4DA;;;;;EAtDf,UAAA;EA0DS;;;;AAOA;;;;;;EAtDT,mBAAA;;EAEA,OAAA,GAAU,MAAM;;;;;;EAMhB,SAAA;;;;;;EAMA,QAAA;AAAA;;;;UAMe,gBAAA;;;;;EAKf,MAAA,GAAS,UAAA;;;;;EAKT,GAAA;;EAEA,OAAA,GAAU,MAAM;;;;;EAKhB,IAAA;AAAA;;;;UAMe,iBAAA;;EAEf,MAAA;;EAEA,OAAA,EAAS,MAAM;;;;;EAKf,IAAA;;EAEA,SAAA;AAAA"}
|
|
@@ -107,9 +107,21 @@ interface FetchUrlOptions {
|
|
|
107
107
|
extract?: FetchUrlExtract;
|
|
108
108
|
/**
|
|
109
109
|
* Host allowlist. When set, any request to a host not in this list is
|
|
110
|
-
* rejected before the network call (an SSRF guardrail).
|
|
110
|
+
* rejected before the network call (an SSRF guardrail). Redirect
|
|
111
|
+
* targets are held to the same allowlist.
|
|
111
112
|
*/
|
|
112
113
|
allowHosts?: string[];
|
|
114
|
+
/**
|
|
115
|
+
* Permit requests to private / loopback / link-local / cloud-metadata
|
|
116
|
+
* addresses. Off by default: the tool refuses internal targets (and
|
|
117
|
+
* hostnames that resolve to them) even when `allowHosts` is unset, so
|
|
118
|
+
* a prompt-injected URL cannot reach `169.254.169.254` or
|
|
119
|
+
* `localhost`. Enable only for a tool that is meant to call an
|
|
120
|
+
* internal service.
|
|
121
|
+
*
|
|
122
|
+
* @default false
|
|
123
|
+
*/
|
|
124
|
+
allowPrivateNetwork?: boolean;
|
|
113
125
|
}
|
|
114
126
|
/**
|
|
115
127
|
* The validated input shape the model passes to `fetch_url`.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.type.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/contracts/web.type.ts"],"mappings":";;AAaA;;;;AAA0B;AAK1B;;;;;;AAL0B,KAAd,cAAA;;;;UAKK,gBAAA;EA0BA;EAxBf,QAAA,EAAU,cAAc;;;AA4Bd;AAOZ;EA9BE,MAAA;;;;;;;EAOA,UAAA;EA+BK;AAMP;;;;EA/BE,IAAA;AAAA;;;;UAMe,cAAA;EA4CA;EA1Cf,KAAA;;EAEA,UAAU;AAAA;;;;;UAOK,mBAAA;
|
|
1
|
+
{"version":3,"file":"web.type.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/contracts/web.type.ts"],"mappings":";;AAaA;;;;AAA0B;AAK1B;;;;;;AAL0B,KAAd,cAAA;;;;UAKK,gBAAA;EA0BA;EAxBf,QAAA,EAAU,cAAc;;;AA4Bd;AAOZ;EA9BE,MAAA;;;;;;;EAOA,UAAA;EA+BK;AAMP;;;;EA/BE,IAAA;AAAA;;;;UAMe,cAAA;EA4CA;EA1Cf,KAAA;;EAEA,UAAU;AAAA;;;;;UAOK,mBAAA;EA0Ef;EAxEA,KAAA;EAwEmB;EAtEnB,GAAA;EA4E4B;EA1E5B,OAAA;EA4EG;EA1EH,KAAA;AAAA;;;;UAMe,eAAA;EA8Ef;EA5EA,OAAA,EAAS,mBAAmB;AAAA;;AAgFnB;;;;;;;;KApEC,eAAA;;;;UAKK,eAAA;;;;;;EAMf,IAAA;;;;;;EAMA,QAAA;;;;;;EAMA,SAAA;;;;;;EAMA,OAAA,GAAU,eAAe;;;;;;EAMzB,UAAA;;;;;;;;;;;EAWA,mBAAA;AAAA;;;;UAMe,aAAA;;EAEf,GAAG;AAAA;;;;UAMY,cAAA;;EAEf,GAAA;;EAEA,MAAA;;EAEA,OAAA;;EAEA,SAAA;AAAA"}
|
package/esm/errors.d.mts
CHANGED
|
@@ -136,8 +136,11 @@ declare class McpTransportError extends AIError {
|
|
|
136
136
|
* `npm install` string for the developer.
|
|
137
137
|
* - `"missing-key"` — no API key was supplied via options or the
|
|
138
138
|
* provider's environment variable.
|
|
139
|
-
* - `"denied-host"` — the requested URL
|
|
140
|
-
*
|
|
139
|
+
* - `"denied-host"` — the requested URL (or a redirect target) was
|
|
140
|
+
* refused by the outbound policy: a host outside the configured
|
|
141
|
+
* `allowHosts` allowlist, or a private / loopback / link-local /
|
|
142
|
+
* cloud-metadata address blocked by the default private-network deny
|
|
143
|
+
* (an SSRF guardrail).
|
|
141
144
|
* - `"invalid-url"` — the supplied URL could not be parsed, or used a
|
|
142
145
|
* non-`http(s)` scheme.
|
|
143
146
|
* - `"request-failed"` — the network call itself failed (DNS, connection
|
|
@@ -181,8 +184,11 @@ declare class WebToolError extends AIError {
|
|
|
181
184
|
*
|
|
182
185
|
* - `"method-not-allowed"` — the model requested an HTTP method that is
|
|
183
186
|
* not on the tool's `allowMethods` allowlist (defaults to `["GET"]`).
|
|
184
|
-
* - `"host-not-allowed"` — the resolved request host
|
|
185
|
-
*
|
|
187
|
+
* - `"host-not-allowed"` — the resolved request host (or a redirect
|
|
188
|
+
* target) was refused by the outbound policy: not on the tool's
|
|
189
|
+
* `allowHosts` allowlist, or a private / loopback / link-local /
|
|
190
|
+
* cloud-metadata address blocked by the default private-network deny
|
|
191
|
+
* (an SSRF guardrail).
|
|
186
192
|
* - `"invalid-url"` — the supplied URL (or its join with `baseUrl`)
|
|
187
193
|
* could not be parsed into an absolute `http(s)` URL.
|
|
188
194
|
*/
|
package/esm/errors.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.mts","names":[],"sources":["../../../../../../ai-tools/src/errors.ts"],"mappings":";;;;;AAUA;;;;AAA6B;AAM7B;KANY,iBAAA;;;;;KAMA,sBAAA,GAAyB,cAAA;EAEZ,kDAAvB,IAAA,EAAM,iBAAiB;AAAA;;;;;;;;;;;;;;;;AAyB4C;AAiBrE;;cArBa,eAAA,SAAwB,OAAA;EAqBV;EAAA,SAnBT,IAAA,EAAM,iBAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,sBAAA;AAAA;;;;;;;AA6BxB;AAmBvB;;KA/BY,eAAA;;;;;KAUA,oBAAA,GAAuB,cAAA;EAqBA,iDAnBjC,IAAA,EAAM,eAAe;AAAA;;;;;;AAuB4C;AAoBnE;;;;AAA+B;AAO/B;;;;;cA/Ba,aAAA,SAAsB,OAAA;EAiC3B;EAAA,SA/BU,IAAA,EAAM,eAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,oBAAA;AAAA;AAqD/C;;;;;;;;;;;;AAAA,KAjCY,mBAAA;;;;;AAuC2D;KAhC3D,wBAAA,GAA2B,cAAA;
|
|
1
|
+
{"version":3,"file":"errors.d.mts","names":[],"sources":["../../../../../../ai-tools/src/errors.ts"],"mappings":";;;;;AAUA;;;;AAA6B;AAM7B;KANY,iBAAA;;;;;KAMA,sBAAA,GAAyB,cAAA;EAEZ,kDAAvB,IAAA,EAAM,iBAAiB;AAAA;;;;;;;;;;;;;;;;AAyB4C;AAiBrE;;cArBa,eAAA,SAAwB,OAAA;EAqBV;EAAA,SAnBT,IAAA,EAAM,iBAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,sBAAA;AAAA;;;;;;;AA6BxB;AAmBvB;;KA/BY,eAAA;;;;;KAUA,oBAAA,GAAuB,cAAA;EAqBA,iDAnBjC,IAAA,EAAM,eAAe;AAAA;;;;;;AAuB4C;AAoBnE;;;;AAA+B;AAO/B;;;;;cA/Ba,aAAA,SAAsB,OAAA;EAiC3B;EAAA,SA/BU,IAAA,EAAM,eAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,oBAAA;AAAA;AAqD/C;;;;;;;;;;;;AAAA,KAjCY,mBAAA;;;;;AAuC2D;KAhC3D,wBAAA,GAA2B,cAAA;EA6Db,iDA3DxB,IAAA,EAAM,mBAAmB,EA2DD;EAzDxB,MAAA;AAAA;;;;;;;;AAsEoB;AAsBtB;;;;;;;;;;;cAtEa,iBAAA,SAA0B,OAAA;EA0ElB;EAAA,SAxEH,IAAA,EAAM,mBAAA;EAwEc;EAAA,SAtEpB,MAAA;cAEG,OAAA,UAAiB,OAAA,EAAS,wBAAA;AAAA;;;;AA0FlB;AAM7B;;;;;;;;AAEyB;AAwBzB;;;;;;;KA7FY,cAAA;;;;;KAWA,mBAAA,GAAsB,cAAA;EAsFa,gDApF7C,IAAA,EAAM,cAAc;AAAA;AAoF+C;;;;;;;;;;;;;;;;;;;AAAA,cA9DxD,YAAA,SAAqB,OAAA;;WAEhB,IAAA,EAAM,cAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,mBAAA;AAAA;;;;;;;;;;;;;;;KAsBnC,iBAAA;;;;;KAMA,sBAAA,GAAyB,cAAA;kDAEnC,IAAA,EAAM,iBAAiB;AAAA;;;;;;;;;;;;;;;;;;;;;;cAwBZ,eAAA,SAAwB,OAAA;;WAEnB,IAAA,EAAM,iBAAA;cAEH,OAAA,UAAiB,OAAA,EAAS,sBAAA;AAAA"}
|
package/esm/errors.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.mjs","names":[],"sources":["../../../../../../ai-tools/src/errors.ts"],"sourcesContent":["import { AIError, type AIErrorOptions } from \"@warlock.js/ai\";\n\n/**\n * Why the calculator rejected an expression.\n *\n * - `\"syntax\"` — the expression could not be tokenized or parsed\n * (an unknown character, a misplaced operator, unbalanced parens).\n * - `\"divide-by-zero\"` — evaluation divided (or took a modulo) by zero.\n * - `\"overflow\"` — the computed result was not a finite number.\n */\nexport type CalculatorFailure = \"syntax\" | \"divide-by-zero\" | \"overflow\";\n\n/**\n * Options for {@link CalculatorError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type CalculatorErrorOptions = AIErrorOptions & {\n /** Which class of calculator failure occurred. */\n type: CalculatorFailure;\n};\n\n/**\n * The `calculator` tool could not evaluate an expression — it was not\n * valid arithmetic, divided by zero, or overflowed to a non-finite\n * value.\n *\n * **Surface.** Thrown inside the tool handler, where `tool()` wraps it\n * into a `ToolExecutionError` whose message is preserved verbatim and\n * reaches the model as `{ error }` data, so the agent self-corrects\n * rather than crashing. Extends the framework {@link AIError} (category\n * `\"tool\"` via code `TOOL_EXEC_FAILED`) so it flows through the same\n * typed error contract as every other AI error; branch on `error.type`\n * for the specific failure.\n *\n * @example\n * if (error instanceof CalculatorError && error.type === \"divide-by-zero\") {\n * // the expression divided by zero — ask the model to revise it\n * }\n */\nexport class CalculatorError extends AIError {\n /** Which class of calculator failure occurred. */\n public readonly type: CalculatorFailure;\n\n public constructor(message: string, options: CalculatorErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"CalculatorError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why the date-time tool rejected a call.\n *\n * - `\"invalid-input\"` — a required field for the chosen `op` was\n * missing or malformed (an unparsable ISO instant, a bad `amount`).\n * - `\"invalid-unit\"` — `unit` was not one of the supported units.\n * - `\"invalid-time-zone\"` — the IANA time zone was not recognized.\n * - `\"unsupported-op\"` — the `op` was not one this tool implements.\n */\nexport type DateTimeFailure =\n | \"invalid-input\"\n | \"invalid-unit\"\n | \"invalid-time-zone\"\n | \"unsupported-op\";\n\n/**\n * Options for {@link DateTimeError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type DateTimeErrorOptions = AIErrorOptions & {\n /** Which class of date-time failure occurred. */\n type: DateTimeFailure;\n};\n\n/**\n * The `date_time` tool could not complete a call — a required field was\n * missing or malformed, the unit/time zone was unrecognized, or the\n * operation is unsupported.\n *\n * **Surface.** Thrown inside the tool handler, where `tool()` wraps it\n * into a `ToolExecutionError` whose message reaches the model as\n * `{ error }` data so the agent self-corrects. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`); branch\n * on `error.type` for the specific failure.\n *\n * @example\n * if (error instanceof DateTimeError && error.type === \"invalid-unit\") {\n * // the model passed an unknown unit — re-prompt with the allowed set\n * }\n */\nexport class DateTimeError extends AIError {\n /** Which class of date-time failure occurred. */\n public readonly type: DateTimeFailure;\n\n public constructor(message: string, options: DateTimeErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"DateTimeError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why an MCP transport operation failed.\n *\n * - `\"connect\"` — the transport could not be opened (child process\n * failed to spawn, HTTP endpoint unreachable) or the `initialize`\n * handshake failed.\n * - `\"protocol\"` — a malformed / unexpected JSON-RPC message, a\n * response that matched no in-flight request, or a missing field.\n * - `\"timeout\"` — a request exceeded its per-call deadline.\n * - `\"closed\"` — the transport was used after it was closed, or the\n * peer closed it mid-call.\n */\nexport type McpTransportFailure = \"connect\" | \"protocol\" | \"timeout\" | \"closed\";\n\n/**\n * Options for {@link McpTransportError} — the structured `type`\n * discriminator plus an optional JSON-RPC method name for branchable\n * diagnostics without parsing the message.\n */\nexport type McpTransportErrorOptions = AIErrorOptions & {\n /** Which class of transport failure occurred. */\n type: McpTransportFailure;\n /** The JSON-RPC method in flight when the failure occurred, if any. */\n method?: string;\n};\n\n/**\n * The MCP client's transport layer failed — it could not connect, the\n * peer spoke malformed JSON-RPC, a call timed out, or the transport was\n * already closed.\n *\n * **Surface.** Connection / handshake failures surface at\n * agent-construction time (the caller `await`s `client.tools()`). A\n * `tools/call` failure raised mid-run is wrapped by `tool()` into a\n * `ToolExecutionError` and reaches the model as `{ error }` data, so the\n * agent self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"`, code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * if (error instanceof McpTransportError && error.type === \"timeout\") {\n * // the remote call exceeded its deadline — retry or escalate\n * }\n */\nexport class McpTransportError extends AIError {\n /** Which class of transport failure occurred. */\n public readonly type: McpTransportFailure;\n /** The JSON-RPC method in flight when the failure occurred, if any. */\n public readonly method?: string;\n\n public constructor(message: string, options: McpTransportErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"McpTransportError\";\n this.type = options.type;\n this.method = options.method;\n }\n}\n\n/**\n * Why a web tool (`ai.tools.webSearch` / `ai.tools.fetchUrl`) failed\n * before or during a network call.\n *\n * - `\"missing-peer\"` — an optional peer dependency the chosen mode needs\n * (`@mozilla/readability` + `jsdom` for text/markdown extraction, a\n * search provider SDK) is not installed. The message carries a curated\n * `npm install` string for the developer.\n * - `\"missing-key\"` — no API key was supplied via options or the\n * provider's environment variable.\n * - `\"denied-host\"` — the requested URL's host is not in the configured\n * `allowHosts` allowlist (an SSRF guardrail), rejected before any fetch.\n * - `\"invalid-url\"` — the supplied URL could not be parsed, or used a\n * non-`http(s)` scheme.\n * - `\"request-failed\"` — the network call itself failed (DNS, connection\n * reset, timeout) or the provider returned a non-OK status.\n */\nexport type WebToolFailure =\n | \"missing-peer\"\n | \"missing-key\"\n | \"denied-host\"\n | \"invalid-url\"\n | \"request-failed\";\n\n/**\n * Options for {@link WebToolError} — the structured `type` discriminator\n * so a caller can branch without parsing the message.\n */\nexport type WebToolErrorOptions = AIErrorOptions & {\n /** Which class of web-tool failure occurred. */\n type: WebToolFailure;\n};\n\n/**\n * A web tool failed — a missing optional peer, an absent API key, a host\n * rejected by the `allowHosts` guardrail, an unparseable URL, or a failed\n * network call.\n *\n * **Surface.** Thrown from inside a tool's `execute`, so the framework's\n * `tool()` wrapper catches it and surfaces it in the returned `{ error }`\n * field (`invoke()` never throws) — the agent reads the failure as data\n * and self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * const { error } = await fetchTool.invoke({ url: \"http://evil.test\" });\n * if (error instanceof WebToolError && error.type === \"denied-host\") {\n * // the host was not in allowHosts — surfaced before any network call\n * }\n */\nexport class WebToolError extends AIError {\n /** Which class of web-tool failure occurred. */\n public readonly type: WebToolFailure;\n\n public constructor(message: string, options: WebToolErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"WebToolError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why an `http_request` call was rejected by its own guardrails, before\n * the network request was ever issued.\n *\n * - `\"method-not-allowed\"` — the model requested an HTTP method that is\n * not on the tool's `allowMethods` allowlist (defaults to `[\"GET\"]`).\n * - `\"host-not-allowed\"` — the resolved request host is not on the\n * tool's `allowHosts` allowlist (an SSRF guardrail).\n * - `\"invalid-url\"` — the supplied URL (or its join with `baseUrl`)\n * could not be parsed into an absolute `http(s)` URL.\n */\nexport type HttpPolicyFailure = \"method-not-allowed\" | \"host-not-allowed\" | \"invalid-url\";\n\n/**\n * Options for {@link HttpPolicyError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type HttpPolicyErrorOptions = AIErrorOptions & {\n /** Which class of policy rejection occurred. */\n type: HttpPolicyFailure;\n};\n\n/**\n * The `http_request` tool refused a call its construction-time policy\n * does not permit — a disallowed method, a host outside the allowlist,\n * or an unparseable URL. The rejection happens *before* any network\n * request, so a guarded tool can never be coaxed into reaching an\n * off-allowlist host (an SSRF guardrail).\n *\n * **Surface.** Thrown from inside the tool's `execute`, so the framework's\n * `tool()` wrapper catches it and surfaces it in the returned `{ error }`\n * field (`invoke()` never throws) — the agent reads the typed failure as\n * data and self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * const { error } = await httpTool.invoke({ url: \"https://evil.test\" });\n * if (error instanceof HttpPolicyError && error.type === \"host-not-allowed\") {\n * // the model tried to reach a host outside the configured allowlist\n * }\n */\nexport class HttpPolicyError extends AIError {\n /** Which class of policy rejection occurred. */\n public readonly type: HttpPolicyFailure;\n\n public constructor(message: string, options: HttpPolicyErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"HttpPolicyError\";\n this.type = options.type;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,kBAAb,cAAqC,QAAQ;CAI3C,AAAO,YAAY,SAAiB,SAAiC;EACnE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;AA0CA,IAAa,gBAAb,cAAmC,QAAQ;CAIzC,AAAO,YAAY,SAAiB,SAA+B;EACjE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;AA+CA,IAAa,oBAAb,cAAuC,QAAQ;CAM7C,AAAO,YAAY,SAAiB,SAAmC;EACrE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;EACpB,KAAK,SAAS,QAAQ;CACxB;AACF;;;;;;;;;;;;;;;;;;;;AAsDA,IAAa,eAAb,cAAkC,QAAQ;CAIxC,AAAO,YAAY,SAAiB,SAA8B;EAChE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;AA6CA,IAAa,kBAAb,cAAqC,QAAQ;CAI3C,AAAO,YAAY,SAAiB,SAAiC;EACnE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF"}
|
|
1
|
+
{"version":3,"file":"errors.mjs","names":[],"sources":["../../../../../../ai-tools/src/errors.ts"],"sourcesContent":["import { AIError, type AIErrorOptions } from \"@warlock.js/ai\";\n\n/**\n * Why the calculator rejected an expression.\n *\n * - `\"syntax\"` — the expression could not be tokenized or parsed\n * (an unknown character, a misplaced operator, unbalanced parens).\n * - `\"divide-by-zero\"` — evaluation divided (or took a modulo) by zero.\n * - `\"overflow\"` — the computed result was not a finite number.\n */\nexport type CalculatorFailure = \"syntax\" | \"divide-by-zero\" | \"overflow\";\n\n/**\n * Options for {@link CalculatorError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type CalculatorErrorOptions = AIErrorOptions & {\n /** Which class of calculator failure occurred. */\n type: CalculatorFailure;\n};\n\n/**\n * The `calculator` tool could not evaluate an expression — it was not\n * valid arithmetic, divided by zero, or overflowed to a non-finite\n * value.\n *\n * **Surface.** Thrown inside the tool handler, where `tool()` wraps it\n * into a `ToolExecutionError` whose message is preserved verbatim and\n * reaches the model as `{ error }` data, so the agent self-corrects\n * rather than crashing. Extends the framework {@link AIError} (category\n * `\"tool\"` via code `TOOL_EXEC_FAILED`) so it flows through the same\n * typed error contract as every other AI error; branch on `error.type`\n * for the specific failure.\n *\n * @example\n * if (error instanceof CalculatorError && error.type === \"divide-by-zero\") {\n * // the expression divided by zero — ask the model to revise it\n * }\n */\nexport class CalculatorError extends AIError {\n /** Which class of calculator failure occurred. */\n public readonly type: CalculatorFailure;\n\n public constructor(message: string, options: CalculatorErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"CalculatorError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why the date-time tool rejected a call.\n *\n * - `\"invalid-input\"` — a required field for the chosen `op` was\n * missing or malformed (an unparsable ISO instant, a bad `amount`).\n * - `\"invalid-unit\"` — `unit` was not one of the supported units.\n * - `\"invalid-time-zone\"` — the IANA time zone was not recognized.\n * - `\"unsupported-op\"` — the `op` was not one this tool implements.\n */\nexport type DateTimeFailure =\n | \"invalid-input\"\n | \"invalid-unit\"\n | \"invalid-time-zone\"\n | \"unsupported-op\";\n\n/**\n * Options for {@link DateTimeError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type DateTimeErrorOptions = AIErrorOptions & {\n /** Which class of date-time failure occurred. */\n type: DateTimeFailure;\n};\n\n/**\n * The `date_time` tool could not complete a call — a required field was\n * missing or malformed, the unit/time zone was unrecognized, or the\n * operation is unsupported.\n *\n * **Surface.** Thrown inside the tool handler, where `tool()` wraps it\n * into a `ToolExecutionError` whose message reaches the model as\n * `{ error }` data so the agent self-corrects. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`); branch\n * on `error.type` for the specific failure.\n *\n * @example\n * if (error instanceof DateTimeError && error.type === \"invalid-unit\") {\n * // the model passed an unknown unit — re-prompt with the allowed set\n * }\n */\nexport class DateTimeError extends AIError {\n /** Which class of date-time failure occurred. */\n public readonly type: DateTimeFailure;\n\n public constructor(message: string, options: DateTimeErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"DateTimeError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why an MCP transport operation failed.\n *\n * - `\"connect\"` — the transport could not be opened (child process\n * failed to spawn, HTTP endpoint unreachable) or the `initialize`\n * handshake failed.\n * - `\"protocol\"` — a malformed / unexpected JSON-RPC message, a\n * response that matched no in-flight request, or a missing field.\n * - `\"timeout\"` — a request exceeded its per-call deadline.\n * - `\"closed\"` — the transport was used after it was closed, or the\n * peer closed it mid-call.\n */\nexport type McpTransportFailure = \"connect\" | \"protocol\" | \"timeout\" | \"closed\";\n\n/**\n * Options for {@link McpTransportError} — the structured `type`\n * discriminator plus an optional JSON-RPC method name for branchable\n * diagnostics without parsing the message.\n */\nexport type McpTransportErrorOptions = AIErrorOptions & {\n /** Which class of transport failure occurred. */\n type: McpTransportFailure;\n /** The JSON-RPC method in flight when the failure occurred, if any. */\n method?: string;\n};\n\n/**\n * The MCP client's transport layer failed — it could not connect, the\n * peer spoke malformed JSON-RPC, a call timed out, or the transport was\n * already closed.\n *\n * **Surface.** Connection / handshake failures surface at\n * agent-construction time (the caller `await`s `client.tools()`). A\n * `tools/call` failure raised mid-run is wrapped by `tool()` into a\n * `ToolExecutionError` and reaches the model as `{ error }` data, so the\n * agent self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"`, code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * if (error instanceof McpTransportError && error.type === \"timeout\") {\n * // the remote call exceeded its deadline — retry or escalate\n * }\n */\nexport class McpTransportError extends AIError {\n /** Which class of transport failure occurred. */\n public readonly type: McpTransportFailure;\n /** The JSON-RPC method in flight when the failure occurred, if any. */\n public readonly method?: string;\n\n public constructor(message: string, options: McpTransportErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"McpTransportError\";\n this.type = options.type;\n this.method = options.method;\n }\n}\n\n/**\n * Why a web tool (`ai.tools.webSearch` / `ai.tools.fetchUrl`) failed\n * before or during a network call.\n *\n * - `\"missing-peer\"` — an optional peer dependency the chosen mode needs\n * (`@mozilla/readability` + `jsdom` for text/markdown extraction, a\n * search provider SDK) is not installed. The message carries a curated\n * `npm install` string for the developer.\n * - `\"missing-key\"` — no API key was supplied via options or the\n * provider's environment variable.\n * - `\"denied-host\"` — the requested URL (or a redirect target) was\n * refused by the outbound policy: a host outside the configured\n * `allowHosts` allowlist, or a private / loopback / link-local /\n * cloud-metadata address blocked by the default private-network deny\n * (an SSRF guardrail).\n * - `\"invalid-url\"` — the supplied URL could not be parsed, or used a\n * non-`http(s)` scheme.\n * - `\"request-failed\"` — the network call itself failed (DNS, connection\n * reset, timeout) or the provider returned a non-OK status.\n */\nexport type WebToolFailure =\n | \"missing-peer\"\n | \"missing-key\"\n | \"denied-host\"\n | \"invalid-url\"\n | \"request-failed\";\n\n/**\n * Options for {@link WebToolError} — the structured `type` discriminator\n * so a caller can branch without parsing the message.\n */\nexport type WebToolErrorOptions = AIErrorOptions & {\n /** Which class of web-tool failure occurred. */\n type: WebToolFailure;\n};\n\n/**\n * A web tool failed — a missing optional peer, an absent API key, a host\n * rejected by the `allowHosts` guardrail, an unparseable URL, or a failed\n * network call.\n *\n * **Surface.** Thrown from inside a tool's `execute`, so the framework's\n * `tool()` wrapper catches it and surfaces it in the returned `{ error }`\n * field (`invoke()` never throws) — the agent reads the failure as data\n * and self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * const { error } = await fetchTool.invoke({ url: \"http://evil.test\" });\n * if (error instanceof WebToolError && error.type === \"denied-host\") {\n * // the host was not in allowHosts — surfaced before any network call\n * }\n */\nexport class WebToolError extends AIError {\n /** Which class of web-tool failure occurred. */\n public readonly type: WebToolFailure;\n\n public constructor(message: string, options: WebToolErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"WebToolError\";\n this.type = options.type;\n }\n}\n\n/**\n * Why an `http_request` call was rejected by its own guardrails, before\n * the network request was ever issued.\n *\n * - `\"method-not-allowed\"` — the model requested an HTTP method that is\n * not on the tool's `allowMethods` allowlist (defaults to `[\"GET\"]`).\n * - `\"host-not-allowed\"` — the resolved request host (or a redirect\n * target) was refused by the outbound policy: not on the tool's\n * `allowHosts` allowlist, or a private / loopback / link-local /\n * cloud-metadata address blocked by the default private-network deny\n * (an SSRF guardrail).\n * - `\"invalid-url\"` — the supplied URL (or its join with `baseUrl`)\n * could not be parsed into an absolute `http(s)` URL.\n */\nexport type HttpPolicyFailure = \"method-not-allowed\" | \"host-not-allowed\" | \"invalid-url\";\n\n/**\n * Options for {@link HttpPolicyError} — the structured `type`\n * discriminator so a caller can branch without parsing the message.\n */\nexport type HttpPolicyErrorOptions = AIErrorOptions & {\n /** Which class of policy rejection occurred. */\n type: HttpPolicyFailure;\n};\n\n/**\n * The `http_request` tool refused a call its construction-time policy\n * does not permit — a disallowed method, a host outside the allowlist,\n * or an unparseable URL. The rejection happens *before* any network\n * request, so a guarded tool can never be coaxed into reaching an\n * off-allowlist host (an SSRF guardrail).\n *\n * **Surface.** Thrown from inside the tool's `execute`, so the framework's\n * `tool()` wrapper catches it and surfaces it in the returned `{ error }`\n * field (`invoke()` never throws) — the agent reads the typed failure as\n * data and self-corrects rather than crashing. Extends the framework\n * {@link AIError} (category `\"tool\"` via code `TOOL_EXEC_FAILED`) so it\n * flows through the same typed error contract as every other AI error;\n * branch on `error.type` for the specific failure.\n *\n * @example\n * const { error } = await httpTool.invoke({ url: \"https://evil.test\" });\n * if (error instanceof HttpPolicyError && error.type === \"host-not-allowed\") {\n * // the model tried to reach a host outside the configured allowlist\n * }\n */\nexport class HttpPolicyError extends AIError {\n /** Which class of policy rejection occurred. */\n public readonly type: HttpPolicyFailure;\n\n public constructor(message: string, options: HttpPolicyErrorOptions) {\n super(\"TOOL_EXEC_FAILED\", message, options);\n\n this.name = \"HttpPolicyError\";\n this.type = options.type;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAuCA,IAAa,kBAAb,cAAqC,QAAQ;CAI3C,AAAO,YAAY,SAAiB,SAAiC;EACnE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;AA0CA,IAAa,gBAAb,cAAmC,QAAQ;CAIzC,AAAO,YAAY,SAAiB,SAA+B;EACjE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;AA+CA,IAAa,oBAAb,cAAuC,QAAQ;CAM7C,AAAO,YAAY,SAAiB,SAAmC;EACrE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;EACpB,KAAK,SAAS,QAAQ;CACxB;AACF;;;;;;;;;;;;;;;;;;;;AAyDA,IAAa,eAAb,cAAkC,QAAQ;CAIxC,AAAO,YAAY,SAAiB,SAA8B;EAChE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF;;;;;;;;;;;;;;;;;;;;;;AAgDA,IAAa,kBAAb,cAAqC,QAAQ;CAI3C,AAAO,YAAY,SAAiB,SAAiC;EACnE,MAAM,oBAAoB,SAAS,OAAO;EAE1C,KAAK,OAAO;EACZ,KAAK,OAAO,QAAQ;CACtB;AACF"}
|
|
@@ -12,8 +12,16 @@ import { ToolContract } from "@warlock.js/ai";
|
|
|
12
12
|
* - **Method allowlist** — `allowMethods` (default `["GET"]`). A method
|
|
13
13
|
* outside the list is rejected with a typed
|
|
14
14
|
* {@link HttpPolicyError} (`type: "method-not-allowed"`).
|
|
15
|
+
* - **Private-network deny (default).** The request — and every redirect
|
|
16
|
+
* hop — goes through the framework's `guardedFetch` outbound policy,
|
|
17
|
+
* which refuses private / loopback / link-local / cloud-metadata
|
|
18
|
+
* addresses (and hostnames resolving to them) unless
|
|
19
|
+
* `allowPrivateNetwork: true` is set. This applies even when
|
|
20
|
+
* `allowHosts` is not configured, so a bare `ai.tools.http()` is not
|
|
21
|
+
* an SSRF primitive (`type: "host-not-allowed"`).
|
|
15
22
|
* - **Host allowlist** — when `allowHosts` is set, any other host is
|
|
16
|
-
* rejected (`type: "host-not-allowed"`), an SSRF guardrail
|
|
23
|
+
* rejected (`type: "host-not-allowed"`), an SSRF guardrail; redirect
|
|
24
|
+
* targets are held to the same allowlist.
|
|
17
25
|
* - **`baseUrl` join** — when configured, the model passes a path that
|
|
18
26
|
* is resolved against `baseUrl`; otherwise it must pass an absolute
|
|
19
27
|
* `http(s)` URL. An unresolvable URL is rejected
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-request.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/http/http-request.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"http-request.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/http/http-request.ts"],"mappings":";;;;;;;AAqNA;;;;;;;;;;;;;;;AAEmD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAFnC,eAAA,CACd,OAAA,GAAS,kBAAA,GACR,YAAA,CAAa,gBAAA,EAAkB,iBAAA"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { HttpPolicyError } from "../errors.mjs";
|
|
2
2
|
import { objectSchema, optionalStringEnumField, optionalStringRecordField, passthroughField, stringField } from "../schema.mjs";
|
|
3
|
-
import { tool } from "@warlock.js/ai";
|
|
3
|
+
import { OutboundPolicyError, guardedFetch, tool } from "@warlock.js/ai";
|
|
4
4
|
|
|
5
5
|
//#region ../ai-tools/src/http/http-request.ts
|
|
6
6
|
/** Default tool name exposed to the LLM. */
|
|
@@ -112,8 +112,16 @@ function isJsonContentType(contentType) {
|
|
|
112
112
|
* - **Method allowlist** — `allowMethods` (default `["GET"]`). A method
|
|
113
113
|
* outside the list is rejected with a typed
|
|
114
114
|
* {@link HttpPolicyError} (`type: "method-not-allowed"`).
|
|
115
|
+
* - **Private-network deny (default).** The request — and every redirect
|
|
116
|
+
* hop — goes through the framework's `guardedFetch` outbound policy,
|
|
117
|
+
* which refuses private / loopback / link-local / cloud-metadata
|
|
118
|
+
* addresses (and hostnames resolving to them) unless
|
|
119
|
+
* `allowPrivateNetwork: true` is set. This applies even when
|
|
120
|
+
* `allowHosts` is not configured, so a bare `ai.tools.http()` is not
|
|
121
|
+
* an SSRF primitive (`type: "host-not-allowed"`).
|
|
115
122
|
* - **Host allowlist** — when `allowHosts` is set, any other host is
|
|
116
|
-
* rejected (`type: "host-not-allowed"`), an SSRF guardrail
|
|
123
|
+
* rejected (`type: "host-not-allowed"`), an SSRF guardrail; redirect
|
|
124
|
+
* targets are held to the same allowlist.
|
|
117
125
|
* - **`baseUrl` join** — when configured, the model passes a path that
|
|
118
126
|
* is resolved against `baseUrl`; otherwise it must pass an absolute
|
|
119
127
|
* `http(s)` URL. An unresolvable URL is rejected
|
|
@@ -155,6 +163,7 @@ function httpRequestTool(options = {}) {
|
|
|
155
163
|
const allowMethods = options.allowMethods ?? ["GET"];
|
|
156
164
|
const allowedMethodSet = new Set(allowMethods);
|
|
157
165
|
const allowHostSet = options.allowHosts ? new Set(options.allowHosts) : void 0;
|
|
166
|
+
const allowPrivateNetwork = options.allowPrivateNetwork ?? false;
|
|
158
167
|
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
159
168
|
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
160
169
|
const staticHeaders = options.headers;
|
|
@@ -178,22 +187,25 @@ function httpRequestTool(options = {}) {
|
|
|
178
187
|
body = JSON.stringify(input.body);
|
|
179
188
|
if (!Object.keys(headers).some((key) => key.toLowerCase() === "content-type")) headers["content-type"] = "application/json";
|
|
180
189
|
}
|
|
181
|
-
const controller = new AbortController();
|
|
182
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
183
|
-
const onAbort = () => controller.abort();
|
|
184
|
-
if (ctx?.signal) if (ctx.signal.aborted) controller.abort();
|
|
185
|
-
else ctx.signal.addEventListener("abort", onAbort, { once: true });
|
|
186
190
|
let response;
|
|
187
191
|
try {
|
|
188
|
-
response = await
|
|
192
|
+
response = await guardedFetch(url.toString(), {
|
|
193
|
+
allowedSchemes: ["http", "https"],
|
|
194
|
+
hostAllowlist: options.allowHosts,
|
|
195
|
+
denyPrivateIPsAfterDNS: !allowPrivateNetwork,
|
|
196
|
+
timeoutMs,
|
|
197
|
+
signal: ctx?.signal
|
|
198
|
+
}, {
|
|
189
199
|
method,
|
|
190
200
|
headers,
|
|
191
|
-
body
|
|
192
|
-
signal: controller.signal
|
|
201
|
+
body
|
|
193
202
|
});
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
|
|
203
|
+
} catch (cause) {
|
|
204
|
+
if (cause instanceof OutboundPolicyError) throw new HttpPolicyError(`http_request blocked: ${cause.message}`, {
|
|
205
|
+
type: "host-not-allowed",
|
|
206
|
+
cause
|
|
207
|
+
});
|
|
208
|
+
throw cause;
|
|
197
209
|
}
|
|
198
210
|
const responseHeaders = {};
|
|
199
211
|
response.headers.forEach((value, key) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-request.mjs","names":[],"sources":["../../../../../../../ai-tools/src/http/http-request.ts"],"sourcesContent":["import { type ToolContract, tool } from \"@warlock.js/ai\";\nimport type {\n HttpMethod,\n HttpRequestInput,\n HttpRequestOptions,\n HttpRequestResult,\n} from \"../contracts\";\nimport { HttpPolicyError } from \"../errors\";\nimport {\n objectSchema,\n optionalStringEnumField,\n optionalStringRecordField,\n passthroughField,\n stringField,\n} from \"../schema\";\n\n/** Default tool name exposed to the LLM. */\nconst DEFAULT_NAME = \"http_request\";\n\n/** Default per-request wall-clock timeout, in milliseconds. */\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/** Default hard cap on response-body bytes before truncation. */\nconst DEFAULT_MAX_BYTES = 1_000_000;\n\n/** The full set of HTTP methods, in the order they appear in {@link HttpMethod}. */\nconst ALL_METHODS: readonly HttpMethod[] = [\"GET\", \"POST\", \"PUT\", \"PATCH\", \"DELETE\"];\n\n/** Methods that conventionally carry no request body — `body` is dropped for these. */\nconst BODYLESS_METHODS: ReadonlySet<HttpMethod> = new Set<HttpMethod>([\"GET\"]);\n\n/**\n * Standard Schema for {@link HttpRequestInput}. `url` is required;\n * `method` is constrained to the canonical HTTP verb set (further\n * narrowed to the tool's `allowMethods` at runtime); `headers` is an\n * optional string-to-string record; `body` is an opaque passthrough the\n * handler serializes based on its runtime type.\n */\nconst httpRequestInputSchema = objectSchema<HttpRequestInput>({\n method: optionalStringEnumField<HttpMethod>(ALL_METHODS),\n url: stringField(),\n headers: optionalStringRecordField(),\n body: passthroughField(),\n});\n\n/**\n * Resolve the request target. With a `baseUrl` configured the model\n * supplies a path joined against it; otherwise the model's `url` must be\n * an absolute `http(s)` URL. Throws a typed {@link HttpPolicyError} of\n * type `\"invalid-url\"` when the result cannot be parsed or is not an\n * `http`/`https` URL — surfaced as `{ error }` data, never a crash.\n */\nfunction resolveUrl(rawUrl: string, baseUrl: string | undefined): URL {\n let resolved: URL;\n\n try {\n // `new URL(input, base)` joins relative paths against `base` and\n // ignores `base` when `input` is already absolute, which is exactly\n // the \"path vs full URL\" behavior the design specifies.\n resolved = baseUrl !== undefined ? new URL(rawUrl, baseUrl) : new URL(rawUrl);\n } catch {\n throw new HttpPolicyError(\n `http_request could not resolve a valid URL from \"${rawUrl}\"` +\n (baseUrl !== undefined ? ` against base \"${baseUrl}\".` : \".\"),\n { type: \"invalid-url\" },\n );\n }\n\n if (resolved.protocol !== \"http:\" && resolved.protocol !== \"https:\") {\n throw new HttpPolicyError(\n `http_request only permits http(s) URLs; got \"${resolved.protocol}\".`,\n { type: \"invalid-url\" },\n );\n }\n\n return resolved;\n}\n\n/**\n * Read a `Response` body, capping at `maxBytes`. Returns the decoded text\n * and whether it was cut off. Streams chunk-by-chunk so an oversized body\n * is abandoned at the cap rather than fully buffered; falls back to\n * `response.text()` (then a post-hoc byte slice) when the body is not a\n * readable stream (e.g. a stubbed `Response` in tests).\n */\nasync function readCappedBody(\n response: Response,\n maxBytes: number,\n): Promise<{ text: string; truncated: boolean }> {\n const body = response.body;\n\n if (!body) {\n return { text: \"\", truncated: false };\n }\n\n const decoder = new TextDecoder();\n const reader = body.getReader();\n let received = 0;\n let truncated = false;\n let text = \"\";\n\n try {\n for (;;) {\n const { done, value } = await reader.read();\n\n if (done) {\n break;\n }\n\n if (!value) {\n continue;\n }\n\n const remaining = maxBytes - received;\n\n if (value.byteLength > remaining) {\n text += decoder.decode(value.subarray(0, remaining), { stream: true });\n received = maxBytes;\n truncated = true;\n break;\n }\n\n text += decoder.decode(value, { stream: true });\n received += value.byteLength;\n }\n } finally {\n // Release the lock and abandon any unread remainder.\n await reader.cancel().catch(() => undefined);\n reader.releaseLock();\n }\n\n text += decoder.decode();\n\n return { text, truncated };\n}\n\n/**\n * Decide whether a response's `content-type` indicates JSON. Matches\n * `application/json` and the `+json` structured-suffix convention\n * (e.g. `application/vnd.api+json`), case-insensitively.\n */\nfunction isJsonContentType(contentType: string | undefined): boolean {\n if (!contentType) {\n return false;\n }\n\n const value = contentType.toLowerCase();\n\n return value.includes(\"application/json\") || value.includes(\"+json\");\n}\n\n/**\n * Build the agent-facing `http_request` tool — a guarded HTTP/REST client\n * over the global `fetch`. The `options` bound what the model may do; the\n * model supplies the per-call URL / method / headers / body within those\n * rails.\n *\n * **Guardrails (all enforced before the network call).**\n * - **Method allowlist** — `allowMethods` (default `[\"GET\"]`). A method\n * outside the list is rejected with a typed\n * {@link HttpPolicyError} (`type: \"method-not-allowed\"`).\n * - **Host allowlist** — when `allowHosts` is set, any other host is\n * rejected (`type: \"host-not-allowed\"`), an SSRF guardrail.\n * - **`baseUrl` join** — when configured, the model passes a path that\n * is resolved against `baseUrl`; otherwise it must pass an absolute\n * `http(s)` URL. An unresolvable URL is rejected\n * (`type: \"invalid-url\"`).\n *\n * **Request shaping.** Static `options.headers` are merged under the\n * per-call `headers` (the per-call value wins). An object `body` is\n * JSON-serialized with a `content-type: application/json` default; a\n * string `body` is sent verbatim; `body` is dropped for bodyless methods\n * (`GET`). The call is bounded by `timeoutMs` (default `15_000`) via an\n * `AbortController`, also wired to `ctx.signal` for cooperative\n * cancellation.\n *\n * **Response shaping.** Headers are returned with lower-cased keys. The\n * body is read up to `maxBytes` (default `1_000_000`) and JSON-parsed\n * when the response `content-type` is JSON, otherwise returned as text;\n * `truncated` is `true` when the body was cut off at the cap (a truncated\n * JSON body is returned as the raw partial string, since it can no longer\n * be parsed).\n *\n * **Errors flow as data.** Every guardrail rejection and network failure\n * is thrown inside `execute`; the framework's `tool()` wrapper catches it\n * and surfaces it in the returned `{ error }` field, so the agent reads\n * the failure and self-corrects rather than crashing.\n *\n * @param options - Construction-time policy bounding the tool.\n * @returns A {@link ToolContract} the agent can call as `http_request`.\n *\n * @example\n * const stripe = httpRequestTool({\n * baseUrl: \"https://api.stripe.com\",\n * allowHosts: [\"api.stripe.com\"],\n * allowMethods: [\"GET\", \"POST\"],\n * headers: { authorization: `Bearer ${process.env.STRIPE_KEY}` },\n * });\n * const { data } = await stripe.invoke({ method: \"GET\", url: \"/v1/charges\" });\n */\nexport function httpRequestTool(\n options: HttpRequestOptions = {},\n): ToolContract<HttpRequestInput, HttpRequestResult> {\n const allowMethods = options.allowMethods ?? [\"GET\"];\n const allowedMethodSet = new Set<HttpMethod>(allowMethods);\n const allowHostSet = options.allowHosts ? new Set(options.allowHosts) : undefined;\n const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;\n const staticHeaders = options.headers;\n\n return tool<HttpRequestInput, HttpRequestResult>({\n name: options.name ?? DEFAULT_NAME,\n description:\n \"Issue an HTTP request and return the status, response headers, and \" +\n \"parsed body. Allowed methods and hosts are restricted by the tool's \" +\n \"configuration; a request outside those rails is rejected before any \" +\n \"network call. Pass an object body to send JSON, or a string to send \" +\n \"it verbatim. The response body is JSON-parsed when the content-type \" +\n \"is JSON, otherwise returned as text, and is capped — `truncated` is \" +\n \"true when the body was cut off.\",\n action: (input) => `Requesting ${input.method ?? \"GET\"} ${input.url}`,\n input: httpRequestInputSchema,\n async execute(input, ctx) {\n const method: HttpMethod = input.method ?? \"GET\";\n\n // 1. Method allowlist — rejected before anything else.\n if (!allowedMethodSet.has(method)) {\n throw new HttpPolicyError(\n `http_request method \"${method}\" is not allowed. ` +\n `Permitted methods: ${[...allowedMethodSet].join(\", \")}.`,\n { type: \"method-not-allowed\" },\n );\n }\n\n // 2. URL resolution (baseUrl join when configured).\n const url = resolveUrl(input.url, options.baseUrl);\n\n // 3. Host allowlist — SSRF guardrail, before the fetch.\n if (allowHostSet && !allowHostSet.has(url.hostname)) {\n throw new HttpPolicyError(\n `http_request host \"${url.hostname}\" is not in the allowlist. ` +\n `Permitted hosts: ${[...allowHostSet].join(\", \")}.`,\n { type: \"host-not-allowed\" },\n );\n }\n\n // 4. Merge headers — static option headers under the per-call ones,\n // so a per-call header overrides a static default of the same name.\n const headers: Record<string, string> = { ...staticHeaders, ...input.headers };\n\n // 5. Shape the body. Dropped for bodyless methods; objects become\n // JSON (with a default content-type); strings are sent verbatim.\n let body: string | undefined;\n\n if (!BODYLESS_METHODS.has(method) && input.body !== undefined) {\n if (typeof input.body === \"string\") {\n body = input.body;\n } else {\n body = JSON.stringify(input.body);\n\n const hasContentType = Object.keys(headers).some(\n (key) => key.toLowerCase() === \"content-type\",\n );\n\n if (!hasContentType) {\n headers[\"content-type\"] = \"application/json\";\n }\n }\n }\n\n // 6. Bound the call by timeout, chained to the caller's signal.\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), timeoutMs);\n\n const onAbort = () => controller.abort();\n\n if (ctx?.signal) {\n if (ctx.signal.aborted) {\n controller.abort();\n } else {\n ctx.signal.addEventListener(\"abort\", onAbort, { once: true });\n }\n }\n\n let response: Response;\n\n try {\n response = await fetch(url, { method, headers, body, signal: controller.signal });\n } finally {\n clearTimeout(timer);\n ctx?.signal?.removeEventListener(\"abort\", onAbort);\n }\n\n // 7. Collect response headers with lower-cased keys.\n const responseHeaders: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n responseHeaders[key.toLowerCase()] = value;\n });\n\n // 8. Read the body up to the cap, then parse-or-pass.\n const { text, truncated } = await readCappedBody(response, maxBytes);\n\n let parsedBody: unknown = text;\n\n // A truncated body can no longer be valid JSON, so only attempt a\n // parse on a complete JSON response; otherwise hand back the raw text.\n if (!truncated && isJsonContentType(responseHeaders[\"content-type\"]) && text.length > 0) {\n try {\n parsedBody = JSON.parse(text);\n } catch {\n // Content-type claimed JSON but the body was not — fall back to\n // the raw text rather than failing the whole call.\n parsedBody = text;\n }\n }\n\n return {\n status: response.status,\n headers: responseHeaders,\n body: parsedBody,\n truncated,\n };\n },\n });\n}\n"],"mappings":";;;;;;AAiBA,MAAM,eAAe;;AAGrB,MAAM,qBAAqB;;AAG3B,MAAM,oBAAoB;;AAG1B,MAAM,cAAqC;CAAC;CAAO;CAAQ;CAAO;CAAS;AAAQ;;AAGnF,MAAM,mBAA4C,IAAI,IAAgB,CAAC,KAAK,CAAC;;;;;;;;AAS7E,MAAM,yBAAyB,aAA+B;CAC5D,QAAQ,wBAAoC,WAAW;CACvD,KAAK,YAAY;CACjB,SAAS,0BAA0B;CACnC,MAAM,iBAAiB;AACzB,CAAC;;;;;;;;AASD,SAAS,WAAW,QAAgB,SAAkC;CACpE,IAAI;CAEJ,IAAI;EAIF,WAAW,YAAY,SAAY,IAAI,IAAI,QAAQ,OAAO,IAAI,IAAI,IAAI,MAAM;CAC9E,QAAQ;EACN,MAAM,IAAI,gBACR,oDAAoD,OAAO,MACxD,YAAY,SAAY,kBAAkB,QAAQ,MAAM,MAC3D,EAAE,MAAM,cAAc,CACxB;CACF;CAEA,IAAI,SAAS,aAAa,WAAW,SAAS,aAAa,UACzD,MAAM,IAAI,gBACR,gDAAgD,SAAS,SAAS,KAClE,EAAE,MAAM,cAAc,CACxB;CAGF,OAAO;AACT;;;;;;;;AASA,eAAe,eACb,UACA,UAC+C;CAC/C,MAAM,OAAO,SAAS;CAEtB,IAAI,CAAC,MACH,OAAO;EAAE,MAAM;EAAI,WAAW;CAAM;CAGtC,MAAM,UAAU,IAAI,YAAY;CAChC,MAAM,SAAS,KAAK,UAAU;CAC9B,IAAI,WAAW;CACf,IAAI,YAAY;CAChB,IAAI,OAAO;CAEX,IAAI;EACF,SAAS;GACP,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;GAE1C,IAAI,MACF;GAGF,IAAI,CAAC,OACH;GAGF,MAAM,YAAY,WAAW;GAE7B,IAAI,MAAM,aAAa,WAAW;IAChC,QAAQ,QAAQ,OAAO,MAAM,SAAS,GAAG,SAAS,GAAG,EAAE,QAAQ,KAAK,CAAC;IACrE,WAAW;IACX,YAAY;IACZ;GACF;GAEA,QAAQ,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;GAC9C,YAAY,MAAM;EACpB;CACF,UAAU;EAER,MAAM,OAAO,OAAO,CAAC,CAAC,YAAY,MAAS;EAC3C,OAAO,YAAY;CACrB;CAEA,QAAQ,QAAQ,OAAO;CAEvB,OAAO;EAAE;EAAM;CAAU;AAC3B;;;;;;AAOA,SAAS,kBAAkB,aAA0C;CACnE,IAAI,CAAC,aACH,OAAO;CAGT,MAAM,QAAQ,YAAY,YAAY;CAEtC,OAAO,MAAM,SAAS,kBAAkB,KAAK,MAAM,SAAS,OAAO;AACrE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDA,SAAgB,gBACd,UAA8B,CAAC,GACoB;CACnD,MAAM,eAAe,QAAQ,gBAAgB,CAAC,KAAK;CACnD,MAAM,mBAAmB,IAAI,IAAgB,YAAY;CACzD,MAAM,eAAe,QAAQ,aAAa,IAAI,IAAI,QAAQ,UAAU,IAAI;CACxE,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,gBAAgB,QAAQ;CAE9B,OAAO,KAA0C;EAC/C,MAAM,QAAQ,QAAQ;EACtB,aACE;EAOF,SAAS,UAAU,cAAc,MAAM,UAAU,MAAM,GAAG,MAAM;EAChE,OAAO;EACP,MAAM,QAAQ,OAAO,KAAK;GACxB,MAAM,SAAqB,MAAM,UAAU;GAG3C,IAAI,CAAC,iBAAiB,IAAI,MAAM,GAC9B,MAAM,IAAI,gBACR,wBAAwB,OAAO,uCACP,CAAC,GAAG,gBAAgB,CAAC,CAAC,KAAK,IAAI,EAAE,IACzD,EAAE,MAAM,qBAAqB,CAC/B;GAIF,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ,OAAO;GAGjD,IAAI,gBAAgB,CAAC,aAAa,IAAI,IAAI,QAAQ,GAChD,MAAM,IAAI,gBACR,sBAAsB,IAAI,SAAS,8CACb,CAAC,GAAG,YAAY,CAAC,CAAC,KAAK,IAAI,EAAE,IACnD,EAAE,MAAM,mBAAmB,CAC7B;GAKF,MAAM,UAAkC;IAAE,GAAG;IAAe,GAAG,MAAM;GAAQ;GAI7E,IAAI;GAEJ,IAAI,CAAC,iBAAiB,IAAI,MAAM,KAAK,MAAM,SAAS,QAClD,IAAI,OAAO,MAAM,SAAS,UACxB,OAAO,MAAM;QACR;IACL,OAAO,KAAK,UAAU,MAAM,IAAI;IAMhC,IAAI,CAJmB,OAAO,KAAK,OAAO,CAAC,CAAC,MACzC,QAAQ,IAAI,YAAY,MAAM,cAGf,GAChB,QAAQ,kBAAkB;GAE9B;GAIF,MAAM,aAAa,IAAI,gBAAgB;GACvC,MAAM,QAAQ,iBAAiB,WAAW,MAAM,GAAG,SAAS;GAE5D,MAAM,gBAAgB,WAAW,MAAM;GAEvC,IAAI,KAAK,QACP,IAAI,IAAI,OAAO,SACb,WAAW,MAAM;QAEjB,IAAI,OAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;GAIhE,IAAI;GAEJ,IAAI;IACF,WAAW,MAAM,MAAM,KAAK;KAAE;KAAQ;KAAS;KAAM,QAAQ,WAAW;IAAO,CAAC;GAClF,UAAU;IACR,aAAa,KAAK;IAClB,KAAK,QAAQ,oBAAoB,SAAS,OAAO;GACnD;GAGA,MAAM,kBAA0C,CAAC;GACjD,SAAS,QAAQ,SAAS,OAAO,QAAQ;IACvC,gBAAgB,IAAI,YAAY,KAAK;GACvC,CAAC;GAGD,MAAM,EAAE,MAAM,cAAc,MAAM,eAAe,UAAU,QAAQ;GAEnE,IAAI,aAAsB;GAI1B,IAAI,CAAC,aAAa,kBAAkB,gBAAgB,eAAe,KAAK,KAAK,SAAS,GACpF,IAAI;IACF,aAAa,KAAK,MAAM,IAAI;GAC9B,QAAQ;IAGN,aAAa;GACf;GAGF,OAAO;IACL,QAAQ,SAAS;IACjB,SAAS;IACT,MAAM;IACN;GACF;EACF;CACF,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"http-request.mjs","names":[],"sources":["../../../../../../../ai-tools/src/http/http-request.ts"],"sourcesContent":["import {\n guardedFetch,\n OutboundPolicyError,\n tool,\n type ToolContract,\n} from \"@warlock.js/ai\";\nimport type {\n HttpMethod,\n HttpRequestInput,\n HttpRequestOptions,\n HttpRequestResult,\n} from \"../contracts\";\nimport { HttpPolicyError } from \"../errors\";\nimport {\n objectSchema,\n optionalStringEnumField,\n optionalStringRecordField,\n passthroughField,\n stringField,\n} from \"../schema\";\n\n/** Default tool name exposed to the LLM. */\nconst DEFAULT_NAME = \"http_request\";\n\n/** Default per-request wall-clock timeout, in milliseconds. */\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/** Default hard cap on response-body bytes before truncation. */\nconst DEFAULT_MAX_BYTES = 1_000_000;\n\n/** The full set of HTTP methods, in the order they appear in {@link HttpMethod}. */\nconst ALL_METHODS: readonly HttpMethod[] = [\"GET\", \"POST\", \"PUT\", \"PATCH\", \"DELETE\"];\n\n/** Methods that conventionally carry no request body — `body` is dropped for these. */\nconst BODYLESS_METHODS: ReadonlySet<HttpMethod> = new Set<HttpMethod>([\"GET\"]);\n\n/**\n * Standard Schema for {@link HttpRequestInput}. `url` is required;\n * `method` is constrained to the canonical HTTP verb set (further\n * narrowed to the tool's `allowMethods` at runtime); `headers` is an\n * optional string-to-string record; `body` is an opaque passthrough the\n * handler serializes based on its runtime type.\n */\nconst httpRequestInputSchema = objectSchema<HttpRequestInput>({\n method: optionalStringEnumField<HttpMethod>(ALL_METHODS),\n url: stringField(),\n headers: optionalStringRecordField(),\n body: passthroughField(),\n});\n\n/**\n * Resolve the request target. With a `baseUrl` configured the model\n * supplies a path joined against it; otherwise the model's `url` must be\n * an absolute `http(s)` URL. Throws a typed {@link HttpPolicyError} of\n * type `\"invalid-url\"` when the result cannot be parsed or is not an\n * `http`/`https` URL — surfaced as `{ error }` data, never a crash.\n */\nfunction resolveUrl(rawUrl: string, baseUrl: string | undefined): URL {\n let resolved: URL;\n\n try {\n // `new URL(input, base)` joins relative paths against `base` and\n // ignores `base` when `input` is already absolute, which is exactly\n // the \"path vs full URL\" behavior the design specifies.\n resolved = baseUrl !== undefined ? new URL(rawUrl, baseUrl) : new URL(rawUrl);\n } catch {\n throw new HttpPolicyError(\n `http_request could not resolve a valid URL from \"${rawUrl}\"` +\n (baseUrl !== undefined ? ` against base \"${baseUrl}\".` : \".\"),\n { type: \"invalid-url\" },\n );\n }\n\n if (resolved.protocol !== \"http:\" && resolved.protocol !== \"https:\") {\n throw new HttpPolicyError(\n `http_request only permits http(s) URLs; got \"${resolved.protocol}\".`,\n { type: \"invalid-url\" },\n );\n }\n\n return resolved;\n}\n\n/**\n * Read a `Response` body, capping at `maxBytes`. Returns the decoded text\n * and whether it was cut off. Streams chunk-by-chunk so an oversized body\n * is abandoned at the cap rather than fully buffered; falls back to\n * `response.text()` (then a post-hoc byte slice) when the body is not a\n * readable stream (e.g. a stubbed `Response` in tests).\n */\nasync function readCappedBody(\n response: Response,\n maxBytes: number,\n): Promise<{ text: string; truncated: boolean }> {\n const body = response.body;\n\n if (!body) {\n return { text: \"\", truncated: false };\n }\n\n const decoder = new TextDecoder();\n const reader = body.getReader();\n let received = 0;\n let truncated = false;\n let text = \"\";\n\n try {\n for (;;) {\n const { done, value } = await reader.read();\n\n if (done) {\n break;\n }\n\n if (!value) {\n continue;\n }\n\n const remaining = maxBytes - received;\n\n if (value.byteLength > remaining) {\n text += decoder.decode(value.subarray(0, remaining), { stream: true });\n received = maxBytes;\n truncated = true;\n break;\n }\n\n text += decoder.decode(value, { stream: true });\n received += value.byteLength;\n }\n } finally {\n // Release the lock and abandon any unread remainder.\n await reader.cancel().catch(() => undefined);\n reader.releaseLock();\n }\n\n text += decoder.decode();\n\n return { text, truncated };\n}\n\n/**\n * Decide whether a response's `content-type` indicates JSON. Matches\n * `application/json` and the `+json` structured-suffix convention\n * (e.g. `application/vnd.api+json`), case-insensitively.\n */\nfunction isJsonContentType(contentType: string | undefined): boolean {\n if (!contentType) {\n return false;\n }\n\n const value = contentType.toLowerCase();\n\n return value.includes(\"application/json\") || value.includes(\"+json\");\n}\n\n/**\n * Build the agent-facing `http_request` tool — a guarded HTTP/REST client\n * over the global `fetch`. The `options` bound what the model may do; the\n * model supplies the per-call URL / method / headers / body within those\n * rails.\n *\n * **Guardrails (all enforced before the network call).**\n * - **Method allowlist** — `allowMethods` (default `[\"GET\"]`). A method\n * outside the list is rejected with a typed\n * {@link HttpPolicyError} (`type: \"method-not-allowed\"`).\n * - **Private-network deny (default).** The request — and every redirect\n * hop — goes through the framework's `guardedFetch` outbound policy,\n * which refuses private / loopback / link-local / cloud-metadata\n * addresses (and hostnames resolving to them) unless\n * `allowPrivateNetwork: true` is set. This applies even when\n * `allowHosts` is not configured, so a bare `ai.tools.http()` is not\n * an SSRF primitive (`type: \"host-not-allowed\"`).\n * - **Host allowlist** — when `allowHosts` is set, any other host is\n * rejected (`type: \"host-not-allowed\"`), an SSRF guardrail; redirect\n * targets are held to the same allowlist.\n * - **`baseUrl` join** — when configured, the model passes a path that\n * is resolved against `baseUrl`; otherwise it must pass an absolute\n * `http(s)` URL. An unresolvable URL is rejected\n * (`type: \"invalid-url\"`).\n *\n * **Request shaping.** Static `options.headers` are merged under the\n * per-call `headers` (the per-call value wins). An object `body` is\n * JSON-serialized with a `content-type: application/json` default; a\n * string `body` is sent verbatim; `body` is dropped for bodyless methods\n * (`GET`). The call is bounded by `timeoutMs` (default `15_000`) via an\n * `AbortController`, also wired to `ctx.signal` for cooperative\n * cancellation.\n *\n * **Response shaping.** Headers are returned with lower-cased keys. The\n * body is read up to `maxBytes` (default `1_000_000`) and JSON-parsed\n * when the response `content-type` is JSON, otherwise returned as text;\n * `truncated` is `true` when the body was cut off at the cap (a truncated\n * JSON body is returned as the raw partial string, since it can no longer\n * be parsed).\n *\n * **Errors flow as data.** Every guardrail rejection and network failure\n * is thrown inside `execute`; the framework's `tool()` wrapper catches it\n * and surfaces it in the returned `{ error }` field, so the agent reads\n * the failure and self-corrects rather than crashing.\n *\n * @param options - Construction-time policy bounding the tool.\n * @returns A {@link ToolContract} the agent can call as `http_request`.\n *\n * @example\n * const stripe = httpRequestTool({\n * baseUrl: \"https://api.stripe.com\",\n * allowHosts: [\"api.stripe.com\"],\n * allowMethods: [\"GET\", \"POST\"],\n * headers: { authorization: `Bearer ${process.env.STRIPE_KEY}` },\n * });\n * const { data } = await stripe.invoke({ method: \"GET\", url: \"/v1/charges\" });\n */\nexport function httpRequestTool(\n options: HttpRequestOptions = {},\n): ToolContract<HttpRequestInput, HttpRequestResult> {\n const allowMethods = options.allowMethods ?? [\"GET\"];\n const allowedMethodSet = new Set<HttpMethod>(allowMethods);\n const allowHostSet = options.allowHosts ? new Set(options.allowHosts) : undefined;\n const allowPrivateNetwork = options.allowPrivateNetwork ?? false;\n const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;\n const staticHeaders = options.headers;\n\n return tool<HttpRequestInput, HttpRequestResult>({\n name: options.name ?? DEFAULT_NAME,\n description:\n \"Issue an HTTP request and return the status, response headers, and \" +\n \"parsed body. Allowed methods and hosts are restricted by the tool's \" +\n \"configuration; a request outside those rails is rejected before any \" +\n \"network call. Pass an object body to send JSON, or a string to send \" +\n \"it verbatim. The response body is JSON-parsed when the content-type \" +\n \"is JSON, otherwise returned as text, and is capped — `truncated` is \" +\n \"true when the body was cut off.\",\n action: (input) => `Requesting ${input.method ?? \"GET\"} ${input.url}`,\n input: httpRequestInputSchema,\n async execute(input, ctx) {\n const method: HttpMethod = input.method ?? \"GET\";\n\n // 1. Method allowlist — rejected before anything else.\n if (!allowedMethodSet.has(method)) {\n throw new HttpPolicyError(\n `http_request method \"${method}\" is not allowed. ` +\n `Permitted methods: ${[...allowedMethodSet].join(\", \")}.`,\n { type: \"method-not-allowed\" },\n );\n }\n\n // 2. URL resolution (baseUrl join when configured).\n const url = resolveUrl(input.url, options.baseUrl);\n\n // 3. Host allowlist — SSRF guardrail, before the fetch.\n if (allowHostSet && !allowHostSet.has(url.hostname)) {\n throw new HttpPolicyError(\n `http_request host \"${url.hostname}\" is not in the allowlist. ` +\n `Permitted hosts: ${[...allowHostSet].join(\", \")}.`,\n { type: \"host-not-allowed\" },\n );\n }\n\n // 4. Merge headers — static option headers under the per-call ones,\n // so a per-call header overrides a static default of the same name.\n const headers: Record<string, string> = { ...staticHeaders, ...input.headers };\n\n // 5. Shape the body. Dropped for bodyless methods; objects become\n // JSON (with a default content-type); strings are sent verbatim.\n let body: string | undefined;\n\n if (!BODYLESS_METHODS.has(method) && input.body !== undefined) {\n if (typeof input.body === \"string\") {\n body = input.body;\n } else {\n body = JSON.stringify(input.body);\n\n const hasContentType = Object.keys(headers).some(\n (key) => key.toLowerCase() === \"content-type\",\n );\n\n if (!hasContentType) {\n headers[\"content-type\"] = \"application/json\";\n }\n }\n }\n\n // 6. Issue the request through the shared hardened outbound path:\n // private-IP deny by default, per-hop redirect re-validation,\n // timeout, and the caller's signal — never a raw fetch.\n let response: Response;\n\n try {\n response = await guardedFetch(\n url.toString(),\n {\n allowedSchemes: [\"http\", \"https\"],\n hostAllowlist: options.allowHosts,\n denyPrivateIPsAfterDNS: !allowPrivateNetwork,\n timeoutMs,\n signal: ctx?.signal,\n },\n { method, headers, body },\n );\n } catch (cause) {\n if (cause instanceof OutboundPolicyError) {\n throw new HttpPolicyError(`http_request blocked: ${cause.message}`, {\n type: \"host-not-allowed\",\n cause,\n });\n }\n\n throw cause;\n }\n\n // 7. Collect response headers with lower-cased keys.\n const responseHeaders: Record<string, string> = {};\n response.headers.forEach((value, key) => {\n responseHeaders[key.toLowerCase()] = value;\n });\n\n // 8. Read the body up to the cap, then parse-or-pass.\n const { text, truncated } = await readCappedBody(response, maxBytes);\n\n let parsedBody: unknown = text;\n\n // A truncated body can no longer be valid JSON, so only attempt a\n // parse on a complete JSON response; otherwise hand back the raw text.\n if (!truncated && isJsonContentType(responseHeaders[\"content-type\"]) && text.length > 0) {\n try {\n parsedBody = JSON.parse(text);\n } catch {\n // Content-type claimed JSON but the body was not — fall back to\n // the raw text rather than failing the whole call.\n parsedBody = text;\n }\n }\n\n return {\n status: response.status,\n headers: responseHeaders,\n body: parsedBody,\n truncated,\n };\n },\n });\n}\n"],"mappings":";;;;;;AAsBA,MAAM,eAAe;;AAGrB,MAAM,qBAAqB;;AAG3B,MAAM,oBAAoB;;AAG1B,MAAM,cAAqC;CAAC;CAAO;CAAQ;CAAO;CAAS;AAAQ;;AAGnF,MAAM,mBAA4C,IAAI,IAAgB,CAAC,KAAK,CAAC;;;;;;;;AAS7E,MAAM,yBAAyB,aAA+B;CAC5D,QAAQ,wBAAoC,WAAW;CACvD,KAAK,YAAY;CACjB,SAAS,0BAA0B;CACnC,MAAM,iBAAiB;AACzB,CAAC;;;;;;;;AASD,SAAS,WAAW,QAAgB,SAAkC;CACpE,IAAI;CAEJ,IAAI;EAIF,WAAW,YAAY,SAAY,IAAI,IAAI,QAAQ,OAAO,IAAI,IAAI,IAAI,MAAM;CAC9E,QAAQ;EACN,MAAM,IAAI,gBACR,oDAAoD,OAAO,MACxD,YAAY,SAAY,kBAAkB,QAAQ,MAAM,MAC3D,EAAE,MAAM,cAAc,CACxB;CACF;CAEA,IAAI,SAAS,aAAa,WAAW,SAAS,aAAa,UACzD,MAAM,IAAI,gBACR,gDAAgD,SAAS,SAAS,KAClE,EAAE,MAAM,cAAc,CACxB;CAGF,OAAO;AACT;;;;;;;;AASA,eAAe,eACb,UACA,UAC+C;CAC/C,MAAM,OAAO,SAAS;CAEtB,IAAI,CAAC,MACH,OAAO;EAAE,MAAM;EAAI,WAAW;CAAM;CAGtC,MAAM,UAAU,IAAI,YAAY;CAChC,MAAM,SAAS,KAAK,UAAU;CAC9B,IAAI,WAAW;CACf,IAAI,YAAY;CAChB,IAAI,OAAO;CAEX,IAAI;EACF,SAAS;GACP,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;GAE1C,IAAI,MACF;GAGF,IAAI,CAAC,OACH;GAGF,MAAM,YAAY,WAAW;GAE7B,IAAI,MAAM,aAAa,WAAW;IAChC,QAAQ,QAAQ,OAAO,MAAM,SAAS,GAAG,SAAS,GAAG,EAAE,QAAQ,KAAK,CAAC;IACrE,WAAW;IACX,YAAY;IACZ;GACF;GAEA,QAAQ,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;GAC9C,YAAY,MAAM;EACpB;CACF,UAAU;EAER,MAAM,OAAO,OAAO,CAAC,CAAC,YAAY,MAAS;EAC3C,OAAO,YAAY;CACrB;CAEA,QAAQ,QAAQ,OAAO;CAEvB,OAAO;EAAE;EAAM;CAAU;AAC3B;;;;;;AAOA,SAAS,kBAAkB,aAA0C;CACnE,IAAI,CAAC,aACH,OAAO;CAGT,MAAM,QAAQ,YAAY,YAAY;CAEtC,OAAO,MAAM,SAAS,kBAAkB,KAAK,MAAM,SAAS,OAAO;AACrE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,SAAgB,gBACd,UAA8B,CAAC,GACoB;CACnD,MAAM,eAAe,QAAQ,gBAAgB,CAAC,KAAK;CACnD,MAAM,mBAAmB,IAAI,IAAgB,YAAY;CACzD,MAAM,eAAe,QAAQ,aAAa,IAAI,IAAI,QAAQ,UAAU,IAAI;CACxE,MAAM,sBAAsB,QAAQ,uBAAuB;CAC3D,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,gBAAgB,QAAQ;CAE9B,OAAO,KAA0C;EAC/C,MAAM,QAAQ,QAAQ;EACtB,aACE;EAOF,SAAS,UAAU,cAAc,MAAM,UAAU,MAAM,GAAG,MAAM;EAChE,OAAO;EACP,MAAM,QAAQ,OAAO,KAAK;GACxB,MAAM,SAAqB,MAAM,UAAU;GAG3C,IAAI,CAAC,iBAAiB,IAAI,MAAM,GAC9B,MAAM,IAAI,gBACR,wBAAwB,OAAO,uCACP,CAAC,GAAG,gBAAgB,CAAC,CAAC,KAAK,IAAI,EAAE,IACzD,EAAE,MAAM,qBAAqB,CAC/B;GAIF,MAAM,MAAM,WAAW,MAAM,KAAK,QAAQ,OAAO;GAGjD,IAAI,gBAAgB,CAAC,aAAa,IAAI,IAAI,QAAQ,GAChD,MAAM,IAAI,gBACR,sBAAsB,IAAI,SAAS,8CACb,CAAC,GAAG,YAAY,CAAC,CAAC,KAAK,IAAI,EAAE,IACnD,EAAE,MAAM,mBAAmB,CAC7B;GAKF,MAAM,UAAkC;IAAE,GAAG;IAAe,GAAG,MAAM;GAAQ;GAI7E,IAAI;GAEJ,IAAI,CAAC,iBAAiB,IAAI,MAAM,KAAK,MAAM,SAAS,QAClD,IAAI,OAAO,MAAM,SAAS,UACxB,OAAO,MAAM;QACR;IACL,OAAO,KAAK,UAAU,MAAM,IAAI;IAMhC,IAAI,CAJmB,OAAO,KAAK,OAAO,CAAC,CAAC,MACzC,QAAQ,IAAI,YAAY,MAAM,cAGf,GAChB,QAAQ,kBAAkB;GAE9B;GAMF,IAAI;GAEJ,IAAI;IACF,WAAW,MAAM,aACf,IAAI,SAAS,GACb;KACE,gBAAgB,CAAC,QAAQ,OAAO;KAChC,eAAe,QAAQ;KACvB,wBAAwB,CAAC;KACzB;KACA,QAAQ,KAAK;IACf,GACA;KAAE;KAAQ;KAAS;IAAK,CAC1B;GACF,SAAS,OAAO;IACd,IAAI,iBAAiB,qBACnB,MAAM,IAAI,gBAAgB,yBAAyB,MAAM,WAAW;KAClE,MAAM;KACN;IACF,CAAC;IAGH,MAAM;GACR;GAGA,MAAM,kBAA0C,CAAC;GACjD,SAAS,QAAQ,SAAS,OAAO,QAAQ;IACvC,gBAAgB,IAAI,YAAY,KAAK;GACvC,CAAC;GAGD,MAAM,EAAE,MAAM,cAAc,MAAM,eAAe,UAAU,QAAQ;GAEnE,IAAI,aAAsB;GAI1B,IAAI,CAAC,aAAa,kBAAkB,gBAAgB,eAAe,KAAK,KAAK,SAAS,GACpF,IAAI;IACF,aAAa,KAAK,MAAM,IAAI;GAC9B,QAAQ;IAGN,aAAa;GACf;GAGF,OAAO;IACL,QAAQ,SAAS;IACjB,SAAS;IACT,MAAM;IACN;GACF;EACF;CACF,CAAC;AACH"}
|
package/esm/web/fetch-url.d.mts
CHANGED
|
@@ -7,8 +7,16 @@ import { ToolContract } from "@warlock.js/ai";
|
|
|
7
7
|
* `fetch` (Node 18+) and hand the model back rendered `content`.
|
|
8
8
|
*
|
|
9
9
|
* Guardrails, applied in order before/around the network call:
|
|
10
|
+
* - **Private-network deny (default).** Every request — and every
|
|
11
|
+
* redirect hop — goes through the framework's `guardedFetch` outbound
|
|
12
|
+
* policy, which refuses private / loopback / link-local /
|
|
13
|
+
* cloud-metadata addresses (and hostnames resolving to them) unless
|
|
14
|
+
* `allowPrivateNetwork: true` is set. This applies even when
|
|
15
|
+
* `allowHosts` is not configured, so a bare `ai.tools.fetchUrl()` is
|
|
16
|
+
* not an SSRF primitive.
|
|
10
17
|
* - **`allowHosts`** — when set, a request to any host not in the list is
|
|
11
|
-
* rejected before the fetch
|
|
18
|
+
* rejected before the fetch, and redirect targets are held to the same
|
|
19
|
+
* allowlist.
|
|
12
20
|
* - **`timeoutMs`** — the request is aborted via `AbortSignal.timeout`
|
|
13
21
|
* (default {@link DEFAULT_TIMEOUT_MS}).
|
|
14
22
|
* - **`maxBytes`** — the response body is read up to this cap and the
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/web/fetch-url.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"fetch-url.d.mts","names":[],"sources":["../../../../../../../ai-tools/src/web/fetch-url.ts"],"mappings":";;;;;;;AA0TA;;;;;;;;;;;;;;;AAE6C;;;;;;;;;;;;;;;;;;;;;iBAF7B,YAAA,CACd,OAAA,GAAU,eAAA,GACT,YAAA,CAAa,aAAA,EAAe,cAAA"}
|
package/esm/web/fetch-url.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { WebToolError } from "../errors.mjs";
|
|
2
2
|
import { objectSchema, stringField } from "./schema.mjs";
|
|
3
|
-
import { tool } from "@warlock.js/ai";
|
|
3
|
+
import { OutboundPolicyError, guardedFetch, tool } from "@warlock.js/ai";
|
|
4
4
|
|
|
5
5
|
//#region ../ai-tools/src/web/fetch-url.ts
|
|
6
6
|
/** Default tool name exposed to the LLM. */
|
|
@@ -163,8 +163,16 @@ function stripTags(html) {
|
|
|
163
163
|
* `fetch` (Node 18+) and hand the model back rendered `content`.
|
|
164
164
|
*
|
|
165
165
|
* Guardrails, applied in order before/around the network call:
|
|
166
|
+
* - **Private-network deny (default).** Every request — and every
|
|
167
|
+
* redirect hop — goes through the framework's `guardedFetch` outbound
|
|
168
|
+
* policy, which refuses private / loopback / link-local /
|
|
169
|
+
* cloud-metadata addresses (and hostnames resolving to them) unless
|
|
170
|
+
* `allowPrivateNetwork: true` is set. This applies even when
|
|
171
|
+
* `allowHosts` is not configured, so a bare `ai.tools.fetchUrl()` is
|
|
172
|
+
* not an SSRF primitive.
|
|
166
173
|
* - **`allowHosts`** — when set, a request to any host not in the list is
|
|
167
|
-
* rejected before the fetch
|
|
174
|
+
* rejected before the fetch, and redirect targets are held to the same
|
|
175
|
+
* allowlist.
|
|
168
176
|
* - **`timeoutMs`** — the request is aborted via `AbortSignal.timeout`
|
|
169
177
|
* (default {@link DEFAULT_TIMEOUT_MS}).
|
|
170
178
|
* - **`maxBytes`** — the response body is read up to this cap and the
|
|
@@ -194,6 +202,7 @@ function fetchUrlTool(options) {
|
|
|
194
202
|
const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
195
203
|
const extract = options?.extract ?? DEFAULT_EXTRACT;
|
|
196
204
|
const allowHosts = options?.allowHosts;
|
|
205
|
+
const allowPrivateNetwork = options?.allowPrivateNetwork ?? false;
|
|
197
206
|
if (extract !== "html") loadReadability();
|
|
198
207
|
return tool({
|
|
199
208
|
name: options?.name ?? DEFAULT_NAME,
|
|
@@ -204,8 +213,17 @@ function fetchUrlTool(options) {
|
|
|
204
213
|
assertHostAllowed(url, allowHosts);
|
|
205
214
|
let response;
|
|
206
215
|
try {
|
|
207
|
-
response = await
|
|
216
|
+
response = await guardedFetch(url.toString(), {
|
|
217
|
+
allowedSchemes: ["http", "https"],
|
|
218
|
+
hostAllowlist: allowHosts,
|
|
219
|
+
denyPrivateIPsAfterDNS: !allowPrivateNetwork,
|
|
220
|
+
timeoutMs
|
|
221
|
+
});
|
|
208
222
|
} catch (cause) {
|
|
223
|
+
if (cause instanceof OutboundPolicyError) throw new WebToolError(`fetch_url blocked: ${cause.message}`, {
|
|
224
|
+
type: "denied-host",
|
|
225
|
+
cause
|
|
226
|
+
});
|
|
209
227
|
throw new WebToolError(`fetch_url request failed: ${cause instanceof Error ? cause.message : String(cause)}`, {
|
|
210
228
|
type: "request-failed",
|
|
211
229
|
cause
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.mjs","names":[],"sources":["../../../../../../../ai-tools/src/web/fetch-url.ts"],"sourcesContent":["import { tool, type ToolContract } from \"@warlock.js/ai\";\nimport type {\n FetchUrlExtract,\n FetchUrlInput,\n FetchUrlOptions,\n FetchUrlResult,\n} from \"../contracts\";\nimport { WebToolError } from \"../errors\";\nimport { objectSchema, stringField } from \"./schema\";\n\n/** Default tool name exposed to the LLM. */\nconst DEFAULT_NAME = \"fetch_url\";\n\n/** Default hard cap on response bytes read before truncation. */\nconst DEFAULT_MAX_BYTES = 1_000_000;\n\n/** Default per-request timeout in milliseconds. */\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/** Default rendering mode for the fetched document. */\nconst DEFAULT_EXTRACT: FetchUrlExtract = \"text\";\n\n/** Input schema: `{ url }`. */\nconst inputSchema = objectSchema<FetchUrlInput>({\n url: stringField(),\n});\n\n// ============================================================\n// Lazily-loaded @mozilla/readability + jsdom (OPTIONAL peers)\n// ============================================================\n\n/**\n * Minimal structural shapes of the optional `@mozilla/readability` + `jsdom`\n * peers — only the members this file actually uses. Declared locally so the\n * package type-checks even when the peers are NOT installed (they are lazy\n * optional peers, loaded below via a `string` specifier so `tsc` never tries\n * to statically resolve them).\n */\ninterface ReadabilityArticleLike {\n readonly content?: string | null;\n readonly textContent?: string | null;\n}\ninterface ReadabilityInstanceLike {\n parse(): ReadabilityArticleLike | null;\n}\ninterface ReadabilityModuleLike {\n Readability: new (document: unknown) => ReadabilityInstanceLike;\n}\ninterface JsdomInstanceLike {\n readonly window: { readonly document: unknown };\n}\ninterface JsdomModuleLike {\n JSDOM: new (html: string, options?: { url?: string }) => JsdomInstanceLike;\n}\n\n// String specifiers typed `string` (not a literal) so `tsc` does not attempt\n// to resolve these optional peers at build time.\nconst READABILITY_MODULE_ID: string = \"@mozilla/readability\";\nconst JSDOM_MODULE_ID: string = \"jsdom\";\n\nlet ReadabilitySdk: ReadabilityModuleLike;\nlet JsdomSdk: JsdomModuleLike;\nlet isReadabilityAvailable: boolean | undefined;\nlet loadingPromise: Promise<void> | undefined;\n\nconst READABILITY_INSTALL_INSTRUCTIONS = `\nThe fetch_url text/markdown extractor requires the @mozilla/readability and jsdom packages.\nInstall them with:\n\n npm install @mozilla/readability jsdom\n\nOr with your preferred package manager:\n\n pnpm add @mozilla/readability jsdom\n yarn add @mozilla/readability jsdom\n`.trim();\n\n/**\n * Settle the lazy import of `@mozilla/readability` + `jsdom` once,\n * concurrency-safe. Only needed for the `\"text\"` / `\"markdown\"` extract\n * modes — `\"html\"` returns the raw body and never loads them. A bare\n * `catch` flips the flag to `false`; the curated install string surfaces\n * at use time via {@link WebToolError}, never a raw module-resolution\n * stack trace.\n */\nfunction loadReadability(): Promise<void> {\n if (isReadabilityAvailable !== undefined) {\n return Promise.resolve();\n }\n\n if (loadingPromise) {\n return loadingPromise;\n }\n\n loadingPromise = (async () => {\n try {\n ReadabilitySdk = (await import(READABILITY_MODULE_ID)) as ReadabilityModuleLike;\n JsdomSdk = (await import(JSDOM_MODULE_ID)) as JsdomModuleLike;\n isReadabilityAvailable = true;\n } catch {\n isReadabilityAvailable = false;\n }\n })();\n\n return loadingPromise;\n}\n\n/**\n * Parse the input URL, rejecting anything unparseable or not over\n * `http`/`https` with a typed {@link WebToolError}.\n */\nfunction parseUrl(raw: string): URL {\n let url: URL;\n\n try {\n url = new URL(raw);\n } catch {\n throw new WebToolError(`fetch_url received an unparseable URL: \"${raw}\".`, {\n type: \"invalid-url\",\n });\n }\n\n if (url.protocol !== \"http:\" && url.protocol !== \"https:\") {\n throw new WebToolError(\n `fetch_url only supports http(s) URLs; got \"${url.protocol}\".`,\n { type: \"invalid-url\" },\n );\n }\n\n return url;\n}\n\n/**\n * Enforce the host allowlist (an SSRF guardrail) before any network call.\n * When `allowHosts` is set, a request to a host not in it is rejected;\n * matching is exact on the URL hostname (case-insensitive).\n */\nfunction assertHostAllowed(url: URL, allowHosts?: string[]): void {\n if (!allowHosts || allowHosts.length === 0) {\n return;\n }\n\n const host = url.hostname.toLowerCase();\n const allowed = allowHosts.some((entry) => entry.toLowerCase() === host);\n\n if (!allowed) {\n throw new WebToolError(\n `fetch_url blocked host \"${url.hostname}\" — it is not in the allowHosts allowlist.`,\n { type: \"denied-host\", context: { host: url.hostname } },\n );\n }\n}\n\n/**\n * Read the response body up to `maxBytes`, stopping early once the cap is\n * reached. Returns the decoded text and whether it was truncated. When\n * the body has no stream (a stubbed `Response`), falls back to `.text()`\n * and truncates the decoded string at `maxBytes`.\n */\nasync function readBody(\n response: Response,\n maxBytes: number,\n): Promise<{ body: string; truncated: boolean }> {\n const stream = response.body;\n\n if (!stream) {\n const text = await response.text();\n\n if (text.length > maxBytes) {\n return { body: text.slice(0, maxBytes), truncated: true };\n }\n\n return { body: text, truncated: false };\n }\n\n const reader = stream.getReader();\n const chunks: Uint8Array[] = [];\n let received = 0;\n let truncated = false;\n\n for (;;) {\n const { done, value } = await reader.read();\n\n if (done) {\n break;\n }\n\n if (value) {\n received += value.byteLength;\n chunks.push(value);\n\n if (received >= maxBytes) {\n truncated = true;\n await reader.cancel();\n break;\n }\n }\n }\n\n const merged = new Uint8Array(received);\n let offset = 0;\n\n for (const chunk of chunks) {\n merged.set(chunk, offset);\n offset += chunk.byteLength;\n }\n\n const sliced = truncated ? merged.subarray(0, maxBytes) : merged;\n const body = new TextDecoder().decode(sliced);\n\n return { body, truncated };\n}\n\n/**\n * Render the fetched HTML into the configured `extract` mode. `\"html\"`\n * returns the raw body untouched; `\"text\"` and `\"markdown\"` run\n * `@mozilla/readability` over a `jsdom` document, throwing a curated\n * {@link WebToolError} when those optional peers are absent. `\"markdown\"`\n * applies a light heading/paragraph conversion over the extracted text.\n */\nasync function render(\n html: string,\n url: string,\n extract: FetchUrlExtract,\n): Promise<string> {\n if (extract === \"html\") {\n return html;\n }\n\n await loadReadability();\n\n if (!isReadabilityAvailable) {\n throw new WebToolError(READABILITY_INSTALL_INSTRUCTIONS, { type: \"missing-peer\" });\n }\n\n const dom = new JsdomSdk.JSDOM(html, { url });\n const article = new ReadabilitySdk.Readability(dom.window.document).parse();\n\n if (extract === \"markdown\") {\n return article?.content ? htmlToMarkdown(article.content) : (article?.textContent ?? \"\");\n }\n\n return article?.textContent ?? \"\";\n}\n\n/**\n * Minimal HTML→Markdown reduction for readability's extracted article\n * HTML — headings become `#` prefixes, paragraphs/line-breaks become\n * blank-line separated blocks, and remaining tags are stripped. This is a\n * pragmatic conversion, not a full CommonMark serializer.\n */\nfunction htmlToMarkdown(html: string): string {\n return html\n .replace(/<h([1-6])[^>]*>(.*?)<\\/h\\1>/gis, (_match, level: string, text: string) => {\n const hashes = \"#\".repeat(Number(level));\n\n return `\\n\\n${hashes} ${stripTags(text).trim()}\\n\\n`;\n })\n .replace(/<\\/(p|div|section|article|li)>/gi, \"\\n\\n\")\n .replace(/<br\\s*\\/?>/gi, \"\\n\")\n .replace(/<[^>]+>/g, \"\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n}\n\n/** Strip any remaining HTML tags from a fragment. */\nfunction stripTags(html: string): string {\n return html.replace(/<[^>]+>/g, \"\");\n}\n\n/**\n * Build the agent-facing `fetch_url` tool — fetch a URL over the global\n * `fetch` (Node 18+) and hand the model back rendered `content`.\n *\n * Guardrails, applied in order before/around the network call:\n * - **`allowHosts`** — when set, a request to any host not in the list is\n * rejected before the fetch (an SSRF guardrail).\n * - **`timeoutMs`** — the request is aborted via `AbortSignal.timeout`\n * (default {@link DEFAULT_TIMEOUT_MS}).\n * - **`maxBytes`** — the response body is read up to this cap and the\n * result flags `truncated` when it was cut off (default\n * {@link DEFAULT_MAX_BYTES}).\n * - **`extract`** — `\"html\"` returns the raw body; `\"text\"` (default) and\n * `\"markdown\"` run `@mozilla/readability` over `jsdom`, lazily imported\n * so they stay OPTIONAL peers — a missing peer throws the curated\n * install string.\n *\n * **Errors flow as data.** Every guardrail rejection and network failure\n * throws a typed {@link WebToolError}; the `tool()` wrapper catches it and\n * surfaces it in the returned `{ error }` field — `invoke()` never throws\n * — so the agent reads the failure and self-corrects.\n *\n * @param options - Tool-name override, byte cap, timeout, extract mode,\n * and host allowlist.\n * @returns A `ToolContract<{ url }, FetchUrlResult>`.\n *\n * @example\n * const fetchUrl = fetchUrlTool({ extract: \"text\", allowHosts: [\"docs.stripe.com\"] });\n * const { data } = await fetchUrl.invoke({ url: \"https://docs.stripe.com/api\" });\n * console.log(data?.content, data?.truncated);\n */\nexport function fetchUrlTool(\n options?: FetchUrlOptions,\n): ToolContract<FetchUrlInput, FetchUrlResult> {\n const maxBytes = options?.maxBytes ?? DEFAULT_MAX_BYTES;\n const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const extract = options?.extract ?? DEFAULT_EXTRACT;\n const allowHosts = options?.allowHosts;\n\n // Warm the readability peers non-blockingly when the configured mode\n // needs them, so the curated install string is ready (and logged at\n // first use) without delaying construction.\n if (extract !== \"html\") {\n void loadReadability();\n }\n\n return tool<FetchUrlInput, FetchUrlResult>({\n name: options?.name ?? DEFAULT_NAME,\n description:\n \"Fetch a web page or document by URL and return its main content as \" +\n \"text (readability-extracted), raw HTML, or Markdown. Use to read a \" +\n \"specific page you already have the URL for.\",\n input: inputSchema,\n async execute(input) {\n const url = parseUrl(input.url);\n assertHostAllowed(url, allowHosts);\n\n let response: Response;\n\n try {\n response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n } catch (cause) {\n const message = cause instanceof Error ? cause.message : String(cause);\n\n throw new WebToolError(`fetch_url request failed: ${message}`, {\n type: \"request-failed\",\n cause,\n });\n }\n\n const { body, truncated } = await readBody(response, maxBytes);\n const content = await render(body, response.url || url.toString(), extract);\n\n return {\n url: response.url || url.toString(),\n status: response.status,\n content,\n truncated,\n };\n },\n });\n}\n"],"mappings":";;;;;;AAWA,MAAM,eAAe;;AAGrB,MAAM,oBAAoB;;AAG1B,MAAM,qBAAqB;;AAG3B,MAAM,kBAAmC;;AAGzC,MAAM,cAAc,aAA4B,EAC9C,KAAK,YAAY,EACnB,CAAC;AAgCD,MAAM,wBAAgC;AACtC,MAAM,kBAA0B;AAEhC,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AAEJ,MAAM,mCAAmC;;;;;;;;;;EAUvC,KAAK;;;;;;;;;AAUP,SAAS,kBAAiC;CACxC,IAAI,2BAA2B,QAC7B,OAAO,QAAQ,QAAQ;CAGzB,IAAI,gBACF,OAAO;CAGT,kBAAkB,YAAY;EAC5B,IAAI;GACF,iBAAkB,MAAM,OAAO;GAC/B,WAAY,MAAM,OAAO;GACzB,yBAAyB;EAC3B,QAAQ;GACN,yBAAyB;EAC3B;CACF,EAAC,CAAE;CAEH,OAAO;AACT;;;;;AAMA,SAAS,SAAS,KAAkB;CAClC,IAAI;CAEJ,IAAI;EACF,MAAM,IAAI,IAAI,GAAG;CACnB,QAAQ;EACN,MAAM,IAAI,aAAa,2CAA2C,IAAI,KAAK,EACzE,MAAM,cACR,CAAC;CACH;CAEA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,MAAM,IAAI,aACR,8CAA8C,IAAI,SAAS,KAC3D,EAAE,MAAM,cAAc,CACxB;CAGF,OAAO;AACT;;;;;;AAOA,SAAS,kBAAkB,KAAU,YAA6B;CAChE,IAAI,CAAC,cAAc,WAAW,WAAW,GACvC;CAGF,MAAM,OAAO,IAAI,SAAS,YAAY;CAGtC,IAAI,CAFY,WAAW,MAAM,UAAU,MAAM,YAAY,MAAM,IAExD,GACT,MAAM,IAAI,aACR,2BAA2B,IAAI,SAAS,6CACxC;EAAE,MAAM;EAAe,SAAS,EAAE,MAAM,IAAI,SAAS;CAAE,CACzD;AAEJ;;;;;;;AAQA,eAAe,SACb,UACA,UAC+C;CAC/C,MAAM,SAAS,SAAS;CAExB,IAAI,CAAC,QAAQ;EACX,MAAM,OAAO,MAAM,SAAS,KAAK;EAEjC,IAAI,KAAK,SAAS,UAChB,OAAO;GAAE,MAAM,KAAK,MAAM,GAAG,QAAQ;GAAG,WAAW;EAAK;EAG1D,OAAO;GAAE,MAAM;GAAM,WAAW;EAAM;CACxC;CAEA,MAAM,SAAS,OAAO,UAAU;CAChC,MAAM,SAAuB,CAAC;CAC9B,IAAI,WAAW;CACf,IAAI,YAAY;CAEhB,SAAS;EACP,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;EAE1C,IAAI,MACF;EAGF,IAAI,OAAO;GACT,YAAY,MAAM;GAClB,OAAO,KAAK,KAAK;GAEjB,IAAI,YAAY,UAAU;IACxB,YAAY;IACZ,MAAM,OAAO,OAAO;IACpB;GACF;EACF;CACF;CAEA,MAAM,SAAS,IAAI,WAAW,QAAQ;CACtC,IAAI,SAAS;CAEb,KAAK,MAAM,SAAS,QAAQ;EAC1B,OAAO,IAAI,OAAO,MAAM;EACxB,UAAU,MAAM;CAClB;CAEA,MAAM,SAAS,YAAY,OAAO,SAAS,GAAG,QAAQ,IAAI;CAG1D,OAAO;EAAE,MAFI,IAAI,YAAY,CAAC,CAAC,OAAO,MAE1B;EAAG;CAAU;AAC3B;;;;;;;;AASA,eAAe,OACb,MACA,KACA,SACiB;CACjB,IAAI,YAAY,QACd,OAAO;CAGT,MAAM,gBAAgB;CAEtB,IAAI,CAAC,wBACH,MAAM,IAAI,aAAa,kCAAkC,EAAE,MAAM,eAAe,CAAC;CAGnF,MAAM,MAAM,IAAI,SAAS,MAAM,MAAM,EAAE,IAAI,CAAC;CAC5C,MAAM,UAAU,IAAI,eAAe,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,MAAM;CAE1E,IAAI,YAAY,YACd,OAAO,SAAS,UAAU,eAAe,QAAQ,OAAO,IAAK,SAAS,eAAe;CAGvF,OAAO,SAAS,eAAe;AACjC;;;;;;;AAQA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KACJ,QAAQ,mCAAmC,QAAQ,OAAe,SAAiB;EAGlF,OAAO,OAFQ,IAAI,OAAO,OAAO,KAAK,CAEnB,EAAE,GAAG,UAAU,IAAI,CAAC,CAAC,KAAK,EAAE;CACjD,CAAC,CAAC,CACD,QAAQ,oCAAoC,MAAM,CAAC,CACnD,QAAQ,gBAAgB,IAAI,CAAC,CAC7B,QAAQ,YAAY,EAAE,CAAC,CACvB,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;AACV;;AAGA,SAAS,UAAU,MAAsB;CACvC,OAAO,KAAK,QAAQ,YAAY,EAAE;AACpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,SAAgB,aACd,SAC6C;CAC7C,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,YAAY,SAAS,aAAa;CACxC,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,aAAa,SAAS;CAK5B,IAAI,YAAY,QACd,AAAK,gBAAgB;CAGvB,OAAO,KAAoC;EACzC,MAAM,SAAS,QAAQ;EACvB,aACE;EAGF,OAAO;EACP,MAAM,QAAQ,OAAO;GACnB,MAAM,MAAM,SAAS,MAAM,GAAG;GAC9B,kBAAkB,KAAK,UAAU;GAEjC,IAAI;GAEJ,IAAI;IACF,WAAW,MAAM,MAAM,KAAK,EAAE,QAAQ,YAAY,QAAQ,SAAS,EAAE,CAAC;GACxE,SAAS,OAAO;IAGd,MAAM,IAAI,aAAa,6BAFP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEN;KAC7D,MAAM;KACN;IACF,CAAC;GACH;GAEA,MAAM,EAAE,MAAM,cAAc,MAAM,SAAS,UAAU,QAAQ;GAC7D,MAAM,UAAU,MAAM,OAAO,MAAM,SAAS,OAAO,IAAI,SAAS,GAAG,OAAO;GAE1E,OAAO;IACL,KAAK,SAAS,OAAO,IAAI,SAAS;IAClC,QAAQ,SAAS;IACjB;IACA;GACF;EACF;CACF,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"fetch-url.mjs","names":[],"sources":["../../../../../../../ai-tools/src/web/fetch-url.ts"],"sourcesContent":["import {\n guardedFetch,\n OutboundPolicyError,\n tool,\n type ToolContract,\n} from \"@warlock.js/ai\";\nimport type {\n FetchUrlExtract,\n FetchUrlInput,\n FetchUrlOptions,\n FetchUrlResult,\n} from \"../contracts\";\nimport { WebToolError } from \"../errors\";\nimport { objectSchema, stringField } from \"./schema\";\n\n/** Default tool name exposed to the LLM. */\nconst DEFAULT_NAME = \"fetch_url\";\n\n/** Default hard cap on response bytes read before truncation. */\nconst DEFAULT_MAX_BYTES = 1_000_000;\n\n/** Default per-request timeout in milliseconds. */\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/** Default rendering mode for the fetched document. */\nconst DEFAULT_EXTRACT: FetchUrlExtract = \"text\";\n\n/** Input schema: `{ url }`. */\nconst inputSchema = objectSchema<FetchUrlInput>({\n url: stringField(),\n});\n\n// ============================================================\n// Lazily-loaded @mozilla/readability + jsdom (OPTIONAL peers)\n// ============================================================\n\n/**\n * Minimal structural shapes of the optional `@mozilla/readability` + `jsdom`\n * peers — only the members this file actually uses. Declared locally so the\n * package type-checks even when the peers are NOT installed (they are lazy\n * optional peers, loaded below via a `string` specifier so `tsc` never tries\n * to statically resolve them).\n */\ninterface ReadabilityArticleLike {\n readonly content?: string | null;\n readonly textContent?: string | null;\n}\ninterface ReadabilityInstanceLike {\n parse(): ReadabilityArticleLike | null;\n}\ninterface ReadabilityModuleLike {\n Readability: new (document: unknown) => ReadabilityInstanceLike;\n}\ninterface JsdomInstanceLike {\n readonly window: { readonly document: unknown };\n}\ninterface JsdomModuleLike {\n JSDOM: new (html: string, options?: { url?: string }) => JsdomInstanceLike;\n}\n\n// String specifiers typed `string` (not a literal) so `tsc` does not attempt\n// to resolve these optional peers at build time.\nconst READABILITY_MODULE_ID: string = \"@mozilla/readability\";\nconst JSDOM_MODULE_ID: string = \"jsdom\";\n\nlet ReadabilitySdk: ReadabilityModuleLike;\nlet JsdomSdk: JsdomModuleLike;\nlet isReadabilityAvailable: boolean | undefined;\nlet loadingPromise: Promise<void> | undefined;\n\nconst READABILITY_INSTALL_INSTRUCTIONS = `\nThe fetch_url text/markdown extractor requires the @mozilla/readability and jsdom packages.\nInstall them with:\n\n npm install @mozilla/readability jsdom\n\nOr with your preferred package manager:\n\n pnpm add @mozilla/readability jsdom\n yarn add @mozilla/readability jsdom\n`.trim();\n\n/**\n * Settle the lazy import of `@mozilla/readability` + `jsdom` once,\n * concurrency-safe. Only needed for the `\"text\"` / `\"markdown\"` extract\n * modes — `\"html\"` returns the raw body and never loads them. A bare\n * `catch` flips the flag to `false`; the curated install string surfaces\n * at use time via {@link WebToolError}, never a raw module-resolution\n * stack trace.\n */\nfunction loadReadability(): Promise<void> {\n if (isReadabilityAvailable !== undefined) {\n return Promise.resolve();\n }\n\n if (loadingPromise) {\n return loadingPromise;\n }\n\n loadingPromise = (async () => {\n try {\n ReadabilitySdk = (await import(READABILITY_MODULE_ID)) as ReadabilityModuleLike;\n JsdomSdk = (await import(JSDOM_MODULE_ID)) as JsdomModuleLike;\n isReadabilityAvailable = true;\n } catch {\n isReadabilityAvailable = false;\n }\n })();\n\n return loadingPromise;\n}\n\n/**\n * Parse the input URL, rejecting anything unparseable or not over\n * `http`/`https` with a typed {@link WebToolError}.\n */\nfunction parseUrl(raw: string): URL {\n let url: URL;\n\n try {\n url = new URL(raw);\n } catch {\n throw new WebToolError(`fetch_url received an unparseable URL: \"${raw}\".`, {\n type: \"invalid-url\",\n });\n }\n\n if (url.protocol !== \"http:\" && url.protocol !== \"https:\") {\n throw new WebToolError(\n `fetch_url only supports http(s) URLs; got \"${url.protocol}\".`,\n { type: \"invalid-url\" },\n );\n }\n\n return url;\n}\n\n/**\n * Enforce the host allowlist (an SSRF guardrail) before any network call.\n * When `allowHosts` is set, a request to a host not in it is rejected;\n * matching is exact on the URL hostname (case-insensitive).\n */\nfunction assertHostAllowed(url: URL, allowHosts?: string[]): void {\n if (!allowHosts || allowHosts.length === 0) {\n return;\n }\n\n const host = url.hostname.toLowerCase();\n const allowed = allowHosts.some((entry) => entry.toLowerCase() === host);\n\n if (!allowed) {\n throw new WebToolError(\n `fetch_url blocked host \"${url.hostname}\" — it is not in the allowHosts allowlist.`,\n { type: \"denied-host\", context: { host: url.hostname } },\n );\n }\n}\n\n/**\n * Read the response body up to `maxBytes`, stopping early once the cap is\n * reached. Returns the decoded text and whether it was truncated. When\n * the body has no stream (a stubbed `Response`), falls back to `.text()`\n * and truncates the decoded string at `maxBytes`.\n */\nasync function readBody(\n response: Response,\n maxBytes: number,\n): Promise<{ body: string; truncated: boolean }> {\n const stream = response.body;\n\n if (!stream) {\n const text = await response.text();\n\n if (text.length > maxBytes) {\n return { body: text.slice(0, maxBytes), truncated: true };\n }\n\n return { body: text, truncated: false };\n }\n\n const reader = stream.getReader();\n const chunks: Uint8Array[] = [];\n let received = 0;\n let truncated = false;\n\n for (;;) {\n const { done, value } = await reader.read();\n\n if (done) {\n break;\n }\n\n if (value) {\n received += value.byteLength;\n chunks.push(value);\n\n if (received >= maxBytes) {\n truncated = true;\n await reader.cancel();\n break;\n }\n }\n }\n\n const merged = new Uint8Array(received);\n let offset = 0;\n\n for (const chunk of chunks) {\n merged.set(chunk, offset);\n offset += chunk.byteLength;\n }\n\n const sliced = truncated ? merged.subarray(0, maxBytes) : merged;\n const body = new TextDecoder().decode(sliced);\n\n return { body, truncated };\n}\n\n/**\n * Render the fetched HTML into the configured `extract` mode. `\"html\"`\n * returns the raw body untouched; `\"text\"` and `\"markdown\"` run\n * `@mozilla/readability` over a `jsdom` document, throwing a curated\n * {@link WebToolError} when those optional peers are absent. `\"markdown\"`\n * applies a light heading/paragraph conversion over the extracted text.\n */\nasync function render(\n html: string,\n url: string,\n extract: FetchUrlExtract,\n): Promise<string> {\n if (extract === \"html\") {\n return html;\n }\n\n await loadReadability();\n\n if (!isReadabilityAvailable) {\n throw new WebToolError(READABILITY_INSTALL_INSTRUCTIONS, { type: \"missing-peer\" });\n }\n\n const dom = new JsdomSdk.JSDOM(html, { url });\n const article = new ReadabilitySdk.Readability(dom.window.document).parse();\n\n if (extract === \"markdown\") {\n return article?.content ? htmlToMarkdown(article.content) : (article?.textContent ?? \"\");\n }\n\n return article?.textContent ?? \"\";\n}\n\n/**\n * Minimal HTML→Markdown reduction for readability's extracted article\n * HTML — headings become `#` prefixes, paragraphs/line-breaks become\n * blank-line separated blocks, and remaining tags are stripped. This is a\n * pragmatic conversion, not a full CommonMark serializer.\n */\nfunction htmlToMarkdown(html: string): string {\n return html\n .replace(/<h([1-6])[^>]*>(.*?)<\\/h\\1>/gis, (_match, level: string, text: string) => {\n const hashes = \"#\".repeat(Number(level));\n\n return `\\n\\n${hashes} ${stripTags(text).trim()}\\n\\n`;\n })\n .replace(/<\\/(p|div|section|article|li)>/gi, \"\\n\\n\")\n .replace(/<br\\s*\\/?>/gi, \"\\n\")\n .replace(/<[^>]+>/g, \"\")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n}\n\n/** Strip any remaining HTML tags from a fragment. */\nfunction stripTags(html: string): string {\n return html.replace(/<[^>]+>/g, \"\");\n}\n\n/**\n * Build the agent-facing `fetch_url` tool — fetch a URL over the global\n * `fetch` (Node 18+) and hand the model back rendered `content`.\n *\n * Guardrails, applied in order before/around the network call:\n * - **Private-network deny (default).** Every request — and every\n * redirect hop — goes through the framework's `guardedFetch` outbound\n * policy, which refuses private / loopback / link-local /\n * cloud-metadata addresses (and hostnames resolving to them) unless\n * `allowPrivateNetwork: true` is set. This applies even when\n * `allowHosts` is not configured, so a bare `ai.tools.fetchUrl()` is\n * not an SSRF primitive.\n * - **`allowHosts`** — when set, a request to any host not in the list is\n * rejected before the fetch, and redirect targets are held to the same\n * allowlist.\n * - **`timeoutMs`** — the request is aborted via `AbortSignal.timeout`\n * (default {@link DEFAULT_TIMEOUT_MS}).\n * - **`maxBytes`** — the response body is read up to this cap and the\n * result flags `truncated` when it was cut off (default\n * {@link DEFAULT_MAX_BYTES}).\n * - **`extract`** — `\"html\"` returns the raw body; `\"text\"` (default) and\n * `\"markdown\"` run `@mozilla/readability` over `jsdom`, lazily imported\n * so they stay OPTIONAL peers — a missing peer throws the curated\n * install string.\n *\n * **Errors flow as data.** Every guardrail rejection and network failure\n * throws a typed {@link WebToolError}; the `tool()` wrapper catches it and\n * surfaces it in the returned `{ error }` field — `invoke()` never throws\n * — so the agent reads the failure and self-corrects.\n *\n * @param options - Tool-name override, byte cap, timeout, extract mode,\n * and host allowlist.\n * @returns A `ToolContract<{ url }, FetchUrlResult>`.\n *\n * @example\n * const fetchUrl = fetchUrlTool({ extract: \"text\", allowHosts: [\"docs.stripe.com\"] });\n * const { data } = await fetchUrl.invoke({ url: \"https://docs.stripe.com/api\" });\n * console.log(data?.content, data?.truncated);\n */\nexport function fetchUrlTool(\n options?: FetchUrlOptions,\n): ToolContract<FetchUrlInput, FetchUrlResult> {\n const maxBytes = options?.maxBytes ?? DEFAULT_MAX_BYTES;\n const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n const extract = options?.extract ?? DEFAULT_EXTRACT;\n const allowHosts = options?.allowHosts;\n const allowPrivateNetwork = options?.allowPrivateNetwork ?? false;\n\n // Warm the readability peers non-blockingly when the configured mode\n // needs them, so the curated install string is ready (and logged at\n // first use) without delaying construction.\n if (extract !== \"html\") {\n void loadReadability();\n }\n\n return tool<FetchUrlInput, FetchUrlResult>({\n name: options?.name ?? DEFAULT_NAME,\n description:\n \"Fetch a web page or document by URL and return its main content as \" +\n \"text (readability-extracted), raw HTML, or Markdown. Use to read a \" +\n \"specific page you already have the URL for.\",\n input: inputSchema,\n async execute(input) {\n const url = parseUrl(input.url);\n assertHostAllowed(url, allowHosts);\n\n let response: Response;\n\n try {\n // The shared hardened outbound path: private-IP deny by default,\n // per-hop redirect re-validation, timeout — never a raw fetch.\n response = await guardedFetch(url.toString(), {\n allowedSchemes: [\"http\", \"https\"],\n hostAllowlist: allowHosts,\n denyPrivateIPsAfterDNS: !allowPrivateNetwork,\n timeoutMs,\n });\n } catch (cause) {\n if (cause instanceof OutboundPolicyError) {\n throw new WebToolError(`fetch_url blocked: ${cause.message}`, {\n type: \"denied-host\",\n cause,\n });\n }\n\n const message = cause instanceof Error ? cause.message : String(cause);\n\n throw new WebToolError(`fetch_url request failed: ${message}`, {\n type: \"request-failed\",\n cause,\n });\n }\n\n const { body, truncated } = await readBody(response, maxBytes);\n const content = await render(body, response.url || url.toString(), extract);\n\n return {\n url: response.url || url.toString(),\n status: response.status,\n content,\n truncated,\n };\n },\n });\n}\n"],"mappings":";;;;;;AAgBA,MAAM,eAAe;;AAGrB,MAAM,oBAAoB;;AAG1B,MAAM,qBAAqB;;AAG3B,MAAM,kBAAmC;;AAGzC,MAAM,cAAc,aAA4B,EAC9C,KAAK,YAAY,EACnB,CAAC;AAgCD,MAAM,wBAAgC;AACtC,MAAM,kBAA0B;AAEhC,IAAI;AACJ,IAAI;AACJ,IAAI;AACJ,IAAI;AAEJ,MAAM,mCAAmC;;;;;;;;;;EAUvC,KAAK;;;;;;;;;AAUP,SAAS,kBAAiC;CACxC,IAAI,2BAA2B,QAC7B,OAAO,QAAQ,QAAQ;CAGzB,IAAI,gBACF,OAAO;CAGT,kBAAkB,YAAY;EAC5B,IAAI;GACF,iBAAkB,MAAM,OAAO;GAC/B,WAAY,MAAM,OAAO;GACzB,yBAAyB;EAC3B,QAAQ;GACN,yBAAyB;EAC3B;CACF,EAAC,CAAE;CAEH,OAAO;AACT;;;;;AAMA,SAAS,SAAS,KAAkB;CAClC,IAAI;CAEJ,IAAI;EACF,MAAM,IAAI,IAAI,GAAG;CACnB,QAAQ;EACN,MAAM,IAAI,aAAa,2CAA2C,IAAI,KAAK,EACzE,MAAM,cACR,CAAC;CACH;CAEA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,MAAM,IAAI,aACR,8CAA8C,IAAI,SAAS,KAC3D,EAAE,MAAM,cAAc,CACxB;CAGF,OAAO;AACT;;;;;;AAOA,SAAS,kBAAkB,KAAU,YAA6B;CAChE,IAAI,CAAC,cAAc,WAAW,WAAW,GACvC;CAGF,MAAM,OAAO,IAAI,SAAS,YAAY;CAGtC,IAAI,CAFY,WAAW,MAAM,UAAU,MAAM,YAAY,MAAM,IAExD,GACT,MAAM,IAAI,aACR,2BAA2B,IAAI,SAAS,6CACxC;EAAE,MAAM;EAAe,SAAS,EAAE,MAAM,IAAI,SAAS;CAAE,CACzD;AAEJ;;;;;;;AAQA,eAAe,SACb,UACA,UAC+C;CAC/C,MAAM,SAAS,SAAS;CAExB,IAAI,CAAC,QAAQ;EACX,MAAM,OAAO,MAAM,SAAS,KAAK;EAEjC,IAAI,KAAK,SAAS,UAChB,OAAO;GAAE,MAAM,KAAK,MAAM,GAAG,QAAQ;GAAG,WAAW;EAAK;EAG1D,OAAO;GAAE,MAAM;GAAM,WAAW;EAAM;CACxC;CAEA,MAAM,SAAS,OAAO,UAAU;CAChC,MAAM,SAAuB,CAAC;CAC9B,IAAI,WAAW;CACf,IAAI,YAAY;CAEhB,SAAS;EACP,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;EAE1C,IAAI,MACF;EAGF,IAAI,OAAO;GACT,YAAY,MAAM;GAClB,OAAO,KAAK,KAAK;GAEjB,IAAI,YAAY,UAAU;IACxB,YAAY;IACZ,MAAM,OAAO,OAAO;IACpB;GACF;EACF;CACF;CAEA,MAAM,SAAS,IAAI,WAAW,QAAQ;CACtC,IAAI,SAAS;CAEb,KAAK,MAAM,SAAS,QAAQ;EAC1B,OAAO,IAAI,OAAO,MAAM;EACxB,UAAU,MAAM;CAClB;CAEA,MAAM,SAAS,YAAY,OAAO,SAAS,GAAG,QAAQ,IAAI;CAG1D,OAAO;EAAE,MAFI,IAAI,YAAY,CAAC,CAAC,OAAO,MAE1B;EAAG;CAAU;AAC3B;;;;;;;;AASA,eAAe,OACb,MACA,KACA,SACiB;CACjB,IAAI,YAAY,QACd,OAAO;CAGT,MAAM,gBAAgB;CAEtB,IAAI,CAAC,wBACH,MAAM,IAAI,aAAa,kCAAkC,EAAE,MAAM,eAAe,CAAC;CAGnF,MAAM,MAAM,IAAI,SAAS,MAAM,MAAM,EAAE,IAAI,CAAC;CAC5C,MAAM,UAAU,IAAI,eAAe,YAAY,IAAI,OAAO,QAAQ,CAAC,CAAC,MAAM;CAE1E,IAAI,YAAY,YACd,OAAO,SAAS,UAAU,eAAe,QAAQ,OAAO,IAAK,SAAS,eAAe;CAGvF,OAAO,SAAS,eAAe;AACjC;;;;;;;AAQA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KACJ,QAAQ,mCAAmC,QAAQ,OAAe,SAAiB;EAGlF,OAAO,OAFQ,IAAI,OAAO,OAAO,KAAK,CAEnB,EAAE,GAAG,UAAU,IAAI,CAAC,CAAC,KAAK,EAAE;CACjD,CAAC,CAAC,CACD,QAAQ,oCAAoC,MAAM,CAAC,CACnD,QAAQ,gBAAgB,IAAI,CAAC,CAC7B,QAAQ,YAAY,EAAE,CAAC,CACvB,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;AACV;;AAGA,SAAS,UAAU,MAAsB;CACvC,OAAO,KAAK,QAAQ,YAAY,EAAE;AACpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyCA,SAAgB,aACd,SAC6C;CAC7C,MAAM,WAAW,SAAS,YAAY;CACtC,MAAM,YAAY,SAAS,aAAa;CACxC,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,aAAa,SAAS;CAC5B,MAAM,sBAAsB,SAAS,uBAAuB;CAK5D,IAAI,YAAY,QACd,AAAK,gBAAgB;CAGvB,OAAO,KAAoC;EACzC,MAAM,SAAS,QAAQ;EACvB,aACE;EAGF,OAAO;EACP,MAAM,QAAQ,OAAO;GACnB,MAAM,MAAM,SAAS,MAAM,GAAG;GAC9B,kBAAkB,KAAK,UAAU;GAEjC,IAAI;GAEJ,IAAI;IAGF,WAAW,MAAM,aAAa,IAAI,SAAS,GAAG;KAC5C,gBAAgB,CAAC,QAAQ,OAAO;KAChC,eAAe;KACf,wBAAwB,CAAC;KACzB;IACF,CAAC;GACH,SAAS,OAAO;IACd,IAAI,iBAAiB,qBACnB,MAAM,IAAI,aAAa,sBAAsB,MAAM,WAAW;KAC5D,MAAM;KACN;IACF,CAAC;IAKH,MAAM,IAAI,aAAa,6BAFP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEN;KAC7D,MAAM;KACN;IACF,CAAC;GACH;GAEA,MAAM,EAAE,MAAM,cAAc,MAAM,SAAS,UAAU,QAAQ;GAC7D,MAAM,UAAU,MAAM,OAAO,MAAM,SAAS,OAAO,IAAI,SAAS,GAAG,OAAO;GAE1E,OAAO;IACL,KAAK,SAAS,OAAO,IAAI,SAAS;IAClC,QAAQ,SAAS;IACjB;IACA;GACF;EACF;CACF,CAAC;AACH"}
|
package/llms-full.txt
CHANGED
|
@@ -199,7 +199,7 @@ const response = await handler.handle(jsonRpcRequest); // request → response,
|
|
|
199
199
|
|
|
200
200
|
---
|
|
201
201
|
name: use-web-and-http-tools
|
|
202
|
-
description: 'Wire the @warlock.js/ai-tools belt — web search, fetch/scrape, HTTP/REST, calculator, date-time — into an agent under their guardrails. Triggers: `ai.tools.webSearch`, `ai.tools.fetchUrl`, `ai.tools.http`, `ai.tools.calculator`, `ai.tools.dateTime`, `webSearchTool`, `fetchUrlTool`, `httpRequestTool`, `calculatorTool`, `dateTimeTool`, `web_search`, `fetch_url`, `http_request`, `calculator`, `date_time`, `WebToolError`, `HttpPolicyError`, `CalculatorError`, `DateTimeError`, `allowHosts`, `allowMethods`, `baseUrl`, `maxBytes`, `SearchProvider`, `TAVILY_API_KEY`; ''give an agent web search'', ''let an agent fetch/scrape a page'', ''let an agent call a REST API'', ''restrict which hosts/methods an agent can hit'', ''add a calculator/date tool to an agent'', ''SSRF guardrail for an agent tool''; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: connecting to an external MCP server — `@warlock.js/ai-tools/connect-mcp-server/SKILL.md`; exposing tools AS an MCP server — `@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md`; filesystem/shell tools — `@warlock.js/ai-workspace/use-a-workspace/SKILL.md`; defining your own custom tool — `@warlock.js/ai/define-ai-tool/SKILL.md`.'
|
|
202
|
+
description: 'Wire the @warlock.js/ai-tools belt — web search, fetch/scrape, HTTP/REST, calculator, date-time — into an agent under their guardrails. Triggers: `ai.tools.webSearch`, `ai.tools.fetchUrl`, `ai.tools.http`, `ai.tools.calculator`, `ai.tools.dateTime`, `webSearchTool`, `fetchUrlTool`, `httpRequestTool`, `calculatorTool`, `dateTimeTool`, `web_search`, `fetch_url`, `http_request`, `calculator`, `date_time`, `WebToolError`, `HttpPolicyError`, `CalculatorError`, `DateTimeError`, `allowHosts`, `allowMethods`, `allowPrivateNetwork`, `baseUrl`, `maxBytes`, `guardedFetch`, `SearchProvider`, `TAVILY_API_KEY`; ''give an agent web search'', ''let an agent fetch/scrape a page'', ''let an agent call a REST API'', ''restrict which hosts/methods an agent can hit'', ''add a calculator/date tool to an agent'', ''SSRF guardrail for an agent tool'', ''let a tool reach an internal/dev service''; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: connecting to an external MCP server — `@warlock.js/ai-tools/connect-mcp-server/SKILL.md`; exposing tools AS an MCP server — `@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md`; filesystem/shell tools — `@warlock.js/ai-workspace/use-a-workspace/SKILL.md`; defining your own custom tool — `@warlock.js/ai/define-ai-tool/SKILL.md`.'
|
|
203
203
|
---
|
|
204
204
|
|
|
205
205
|
# Use the web + HTTP tool belt
|
|
@@ -256,7 +256,9 @@ ai.tools.fetchUrl({ extract: "text", allowHosts: ["docs.stripe.com"], maxBytes:
|
|
|
256
256
|
```
|
|
257
257
|
|
|
258
258
|
- `extract`: `"text"` (default — readability-extracted main text), `"html"` (raw body), or `"markdown"`. **`"text"`/`"markdown"` lazily import the `@mozilla/readability` + `jsdom` optional peers** — a missing peer is a `WebToolError` of type `"missing-peer"` carrying the `npm install @mozilla/readability jsdom` string. `"html"` needs nothing.
|
|
259
|
-
- `allowHosts` (when set) rejects any other host **before the fetch** (an SSRF guardrail) → `"denied-host"
|
|
259
|
+
- `allowHosts` (when set) rejects any other host **before the fetch** (an SSRF guardrail) → `"denied-host"`, and redirect targets are held to the same allowlist.
|
|
260
|
+
- **Private-network deny by default.** Every request — and every redirect hop — now routes through `@warlock.js/ai`'s hardened `guardedFetch` outbound policy instead of a local host check. It refuses private / loopback / link-local / CGNAT / cloud-metadata addresses (`169.254.169.254`, RFC1918, `localhost`, …) by resolving the hostname via DNS and checking every returned address, failing closed on resolution failure — and re-validates every redirect `Location` (scheme, allowlist, private-IP deny) before following it. This applies even with no `allowHosts` set, so a bare `ai.tools.fetchUrl()` is no longer an SSRF primitive; blocks surface as `WebToolError` `type: "denied-host"`.
|
|
261
|
+
- `allowPrivateNetwork` (default `false`) — set `true` to deliberately permit a private/internal target, e.g. a tool that must call a local dev server. Pair with `allowHosts` to scope it rather than opening the whole private range.
|
|
260
262
|
- `maxBytes` caps the body and flags `truncated`; `timeoutMs` aborts via `AbortSignal.timeout`.
|
|
261
263
|
|
|
262
264
|
## `http_request` — a guarded REST client
|
|
@@ -269,6 +271,7 @@ ai.tools.http({
|
|
|
269
271
|
headers: { authorization: `Bearer ${process.env.STRIPE_KEY}` },
|
|
270
272
|
timeoutMs: 15_000,
|
|
271
273
|
maxBytes: 1_000_000,
|
|
274
|
+
// allowPrivateNetwork: true, // only for a deliberate internal/dev target
|
|
272
275
|
});
|
|
273
276
|
```
|
|
274
277
|
|
|
@@ -276,6 +279,7 @@ All guardrails are enforced **before the network call** and surface as a typed `
|
|
|
276
279
|
- a method outside `allowMethods` → `"method-not-allowed"`;
|
|
277
280
|
- a host outside `allowHosts` → `"host-not-allowed"` (SSRF guardrail);
|
|
278
281
|
- an unresolvable URL → `"invalid-url"`.
|
|
282
|
+
- **Private-network deny by default** — like `fetch_url`, every request and redirect hop routes through `guardedFetch`, refusing private/loopback/link-local/CGNAT/cloud-metadata addresses (DNS-resolved and re-checked per redirect hop) even when `allowHosts` isn't set → `"host-not-allowed"`. Set `allowPrivateNetwork: true` (default `false`) to permit a deliberate internal target.
|
|
279
283
|
|
|
280
284
|
When `baseUrl` is set the model passes a **path** joined against it; otherwise it must pass an absolute `http(s)` URL. Static `headers` merge **under** the per-call headers (per-call wins). An object `body` is JSON-serialized (with a default `content-type: application/json`); a string `body` is sent verbatim; `body` is dropped for `GET`. The response body is JSON-parsed when the `content-type` is JSON, else returned as text.
|
|
281
285
|
|
package/llms.txt
CHANGED
|
@@ -8,4 +8,4 @@
|
|
|
8
8
|
|
|
9
9
|
- [connect-mcp-server](@warlock.js/ai-tools/connect-mcp-server/SKILL.md): Connect a @warlock.js/ai agent to an external Model Context Protocol (MCP) server and adapt its tools as native agent tools (Direction A) with `ai.mcp(server)`. Triggers: `ai.mcp(`, `ai.mcp.serve` (the other direction), `mcp(`, `McpClient`, `McpClientOptions`, `McpTransport`, `client.tools()`, `client.close()`, `namePrefix`, `filter`, `McpTransportError`, `{ type: "stdio" }`, `{ type: "http" }`, `tools/list`, `tools/call`, `jsonSchemaToStandard`, `@modelcontextprotocol/sdk`, `ajv`; 'consume an MCP server', 'use an external MCP server's tools in my agent', 'connect to the GitHub/Slack/Filesystem MCP server', 'adapt MCP tools as agent tools', 'bridge my agent to the MCP ecosystem', 'run an MCP server over stdio/http'; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: exposing YOUR agent/tools AS an MCP server (the reverse direction) — `@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md`; the built-in web/HTTP/calc/date tools — `@warlock.js/ai-tools/use-web-and-http-tools/SKILL.md`.
|
|
10
10
|
- [expose-as-mcp-server](@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md): Expose a @warlock.js/ai agent / supervisor / orchestrator (or a raw ToolContract[]) AS a Model Context Protocol (MCP) server other clients consume (Direction B) with `ai.mcp.serve(source, options)`. Triggers: `ai.mcp.serve(`, `serve(`, `createServeHandler`, `McpServer`, `McpServeOptions`, `McpServeSource`, `McpServeTransport`, `server.start()`, `server.stop()`, `schemaTarget`, `draft-2020-12`, `extractJsonSchema`, `tools/list`, `tools/call`, `{ type: "stdio" }`, `{ type: "http"; port }`; 'expose my agent as an MCP server', 'publish my tools to Cursor/Claude Desktop', 'make my warlock workspace an MCP server', 'serve my ToolContract[] over MCP', 'let other MCP clients call my agent'; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: CONSUMING an external MCP server (the reverse direction) — `@warlock.js/ai-tools/connect-mcp-server/SKILL.md`; the built-in web/HTTP/calc/date tools — `@warlock.js/ai-tools/use-web-and-http-tools/SKILL.md`.
|
|
11
|
-
- [use-web-and-http-tools](@warlock.js/ai-tools/use-web-and-http-tools/SKILL.md): Wire the @warlock.js/ai-tools belt — web search, fetch/scrape, HTTP/REST, calculator, date-time — into an agent under their guardrails. Triggers: `ai.tools.webSearch`, `ai.tools.fetchUrl`, `ai.tools.http`, `ai.tools.calculator`, `ai.tools.dateTime`, `webSearchTool`, `fetchUrlTool`, `httpRequestTool`, `calculatorTool`, `dateTimeTool`, `web_search`, `fetch_url`, `http_request`, `calculator`, `date_time`, `WebToolError`, `HttpPolicyError`, `CalculatorError`, `DateTimeError`, `allowHosts`, `allowMethods`, `baseUrl`, `maxBytes`, `SearchProvider`, `TAVILY_API_KEY`; 'give an agent web search', 'let an agent fetch/scrape a page', 'let an agent call a REST API', 'restrict which hosts/methods an agent can hit', 'add a calculator/date tool to an agent', 'SSRF guardrail for an agent tool'; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: connecting to an external MCP server — `@warlock.js/ai-tools/connect-mcp-server/SKILL.md`; exposing tools AS an MCP server — `@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md`; filesystem/shell tools — `@warlock.js/ai-workspace/use-a-workspace/SKILL.md`; defining your own custom tool — `@warlock.js/ai/define-ai-tool/SKILL.md`.
|
|
11
|
+
- [use-web-and-http-tools](@warlock.js/ai-tools/use-web-and-http-tools/SKILL.md): Wire the @warlock.js/ai-tools belt — web search, fetch/scrape, HTTP/REST, calculator, date-time — into an agent under their guardrails. Triggers: `ai.tools.webSearch`, `ai.tools.fetchUrl`, `ai.tools.http`, `ai.tools.calculator`, `ai.tools.dateTime`, `webSearchTool`, `fetchUrlTool`, `httpRequestTool`, `calculatorTool`, `dateTimeTool`, `web_search`, `fetch_url`, `http_request`, `calculator`, `date_time`, `WebToolError`, `HttpPolicyError`, `CalculatorError`, `DateTimeError`, `allowHosts`, `allowMethods`, `allowPrivateNetwork`, `baseUrl`, `maxBytes`, `guardedFetch`, `SearchProvider`, `TAVILY_API_KEY`; 'give an agent web search', 'let an agent fetch/scrape a page', 'let an agent call a REST API', 'restrict which hosts/methods an agent can hit', 'add a calculator/date tool to an agent', 'SSRF guardrail for an agent tool', 'let a tool reach an internal/dev service'; typical import `import "@warlock.js/ai-tools"; import { ai } from "@warlock.js/ai"`. Skip: connecting to an external MCP server — `@warlock.js/ai-tools/connect-mcp-server/SKILL.md`; exposing tools AS an MCP server — `@warlock.js/ai-tools/expose-as-mcp-server/SKILL.md`; filesystem/shell tools — `@warlock.js/ai-workspace/use-a-workspace/SKILL.md`; defining your own custom tool — `@warlock.js/ai/define-ai-tool/SKILL.md`.
|
package/package.json
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"@modelcontextprotocol/sdk": "*",
|
|
23
23
|
"@mozilla/readability": "*",
|
|
24
24
|
"@tavily/core": "*",
|
|
25
|
-
"@warlock.js/ai": "4.
|
|
25
|
+
"@warlock.js/ai": "4.16.0",
|
|
26
26
|
"ajv": "*",
|
|
27
27
|
"jsdom": "*"
|
|
28
28
|
},
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"optional": true
|
|
44
44
|
}
|
|
45
45
|
},
|
|
46
|
-
"version": "4.
|
|
46
|
+
"version": "4.16.0",
|
|
47
47
|
"main": "./cjs/index.cjs",
|
|
48
48
|
"module": "./esm/index.mjs",
|
|
49
49
|
"types": "./esm/index.d.mts",
|