@crewhaus/tool-web 0.5.8 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -17,6 +17,14 @@ export type WebFetchConfigInput = {
17
17
  readonly allowedDomains?: readonly string[];
18
18
  };
19
19
  export declare function registerWebFetchConfig(input: WebFetchConfigInput): void;
20
+ /**
21
+ * 0.6.0 §4.4 — the config ONE call runs under: the serving candidate's
22
+ * `tool_config.webFetch` block when its profile declares one
23
+ * (`ToolExecuteContext.toolConfig`, REPLACING the registered block for this
24
+ * call exactly as `registerWebFetchConfig` replaces it at boot), else the
25
+ * process-global registration. A non-object override is ignored.
26
+ */
27
+ export declare function resolveWebFetchConfig(override: unknown): WebFetchConfig;
20
28
  export declare function getWebFetchConfig(): WebFetchConfig;
21
29
  export declare function _resetWebFetchConfig(): void;
22
30
  /**
@@ -29,6 +37,30 @@ export type DnsLookupFn = (host: string) => Promise<{
29
37
  readonly family: number;
30
38
  }>;
31
39
  export declare function _setDnsLookup(fn: DnsLookupFn | undefined): void;
40
+ /**
41
+ * Canonicalise an IPv4 literal.
42
+ *
43
+ * `inet_aton` forms are the classic allow-list bypass: `0177.0.0.1`,
44
+ * `0x7f.0.0.1`, `2130706433` and `127.1` are all 127.0.0.1, and a check that
45
+ * only understands dotted-decimal waves every one of them through. Returns
46
+ * `null` when the string is not an IPv4 literal at all.
47
+ */
48
+ export declare function normalizeIpv4(raw: string): string | null;
49
+ /**
50
+ * Expand an IPv6 literal to its eight 16-bit groups, or `null` when the string
51
+ * is not one.
52
+ *
53
+ * Classifying IPv6 by its TEXT is where the bypasses live, because one address
54
+ * has many spellings and the one a check was written against is rarely the one
55
+ * that arrives. `http://[::ffff:169.254.169.254]/` never reaches a guard in
56
+ * that form: the WHATWG URL parser re-serialises the embedded quad as hex
57
+ * pieces, so what the guard sees is `::ffff:a9fe:a9fe`. Normalising to numbers
58
+ * first means the prefix tests below are arithmetic, and spelling stops
59
+ * mattering.
60
+ */
61
+ export declare function parseIpv6(raw: string): ReadonlyArray<number> | null;
62
+ /** Ranges that are never a public RPC endpoint. */
63
+ export declare function isPrivateIp(address: string): boolean;
32
64
  /**
33
65
  * `pinnedIp` is the address `assertNotSsrf` validated for the request's host
34
66
  * (empty string ⇒ no pin, e.g. fixed trusted API hosts). The production
package/dist/index.js CHANGED
@@ -51,8 +51,24 @@ export class WebFetchPermissionError extends CrewhausError {
51
51
  }
52
52
  let webFetchConfig = { allowedDomains: [] };
53
53
  export function registerWebFetchConfig(input) {
54
+ webFetchConfig = buildWebFetchConfig(input);
55
+ }
56
+ function buildWebFetchConfig(input) {
54
57
  const raw = input.allowedDomains ?? input.allowed_domains ?? [];
55
- webFetchConfig = { allowedDomains: raw.map((d) => d.toLowerCase()) };
58
+ return { allowedDomains: raw.map((d) => d.toLowerCase()) };
59
+ }
60
+ /**
61
+ * 0.6.0 §4.4 — the config ONE call runs under: the serving candidate's
62
+ * `tool_config.webFetch` block when its profile declares one
63
+ * (`ToolExecuteContext.toolConfig`, REPLACING the registered block for this
64
+ * call exactly as `registerWebFetchConfig` replaces it at boot), else the
65
+ * process-global registration. A non-object override is ignored.
66
+ */
67
+ export function resolveWebFetchConfig(override) {
68
+ if (typeof override === "object" && override !== null && !Array.isArray(override)) {
69
+ return buildWebFetchConfig(override);
70
+ }
71
+ return webFetchConfig;
56
72
  }
57
73
  export function getWebFetchConfig() {
58
74
  return webFetchConfig;
@@ -60,11 +76,11 @@ export function getWebFetchConfig() {
60
76
  export function _resetWebFetchConfig() {
61
77
  webFetchConfig = { allowedDomains: [] };
62
78
  }
63
- function isHostAllowed(host) {
64
- if (webFetchConfig.allowedDomains.length === 0)
79
+ function isHostAllowed(host, cfg) {
80
+ if (cfg.allowedDomains.length === 0)
65
81
  return true;
66
82
  const lower = host.toLowerCase();
67
- return webFetchConfig.allowedDomains.some((entry) => lower === entry || lower.endsWith(`.${entry}`));
83
+ return cfg.allowedDomains.some((entry) => lower === entry || lower.endsWith(`.${entry}`));
68
84
  }
69
85
  // Production default — a single named function shared by both the initial
70
86
  // binding and the `_setDnsLookup(undefined)` reset path, so there is one
@@ -75,109 +91,206 @@ let dnsLookupFn = defaultDnsLookup;
75
91
  export function _setDnsLookup(fn) {
76
92
  dnsLookupFn = fn ?? defaultDnsLookup;
77
93
  }
78
- // Normalise octal/hex/integer IPv4 literals (e.g. 0177.0.0.1, 0x7f000001,
79
- // 2130706433) to dotted-decimal so isPrivateIp can classify them — browsers
80
- // and many HTTP stacks accept these forms, which would otherwise bypass a
81
- // naive dotted-only check. Mirrors @crewhaus/tool-fetch / crawler.
82
- function normalizeIpv4(raw) {
94
+ // ---------------------------------------------------------------------------
95
+ // BEGIN SYNCHRONISED BLOCK — private-address classifier
96
+ //
97
+ // This block is BYTE-IDENTICAL across every package that guards an outbound
98
+ // request. Do not edit one copy: `apps/cli/src/tool-registry.test.ts` hashes
99
+ // them all and fails if any differs, and it asserts how many it found, because
100
+ // a copy-scanning guard that matches nothing reports green.
101
+ //
102
+ // It exists in copies rather than a shared package because these files are
103
+ // otherwise independent per-package networking layers; a guard proving the
104
+ // copies are identical is cheaper and safer than the import graph a shared
105
+ // package would need across `crawler`, `computer-use-driver` and ten tools.
106
+ //
107
+ // WHY IT PARSES INSTEAD OF MATCHING TEXT. Six copies were confirmed
108
+ // exploitable on 2026-09-18 because they compared address STRINGS. The WHATWG
109
+ // URL parser rewrites `[::ffff:169.254.169.254]` to `[::ffff:a9fe:a9fe]`, so a
110
+ // text check never sees the spelling it was written for; and `64:ff9b::a9fe:a9fe`
111
+ // IS 169.254.169.254 on any network running DNS64/NAT64. Parsing numerically and
112
+ // recursing into the embedded IPv4 is the only form that holds.
113
+ //
114
+ // WHAT IT CANNOT DO. RFC 6052 lets an operator choose any Network-Specific
115
+ // Prefix for NAT64, so an embedded IPv4 behind an arbitrary NSP is undecidable
116
+ // from the address alone. That case is configuration, and the callers that need
117
+ // it resolve the host and re-check the ANSWER before dialling.
118
+ // ---------------------------------------------------------------------------
119
+ /**
120
+ * Canonicalise an IPv4 literal.
121
+ *
122
+ * `inet_aton` forms are the classic allow-list bypass: `0177.0.0.1`,
123
+ * `0x7f.0.0.1`, `2130706433` and `127.1` are all 127.0.0.1, and a check that
124
+ * only understands dotted-decimal waves every one of them through. Returns
125
+ * `null` when the string is not an IPv4 literal at all.
126
+ */
127
+ export function normalizeIpv4(raw) {
83
128
  const trimmed = raw.trim();
84
- if (trimmed === "")
129
+ if (trimmed === "" || /[^0-9a-fA-FxX.]/.test(trimmed))
85
130
  return null;
86
- const parseComponent = (s) => {
87
- if (s === "")
131
+ const parts = trimmed.split(".");
132
+ if (parts.length === 0 || parts.length > 4)
133
+ return null;
134
+ const values = [];
135
+ for (const part of parts) {
136
+ if (part === "")
88
137
  return null;
89
138
  let value;
90
- if (/^0[xX][0-9a-fA-F]+$/.test(s))
91
- value = Number.parseInt(s.slice(2), 16);
92
- else if (/^0[0-7]+$/.test(s))
93
- value = Number.parseInt(s, 8);
94
- else if (/^[0-9]+$/.test(s))
95
- value = Number.parseInt(s, 10);
139
+ if (/^0[xX][0-9a-fA-F]+$/.test(part))
140
+ value = Number.parseInt(part.slice(2), 16);
141
+ else if (/^0[0-7]+$/.test(part))
142
+ value = Number.parseInt(part.slice(1), 8);
143
+ else if (/^\d+$/.test(part))
144
+ value = Number.parseInt(part, 10);
96
145
  else
97
146
  return null;
98
- return Number.isNaN(value) ? null : value;
99
- };
100
- const segments = trimmed.split(".");
101
- if (segments.length > 4)
102
- return null;
103
- const components = [];
104
- for (const seg of segments) {
105
- const value = parseComponent(seg);
106
- if (value === null || value < 0)
147
+ if (!Number.isFinite(value) || value < 0)
107
148
  return null;
108
- components.push(value);
149
+ values.push(value);
109
150
  }
110
- const n = components.length;
111
- const octets = [0, 0, 0, 0];
112
- for (let i = 0; i < n - 1; i++) {
113
- const c = components[i];
114
- if (c > 255)
151
+ // The short forms pack the remaining octets into the last part: `127.1` is
152
+ // 127.0.0.1, not 127.1.0.0. Getting this backwards is how a bypass survives.
153
+ const last = values[values.length - 1];
154
+ const leading = values.slice(0, -1);
155
+ if (leading.some((v) => v > 255))
156
+ return null;
157
+ const remaining = 4 - leading.length;
158
+ if (last >= 2 ** (8 * remaining))
159
+ return null;
160
+ const octets = [...leading];
161
+ for (let i = remaining - 1; i >= 0; i--)
162
+ octets.push((last >>> (8 * i)) & 0xff);
163
+ return octets.join(".");
164
+ }
165
+ /**
166
+ * Expand an IPv6 literal to its eight 16-bit groups, or `null` when the string
167
+ * is not one.
168
+ *
169
+ * Classifying IPv6 by its TEXT is where the bypasses live, because one address
170
+ * has many spellings and the one a check was written against is rarely the one
171
+ * that arrives. `http://[::ffff:169.254.169.254]/` never reaches a guard in
172
+ * that form: the WHATWG URL parser re-serialises the embedded quad as hex
173
+ * pieces, so what the guard sees is `::ffff:a9fe:a9fe`. Normalising to numbers
174
+ * first means the prefix tests below are arithmetic, and spelling stops
175
+ * mattering.
176
+ */
177
+ export function parseIpv6(raw) {
178
+ let text = raw.trim().toLowerCase();
179
+ if (text.startsWith("["))
180
+ text = text.slice(1);
181
+ if (text.endsWith("]"))
182
+ text = text.slice(0, -1);
183
+ const zone = text.indexOf("%"); // fe80::1%eth0
184
+ if (zone !== -1)
185
+ text = text.slice(0, zone);
186
+ if (!text.includes(":"))
187
+ return null;
188
+ // A trailing dotted quad is the last two groups written in IPv4. Require all
189
+ // three dots: without that, `::1` parses as a one-part inet_aton address and
190
+ // takes a path that has nothing to do with what was written.
191
+ const dotted = /^(.*:)(\d+(?:\.\d+){3})$/.exec(text);
192
+ if (dotted !== null) {
193
+ const quad = normalizeIpv4(dotted[2]);
194
+ if (quad === null)
115
195
  return null;
116
- octets[i] = c;
196
+ const o = quad.split(".").map((n) => Number.parseInt(n, 10));
197
+ const hi = (o[0] << 8) | o[1];
198
+ const lo = (o[2] << 8) | o[3];
199
+ text = `${dotted[1]}${hi.toString(16)}:${lo.toString(16)}`;
117
200
  }
118
- const last = components[n - 1];
119
- const maxLast = 2 ** (8 * (4 - (n - 1)));
120
- if (last >= maxLast)
201
+ const halves = text.split("::");
202
+ if (halves.length > 2)
203
+ return null;
204
+ const pieces = (part) => (part === "" ? [] : part.split(":"));
205
+ const head = pieces(halves[0]);
206
+ const tail = halves.length === 2 ? pieces(halves[1]) : [];
207
+ if (halves.length === 1 && head.length !== 8)
208
+ return null;
209
+ if (head.length + tail.length > 8)
121
210
  return null;
122
- let rest = last;
123
- for (let i = 3; i >= n - 1; i--) {
124
- octets[i] = rest & 0xff;
125
- rest = Math.floor(rest / 256);
211
+ const groups = [];
212
+ for (const piece of head) {
213
+ if (!/^[0-9a-f]{1,4}$/.test(piece))
214
+ return null;
215
+ groups.push(Number.parseInt(piece, 16));
126
216
  }
127
- return octets.join(".");
217
+ for (let i = head.length + tail.length; i < 8; i++)
218
+ groups.push(0);
219
+ for (const piece of tail) {
220
+ if (!/^[0-9a-f]{1,4}$/.test(piece))
221
+ return null;
222
+ groups.push(Number.parseInt(piece, 16));
223
+ }
224
+ return groups.length === 8 ? groups : null;
128
225
  }
129
- function isPrivateIp(addr) {
130
- const ip = addr.replace(/^\[/, "").replace(/\]$/, "");
131
- const lowerIp = ip.toLowerCase();
132
- if (lowerIp === "::" || lowerIp === "::1")
133
- return true; // IPv6 unspecified / loopback
134
- if (lowerIp.startsWith("fe80:"))
135
- return true; // IPv6 link-local
136
- if (lowerIp.startsWith("fc") || lowerIp.startsWith("fd"))
137
- return true; // IPv6 unique-local
138
- // IPv4-mapped IPv6 — both dotted (::ffff:127.0.0.1) AND hex-compressed
139
- // (::ffff:7f00:1, which `new URL` normalizes [::ffff:127.0.0.1] into).
140
- const mapped = lowerIp.match(/^::ffff:(.+)$/);
141
- if (mapped) {
142
- const tail = mapped[1];
143
- if (tail.includes("."))
144
- return isPrivateIp(tail);
145
- const hexGroups = tail.split(":");
146
- if (hexGroups.length === 2 && hexGroups.every((g) => /^[0-9a-f]{1,4}$/.test(g))) {
147
- const hi = Number.parseInt(hexGroups[0], 16);
148
- const lo = Number.parseInt(hexGroups[1], 16);
149
- return isPrivateIp([(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff].join("."));
150
- }
151
- return false;
226
+ /**
227
+ * The IPv4 address an IPv6 address carries, when it carries one.
228
+ *
229
+ * Every transition mechanism embeds a v4 address somewhere, and every one of
230
+ * them is a way to reach a v4 destination while wearing a v6 spelling that no
231
+ * v4 range check looks at. The NAT64 well-known prefix is the sharpest: a
232
+ * DNS64 resolver answers an IPv4-only name with `64:ff9b::<the v4>`, so
233
+ * `64:ff9b::a9fe:a9fe` IS the metadata service on any network that runs one.
234
+ */
235
+ function embeddedIpv4(g) {
236
+ const quad = (hi, lo) => `${(hi >>> 8) & 0xff}.${hi & 0xff}.${(lo >>> 8) & 0xff}.${lo & 0xff}`;
237
+ const zeros = (from, to) => g.slice(from, to).every((x) => x === 0);
238
+ const last = quad(g[6], g[7]);
239
+ if (zeros(0, 5) && g[5] === 0xffff)
240
+ return last; // ::ffff:0:0/96, IPv4-mapped
241
+ if (zeros(0, 4) && g[4] === 0xffff && g[5] === 0)
242
+ return last; // ::ffff:0:0:0/96, translated
243
+ if (g[0] === 0x64 && g[1] === 0xff9b)
244
+ return last; // 64:ff9b::/96 and 64:ff9b:1::/48, NAT64
245
+ if (g[0] === 0x2002)
246
+ return quad(g[1], g[2]); // 2002::/16, 6to4
247
+ // ::a.b.c.d, IPv4-compatible: deprecated, still routed by some stacks, and
248
+ // not to be confused with `::` or `::1`, which are handled before this.
249
+ if (zeros(0, 6))
250
+ return last;
251
+ return null;
252
+ }
253
+ /** Ranges that are never a public RPC endpoint. */
254
+ export function isPrivateIp(address) {
255
+ const v4 = normalizeIpv4(address);
256
+ if (v4 !== null) {
257
+ const [a = 0, b = 0, c = 0, d = 0] = v4.split(".").map((n) => Number.parseInt(n, 10));
258
+ if (a === 0 || a === 10 || a === 127)
259
+ return true;
260
+ if (a === 169 && b === 254)
261
+ return true; // link-local, and the metadata service
262
+ if (a === 172 && b >= 16 && b <= 31)
263
+ return true;
264
+ if (a === 192 && b === 168)
265
+ return true;
266
+ if (a === 192 && b === 0 && c === 0)
267
+ return true;
268
+ if (a === 100 && b >= 64 && b <= 127)
269
+ return true; // carrier-grade NAT
270
+ if (a === 198 && (b === 18 || b === 19))
271
+ return true; // benchmarking
272
+ if (a >= 224)
273
+ return true; // multicast, reserved, broadcast
274
+ return a === 255 && b === 255 && c === 255 && d === 255;
152
275
  }
153
- const normalized = normalizeIpv4(ip);
154
- if (normalized === null)
276
+ const groups = parseIpv6(address);
277
+ if (groups === null)
155
278
  return false;
156
- const parts = normalized.split(".").map((p) => Number.parseInt(p, 10));
157
- if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {
158
- return false;
159
- }
160
- const [a, b] = parts;
161
- if (a === 127)
162
- return true; // 127.0.0.0/8 loopback
163
- if (a === 10)
164
- return true; // 10.0.0.0/8 RFC1918
165
- if (a === 169 && b === 254)
166
- return true; // 169.254.0.0/16 link-local + cloud metadata
167
- if (a === 172 && b >= 16 && b <= 31)
168
- return true; // 172.16.0.0/12 RFC1918
169
- if (a === 192 && b === 168)
170
- return true; // 192.168.0.0/16 RFC1918
171
- if (a === 100 && b >= 64 && b <= 127)
172
- return true; // 100.64.0.0/10 CGNAT (RFC6598)
173
- if (a === 192 && b === 0 && parts[2] === 0)
174
- return true; // 192.0.0.0/24 (RFC6890)
175
- if (a === 198 && (b === 18 || b === 19))
176
- return true; // 198.18.0.0/15 (RFC2544)
177
- if (a === 0)
178
- return true; // 0.0.0.0/8
179
- return false;
279
+ if (groups.every((g) => g === 0))
280
+ return true; // ::, the unspecified address
281
+ if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1)
282
+ return true; // ::1
283
+ const carried = embeddedIpv4(groups);
284
+ if (carried !== null)
285
+ return isPrivateIp(carried);
286
+ const head = groups[0];
287
+ if ((head & 0xfe00) === 0xfc00)
288
+ return true; // fc00::/7, unique-local
289
+ if ((head & 0xffc0) === 0xfe80)
290
+ return true; // fe80::/10, link-local
291
+ return (head & 0xff00) === 0xff00; // ff00::/8, multicast
180
292
  }
293
+ // END SYNCHRONISED BLOCK
181
294
  /**
182
295
  * Reject any host that is — or resolves to — a private/loopback/link-local
183
296
  * address. Applied at the initial URL and at every redirect hop, regardless
@@ -299,7 +412,7 @@ async function readBodyCapped(res) {
299
412
  }
300
413
  return new TextDecoder("utf-8", { fatal: false }).decode(merged);
301
414
  }
302
- async function performWebFetch(initialUrl, signal) {
415
+ async function performWebFetch(initialUrl, signal, cfg) {
303
416
  let currentUrl = initialUrl;
304
417
  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
305
418
  if (currentUrl.protocol !== "http:" && currentUrl.protocol !== "https:") {
@@ -309,7 +422,7 @@ async function performWebFetch(initialUrl, signal) {
309
422
  // The returned IP pins the socket so a rebinding resolver can't swap in a
310
423
  // private address between this check and connect.
311
424
  const pinnedIp = await assertNotSsrf(currentUrl.hostname);
312
- if (!isHostAllowed(currentUrl.hostname)) {
425
+ if (!isHostAllowed(currentUrl.hostname, cfg)) {
313
426
  throw new WebFetchPermissionError(`WebFetch denied: host "${currentUrl.hostname}" is not in allowed_domains`);
314
427
  }
315
428
  const res = await rawFetch(new Request(currentUrl.toString(), {
@@ -394,7 +507,7 @@ export const webFetch = buildTool({
394
507
  }
395
508
  let res;
396
509
  try {
397
- res = await performWebFetch(url, ctrl.signal);
510
+ res = await performWebFetch(url, ctrl.signal, resolveWebFetchConfig(ctx?.toolConfig));
398
511
  }
399
512
  finally {
400
513
  clearTimeout(timer);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crewhaus/tool-web",
3
- "version": "0.5.8",
3
+ "version": "0.7.0",
4
4
  "type": "module",
5
5
  "description": "WebFetch (URL → markdown via cheerio + turndown) and WebSearch (provider dispatch)",
6
6
  "main": "dist/index.js",
@@ -15,9 +15,9 @@
15
15
  "test": "bun test src"
16
16
  },
17
17
  "dependencies": {
18
- "@crewhaus/errors": "0.5.8",
19
- "@crewhaus/tool-builder": "0.5.8",
20
- "@crewhaus/tool-catalog": "0.5.8",
18
+ "@crewhaus/errors": "0.7.0",
19
+ "@crewhaus/tool-builder": "0.7.0",
20
+ "@crewhaus/tool-catalog": "0.7.0",
21
21
  "cheerio": "^1.0.0",
22
22
  "turndown": "^7.2.0",
23
23
  "zod": "^3.23.8"