@crewhaus/tool-web 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +24 -0
- package/dist/index.js +184 -87
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -37,6 +37,30 @@ export type DnsLookupFn = (host: string) => Promise<{
|
|
|
37
37
|
readonly family: number;
|
|
38
38
|
}>;
|
|
39
39
|
export declare function _setDnsLookup(fn: DnsLookupFn | undefined): void;
|
|
40
|
+
/**
|
|
41
|
+
* Canonicalise an IPv4 literal.
|
|
42
|
+
*
|
|
43
|
+
* `inet_aton` forms are the classic allow-list bypass: `0177.0.0.1`,
|
|
44
|
+
* `0x7f.0.0.1`, `2130706433` and `127.1` are all 127.0.0.1, and a check that
|
|
45
|
+
* only understands dotted-decimal waves every one of them through. Returns
|
|
46
|
+
* `null` when the string is not an IPv4 literal at all.
|
|
47
|
+
*/
|
|
48
|
+
export declare function normalizeIpv4(raw: string): string | null;
|
|
49
|
+
/**
|
|
50
|
+
* Expand an IPv6 literal to its eight 16-bit groups, or `null` when the string
|
|
51
|
+
* is not one.
|
|
52
|
+
*
|
|
53
|
+
* Classifying IPv6 by its TEXT is where the bypasses live, because one address
|
|
54
|
+
* has many spellings and the one a check was written against is rarely the one
|
|
55
|
+
* that arrives. `http://[::ffff:169.254.169.254]/` never reaches a guard in
|
|
56
|
+
* that form: the WHATWG URL parser re-serialises the embedded quad as hex
|
|
57
|
+
* pieces, so what the guard sees is `::ffff:a9fe:a9fe`. Normalising to numbers
|
|
58
|
+
* first means the prefix tests below are arithmetic, and spelling stops
|
|
59
|
+
* mattering.
|
|
60
|
+
*/
|
|
61
|
+
export declare function parseIpv6(raw: string): ReadonlyArray<number> | null;
|
|
62
|
+
/** Ranges that are never a public RPC endpoint. */
|
|
63
|
+
export declare function isPrivateIp(address: string): boolean;
|
|
40
64
|
/**
|
|
41
65
|
* `pinnedIp` is the address `assertNotSsrf` validated for the request's host
|
|
42
66
|
* (empty string ⇒ no pin, e.g. fixed trusted API hosts). The production
|
package/dist/index.js
CHANGED
|
@@ -91,109 +91,206 @@ let dnsLookupFn = defaultDnsLookup;
|
|
|
91
91
|
export function _setDnsLookup(fn) {
|
|
92
92
|
dnsLookupFn = fn ?? defaultDnsLookup;
|
|
93
93
|
}
|
|
94
|
-
//
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
//
|
|
98
|
-
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// BEGIN SYNCHRONISED BLOCK — private-address classifier
|
|
96
|
+
//
|
|
97
|
+
// This block is BYTE-IDENTICAL across every package that guards an outbound
|
|
98
|
+
// request. Do not edit one copy: `apps/cli/src/tool-registry.test.ts` hashes
|
|
99
|
+
// them all and fails if any differs, and it asserts how many it found, because
|
|
100
|
+
// a copy-scanning guard that matches nothing reports green.
|
|
101
|
+
//
|
|
102
|
+
// It exists in copies rather than a shared package because these files are
|
|
103
|
+
// otherwise independent per-package networking layers; a guard proving the
|
|
104
|
+
// copies are identical is cheaper and safer than the import graph a shared
|
|
105
|
+
// package would need across `crawler`, `computer-use-driver` and ten tools.
|
|
106
|
+
//
|
|
107
|
+
// WHY IT PARSES INSTEAD OF MATCHING TEXT. Six copies were confirmed
|
|
108
|
+
// exploitable on 2026-09-18 because they compared address STRINGS. The WHATWG
|
|
109
|
+
// URL parser rewrites `[::ffff:169.254.169.254]` to `[::ffff:a9fe:a9fe]`, so a
|
|
110
|
+
// text check never sees the spelling it was written for; and `64:ff9b::a9fe:a9fe`
|
|
111
|
+
// IS 169.254.169.254 on any network running DNS64/NAT64. Parsing numerically and
|
|
112
|
+
// recursing into the embedded IPv4 is the only form that holds.
|
|
113
|
+
//
|
|
114
|
+
// WHAT IT CANNOT DO. RFC 6052 lets an operator choose any Network-Specific
|
|
115
|
+
// Prefix for NAT64, so an embedded IPv4 behind an arbitrary NSP is undecidable
|
|
116
|
+
// from the address alone. That case is configuration, and the callers that need
|
|
117
|
+
// it resolve the host and re-check the ANSWER before dialling.
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
/**
|
|
120
|
+
* Canonicalise an IPv4 literal.
|
|
121
|
+
*
|
|
122
|
+
* `inet_aton` forms are the classic allow-list bypass: `0177.0.0.1`,
|
|
123
|
+
* `0x7f.0.0.1`, `2130706433` and `127.1` are all 127.0.0.1, and a check that
|
|
124
|
+
* only understands dotted-decimal waves every one of them through. Returns
|
|
125
|
+
* `null` when the string is not an IPv4 literal at all.
|
|
126
|
+
*/
|
|
127
|
+
export function normalizeIpv4(raw) {
|
|
99
128
|
const trimmed = raw.trim();
|
|
100
|
-
if (trimmed === "")
|
|
129
|
+
if (trimmed === "" || /[^0-9a-fA-FxX.]/.test(trimmed))
|
|
130
|
+
return null;
|
|
131
|
+
const parts = trimmed.split(".");
|
|
132
|
+
if (parts.length === 0 || parts.length > 4)
|
|
101
133
|
return null;
|
|
102
|
-
const
|
|
103
|
-
|
|
134
|
+
const values = [];
|
|
135
|
+
for (const part of parts) {
|
|
136
|
+
if (part === "")
|
|
104
137
|
return null;
|
|
105
138
|
let value;
|
|
106
|
-
if (/^0[xX][0-9a-fA-F]+$/.test(
|
|
107
|
-
value = Number.parseInt(
|
|
108
|
-
else if (/^0[0-7]+$/.test(
|
|
109
|
-
value = Number.parseInt(
|
|
110
|
-
else if (
|
|
111
|
-
value = Number.parseInt(
|
|
139
|
+
if (/^0[xX][0-9a-fA-F]+$/.test(part))
|
|
140
|
+
value = Number.parseInt(part.slice(2), 16);
|
|
141
|
+
else if (/^0[0-7]+$/.test(part))
|
|
142
|
+
value = Number.parseInt(part.slice(1), 8);
|
|
143
|
+
else if (/^\d+$/.test(part))
|
|
144
|
+
value = Number.parseInt(part, 10);
|
|
112
145
|
else
|
|
113
146
|
return null;
|
|
114
|
-
|
|
115
|
-
};
|
|
116
|
-
const segments = trimmed.split(".");
|
|
117
|
-
if (segments.length > 4)
|
|
118
|
-
return null;
|
|
119
|
-
const components = [];
|
|
120
|
-
for (const seg of segments) {
|
|
121
|
-
const value = parseComponent(seg);
|
|
122
|
-
if (value === null || value < 0)
|
|
147
|
+
if (!Number.isFinite(value) || value < 0)
|
|
123
148
|
return null;
|
|
124
|
-
|
|
149
|
+
values.push(value);
|
|
125
150
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
151
|
+
// The short forms pack the remaining octets into the last part: `127.1` is
|
|
152
|
+
// 127.0.0.1, not 127.1.0.0. Getting this backwards is how a bypass survives.
|
|
153
|
+
const last = values[values.length - 1];
|
|
154
|
+
const leading = values.slice(0, -1);
|
|
155
|
+
if (leading.some((v) => v > 255))
|
|
156
|
+
return null;
|
|
157
|
+
const remaining = 4 - leading.length;
|
|
158
|
+
if (last >= 2 ** (8 * remaining))
|
|
159
|
+
return null;
|
|
160
|
+
const octets = [...leading];
|
|
161
|
+
for (let i = remaining - 1; i >= 0; i--)
|
|
162
|
+
octets.push((last >>> (8 * i)) & 0xff);
|
|
163
|
+
return octets.join(".");
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Expand an IPv6 literal to its eight 16-bit groups, or `null` when the string
|
|
167
|
+
* is not one.
|
|
168
|
+
*
|
|
169
|
+
* Classifying IPv6 by its TEXT is where the bypasses live, because one address
|
|
170
|
+
* has many spellings and the one a check was written against is rarely the one
|
|
171
|
+
* that arrives. `http://[::ffff:169.254.169.254]/` never reaches a guard in
|
|
172
|
+
* that form: the WHATWG URL parser re-serialises the embedded quad as hex
|
|
173
|
+
* pieces, so what the guard sees is `::ffff:a9fe:a9fe`. Normalising to numbers
|
|
174
|
+
* first means the prefix tests below are arithmetic, and spelling stops
|
|
175
|
+
* mattering.
|
|
176
|
+
*/
|
|
177
|
+
export function parseIpv6(raw) {
|
|
178
|
+
let text = raw.trim().toLowerCase();
|
|
179
|
+
if (text.startsWith("["))
|
|
180
|
+
text = text.slice(1);
|
|
181
|
+
if (text.endsWith("]"))
|
|
182
|
+
text = text.slice(0, -1);
|
|
183
|
+
const zone = text.indexOf("%"); // fe80::1%eth0
|
|
184
|
+
if (zone !== -1)
|
|
185
|
+
text = text.slice(0, zone);
|
|
186
|
+
if (!text.includes(":"))
|
|
187
|
+
return null;
|
|
188
|
+
// A trailing dotted quad is the last two groups written in IPv4. Require all
|
|
189
|
+
// three dots: without that, `::1` parses as a one-part inet_aton address and
|
|
190
|
+
// takes a path that has nothing to do with what was written.
|
|
191
|
+
const dotted = /^(.*:)(\d+(?:\.\d+){3})$/.exec(text);
|
|
192
|
+
if (dotted !== null) {
|
|
193
|
+
const quad = normalizeIpv4(dotted[2]);
|
|
194
|
+
if (quad === null)
|
|
131
195
|
return null;
|
|
132
|
-
|
|
196
|
+
const o = quad.split(".").map((n) => Number.parseInt(n, 10));
|
|
197
|
+
const hi = (o[0] << 8) | o[1];
|
|
198
|
+
const lo = (o[2] << 8) | o[3];
|
|
199
|
+
text = `${dotted[1]}${hi.toString(16)}:${lo.toString(16)}`;
|
|
133
200
|
}
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
if (last >= maxLast)
|
|
201
|
+
const halves = text.split("::");
|
|
202
|
+
if (halves.length > 2)
|
|
137
203
|
return null;
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
204
|
+
const pieces = (part) => (part === "" ? [] : part.split(":"));
|
|
205
|
+
const head = pieces(halves[0]);
|
|
206
|
+
const tail = halves.length === 2 ? pieces(halves[1]) : [];
|
|
207
|
+
if (halves.length === 1 && head.length !== 8)
|
|
208
|
+
return null;
|
|
209
|
+
if (head.length + tail.length > 8)
|
|
210
|
+
return null;
|
|
211
|
+
const groups = [];
|
|
212
|
+
for (const piece of head) {
|
|
213
|
+
if (!/^[0-9a-f]{1,4}$/.test(piece))
|
|
214
|
+
return null;
|
|
215
|
+
groups.push(Number.parseInt(piece, 16));
|
|
142
216
|
}
|
|
143
|
-
|
|
217
|
+
for (let i = head.length + tail.length; i < 8; i++)
|
|
218
|
+
groups.push(0);
|
|
219
|
+
for (const piece of tail) {
|
|
220
|
+
if (!/^[0-9a-f]{1,4}$/.test(piece))
|
|
221
|
+
return null;
|
|
222
|
+
groups.push(Number.parseInt(piece, 16));
|
|
223
|
+
}
|
|
224
|
+
return groups.length === 8 ? groups : null;
|
|
144
225
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
226
|
+
/**
|
|
227
|
+
* The IPv4 address an IPv6 address carries, when it carries one.
|
|
228
|
+
*
|
|
229
|
+
* Every transition mechanism embeds a v4 address somewhere, and every one of
|
|
230
|
+
* them is a way to reach a v4 destination while wearing a v6 spelling that no
|
|
231
|
+
* v4 range check looks at. The NAT64 well-known prefix is the sharpest: a
|
|
232
|
+
* DNS64 resolver answers an IPv4-only name with `64:ff9b::<the v4>`, so
|
|
233
|
+
* `64:ff9b::a9fe:a9fe` IS the metadata service on any network that runs one.
|
|
234
|
+
*/
|
|
235
|
+
function embeddedIpv4(g) {
|
|
236
|
+
const quad = (hi, lo) => `${(hi >>> 8) & 0xff}.${hi & 0xff}.${(lo >>> 8) & 0xff}.${lo & 0xff}`;
|
|
237
|
+
const zeros = (from, to) => g.slice(from, to).every((x) => x === 0);
|
|
238
|
+
const last = quad(g[6], g[7]);
|
|
239
|
+
if (zeros(0, 5) && g[5] === 0xffff)
|
|
240
|
+
return last; // ::ffff:0:0/96, IPv4-mapped
|
|
241
|
+
if (zeros(0, 4) && g[4] === 0xffff && g[5] === 0)
|
|
242
|
+
return last; // ::ffff:0:0:0/96, translated
|
|
243
|
+
if (g[0] === 0x64 && g[1] === 0xff9b)
|
|
244
|
+
return last; // 64:ff9b::/96 and 64:ff9b:1::/48, NAT64
|
|
245
|
+
if (g[0] === 0x2002)
|
|
246
|
+
return quad(g[1], g[2]); // 2002::/16, 6to4
|
|
247
|
+
// ::a.b.c.d, IPv4-compatible: deprecated, still routed by some stacks, and
|
|
248
|
+
// not to be confused with `::` or `::1`, which are handled before this.
|
|
249
|
+
if (zeros(0, 6))
|
|
250
|
+
return last;
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
/** Ranges that are never a public RPC endpoint. */
|
|
254
|
+
export function isPrivateIp(address) {
|
|
255
|
+
const v4 = normalizeIpv4(address);
|
|
256
|
+
if (v4 !== null) {
|
|
257
|
+
const [a = 0, b = 0, c = 0, d = 0] = v4.split(".").map((n) => Number.parseInt(n, 10));
|
|
258
|
+
if (a === 0 || a === 10 || a === 127)
|
|
259
|
+
return true;
|
|
260
|
+
if (a === 169 && b === 254)
|
|
261
|
+
return true; // link-local, and the metadata service
|
|
262
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
263
|
+
return true;
|
|
264
|
+
if (a === 192 && b === 168)
|
|
265
|
+
return true;
|
|
266
|
+
if (a === 192 && b === 0 && c === 0)
|
|
267
|
+
return true;
|
|
268
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
269
|
+
return true; // carrier-grade NAT
|
|
270
|
+
if (a === 198 && (b === 18 || b === 19))
|
|
271
|
+
return true; // benchmarking
|
|
272
|
+
if (a >= 224)
|
|
273
|
+
return true; // multicast, reserved, broadcast
|
|
274
|
+
return a === 255 && b === 255 && c === 255 && d === 255;
|
|
168
275
|
}
|
|
169
|
-
const
|
|
170
|
-
if (
|
|
171
|
-
return false;
|
|
172
|
-
const parts = normalized.split(".").map((p) => Number.parseInt(p, 10));
|
|
173
|
-
if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {
|
|
276
|
+
const groups = parseIpv6(address);
|
|
277
|
+
if (groups === null)
|
|
174
278
|
return false;
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
if (
|
|
178
|
-
return true; //
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
if (
|
|
184
|
-
return true; //
|
|
185
|
-
if (
|
|
186
|
-
return true; //
|
|
187
|
-
|
|
188
|
-
return true; // 100.64.0.0/10 CGNAT (RFC6598)
|
|
189
|
-
if (a === 192 && b === 0 && parts[2] === 0)
|
|
190
|
-
return true; // 192.0.0.0/24 (RFC6890)
|
|
191
|
-
if (a === 198 && (b === 18 || b === 19))
|
|
192
|
-
return true; // 198.18.0.0/15 (RFC2544)
|
|
193
|
-
if (a === 0)
|
|
194
|
-
return true; // 0.0.0.0/8
|
|
195
|
-
return false;
|
|
279
|
+
if (groups.every((g) => g === 0))
|
|
280
|
+
return true; // ::, the unspecified address
|
|
281
|
+
if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1)
|
|
282
|
+
return true; // ::1
|
|
283
|
+
const carried = embeddedIpv4(groups);
|
|
284
|
+
if (carried !== null)
|
|
285
|
+
return isPrivateIp(carried);
|
|
286
|
+
const head = groups[0];
|
|
287
|
+
if ((head & 0xfe00) === 0xfc00)
|
|
288
|
+
return true; // fc00::/7, unique-local
|
|
289
|
+
if ((head & 0xffc0) === 0xfe80)
|
|
290
|
+
return true; // fe80::/10, link-local
|
|
291
|
+
return (head & 0xff00) === 0xff00; // ff00::/8, multicast
|
|
196
292
|
}
|
|
293
|
+
// END SYNCHRONISED BLOCK
|
|
197
294
|
/**
|
|
198
295
|
* Reject any host that is — or resolves to — a private/loopback/link-local
|
|
199
296
|
* address. Applied at the initial URL and at every redirect hop, regardless
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/tool-web",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "WebFetch (URL → markdown via cheerio + turndown) and WebSearch (provider dispatch)",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
"test": "bun test src"
|
|
16
16
|
},
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@crewhaus/errors": "0.
|
|
19
|
-
"@crewhaus/tool-builder": "0.
|
|
20
|
-
"@crewhaus/tool-catalog": "0.
|
|
18
|
+
"@crewhaus/errors": "0.7.0",
|
|
19
|
+
"@crewhaus/tool-builder": "0.7.0",
|
|
20
|
+
"@crewhaus/tool-catalog": "0.7.0",
|
|
21
21
|
"cheerio": "^1.0.0",
|
|
22
22
|
"turndown": "^7.2.0",
|
|
23
23
|
"zod": "^3.23.8"
|