@crewhaus/tool-web 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +48 -0
- package/dist/index.js +545 -0
- package/package.json +14 -11
- package/src/index.test.ts +0 -774
- package/src/index.ts +0 -638
package/src/index.ts
DELETED
|
@@ -1,638 +0,0 @@
|
|
|
1
|
-
import { lookup as dnsLookup } from "node:dns/promises";
|
|
2
|
-
import { CrewhausError } from "@crewhaus/errors";
|
|
3
|
-
import { buildTool } from "@crewhaus/tool-builder";
|
|
4
|
-
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
5
|
-
import { load as loadHtml } from "cheerio";
|
|
6
|
-
import TurndownService from "turndown";
|
|
7
|
-
import { z } from "zod";
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Section 14 — `WebFetch` and `WebSearch`.
|
|
11
|
-
*
|
|
12
|
-
* `WebFetch(url, prompt?)`:
|
|
13
|
-
* - Validates http/https scheme.
|
|
14
|
-
* - Optional allow-list via `getWebFetchConfig().allowedDomains`. When
|
|
15
|
-
* non-empty, host must equal an entry or be a subdomain of one.
|
|
16
|
-
* - SSRF guard (independent of the allow-list, re-checked on every
|
|
17
|
-
* redirect hop): rejects loopback, link-local / 169.254.0.0/16,
|
|
18
|
-
* RFC1918, CGNAT, *.local / *.localhost, and DNS-rebinding targets.
|
|
19
|
-
* - Manual redirect handling, max 5; allow-list re-checked at every hop.
|
|
20
|
-
* - 30 s default timeout (`AbortController`, honours `ctx.signal`).
|
|
21
|
-
* - 5 MB response body cap.
|
|
22
|
-
* - HTML pages run through cheerio + turndown to produce markdown that
|
|
23
|
-
* compresses well in the model's context. Plain text and JSON pass
|
|
24
|
-
* through. Other content types are summarised in one line.
|
|
25
|
-
* - When `prompt` is supplied, we DO NOT recursively call the model
|
|
26
|
-
* from inside the tool — that would fight runtime-core's loop. The
|
|
27
|
-
* prompt is prepended to the markdown ("[user prompt: ...]") so the
|
|
28
|
-
* parent agent sees both pieces and can reason about them in its
|
|
29
|
-
* next assistant turn. Documented behaviour, matching the
|
|
30
|
-
* claude-code/tools/WebFetchTool design.
|
|
31
|
-
*
|
|
32
|
-
* `WebSearch(query, allowed_domains?, blocked_domains?)`:
|
|
33
|
-
* - Anthropic server-side `web_search` is the preferred path. The
|
|
34
|
-
* model adapter doesn't yet expose `model.features.web_search`
|
|
35
|
-
* (Section 17), so for now this tool dispatches to a configurable
|
|
36
|
-
* third-party provider via env: `CREWHAUS_SEARCH_PROVIDER` ∈
|
|
37
|
-
* {brave, tavily} plus `CREWHAUS_SEARCH_API_KEY`. When neither is
|
|
38
|
-
* set, returns a clean refusal so the model can either skip search
|
|
39
|
-
* or surface the missing-config to the user.
|
|
40
|
-
*
|
|
41
|
-
* Layer R4. Pairs with the `target-cli` codegen contract — `BUILTIN_TOOL_MAP`
|
|
42
|
-
* has `webFetch: { initSymbol: "registerWebFetchConfig" }` and
|
|
43
|
-
* `webSearch: { ... }` (no init — uses env-only config).
|
|
44
|
-
*/
|
|
45
|
-
|
|
46
|
-
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
47
|
-
const MAX_REDIRECTS = 5;
|
|
48
|
-
const MAX_BODY_BYTES = 5 * 1024 * 1024;
|
|
49
|
-
|
|
50
|
-
export class WebFetchPermissionError extends CrewhausError {
|
|
51
|
-
override readonly name = "WebFetchPermissionError";
|
|
52
|
-
constructor(message: string) {
|
|
53
|
-
super("tool", message);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// ─── WebFetch config registry ──────────────────────────────────────────
|
|
58
|
-
|
|
59
|
-
export type WebFetchConfig = {
|
|
60
|
-
/**
|
|
61
|
-
* Allowed domains (host suffixes). Empty = allow all. A request to host
|
|
62
|
-
* `<h>` is permitted iff `h` equals an entry exactly, or `h` ends with
|
|
63
|
-
* `.<entry>`. Hosts are compared case-insensitively.
|
|
64
|
-
*/
|
|
65
|
-
readonly allowedDomains: readonly string[];
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
let webFetchConfig: WebFetchConfig = { allowedDomains: [] };
|
|
69
|
-
|
|
70
|
-
export type WebFetchConfigInput = {
|
|
71
|
-
readonly allowed_domains?: readonly string[];
|
|
72
|
-
readonly allowedDomains?: readonly string[];
|
|
73
|
-
};
|
|
74
|
-
|
|
75
|
-
export function registerWebFetchConfig(input: WebFetchConfigInput): void {
|
|
76
|
-
const raw = input.allowedDomains ?? input.allowed_domains ?? [];
|
|
77
|
-
webFetchConfig = { allowedDomains: raw.map((d) => d.toLowerCase()) };
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
export function getWebFetchConfig(): WebFetchConfig {
|
|
81
|
-
return webFetchConfig;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export function _resetWebFetchConfig(): void {
|
|
85
|
-
webFetchConfig = { allowedDomains: [] };
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
function isHostAllowed(host: string): boolean {
|
|
89
|
-
if (webFetchConfig.allowedDomains.length === 0) return true;
|
|
90
|
-
const lower = host.toLowerCase();
|
|
91
|
-
return webFetchConfig.allowedDomains.some(
|
|
92
|
-
(entry) => lower === entry || lower.endsWith(`.${entry}`),
|
|
93
|
-
);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// ─── SSRF guard ─────────────────────────────────────────────────────────
|
|
97
|
-
// Defence-in-depth, INDEPENDENT of the allow-list. WebFetch is reachable
|
|
98
|
-
// with model-/prompt-influenced URLs; without this guard an empty allow-list
|
|
99
|
-
// (the documented default) lets an attacker pivot to cloud-metadata
|
|
100
|
-
// (169.254.169.254), loopback, or RFC1918 services. Mirrors tool-fetch's
|
|
101
|
-
// `assertNotSsrf`, including the DNS-resolution backstop for rebinding.
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* DNS resolver injection point used by tests. Production callers leave it at
|
|
105
|
-
* `dnsLookup` from `node:dns/promises`; tests stub it to assert the rebinding
|
|
106
|
-
* path or to keep the suite offline.
|
|
107
|
-
*/
|
|
108
|
-
export type DnsLookupFn = (
|
|
109
|
-
host: string,
|
|
110
|
-
) => Promise<{ readonly address: string; readonly family: number }>;
|
|
111
|
-
// Production default — a single named function shared by both the initial
|
|
112
|
-
// binding and the `_setDnsLookup(undefined)` reset path, so there is one
|
|
113
|
-
// resolver definition to reason about (and to cover) rather than two
|
|
114
|
-
// duplicate closures.
|
|
115
|
-
const defaultDnsLookup: DnsLookupFn = (host) => dnsLookup(host, { verbatim: false });
|
|
116
|
-
let dnsLookupFn: DnsLookupFn = defaultDnsLookup;
|
|
117
|
-
export function _setDnsLookup(fn: DnsLookupFn | undefined): void {
|
|
118
|
-
dnsLookupFn = fn ?? defaultDnsLookup;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
// Normalise octal/hex/integer IPv4 literals (e.g. 0177.0.0.1, 0x7f000001,
|
|
122
|
-
// 2130706433) to dotted-decimal so isPrivateIp can classify them — browsers
|
|
123
|
-
// and many HTTP stacks accept these forms, which would otherwise bypass a
|
|
124
|
-
// naive dotted-only check. Mirrors @crewhaus/tool-fetch / crawler.
|
|
125
|
-
function normalizeIpv4(raw: string): string | null {
|
|
126
|
-
const trimmed = raw.trim();
|
|
127
|
-
if (trimmed === "") return null;
|
|
128
|
-
const parseComponent = (s: string): number | null => {
|
|
129
|
-
if (s === "") return null;
|
|
130
|
-
let value: number;
|
|
131
|
-
if (/^0[xX][0-9a-fA-F]+$/.test(s)) value = Number.parseInt(s.slice(2), 16);
|
|
132
|
-
else if (/^0[0-7]+$/.test(s)) value = Number.parseInt(s, 8);
|
|
133
|
-
else if (/^[0-9]+$/.test(s)) value = Number.parseInt(s, 10);
|
|
134
|
-
else return null;
|
|
135
|
-
return Number.isNaN(value) ? null : value;
|
|
136
|
-
};
|
|
137
|
-
const segments = trimmed.split(".");
|
|
138
|
-
if (segments.length > 4) return null;
|
|
139
|
-
const components: number[] = [];
|
|
140
|
-
for (const seg of segments) {
|
|
141
|
-
const value = parseComponent(seg);
|
|
142
|
-
if (value === null || value < 0) return null;
|
|
143
|
-
components.push(value);
|
|
144
|
-
}
|
|
145
|
-
const n = components.length;
|
|
146
|
-
const octets = [0, 0, 0, 0];
|
|
147
|
-
for (let i = 0; i < n - 1; i++) {
|
|
148
|
-
const c = components[i] as number;
|
|
149
|
-
if (c > 255) return null;
|
|
150
|
-
octets[i] = c;
|
|
151
|
-
}
|
|
152
|
-
const last = components[n - 1] as number;
|
|
153
|
-
const maxLast = 2 ** (8 * (4 - (n - 1)));
|
|
154
|
-
if (last >= maxLast) return null;
|
|
155
|
-
let rest = last;
|
|
156
|
-
for (let i = 3; i >= n - 1; i--) {
|
|
157
|
-
octets[i] = rest & 0xff;
|
|
158
|
-
rest = Math.floor(rest / 256);
|
|
159
|
-
}
|
|
160
|
-
return octets.join(".");
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
function isPrivateIp(addr: string): boolean {
|
|
164
|
-
const ip = addr.replace(/^\[/, "").replace(/\]$/, "");
|
|
165
|
-
const lowerIp = ip.toLowerCase();
|
|
166
|
-
|
|
167
|
-
if (lowerIp === "::" || lowerIp === "::1") return true; // IPv6 unspecified / loopback
|
|
168
|
-
if (lowerIp.startsWith("fe80:")) return true; // IPv6 link-local
|
|
169
|
-
if (lowerIp.startsWith("fc") || lowerIp.startsWith("fd")) return true; // IPv6 unique-local
|
|
170
|
-
|
|
171
|
-
// IPv4-mapped IPv6 — both dotted (::ffff:127.0.0.1) AND hex-compressed
|
|
172
|
-
// (::ffff:7f00:1, which `new URL` normalizes [::ffff:127.0.0.1] into).
|
|
173
|
-
const mapped = lowerIp.match(/^::ffff:(.+)$/);
|
|
174
|
-
if (mapped) {
|
|
175
|
-
const tail = mapped[1] as string;
|
|
176
|
-
if (tail.includes(".")) return isPrivateIp(tail);
|
|
177
|
-
const hexGroups = tail.split(":");
|
|
178
|
-
if (hexGroups.length === 2 && hexGroups.every((g) => /^[0-9a-f]{1,4}$/.test(g))) {
|
|
179
|
-
const hi = Number.parseInt(hexGroups[0] as string, 16);
|
|
180
|
-
const lo = Number.parseInt(hexGroups[1] as string, 16);
|
|
181
|
-
return isPrivateIp([(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff].join("."));
|
|
182
|
-
}
|
|
183
|
-
return false;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
const normalized = normalizeIpv4(ip);
|
|
187
|
-
if (normalized === null) return false;
|
|
188
|
-
const parts = normalized.split(".").map((p) => Number.parseInt(p, 10));
|
|
189
|
-
if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {
|
|
190
|
-
return false;
|
|
191
|
-
}
|
|
192
|
-
const [a, b] = parts as [number, number, number, number];
|
|
193
|
-
if (a === 127) return true; // 127.0.0.0/8 loopback
|
|
194
|
-
if (a === 10) return true; // 10.0.0.0/8 RFC1918
|
|
195
|
-
if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local + cloud metadata
|
|
196
|
-
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 RFC1918
|
|
197
|
-
if (a === 192 && b === 168) return true; // 192.168.0.0/16 RFC1918
|
|
198
|
-
if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGNAT (RFC6598)
|
|
199
|
-
if (a === 192 && b === 0 && parts[2] === 0) return true; // 192.0.0.0/24 (RFC6890)
|
|
200
|
-
if (a === 198 && (b === 18 || b === 19)) return true; // 198.18.0.0/15 (RFC2544)
|
|
201
|
-
if (a === 0) return true; // 0.0.0.0/8
|
|
202
|
-
return false;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
/**
|
|
206
|
-
* Reject any host that is — or resolves to — a private/loopback/link-local
|
|
207
|
-
* address. Applied at the initial URL and at every redirect hop, regardless
|
|
208
|
-
* of the allow-list. Fails closed if the name cannot be resolved.
|
|
209
|
-
*
|
|
210
|
-
* Returns the validated IP so the caller can PIN the connection to that exact
|
|
211
|
-
* address — resolving here and letting the default `fetch` re-resolve at
|
|
212
|
-
* connect time is a DNS-rebinding TOCTOU (CWE-367): a hostile resolver can
|
|
213
|
-
* answer public for this check and private for the socket. For an IP-literal
|
|
214
|
-
* host the pinned value is the (normalized) literal itself.
|
|
215
|
-
*/
|
|
216
|
-
async function assertNotSsrf(hostname: string): Promise<string> {
|
|
217
|
-
const lower = hostname.toLowerCase();
|
|
218
|
-
if (lower === "localhost" || lower.endsWith(".localhost")) {
|
|
219
|
-
throw new WebFetchPermissionError(`WebFetch denied: host "${hostname}" resolves to loopback`);
|
|
220
|
-
}
|
|
221
|
-
if (lower.endsWith(".local")) {
|
|
222
|
-
throw new WebFetchPermissionError(`WebFetch denied: mDNS host "${hostname}" is not allowed`);
|
|
223
|
-
}
|
|
224
|
-
const unbracketed = lower.replace(/^\[/, "").replace(/\]$/, "");
|
|
225
|
-
if (isPrivateIp(unbracketed)) {
|
|
226
|
-
throw new WebFetchPermissionError(
|
|
227
|
-
`WebFetch denied: host "${hostname}" is a private/loopback IP`,
|
|
228
|
-
);
|
|
229
|
-
}
|
|
230
|
-
// An IP literal is its own pinned target — nothing to resolve or rebind.
|
|
231
|
-
const literal = normalizeIpv4(unbracketed) ?? (unbracketed.includes(":") ? unbracketed : null);
|
|
232
|
-
if (literal !== null) return literal;
|
|
233
|
-
// Resolve so a public-looking name pointing at an internal IP (DNS
|
|
234
|
-
// rebinding) is still caught — and return the address so the caller dials it.
|
|
235
|
-
let resolved: { readonly address: string; readonly family: number };
|
|
236
|
-
try {
|
|
237
|
-
resolved = await dnsLookupFn(lower);
|
|
238
|
-
} catch (err) {
|
|
239
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
240
|
-
throw new WebFetchPermissionError(`WebFetch denied: cannot resolve "${hostname}": ${msg}`);
|
|
241
|
-
}
|
|
242
|
-
if (isPrivateIp(resolved.address)) {
|
|
243
|
-
throw new WebFetchPermissionError(
|
|
244
|
-
`WebFetch denied: host "${hostname}" resolves to private IP ${resolved.address}`,
|
|
245
|
-
);
|
|
246
|
-
}
|
|
247
|
-
return resolved.address;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
// ─── WebFetch ─────────────────────────────────────────────────────────
|
|
251
|
-
|
|
252
|
-
const webFetchSchema = z.object({
|
|
253
|
-
url: z.string().min(1),
|
|
254
|
-
prompt: z.string().optional(),
|
|
255
|
-
});
|
|
256
|
-
|
|
257
|
-
/**
|
|
258
|
-
* `pinnedIp` is the address `assertNotSsrf` validated for the request's host
|
|
259
|
-
* (empty string ⇒ no pin, e.g. fixed trusted API hosts). The production
|
|
260
|
-
* fetcher dials that exact IP, preserving the Host header and TLS SNI, so the
|
|
261
|
-
* socket can't be rebound to a private address between the SSRF check and
|
|
262
|
-
* connect. Test stubs may ignore the argument.
|
|
263
|
-
*/
|
|
264
|
-
export type RawFetch = (req: Request, pinnedIp: string) => Promise<Response>;
|
|
265
|
-
|
|
266
|
-
// Dial `pinnedIp` directly while keeping the real host for the Host header and
|
|
267
|
-
// TLS SNI. Mirrors @crewhaus/tool-fetch.
|
|
268
|
-
function pinnedFetch(req: Request, pinnedIp: string): Promise<Response> {
|
|
269
|
-
const original = new URL(req.url);
|
|
270
|
-
const host = original.hostname;
|
|
271
|
-
const hostUnbracketed = host.replace(/^\[/, "").replace(/\]$/, "");
|
|
272
|
-
if (pinnedIp === "" || hostUnbracketed === pinnedIp) {
|
|
273
|
-
return globalThis.fetch(req);
|
|
274
|
-
}
|
|
275
|
-
const hostForUrl = pinnedIp.includes(":") ? `[${pinnedIp}]` : pinnedIp;
|
|
276
|
-
const pinnedUrl = new URL(original.toString());
|
|
277
|
-
pinnedUrl.hostname = hostForUrl;
|
|
278
|
-
const headers = new Headers(req.headers);
|
|
279
|
-
headers.set("host", original.port === "" ? host : `${host}:${original.port}`);
|
|
280
|
-
const init: RequestInit & { tls?: { serverName: string } } = {
|
|
281
|
-
method: req.method,
|
|
282
|
-
headers,
|
|
283
|
-
redirect: "manual",
|
|
284
|
-
signal: req.signal,
|
|
285
|
-
tls: { serverName: host },
|
|
286
|
-
};
|
|
287
|
-
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
288
|
-
init.body = req.body;
|
|
289
|
-
(init as { duplex?: string }).duplex = "half";
|
|
290
|
-
}
|
|
291
|
-
return globalThis.fetch(pinnedUrl.toString(), init);
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
// Production default — one named fetcher shared by the initial binding and the
|
|
295
|
-
// `_setRawFetch(undefined)` reset path (see defaultDnsLookup for rationale).
|
|
296
|
-
const defaultRawFetch: RawFetch = pinnedFetch;
|
|
297
|
-
let rawFetch: RawFetch = defaultRawFetch;
|
|
298
|
-
export function _setRawFetch(fn: RawFetch | undefined): void {
|
|
299
|
-
rawFetch = fn ?? defaultRawFetch;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
async function readBodyCapped(res: Response): Promise<string> {
|
|
303
|
-
if (res.body === null) return "";
|
|
304
|
-
const reader = res.body.getReader();
|
|
305
|
-
const chunks: Uint8Array[] = [];
|
|
306
|
-
let total = 0;
|
|
307
|
-
try {
|
|
308
|
-
while (true) {
|
|
309
|
-
const { done, value } = await reader.read();
|
|
310
|
-
if (done) break;
|
|
311
|
-
total += value.byteLength;
|
|
312
|
-
if (total > MAX_BODY_BYTES) {
|
|
313
|
-
try {
|
|
314
|
-
await reader.cancel();
|
|
315
|
-
} catch {
|
|
316
|
-
/* ignore */
|
|
317
|
-
}
|
|
318
|
-
throw new WebFetchPermissionError(
|
|
319
|
-
`response body exceeded ${MAX_BODY_BYTES} bytes — aborted`,
|
|
320
|
-
);
|
|
321
|
-
}
|
|
322
|
-
chunks.push(value);
|
|
323
|
-
}
|
|
324
|
-
} finally {
|
|
325
|
-
try {
|
|
326
|
-
reader.releaseLock();
|
|
327
|
-
} catch {
|
|
328
|
-
/* ignore */
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
const merged = new Uint8Array(total);
|
|
332
|
-
let offset = 0;
|
|
333
|
-
for (const chunk of chunks) {
|
|
334
|
-
merged.set(chunk, offset);
|
|
335
|
-
offset += chunk.byteLength;
|
|
336
|
-
}
|
|
337
|
-
return new TextDecoder("utf-8", { fatal: false }).decode(merged);
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
async function performWebFetch(initialUrl: URL, signal: AbortSignal): Promise<Response> {
|
|
341
|
-
let currentUrl = initialUrl;
|
|
342
|
-
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
343
|
-
if (currentUrl.protocol !== "http:" && currentUrl.protocol !== "https:") {
|
|
344
|
-
throw new WebFetchPermissionError(
|
|
345
|
-
`WebFetch denied: scheme "${currentUrl.protocol}" — only http/https allowed`,
|
|
346
|
-
);
|
|
347
|
-
}
|
|
348
|
-
// SSRF guard — independent of the allow-list, enforced on every hop.
|
|
349
|
-
// The returned IP pins the socket so a rebinding resolver can't swap in a
|
|
350
|
-
// private address between this check and connect.
|
|
351
|
-
const pinnedIp = await assertNotSsrf(currentUrl.hostname);
|
|
352
|
-
if (!isHostAllowed(currentUrl.hostname)) {
|
|
353
|
-
throw new WebFetchPermissionError(
|
|
354
|
-
`WebFetch denied: host "${currentUrl.hostname}" is not in allowed_domains`,
|
|
355
|
-
);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
const res = await rawFetch(
|
|
359
|
-
new Request(currentUrl.toString(), {
|
|
360
|
-
method: "GET",
|
|
361
|
-
redirect: "manual",
|
|
362
|
-
signal,
|
|
363
|
-
headers: { "user-agent": "crewhaus-tool-web/0.1" },
|
|
364
|
-
}),
|
|
365
|
-
pinnedIp,
|
|
366
|
-
);
|
|
367
|
-
if (res.status >= 300 && res.status < 400 && res.headers.has("location")) {
|
|
368
|
-
const loc = res.headers.get("location") ?? "";
|
|
369
|
-
let next: URL;
|
|
370
|
-
try {
|
|
371
|
-
next = new URL(loc, currentUrl);
|
|
372
|
-
} catch {
|
|
373
|
-
throw new WebFetchPermissionError(`invalid redirect target "${loc}"`);
|
|
374
|
-
}
|
|
375
|
-
currentUrl = next;
|
|
376
|
-
try {
|
|
377
|
-
await res.body?.cancel();
|
|
378
|
-
} catch {
|
|
379
|
-
/* ignore */
|
|
380
|
-
}
|
|
381
|
-
continue;
|
|
382
|
-
}
|
|
383
|
-
return res;
|
|
384
|
-
}
|
|
385
|
-
throw new WebFetchPermissionError(`too many redirects (>${MAX_REDIRECTS})`);
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
/**
|
|
389
|
-
* Convert HTML into compact markdown via cheerio + turndown. Strips
|
|
390
|
-
* script/style/noscript before conversion so non-content noise doesn't
|
|
391
|
-
* blow out the result.
|
|
392
|
-
*/
|
|
393
|
-
export function htmlToMarkdown(html: string): string {
|
|
394
|
-
const $ = loadHtml(html);
|
|
395
|
-
$("script, style, noscript, link, iframe, svg").remove();
|
|
396
|
-
// Prefer <main> / <article> when present so we don't drag headers/footers in.
|
|
397
|
-
let bodyHtml = $("article").first().html();
|
|
398
|
-
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = $("main").first().html();
|
|
399
|
-
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = $("body").html();
|
|
400
|
-
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = html;
|
|
401
|
-
const turndown = new TurndownService({
|
|
402
|
-
headingStyle: "atx",
|
|
403
|
-
codeBlockStyle: "fenced",
|
|
404
|
-
bulletListMarker: "-",
|
|
405
|
-
});
|
|
406
|
-
return turndown.turndown(bodyHtml).trim();
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
export const webFetch: RegisteredTool = buildTool({
|
|
410
|
-
name: "WebFetch",
|
|
411
|
-
description:
|
|
412
|
-
"Fetch an http(s) URL and return its content. HTML is converted to compact markdown; plain text and JSON pass through. ≤5 MB body, ≤5 redirects, 30 s timeout. Optional allow-list via WebFetch.allowed_domains in the spec's tool_config block.",
|
|
413
|
-
inputSchema: webFetchSchema,
|
|
414
|
-
readOnly: true,
|
|
415
|
-
concurrencySafe: true,
|
|
416
|
-
// Pillar 3 sink-side: URL parameters can encode exfiltrated data ("Safe URL"
|
|
417
|
-
// pattern from OpenAI's 2026-05 prompt-injection paper).
|
|
418
|
-
scope: "external",
|
|
419
|
-
// FR-002 — declare the io-capability fact (HTTP fetch) for the audit.
|
|
420
|
-
ioCapability: "network",
|
|
421
|
-
execute: async (input, ctx) => {
|
|
422
|
-
let url: URL;
|
|
423
|
-
try {
|
|
424
|
-
url = new URL(input.url);
|
|
425
|
-
} catch {
|
|
426
|
-
throw new WebFetchPermissionError(`invalid URL "${input.url}"`);
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
const ctrl = new AbortController();
|
|
430
|
-
const timer = setTimeout(() => ctrl.abort(new Error("fetch timeout")), DEFAULT_TIMEOUT_MS);
|
|
431
|
-
if (ctx?.signal !== undefined) {
|
|
432
|
-
if (ctx.signal.aborted) ctrl.abort(ctx.signal.reason);
|
|
433
|
-
else
|
|
434
|
-
ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
|
|
435
|
-
once: true,
|
|
436
|
-
});
|
|
437
|
-
}
|
|
438
|
-
let res: Response;
|
|
439
|
-
try {
|
|
440
|
-
res = await performWebFetch(url, ctrl.signal);
|
|
441
|
-
} finally {
|
|
442
|
-
clearTimeout(timer);
|
|
443
|
-
}
|
|
444
|
-
const body = await readBodyCapped(res);
|
|
445
|
-
const ct = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
446
|
-
let content: string;
|
|
447
|
-
if (ct.includes("text/html") || ct.includes("application/xhtml")) {
|
|
448
|
-
content = htmlToMarkdown(body);
|
|
449
|
-
} else if (
|
|
450
|
-
ct.includes("text/plain") ||
|
|
451
|
-
ct.includes("application/json") ||
|
|
452
|
-
ct === "" ||
|
|
453
|
-
ct.startsWith("text/")
|
|
454
|
-
) {
|
|
455
|
-
content = body;
|
|
456
|
-
} else {
|
|
457
|
-
content = `[non-textual content: ${ct}, ${body.length} bytes — not displayed]`;
|
|
458
|
-
}
|
|
459
|
-
const header = `URL: ${url.toString()}\nStatus: ${res.status}\nContent-Type: ${ct}\n\n`;
|
|
460
|
-
if (input.prompt !== undefined && input.prompt !== "") {
|
|
461
|
-
return `${header}[user prompt: ${input.prompt}]\n\n${content}`;
|
|
462
|
-
}
|
|
463
|
-
return `${header}${content}`;
|
|
464
|
-
},
|
|
465
|
-
});
|
|
466
|
-
|
|
467
|
-
// ─── WebSearch ─────────────────────────────────────────────────────────
|
|
468
|
-
|
|
469
|
-
const webSearchSchema = z.object({
|
|
470
|
-
query: z.string().min(1),
|
|
471
|
-
allowed_domains: z.array(z.string()).optional(),
|
|
472
|
-
blocked_domains: z.array(z.string()).optional(),
|
|
473
|
-
});
|
|
474
|
-
|
|
475
|
-
type SearchHit = {
|
|
476
|
-
readonly title: string;
|
|
477
|
-
readonly url: string;
|
|
478
|
-
readonly snippet: string;
|
|
479
|
-
};
|
|
480
|
-
|
|
481
|
-
/**
|
|
482
|
-
* Provider dispatch. Anthropic server-side `web_search` lands when Section
|
|
483
|
-
* 17 (multi-provider model layer) introduces `model.features.web_search`.
|
|
484
|
-
* Until then, dispatch to a third-party provider via env. When neither
|
|
485
|
-
* env var is set, return a clean refusal so the model handles it
|
|
486
|
-
* gracefully.
|
|
487
|
-
*/
|
|
488
|
-
async function dispatchSearch(
|
|
489
|
-
query: string,
|
|
490
|
-
allowed: readonly string[] | undefined,
|
|
491
|
-
blocked: readonly string[] | undefined,
|
|
492
|
-
signal: AbortSignal,
|
|
493
|
-
): Promise<readonly SearchHit[] | string> {
|
|
494
|
-
const provider = (process.env["CREWHAUS_SEARCH_PROVIDER"] ?? "").toLowerCase();
|
|
495
|
-
const apiKey = process.env["CREWHAUS_SEARCH_API_KEY"] ?? "";
|
|
496
|
-
if (provider === "" || apiKey === "") {
|
|
497
|
-
return "WebSearch unavailable: set CREWHAUS_SEARCH_PROVIDER (brave|tavily) and CREWHAUS_SEARCH_API_KEY in the environment.";
|
|
498
|
-
}
|
|
499
|
-
if (provider === "brave") return braveSearch(query, apiKey, allowed, blocked, signal);
|
|
500
|
-
if (provider === "tavily") return tavilySearch(query, apiKey, allowed, blocked, signal);
|
|
501
|
-
return `WebSearch unavailable: unknown provider "${provider}" — supported: brave, tavily.`;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
async function braveSearch(
|
|
505
|
-
query: string,
|
|
506
|
-
apiKey: string,
|
|
507
|
-
allowed: readonly string[] | undefined,
|
|
508
|
-
blocked: readonly string[] | undefined,
|
|
509
|
-
signal: AbortSignal,
|
|
510
|
-
): Promise<readonly SearchHit[]> {
|
|
511
|
-
const params = new URLSearchParams({ q: query, count: "10" });
|
|
512
|
-
// Brave uses a single `result_filter` and per-request goggles; treat
|
|
513
|
-
// allowed/blocked as advisory hints rather than binding constraints
|
|
514
|
-
// (the provider doesn't enforce them server-side).
|
|
515
|
-
const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
|
|
516
|
-
// Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
|
|
517
|
-
const res = await rawFetch(
|
|
518
|
-
new Request(url, {
|
|
519
|
-
headers: { accept: "application/json", "x-subscription-token": apiKey },
|
|
520
|
-
signal,
|
|
521
|
-
}),
|
|
522
|
-
"",
|
|
523
|
-
);
|
|
524
|
-
if (!res.ok) {
|
|
525
|
-
throw new WebFetchPermissionError(`Brave search failed: HTTP ${res.status}`);
|
|
526
|
-
}
|
|
527
|
-
const json = (await res.json()) as { web?: { results?: BraveResult[] } };
|
|
528
|
-
const raw = json.web?.results ?? [];
|
|
529
|
-
return raw
|
|
530
|
-
.map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.description ?? "" }))
|
|
531
|
-
.filter((h) => filterByDomains(h.url, allowed, blocked));
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
interface BraveResult {
|
|
535
|
-
readonly title?: string;
|
|
536
|
-
readonly url?: string;
|
|
537
|
-
readonly description?: string;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
async function tavilySearch(
|
|
541
|
-
query: string,
|
|
542
|
-
apiKey: string,
|
|
543
|
-
allowed: readonly string[] | undefined,
|
|
544
|
-
blocked: readonly string[] | undefined,
|
|
545
|
-
signal: AbortSignal,
|
|
546
|
-
): Promise<readonly SearchHit[]> {
|
|
547
|
-
const res = await rawFetch(
|
|
548
|
-
new Request("https://api.tavily.com/search", {
|
|
549
|
-
method: "POST",
|
|
550
|
-
headers: { "content-type": "application/json" },
|
|
551
|
-
body: JSON.stringify({
|
|
552
|
-
api_key: apiKey,
|
|
553
|
-
query,
|
|
554
|
-
max_results: 10,
|
|
555
|
-
...(allowed !== undefined ? { include_domains: allowed } : {}),
|
|
556
|
-
...(blocked !== undefined ? { exclude_domains: blocked } : {}),
|
|
557
|
-
}),
|
|
558
|
-
signal,
|
|
559
|
-
}),
|
|
560
|
-
// Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
|
|
561
|
-
"",
|
|
562
|
-
);
|
|
563
|
-
if (!res.ok) {
|
|
564
|
-
throw new WebFetchPermissionError(`Tavily search failed: HTTP ${res.status}`);
|
|
565
|
-
}
|
|
566
|
-
const json = (await res.json()) as { results?: TavilyResult[] };
|
|
567
|
-
const raw = json.results ?? [];
|
|
568
|
-
return raw
|
|
569
|
-
.map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.content ?? "" }))
|
|
570
|
-
.filter((h) => filterByDomains(h.url, allowed, blocked));
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
interface TavilyResult {
|
|
574
|
-
readonly title?: string;
|
|
575
|
-
readonly url?: string;
|
|
576
|
-
readonly content?: string;
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
function filterByDomains(
|
|
580
|
-
hitUrl: string,
|
|
581
|
-
allowed: readonly string[] | undefined,
|
|
582
|
-
blocked: readonly string[] | undefined,
|
|
583
|
-
): boolean {
|
|
584
|
-
let host: string;
|
|
585
|
-
try {
|
|
586
|
-
host = new URL(hitUrl).hostname.toLowerCase();
|
|
587
|
-
} catch {
|
|
588
|
-
return false;
|
|
589
|
-
}
|
|
590
|
-
if (blocked?.some((b) => host === b.toLowerCase() || host.endsWith(`.${b.toLowerCase()}`))) {
|
|
591
|
-
return false;
|
|
592
|
-
}
|
|
593
|
-
if (allowed && allowed.length > 0) {
|
|
594
|
-
return allowed.some((a) => host === a.toLowerCase() || host.endsWith(`.${a.toLowerCase()}`));
|
|
595
|
-
}
|
|
596
|
-
return true;
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
function formatHits(hits: readonly SearchHit[]): string {
|
|
600
|
-
if (hits.length === 0) return "[no results]";
|
|
601
|
-
return hits.map((h, i) => `${i + 1}. ${h.title}\n ${h.url}\n ${h.snippet}`).join("\n\n");
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
export const webSearch: RegisteredTool = buildTool({
|
|
605
|
-
name: "WebSearch",
|
|
606
|
-
description:
|
|
607
|
-
"Search the web for up-to-date information. Provider chosen via the CREWHAUS_SEARCH_PROVIDER env var (brave or tavily); CREWHAUS_SEARCH_API_KEY supplies the credential. Returns a numbered list of {title, url, snippet}. Optional allowed_domains / blocked_domains filter the results.",
|
|
608
|
-
inputSchema: webSearchSchema,
|
|
609
|
-
readOnly: true,
|
|
610
|
-
concurrencySafe: true,
|
|
611
|
-
// Pillar 3 sink-side: search-query parameter is an exfiltration vector.
|
|
612
|
-
scope: "external",
|
|
613
|
-
// FR-002 — declare the io-capability fact (remote search API) for the audit.
|
|
614
|
-
ioCapability: "network",
|
|
615
|
-
execute: async (input, ctx) => {
|
|
616
|
-
const ctrl = new AbortController();
|
|
617
|
-
const timer = setTimeout(() => ctrl.abort(new Error("search timeout")), DEFAULT_TIMEOUT_MS);
|
|
618
|
-
if (ctx?.signal !== undefined) {
|
|
619
|
-
if (ctx.signal.aborted) ctrl.abort(ctx.signal.reason);
|
|
620
|
-
else
|
|
621
|
-
ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
|
|
622
|
-
once: true,
|
|
623
|
-
});
|
|
624
|
-
}
|
|
625
|
-
try {
|
|
626
|
-
const result = await dispatchSearch(
|
|
627
|
-
input.query,
|
|
628
|
-
input.allowed_domains,
|
|
629
|
-
input.blocked_domains,
|
|
630
|
-
ctrl.signal,
|
|
631
|
-
);
|
|
632
|
-
if (typeof result === "string") return result;
|
|
633
|
-
return formatHits(result);
|
|
634
|
-
} finally {
|
|
635
|
-
clearTimeout(timer);
|
|
636
|
-
}
|
|
637
|
-
},
|
|
638
|
-
});
|