@crewhaus/tool-web 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ import { CrewhausError } from "@crewhaus/errors";
2
+ import type { RegisteredTool } from "@crewhaus/tool-catalog";
3
+ export declare class WebFetchPermissionError extends CrewhausError {
4
+ readonly name = "WebFetchPermissionError";
5
+ constructor(message: string);
6
+ }
7
+ export type WebFetchConfig = {
8
+ /**
9
+ * Allowed domains (host suffixes). Empty = allow all. A request to host
10
+ * `<h>` is permitted iff `h` equals an entry exactly, or `h` ends with
11
+ * `.<entry>`. Hosts are compared case-insensitively.
12
+ */
13
+ readonly allowedDomains: readonly string[];
14
+ };
15
+ export type WebFetchConfigInput = {
16
+ readonly allowed_domains?: readonly string[];
17
+ readonly allowedDomains?: readonly string[];
18
+ };
19
+ export declare function registerWebFetchConfig(input: WebFetchConfigInput): void;
20
+ export declare function getWebFetchConfig(): WebFetchConfig;
21
+ export declare function _resetWebFetchConfig(): void;
22
+ /**
23
+ * DNS resolver injection point used by tests. Production callers leave it at
24
+ * `dnsLookup` from `node:dns/promises`; tests stub it to assert the rebinding
25
+ * path or to keep the suite offline.
26
+ */
27
+ export type DnsLookupFn = (host: string) => Promise<{
28
+ readonly address: string;
29
+ readonly family: number;
30
+ }>;
31
+ export declare function _setDnsLookup(fn: DnsLookupFn | undefined): void;
32
+ /**
33
+ * `pinnedIp` is the address `assertNotSsrf` validated for the request's host
34
+ * (empty string ⇒ no pin, e.g. fixed trusted API hosts). The production
35
+ * fetcher dials that exact IP, preserving the Host header and TLS SNI, so the
36
+ * socket can't be rebound to a private address between the SSRF check and
37
+ * connect. Test stubs may ignore the argument.
38
+ */
39
+ export type RawFetch = (req: Request, pinnedIp: string) => Promise<Response>;
40
+ export declare function _setRawFetch(fn: RawFetch | undefined): void;
41
+ /**
42
+ * Convert HTML into compact markdown via cheerio + turndown. Strips
43
+ * script/style/noscript before conversion so non-content noise doesn't
44
+ * blow out the result.
45
+ */
46
+ export declare function htmlToMarkdown(html: string): string;
47
+ export declare const webFetch: RegisteredTool;
48
+ export declare const webSearch: RegisteredTool;
package/dist/index.js ADDED
@@ -0,0 +1,545 @@
1
+ import { lookup as dnsLookup } from "node:dns/promises";
2
+ import { CrewhausError } from "@crewhaus/errors";
3
+ import { buildTool } from "@crewhaus/tool-builder";
4
+ import { load as loadHtml } from "cheerio";
5
+ import TurndownService from "turndown";
6
+ import { z } from "zod";
7
+ /**
8
+ * Section 14 — `WebFetch` and `WebSearch`.
9
+ *
10
+ * `WebFetch(url, prompt?)`:
11
+ * - Validates http/https scheme.
12
+ * - Optional allow-list via `getWebFetchConfig().allowedDomains`. When
13
+ * non-empty, host must equal an entry or be a subdomain of one.
14
+ * - SSRF guard (independent of the allow-list, re-checked on every
15
+ * redirect hop): rejects loopback, link-local / 169.254.0.0/16,
16
+ * RFC1918, CGNAT, *.local / *.localhost, and DNS-rebinding targets.
17
+ * - Manual redirect handling, max 5; allow-list re-checked at every hop.
18
+ * - 30 s default timeout (`AbortController`, honours `ctx.signal`).
19
+ * - 5 MB response body cap.
20
+ * - HTML pages run through cheerio + turndown to produce markdown that
21
+ * compresses well in the model's context. Plain text and JSON pass
22
+ * through. Other content types are summarised in one line.
23
+ * - When `prompt` is supplied, we DO NOT recursively call the model
24
+ * from inside the tool — that would fight runtime-core's loop. The
25
+ * prompt is prepended to the markdown ("[user prompt: ...]") so the
26
+ * parent agent sees both pieces and can reason about them in its
27
+ * next assistant turn. Documented behaviour, matching the
28
+ * claude-code/tools/WebFetchTool design.
29
+ *
30
+ * `WebSearch(query, allowed_domains?, blocked_domains?)`:
31
+ * - Anthropic server-side `web_search` is the preferred path. The
32
+ * model adapter doesn't yet expose `model.features.web_search`
33
+ * (Section 17), so for now this tool dispatches to a configurable
34
+ * third-party provider via env: `CREWHAUS_SEARCH_PROVIDER` ∈
35
+ * {brave, tavily} plus `CREWHAUS_SEARCH_API_KEY`. When neither is
36
+ * set, returns a clean refusal so the model can either skip search
37
+ * or surface the missing-config to the user.
38
+ *
39
+ * Layer R4. Pairs with the `target-cli` codegen contract — `BUILTIN_TOOL_MAP`
40
+ * has `webFetch: { initSymbol: "registerWebFetchConfig" }` and
41
+ * `webSearch: { ... }` (no init — uses env-only config).
42
+ */
43
+ const DEFAULT_TIMEOUT_MS = 30_000;
44
+ const MAX_REDIRECTS = 5;
45
+ const MAX_BODY_BYTES = 5 * 1024 * 1024;
46
+ export class WebFetchPermissionError extends CrewhausError {
47
+ name = "WebFetchPermissionError";
48
+ constructor(message) {
49
+ super("tool", message);
50
+ }
51
+ }
52
+ let webFetchConfig = { allowedDomains: [] };
53
+ export function registerWebFetchConfig(input) {
54
+ const raw = input.allowedDomains ?? input.allowed_domains ?? [];
55
+ webFetchConfig = { allowedDomains: raw.map((d) => d.toLowerCase()) };
56
+ }
57
+ export function getWebFetchConfig() {
58
+ return webFetchConfig;
59
+ }
60
+ export function _resetWebFetchConfig() {
61
+ webFetchConfig = { allowedDomains: [] };
62
+ }
63
+ function isHostAllowed(host) {
64
+ if (webFetchConfig.allowedDomains.length === 0)
65
+ return true;
66
+ const lower = host.toLowerCase();
67
+ return webFetchConfig.allowedDomains.some((entry) => lower === entry || lower.endsWith(`.${entry}`));
68
+ }
69
+ // Production default — a single named function shared by both the initial
70
+ // binding and the `_setDnsLookup(undefined)` reset path, so there is one
71
+ // resolver definition to reason about (and to cover) rather than two
72
+ // duplicate closures.
73
+ const defaultDnsLookup = (host) => dnsLookup(host, { verbatim: false });
74
+ let dnsLookupFn = defaultDnsLookup;
75
+ export function _setDnsLookup(fn) {
76
+ dnsLookupFn = fn ?? defaultDnsLookup;
77
+ }
78
+ // Normalise octal/hex/integer IPv4 literals (e.g. 0177.0.0.1, 0x7f000001,
79
+ // 2130706433) to dotted-decimal so isPrivateIp can classify them — browsers
80
+ // and many HTTP stacks accept these forms, which would otherwise bypass a
81
+ // naive dotted-only check. Mirrors @crewhaus/tool-fetch / crawler.
82
+ function normalizeIpv4(raw) {
83
+ const trimmed = raw.trim();
84
+ if (trimmed === "")
85
+ return null;
86
+ const parseComponent = (s) => {
87
+ if (s === "")
88
+ return null;
89
+ let value;
90
+ if (/^0[xX][0-9a-fA-F]+$/.test(s))
91
+ value = Number.parseInt(s.slice(2), 16);
92
+ else if (/^0[0-7]+$/.test(s))
93
+ value = Number.parseInt(s, 8);
94
+ else if (/^[0-9]+$/.test(s))
95
+ value = Number.parseInt(s, 10);
96
+ else
97
+ return null;
98
+ return Number.isNaN(value) ? null : value;
99
+ };
100
+ const segments = trimmed.split(".");
101
+ if (segments.length > 4)
102
+ return null;
103
+ const components = [];
104
+ for (const seg of segments) {
105
+ const value = parseComponent(seg);
106
+ if (value === null || value < 0)
107
+ return null;
108
+ components.push(value);
109
+ }
110
+ const n = components.length;
111
+ const octets = [0, 0, 0, 0];
112
+ for (let i = 0; i < n - 1; i++) {
113
+ const c = components[i];
114
+ if (c > 255)
115
+ return null;
116
+ octets[i] = c;
117
+ }
118
+ const last = components[n - 1];
119
+ const maxLast = 2 ** (8 * (4 - (n - 1)));
120
+ if (last >= maxLast)
121
+ return null;
122
+ let rest = last;
123
+ for (let i = 3; i >= n - 1; i--) {
124
+ octets[i] = rest & 0xff;
125
+ rest = Math.floor(rest / 256);
126
+ }
127
+ return octets.join(".");
128
+ }
129
+ function isPrivateIp(addr) {
130
+ const ip = addr.replace(/^\[/, "").replace(/\]$/, "");
131
+ const lowerIp = ip.toLowerCase();
132
+ if (lowerIp === "::" || lowerIp === "::1")
133
+ return true; // IPv6 unspecified / loopback
134
+ if (lowerIp.startsWith("fe80:"))
135
+ return true; // IPv6 link-local
136
+ if (lowerIp.startsWith("fc") || lowerIp.startsWith("fd"))
137
+ return true; // IPv6 unique-local
138
+ // IPv4-mapped IPv6 — both dotted (::ffff:127.0.0.1) AND hex-compressed
139
+ // (::ffff:7f00:1, which `new URL` normalizes [::ffff:127.0.0.1] into).
140
+ const mapped = lowerIp.match(/^::ffff:(.+)$/);
141
+ if (mapped) {
142
+ const tail = mapped[1];
143
+ if (tail.includes("."))
144
+ return isPrivateIp(tail);
145
+ const hexGroups = tail.split(":");
146
+ if (hexGroups.length === 2 && hexGroups.every((g) => /^[0-9a-f]{1,4}$/.test(g))) {
147
+ const hi = Number.parseInt(hexGroups[0], 16);
148
+ const lo = Number.parseInt(hexGroups[1], 16);
149
+ return isPrivateIp([(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff].join("."));
150
+ }
151
+ return false;
152
+ }
153
+ const normalized = normalizeIpv4(ip);
154
+ if (normalized === null)
155
+ return false;
156
+ const parts = normalized.split(".").map((p) => Number.parseInt(p, 10));
157
+ if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {
158
+ return false;
159
+ }
160
+ const [a, b] = parts;
161
+ if (a === 127)
162
+ return true; // 127.0.0.0/8 loopback
163
+ if (a === 10)
164
+ return true; // 10.0.0.0/8 RFC1918
165
+ if (a === 169 && b === 254)
166
+ return true; // 169.254.0.0/16 link-local + cloud metadata
167
+ if (a === 172 && b >= 16 && b <= 31)
168
+ return true; // 172.16.0.0/12 RFC1918
169
+ if (a === 192 && b === 168)
170
+ return true; // 192.168.0.0/16 RFC1918
171
+ if (a === 100 && b >= 64 && b <= 127)
172
+ return true; // 100.64.0.0/10 CGNAT (RFC6598)
173
+ if (a === 192 && b === 0 && parts[2] === 0)
174
+ return true; // 192.0.0.0/24 (RFC6890)
175
+ if (a === 198 && (b === 18 || b === 19))
176
+ return true; // 198.18.0.0/15 (RFC2544)
177
+ if (a === 0)
178
+ return true; // 0.0.0.0/8
179
+ return false;
180
+ }
181
+ /**
182
+ * Reject any host that is — or resolves to — a private/loopback/link-local
183
+ * address. Applied at the initial URL and at every redirect hop, regardless
184
+ * of the allow-list. Fails closed if the name cannot be resolved.
185
+ *
186
+ * Returns the validated IP so the caller can PIN the connection to that exact
187
+ * address — resolving here and letting the default `fetch` re-resolve at
188
+ * connect time is a DNS-rebinding TOCTOU (CWE-367): a hostile resolver can
189
+ * answer public for this check and private for the socket. For an IP-literal
190
+ * host the pinned value is the (normalized) literal itself.
191
+ */
192
+ async function assertNotSsrf(hostname) {
193
+ const lower = hostname.toLowerCase();
194
+ if (lower === "localhost" || lower.endsWith(".localhost")) {
195
+ throw new WebFetchPermissionError(`WebFetch denied: host "${hostname}" resolves to loopback`);
196
+ }
197
+ if (lower.endsWith(".local")) {
198
+ throw new WebFetchPermissionError(`WebFetch denied: mDNS host "${hostname}" is not allowed`);
199
+ }
200
+ const unbracketed = lower.replace(/^\[/, "").replace(/\]$/, "");
201
+ if (isPrivateIp(unbracketed)) {
202
+ throw new WebFetchPermissionError(`WebFetch denied: host "${hostname}" is a private/loopback IP`);
203
+ }
204
+ // An IP literal is its own pinned target — nothing to resolve or rebind.
205
+ const literal = normalizeIpv4(unbracketed) ?? (unbracketed.includes(":") ? unbracketed : null);
206
+ if (literal !== null)
207
+ return literal;
208
+ // Resolve so a public-looking name pointing at an internal IP (DNS
209
+ // rebinding) is still caught — and return the address so the caller dials it.
210
+ let resolved;
211
+ try {
212
+ resolved = await dnsLookupFn(lower);
213
+ }
214
+ catch (err) {
215
+ const msg = err instanceof Error ? err.message : String(err);
216
+ throw new WebFetchPermissionError(`WebFetch denied: cannot resolve "${hostname}": ${msg}`);
217
+ }
218
+ if (isPrivateIp(resolved.address)) {
219
+ throw new WebFetchPermissionError(`WebFetch denied: host "${hostname}" resolves to private IP ${resolved.address}`);
220
+ }
221
+ return resolved.address;
222
+ }
223
+ // ─── WebFetch ─────────────────────────────────────────────────────────
224
+ const webFetchSchema = z.object({
225
+ url: z.string().min(1),
226
+ prompt: z.string().optional(),
227
+ });
228
+ // Dial `pinnedIp` directly while keeping the real host for the Host header and
229
+ // TLS SNI. Mirrors @crewhaus/tool-fetch.
230
+ function pinnedFetch(req, pinnedIp) {
231
+ const original = new URL(req.url);
232
+ const host = original.hostname;
233
+ const hostUnbracketed = host.replace(/^\[/, "").replace(/\]$/, "");
234
+ if (pinnedIp === "" || hostUnbracketed === pinnedIp) {
235
+ return globalThis.fetch(req);
236
+ }
237
+ const hostForUrl = pinnedIp.includes(":") ? `[${pinnedIp}]` : pinnedIp;
238
+ const pinnedUrl = new URL(original.toString());
239
+ pinnedUrl.hostname = hostForUrl;
240
+ const headers = new Headers(req.headers);
241
+ headers.set("host", original.port === "" ? host : `${host}:${original.port}`);
242
+ const init = {
243
+ method: req.method,
244
+ headers,
245
+ redirect: "manual",
246
+ signal: req.signal,
247
+ tls: { serverName: host },
248
+ };
249
+ if (req.method !== "GET" && req.method !== "HEAD") {
250
+ init.body = req.body;
251
+ init.duplex = "half";
252
+ }
253
+ return globalThis.fetch(pinnedUrl.toString(), init);
254
+ }
255
+ // Production default — one named fetcher shared by the initial binding and the
256
+ // `_setRawFetch(undefined)` reset path (see defaultDnsLookup for rationale).
257
+ const defaultRawFetch = pinnedFetch;
258
+ let rawFetch = defaultRawFetch;
259
+ export function _setRawFetch(fn) {
260
+ rawFetch = fn ?? defaultRawFetch;
261
+ }
262
+ async function readBodyCapped(res) {
263
+ if (res.body === null)
264
+ return "";
265
+ const reader = res.body.getReader();
266
+ const chunks = [];
267
+ let total = 0;
268
+ try {
269
+ while (true) {
270
+ const { done, value } = await reader.read();
271
+ if (done)
272
+ break;
273
+ total += value.byteLength;
274
+ if (total > MAX_BODY_BYTES) {
275
+ try {
276
+ await reader.cancel();
277
+ }
278
+ catch {
279
+ /* ignore */
280
+ }
281
+ throw new WebFetchPermissionError(`response body exceeded ${MAX_BODY_BYTES} bytes — aborted`);
282
+ }
283
+ chunks.push(value);
284
+ }
285
+ }
286
+ finally {
287
+ try {
288
+ reader.releaseLock();
289
+ }
290
+ catch {
291
+ /* ignore */
292
+ }
293
+ }
294
+ const merged = new Uint8Array(total);
295
+ let offset = 0;
296
+ for (const chunk of chunks) {
297
+ merged.set(chunk, offset);
298
+ offset += chunk.byteLength;
299
+ }
300
+ return new TextDecoder("utf-8", { fatal: false }).decode(merged);
301
+ }
302
+ async function performWebFetch(initialUrl, signal) {
303
+ let currentUrl = initialUrl;
304
+ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
305
+ if (currentUrl.protocol !== "http:" && currentUrl.protocol !== "https:") {
306
+ throw new WebFetchPermissionError(`WebFetch denied: scheme "${currentUrl.protocol}" — only http/https allowed`);
307
+ }
308
+ // SSRF guard — independent of the allow-list, enforced on every hop.
309
+ // The returned IP pins the socket so a rebinding resolver can't swap in a
310
+ // private address between this check and connect.
311
+ const pinnedIp = await assertNotSsrf(currentUrl.hostname);
312
+ if (!isHostAllowed(currentUrl.hostname)) {
313
+ throw new WebFetchPermissionError(`WebFetch denied: host "${currentUrl.hostname}" is not in allowed_domains`);
314
+ }
315
+ const res = await rawFetch(new Request(currentUrl.toString(), {
316
+ method: "GET",
317
+ redirect: "manual",
318
+ signal,
319
+ headers: { "user-agent": "crewhaus-tool-web/0.1" },
320
+ }), pinnedIp);
321
+ if (res.status >= 300 && res.status < 400 && res.headers.has("location")) {
322
+ const loc = res.headers.get("location") ?? "";
323
+ let next;
324
+ try {
325
+ next = new URL(loc, currentUrl);
326
+ }
327
+ catch {
328
+ throw new WebFetchPermissionError(`invalid redirect target "${loc}"`);
329
+ }
330
+ currentUrl = next;
331
+ try {
332
+ await res.body?.cancel();
333
+ }
334
+ catch {
335
+ /* ignore */
336
+ }
337
+ continue;
338
+ }
339
+ return res;
340
+ }
341
+ throw new WebFetchPermissionError(`too many redirects (>${MAX_REDIRECTS})`);
342
+ }
343
+ /**
344
+ * Convert HTML into compact markdown via cheerio + turndown. Strips
345
+ * script/style/noscript before conversion so non-content noise doesn't
346
+ * blow out the result.
347
+ */
348
+ export function htmlToMarkdown(html) {
349
+ const $ = loadHtml(html);
350
+ $("script, style, noscript, link, iframe, svg").remove();
351
+ // Prefer <main> / <article> when present so we don't drag headers/footers in.
352
+ let bodyHtml = $("article").first().html();
353
+ if (bodyHtml === null || bodyHtml === undefined)
354
+ bodyHtml = $("main").first().html();
355
+ if (bodyHtml === null || bodyHtml === undefined)
356
+ bodyHtml = $("body").html();
357
+ if (bodyHtml === null || bodyHtml === undefined)
358
+ bodyHtml = html;
359
+ const turndown = new TurndownService({
360
+ headingStyle: "atx",
361
+ codeBlockStyle: "fenced",
362
+ bulletListMarker: "-",
363
+ });
364
+ return turndown.turndown(bodyHtml).trim();
365
+ }
366
+ export const webFetch = buildTool({
367
+ name: "WebFetch",
368
+ description: "Fetch an http(s) URL and return its content. HTML is converted to compact markdown; plain text and JSON pass through. ≤5 MB body, ≤5 redirects, 30 s timeout. Optional allow-list via WebFetch.allowed_domains in the spec's tool_config block.",
369
+ inputSchema: webFetchSchema,
370
+ readOnly: true,
371
+ concurrencySafe: true,
372
+ // Pillar 3 sink-side: URL parameters can encode exfiltrated data ("Safe URL"
373
+ // pattern from OpenAI's 2026-05 prompt-injection paper).
374
+ scope: "external",
375
+ // FR-002 — declare the io-capability fact (HTTP fetch) for the audit.
376
+ ioCapability: "network",
377
+ execute: async (input, ctx) => {
378
+ let url;
379
+ try {
380
+ url = new URL(input.url);
381
+ }
382
+ catch {
383
+ throw new WebFetchPermissionError(`invalid URL "${input.url}"`);
384
+ }
385
+ const ctrl = new AbortController();
386
+ const timer = setTimeout(() => ctrl.abort(new Error("fetch timeout")), DEFAULT_TIMEOUT_MS);
387
+ if (ctx?.signal !== undefined) {
388
+ if (ctx.signal.aborted)
389
+ ctrl.abort(ctx.signal.reason);
390
+ else
391
+ ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
392
+ once: true,
393
+ });
394
+ }
395
+ let res;
396
+ try {
397
+ res = await performWebFetch(url, ctrl.signal);
398
+ }
399
+ finally {
400
+ clearTimeout(timer);
401
+ }
402
+ const body = await readBodyCapped(res);
403
+ const ct = (res.headers.get("content-type") ?? "").toLowerCase();
404
+ let content;
405
+ if (ct.includes("text/html") || ct.includes("application/xhtml")) {
406
+ content = htmlToMarkdown(body);
407
+ }
408
+ else if (ct.includes("text/plain") ||
409
+ ct.includes("application/json") ||
410
+ ct === "" ||
411
+ ct.startsWith("text/")) {
412
+ content = body;
413
+ }
414
+ else {
415
+ content = `[non-textual content: ${ct}, ${body.length} bytes — not displayed]`;
416
+ }
417
+ const header = `URL: ${url.toString()}\nStatus: ${res.status}\nContent-Type: ${ct}\n\n`;
418
+ if (input.prompt !== undefined && input.prompt !== "") {
419
+ return `${header}[user prompt: ${input.prompt}]\n\n${content}`;
420
+ }
421
+ return `${header}${content}`;
422
+ },
423
+ });
424
+ // ─── WebSearch ─────────────────────────────────────────────────────────
425
+ const webSearchSchema = z.object({
426
+ query: z.string().min(1),
427
+ allowed_domains: z.array(z.string()).optional(),
428
+ blocked_domains: z.array(z.string()).optional(),
429
+ });
430
+ /**
431
+ * Provider dispatch. Anthropic server-side `web_search` lands when Section
432
+ * 17 (multi-provider model layer) introduces `model.features.web_search`.
433
+ * Until then, dispatch to a third-party provider via env. When neither
434
+ * env var is set, return a clean refusal so the model handles it
435
+ * gracefully.
436
+ */
437
+ async function dispatchSearch(query, allowed, blocked, signal) {
438
+ const provider = (process.env["CREWHAUS_SEARCH_PROVIDER"] ?? "").toLowerCase();
439
+ const apiKey = process.env["CREWHAUS_SEARCH_API_KEY"] ?? "";
440
+ if (provider === "" || apiKey === "") {
441
+ return "WebSearch unavailable: set CREWHAUS_SEARCH_PROVIDER (brave|tavily) and CREWHAUS_SEARCH_API_KEY in the environment.";
442
+ }
443
+ if (provider === "brave")
444
+ return braveSearch(query, apiKey, allowed, blocked, signal);
445
+ if (provider === "tavily")
446
+ return tavilySearch(query, apiKey, allowed, blocked, signal);
447
+ return `WebSearch unavailable: unknown provider "${provider}" — supported: brave, tavily.`;
448
+ }
449
+ async function braveSearch(query, apiKey, allowed, blocked, signal) {
450
+ const params = new URLSearchParams({ q: query, count: "10" });
451
+ // Brave uses a single `result_filter` and per-request goggles; treat
452
+ // allowed/blocked as advisory hints rather than binding constraints
453
+ // (the provider doesn't enforce them server-side).
454
+ const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
455
+ // Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
456
+ const res = await rawFetch(new Request(url, {
457
+ headers: { accept: "application/json", "x-subscription-token": apiKey },
458
+ signal,
459
+ }), "");
460
+ if (!res.ok) {
461
+ throw new WebFetchPermissionError(`Brave search failed: HTTP ${res.status}`);
462
+ }
463
+ const json = (await res.json());
464
+ const raw = json.web?.results ?? [];
465
+ return raw
466
+ .map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.description ?? "" }))
467
+ .filter((h) => filterByDomains(h.url, allowed, blocked));
468
+ }
469
+ async function tavilySearch(query, apiKey, allowed, blocked, signal) {
470
+ const res = await rawFetch(new Request("https://api.tavily.com/search", {
471
+ method: "POST",
472
+ headers: { "content-type": "application/json" },
473
+ body: JSON.stringify({
474
+ api_key: apiKey,
475
+ query,
476
+ max_results: 10,
477
+ ...(allowed !== undefined ? { include_domains: allowed } : {}),
478
+ ...(blocked !== undefined ? { exclude_domains: blocked } : {}),
479
+ }),
480
+ signal,
481
+ }),
482
+ // Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
483
+ "");
484
+ if (!res.ok) {
485
+ throw new WebFetchPermissionError(`Tavily search failed: HTTP ${res.status}`);
486
+ }
487
+ const json = (await res.json());
488
+ const raw = json.results ?? [];
489
+ return raw
490
+ .map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.content ?? "" }))
491
+ .filter((h) => filterByDomains(h.url, allowed, blocked));
492
+ }
493
+ function filterByDomains(hitUrl, allowed, blocked) {
494
+ let host;
495
+ try {
496
+ host = new URL(hitUrl).hostname.toLowerCase();
497
+ }
498
+ catch {
499
+ return false;
500
+ }
501
+ if (blocked?.some((b) => host === b.toLowerCase() || host.endsWith(`.${b.toLowerCase()}`))) {
502
+ return false;
503
+ }
504
+ if (allowed && allowed.length > 0) {
505
+ return allowed.some((a) => host === a.toLowerCase() || host.endsWith(`.${a.toLowerCase()}`));
506
+ }
507
+ return true;
508
+ }
509
+ function formatHits(hits) {
510
+ if (hits.length === 0)
511
+ return "[no results]";
512
+ return hits.map((h, i) => `${i + 1}. ${h.title}\n ${h.url}\n ${h.snippet}`).join("\n\n");
513
+ }
514
+ export const webSearch = buildTool({
515
+ name: "WebSearch",
516
+ description: "Search the web for up-to-date information. Provider chosen via the CREWHAUS_SEARCH_PROVIDER env var (brave or tavily); CREWHAUS_SEARCH_API_KEY supplies the credential. Returns a numbered list of {title, url, snippet}. Optional allowed_domains / blocked_domains filter the results.",
517
+ inputSchema: webSearchSchema,
518
+ readOnly: true,
519
+ concurrencySafe: true,
520
+ // Pillar 3 sink-side: search-query parameter is an exfiltration vector.
521
+ scope: "external",
522
+ // FR-002 — declare the io-capability fact (remote search API) for the audit.
523
+ ioCapability: "network",
524
+ execute: async (input, ctx) => {
525
+ const ctrl = new AbortController();
526
+ const timer = setTimeout(() => ctrl.abort(new Error("search timeout")), DEFAULT_TIMEOUT_MS);
527
+ if (ctx?.signal !== undefined) {
528
+ if (ctx.signal.aborted)
529
+ ctrl.abort(ctx.signal.reason);
530
+ else
531
+ ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
532
+ once: true,
533
+ });
534
+ }
535
+ try {
536
+ const result = await dispatchSearch(input.query, input.allowed_domains, input.blocked_domains, ctrl.signal);
537
+ if (typeof result === "string")
538
+ return result;
539
+ return formatHits(result);
540
+ }
541
+ finally {
542
+ clearTimeout(timer);
543
+ }
544
+ },
545
+ });
package/package.json CHANGED
@@ -1,28 +1,31 @@
1
1
  {
2
2
  "name": "@crewhaus/tool-web",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "type": "module",
5
5
  "description": "WebFetch (URL → markdown via cheerio + turndown) and WebSearch (provider dispatch)",
6
- "main": "src/index.ts",
7
- "types": "src/index.ts",
6
+ "main": "dist/index.js",
7
+ "types": "dist/index.d.ts",
8
8
  "exports": {
9
- ".": "./src/index.ts"
9
+ ".": {
10
+ "types": "./dist/index.d.ts",
11
+ "import": "./dist/index.js"
12
+ }
10
13
  },
11
14
  "scripts": {
12
15
  "test": "bun test src"
13
16
  },
14
17
  "dependencies": {
15
- "@crewhaus/errors": "0.1.4",
16
- "@crewhaus/tool-builder": "0.1.4",
17
- "@crewhaus/tool-catalog": "0.1.4",
18
+ "@crewhaus/errors": "0.1.6",
19
+ "@crewhaus/tool-builder": "0.1.6",
20
+ "@crewhaus/tool-catalog": "0.1.6",
18
21
  "cheerio": "^1.0.0",
19
22
  "turndown": "^7.2.0",
20
23
  "zod": "^3.23.8"
21
24
  },
22
25
  "devDependencies": {
23
- "@crewhaus/tool-executor": "0.1.4",
24
- "@crewhaus/tool-permission-matcher": "0.1.4",
25
- "@crewhaus/tool-validate": "0.1.4",
26
+ "@crewhaus/tool-executor": "0.1.6",
27
+ "@crewhaus/tool-permission-matcher": "0.1.6",
28
+ "@crewhaus/tool-validate": "0.1.6",
26
29
  "@types/turndown": "^5.0.5"
27
30
  },
28
31
  "license": "Apache-2.0",
@@ -43,5 +46,5 @@
43
46
  "publishConfig": {
44
47
  "access": "public"
45
48
  },
46
- "files": ["src", "README.md", "LICENSE", "NOTICE"]
49
+ "files": ["dist", "README.md", "LICENSE", "NOTICE"]
47
50
  }