@crewhaus/tool-web 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +52 -0
- package/src/index.test.ts +318 -0
- package/src/index.ts +428 -0
package/package.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@crewhaus/tool-web",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "WebFetch (URL → markdown via cheerio + turndown) and WebSearch (provider dispatch)",
|
|
6
|
+
"main": "src/index.ts",
|
|
7
|
+
"types": "src/index.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": "./src/index.ts"
|
|
10
|
+
},
|
|
11
|
+
"scripts": {
|
|
12
|
+
"test": "bun test src"
|
|
13
|
+
},
|
|
14
|
+
"dependencies": {
|
|
15
|
+
"@crewhaus/errors": "0.0.0",
|
|
16
|
+
"@crewhaus/tool-builder": "0.0.0",
|
|
17
|
+
"@crewhaus/tool-catalog": "0.0.0",
|
|
18
|
+
"cheerio": "^1.0.0",
|
|
19
|
+
"turndown": "^7.2.0",
|
|
20
|
+
"zod": "^3.23.8"
|
|
21
|
+
},
|
|
22
|
+
"devDependencies": {
|
|
23
|
+
"@crewhaus/tool-executor": "0.0.0",
|
|
24
|
+
"@crewhaus/tool-permission-matcher": "0.0.0",
|
|
25
|
+
"@crewhaus/tool-validate": "0.0.0",
|
|
26
|
+
"@types/turndown": "^5.0.5"
|
|
27
|
+
},
|
|
28
|
+
"license": "Apache-2.0",
|
|
29
|
+
"author": {
|
|
30
|
+
"name": "Max Meier",
|
|
31
|
+
"email": "max@studiomax.io",
|
|
32
|
+
"url": "https://studiomax.io"
|
|
33
|
+
},
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "git+https://github.com/crewhaus/factory.git",
|
|
37
|
+
"directory": "packages/tool-web"
|
|
38
|
+
},
|
|
39
|
+
"homepage": "https://github.com/crewhaus/factory/tree/main/packages/tool-web#readme",
|
|
40
|
+
"bugs": {
|
|
41
|
+
"url": "https://github.com/crewhaus/factory/issues"
|
|
42
|
+
},
|
|
43
|
+
"publishConfig": {
|
|
44
|
+
"access": "restricted"
|
|
45
|
+
},
|
|
46
|
+
"files": [
|
|
47
|
+
"src",
|
|
48
|
+
"README.md",
|
|
49
|
+
"LICENSE",
|
|
50
|
+
"NOTICE"
|
|
51
|
+
]
|
|
52
|
+
}
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
WebFetchPermissionError,
|
|
4
|
+
_resetWebFetchConfig,
|
|
5
|
+
_setRawFetch,
|
|
6
|
+
htmlToMarkdown,
|
|
7
|
+
registerWebFetchConfig,
|
|
8
|
+
webFetch,
|
|
9
|
+
webSearch,
|
|
10
|
+
} from "./index";
|
|
11
|
+
|
|
12
|
+
const HTML_PAGE = `<!doctype html>
|
|
13
|
+
<html><head><title>Example Domain</title>
|
|
14
|
+
<style>p{color:red}</style>
|
|
15
|
+
<script>document.write("evil")</script>
|
|
16
|
+
</head><body>
|
|
17
|
+
<main>
|
|
18
|
+
<h1>Example Domain</h1>
|
|
19
|
+
<p>This domain is for use in illustrative examples in documents.</p>
|
|
20
|
+
<p>You may use this domain in literature without prior coordination or asking for permission.</p>
|
|
21
|
+
<a href="https://www.iana.org/domains/example">More information...</a>
|
|
22
|
+
</main>
|
|
23
|
+
</body></html>`;
|
|
24
|
+
|
|
25
|
+
const ORIGINAL_ENV: Record<string, string | undefined> = {};
|
|
26
|
+
|
|
27
|
+
beforeEach(() => {
|
|
28
|
+
_resetWebFetchConfig();
|
|
29
|
+
_setRawFetch(undefined);
|
|
30
|
+
for (const key of ["CREWHAUS_SEARCH_PROVIDER", "CREWHAUS_SEARCH_API_KEY"]) {
|
|
31
|
+
ORIGINAL_ENV[key] = process.env[key];
|
|
32
|
+
delete process.env[key];
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
afterEach(() => {
|
|
37
|
+
_resetWebFetchConfig();
|
|
38
|
+
_setRawFetch(undefined);
|
|
39
|
+
for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
|
|
40
|
+
if (value === undefined) delete process.env[key];
|
|
41
|
+
else process.env[key] = value;
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
describe("WebFetch — registered tool metadata", () => {
|
|
46
|
+
test("name + flags", () => {
|
|
47
|
+
expect(webFetch.name).toBe("WebFetch");
|
|
48
|
+
expect(webFetch.readOnly).toBe(true);
|
|
49
|
+
expect(webFetch.concurrencySafe).toBe(true);
|
|
50
|
+
expect(webFetch.destructive).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe("htmlToMarkdown", () => {
|
|
55
|
+
test("strips script + style and renders main content", () => {
|
|
56
|
+
const md = htmlToMarkdown(HTML_PAGE);
|
|
57
|
+
expect(md).toContain("Example Domain");
|
|
58
|
+
expect(md).toContain("illustrative examples");
|
|
59
|
+
expect(md).not.toContain("color:red");
|
|
60
|
+
expect(md).not.toContain("evil");
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("renders headings as ATX (#)", () => {
|
|
64
|
+
const md = htmlToMarkdown("<h1>Title</h1><p>x</p>");
|
|
65
|
+
expect(md.startsWith("# Title")).toBe(true);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("renders links inline", () => {
|
|
69
|
+
const md = htmlToMarkdown('<p><a href="https://x.com">click</a></p>');
|
|
70
|
+
expect(md).toContain("[click](https://x.com)");
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("WebFetch — happy path", () => {
|
|
75
|
+
test("converts HTML to markdown and includes URL/Status header", async () => {
|
|
76
|
+
_setRawFetch(
|
|
77
|
+
async () =>
|
|
78
|
+
new Response(HTML_PAGE, {
|
|
79
|
+
status: 200,
|
|
80
|
+
headers: { "content-type": "text/html; charset=utf-8" },
|
|
81
|
+
}),
|
|
82
|
+
);
|
|
83
|
+
const result = await webFetch.execute({ url: "https://example.com/" });
|
|
84
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
85
|
+
expect(result).toContain("URL: https://example.com/");
|
|
86
|
+
expect(result).toContain("Status: 200");
|
|
87
|
+
expect(result).toContain("Example Domain");
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("plain text passes through unchanged", async () => {
|
|
91
|
+
_setRawFetch(
|
|
92
|
+
async () =>
|
|
93
|
+
new Response("hello world", { status: 200, headers: { "content-type": "text/plain" } }),
|
|
94
|
+
);
|
|
95
|
+
const result = await webFetch.execute({ url: "https://example.com/" });
|
|
96
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
97
|
+
expect(result).toContain("hello world");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("user prompt is prepended to the body, not sent to the model from the tool", async () => {
|
|
101
|
+
_setRawFetch(
|
|
102
|
+
async () =>
|
|
103
|
+
new Response(HTML_PAGE, { status: 200, headers: { "content-type": "text/html" } }),
|
|
104
|
+
);
|
|
105
|
+
const result = await webFetch.execute({
|
|
106
|
+
url: "https://example.com/",
|
|
107
|
+
prompt: "summarise the page",
|
|
108
|
+
});
|
|
109
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
110
|
+
expect(result).toContain("[user prompt: summarise the page]");
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("T8 — WebFetch scheme + allow-list", () => {
|
|
115
|
+
test("rejects file:// scheme", async () => {
|
|
116
|
+
_setRawFetch(async () => {
|
|
117
|
+
throw new Error("must not call");
|
|
118
|
+
});
|
|
119
|
+
await expect(webFetch.execute({ url: "file:///etc/passwd" })).rejects.toBeInstanceOf(
|
|
120
|
+
WebFetchPermissionError,
|
|
121
|
+
);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("rejects javascript: scheme", async () => {
|
|
125
|
+
_setRawFetch(async () => {
|
|
126
|
+
throw new Error("must not call");
|
|
127
|
+
});
|
|
128
|
+
await expect(webFetch.execute({ url: "javascript:alert(1)" })).rejects.toBeInstanceOf(
|
|
129
|
+
WebFetchPermissionError,
|
|
130
|
+
);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("empty allow-list = allow-all (Section 14 says optional)", async () => {
|
|
134
|
+
_setRawFetch(
|
|
135
|
+
async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
|
|
136
|
+
);
|
|
137
|
+
await expect(webFetch.execute({ url: "https://random.example/" })).resolves.toBeDefined();
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("populated allow-list permits exact host and subdomains", async () => {
|
|
141
|
+
registerWebFetchConfig({ allowed_domains: ["example.com"] });
|
|
142
|
+
_setRawFetch(
|
|
143
|
+
async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
|
|
144
|
+
);
|
|
145
|
+
await expect(webFetch.execute({ url: "https://example.com/" })).resolves.toBeDefined();
|
|
146
|
+
await expect(webFetch.execute({ url: "https://api.example.com/" })).resolves.toBeDefined();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("populated allow-list rejects unrelated hosts", async () => {
|
|
150
|
+
registerWebFetchConfig({ allowed_domains: ["example.com"] });
|
|
151
|
+
_setRawFetch(
|
|
152
|
+
async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
|
|
153
|
+
);
|
|
154
|
+
await expect(webFetch.execute({ url: "https://evil.com/" })).rejects.toBeInstanceOf(
|
|
155
|
+
WebFetchPermissionError,
|
|
156
|
+
);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test("case-mutated host still matches the allow-list (canonicalised)", async () => {
|
|
160
|
+
registerWebFetchConfig({ allowed_domains: ["Example.com"] });
|
|
161
|
+
_setRawFetch(
|
|
162
|
+
async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
|
|
163
|
+
);
|
|
164
|
+
await expect(webFetch.execute({ url: "https://EXAMPLE.com/" })).resolves.toBeDefined();
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
describe("T8 — WebFetch redirects", () => {
|
|
169
|
+
test("follows up to 5 redirects then aborts", async () => {
|
|
170
|
+
let hops = 0;
|
|
171
|
+
_setRawFetch(async () => {
|
|
172
|
+
hops++;
|
|
173
|
+
return new Response(null, {
|
|
174
|
+
status: 302,
|
|
175
|
+
headers: { location: `https://example.com/h${hops}` },
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
await expect(webFetch.execute({ url: "https://example.com/" })).rejects.toBeInstanceOf(
|
|
179
|
+
WebFetchPermissionError,
|
|
180
|
+
);
|
|
181
|
+
expect(hops).toBeGreaterThanOrEqual(6);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test("redirect to a non-allowed host is rejected on the hop", async () => {
|
|
185
|
+
registerWebFetchConfig({ allowed_domains: ["example.com"] });
|
|
186
|
+
let hops = 0;
|
|
187
|
+
_setRawFetch(async () => {
|
|
188
|
+
hops++;
|
|
189
|
+
return new Response(null, {
|
|
190
|
+
status: 302,
|
|
191
|
+
headers: { location: "https://evil.com/" },
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
await expect(webFetch.execute({ url: "https://example.com/" })).rejects.toBeInstanceOf(
|
|
195
|
+
WebFetchPermissionError,
|
|
196
|
+
);
|
|
197
|
+
expect(hops).toBe(1);
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
describe("T8 — WebFetch body cap", () => {
|
|
202
|
+
test("rejects responses over 5 MB", async () => {
|
|
203
|
+
const sixMb = "x".repeat(6 * 1024 * 1024);
|
|
204
|
+
_setRawFetch(
|
|
205
|
+
async () => new Response(sixMb, { status: 200, headers: { "content-type": "text/plain" } }),
|
|
206
|
+
);
|
|
207
|
+
await expect(webFetch.execute({ url: "https://example.com/big" })).rejects.toBeInstanceOf(
|
|
208
|
+
WebFetchPermissionError,
|
|
209
|
+
);
|
|
210
|
+
});
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
describe("WebSearch — registered tool metadata", () => {
|
|
214
|
+
test("name + flags", () => {
|
|
215
|
+
expect(webSearch.name).toBe("WebSearch");
|
|
216
|
+
expect(webSearch.readOnly).toBe(true);
|
|
217
|
+
expect(webSearch.concurrencySafe).toBe(true);
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
describe("WebSearch — env-driven dispatch", () => {
|
|
222
|
+
test("returns a clean refusal when env is unset", async () => {
|
|
223
|
+
_setRawFetch(async () => {
|
|
224
|
+
throw new Error("must not call");
|
|
225
|
+
});
|
|
226
|
+
const result = await webSearch.execute({ query: "anything" });
|
|
227
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
228
|
+
expect(result).toContain("WebSearch unavailable");
|
|
229
|
+
expect(result).toContain("CREWHAUS_SEARCH_PROVIDER");
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
test("returns a clean refusal for an unknown provider", async () => {
|
|
233
|
+
process.env["CREWHAUS_SEARCH_PROVIDER"] = "duckduckgo";
|
|
234
|
+
process.env["CREWHAUS_SEARCH_API_KEY"] = "x";
|
|
235
|
+
_setRawFetch(async () => {
|
|
236
|
+
throw new Error("must not call");
|
|
237
|
+
});
|
|
238
|
+
const result = await webSearch.execute({ query: "anything" });
|
|
239
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
240
|
+
expect(result).toContain("unknown provider");
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
test("contracts a Brave response into uniform hit list", async () => {
|
|
244
|
+
process.env["CREWHAUS_SEARCH_PROVIDER"] = "brave";
|
|
245
|
+
process.env["CREWHAUS_SEARCH_API_KEY"] = "test-token";
|
|
246
|
+
_setRawFetch(async (req) => {
|
|
247
|
+
expect(req.url).toContain("api.search.brave.com");
|
|
248
|
+
expect(req.headers.get("x-subscription-token")).toBe("test-token");
|
|
249
|
+
return new Response(
|
|
250
|
+
JSON.stringify({
|
|
251
|
+
web: {
|
|
252
|
+
results: [
|
|
253
|
+
{
|
|
254
|
+
title: "Bun docs",
|
|
255
|
+
url: "https://bun.sh/docs",
|
|
256
|
+
description: "Fast all-in-one runtime",
|
|
257
|
+
},
|
|
258
|
+
{ title: "Node docs", url: "https://nodejs.org", description: "JS runtime" },
|
|
259
|
+
],
|
|
260
|
+
},
|
|
261
|
+
}),
|
|
262
|
+
{ status: 200, headers: { "content-type": "application/json" } },
|
|
263
|
+
);
|
|
264
|
+
});
|
|
265
|
+
const result = await webSearch.execute({ query: "bun runtime" });
|
|
266
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
267
|
+
expect(result).toContain("Bun docs");
|
|
268
|
+
expect(result).toContain("https://bun.sh/docs");
|
|
269
|
+
expect(result).toContain("Node docs");
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
test("contracts a Tavily response into uniform hit list", async () => {
|
|
273
|
+
process.env["CREWHAUS_SEARCH_PROVIDER"] = "tavily";
|
|
274
|
+
process.env["CREWHAUS_SEARCH_API_KEY"] = "tvly-test";
|
|
275
|
+
_setRawFetch(async (req) => {
|
|
276
|
+
expect(req.url).toContain("api.tavily.com");
|
|
277
|
+
const body = (await req.json()) as { api_key: string };
|
|
278
|
+
expect(body.api_key).toBe("tvly-test");
|
|
279
|
+
return new Response(
|
|
280
|
+
JSON.stringify({
|
|
281
|
+
results: [
|
|
282
|
+
{ title: "TS Handbook", url: "https://www.typescriptlang.org", content: "Docs" },
|
|
283
|
+
],
|
|
284
|
+
}),
|
|
285
|
+
{ status: 200, headers: { "content-type": "application/json" } },
|
|
286
|
+
);
|
|
287
|
+
});
|
|
288
|
+
const result = await webSearch.execute({ query: "typescript" });
|
|
289
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
290
|
+
expect(result).toContain("TS Handbook");
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test("blocked_domains filters out matching hits", async () => {
|
|
294
|
+
process.env["CREWHAUS_SEARCH_PROVIDER"] = "brave";
|
|
295
|
+
process.env["CREWHAUS_SEARCH_API_KEY"] = "x";
|
|
296
|
+
_setRawFetch(
|
|
297
|
+
async () =>
|
|
298
|
+
new Response(
|
|
299
|
+
JSON.stringify({
|
|
300
|
+
web: {
|
|
301
|
+
results: [
|
|
302
|
+
{ title: "good", url: "https://example.com/page", description: "" },
|
|
303
|
+
{ title: "bad", url: "https://evil.com/page", description: "" },
|
|
304
|
+
],
|
|
305
|
+
},
|
|
306
|
+
}),
|
|
307
|
+
{ status: 200, headers: { "content-type": "application/json" } },
|
|
308
|
+
),
|
|
309
|
+
);
|
|
310
|
+
const result = await webSearch.execute({
|
|
311
|
+
query: "x",
|
|
312
|
+
blocked_domains: ["evil.com"],
|
|
313
|
+
});
|
|
314
|
+
if (typeof result !== "string") throw new Error("expected string result");
|
|
315
|
+
expect(result).toContain("good");
|
|
316
|
+
expect(result).not.toContain("evil.com");
|
|
317
|
+
});
|
|
318
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
2
|
+
import { buildTool } from "@crewhaus/tool-builder";
|
|
3
|
+
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
4
|
+
import { load as loadHtml } from "cheerio";
|
|
5
|
+
import TurndownService from "turndown";
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Section 14 — `WebFetch` and `WebSearch`.
|
|
10
|
+
*
|
|
11
|
+
* `WebFetch(url, prompt?)`:
|
|
12
|
+
* - Validates http/https scheme.
|
|
13
|
+
* - Optional allow-list via `getWebFetchConfig().allowedDomains`. When
|
|
14
|
+
* non-empty, host must equal an entry or be a subdomain of one.
|
|
15
|
+
* - Manual redirect handling, max 5; allow-list re-checked at every hop.
|
|
16
|
+
* - 30 s default timeout (`AbortController`, honours `ctx.signal`).
|
|
17
|
+
* - 5 MB response body cap.
|
|
18
|
+
* - HTML pages run through cheerio + turndown to produce markdown that
|
|
19
|
+
* compresses well in the model's context. Plain text and JSON pass
|
|
20
|
+
* through. Other content types are summarised in one line.
|
|
21
|
+
* - When `prompt` is supplied, we DO NOT recursively call the model
|
|
22
|
+
* from inside the tool — that would fight runtime-core's loop. The
|
|
23
|
+
* prompt is prepended to the markdown ("[user prompt: ...]") so the
|
|
24
|
+
* parent agent sees both pieces and can reason about them in its
|
|
25
|
+
* next assistant turn. Documented behaviour, matching the
|
|
26
|
+
* claude-code/tools/WebFetchTool design.
|
|
27
|
+
*
|
|
28
|
+
* `WebSearch(query, allowed_domains?, blocked_domains?)`:
|
|
29
|
+
* - Anthropic server-side `web_search` is the preferred path. The
|
|
30
|
+
* model adapter doesn't yet expose `model.features.web_search`
|
|
31
|
+
* (Section 17), so for now this tool dispatches to a configurable
|
|
32
|
+
* third-party provider via env: `CREWHAUS_SEARCH_PROVIDER` ∈
|
|
33
|
+
* {brave, tavily} plus `CREWHAUS_SEARCH_API_KEY`. When neither is
|
|
34
|
+
* set, returns a clean refusal so the model can either skip search
|
|
35
|
+
* or surface the missing-config to the user.
|
|
36
|
+
*
|
|
37
|
+
* Layer R4. Pairs with the `target-cli` codegen contract — `BUILTIN_TOOL_MAP`
|
|
38
|
+
* has `webFetch: { initSymbol: "registerWebFetchConfig" }` and
|
|
39
|
+
* `webSearch: { ... }` (no init — uses env-only config).
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
43
|
+
const MAX_REDIRECTS = 5;
|
|
44
|
+
const MAX_BODY_BYTES = 5 * 1024 * 1024;
|
|
45
|
+
|
|
46
|
+
export class WebFetchPermissionError extends CrewhausError {
|
|
47
|
+
override readonly name = "WebFetchPermissionError";
|
|
48
|
+
constructor(message: string) {
|
|
49
|
+
super("tool", message);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// ─── WebFetch config registry ──────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
export type WebFetchConfig = {
|
|
56
|
+
/**
|
|
57
|
+
* Allowed domains (host suffixes). Empty = allow all. A request to host
|
|
58
|
+
* `<h>` is permitted iff `h` equals an entry exactly, or `h` ends with
|
|
59
|
+
* `.<entry>`. Hosts are compared case-insensitively.
|
|
60
|
+
*/
|
|
61
|
+
readonly allowedDomains: readonly string[];
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
let webFetchConfig: WebFetchConfig = { allowedDomains: [] };
|
|
65
|
+
|
|
66
|
+
export type WebFetchConfigInput = {
|
|
67
|
+
readonly allowed_domains?: readonly string[];
|
|
68
|
+
readonly allowedDomains?: readonly string[];
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
export function registerWebFetchConfig(input: WebFetchConfigInput): void {
|
|
72
|
+
const raw = input.allowedDomains ?? input.allowed_domains ?? [];
|
|
73
|
+
webFetchConfig = { allowedDomains: raw.map((d) => d.toLowerCase()) };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function getWebFetchConfig(): WebFetchConfig {
|
|
77
|
+
return webFetchConfig;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function _resetWebFetchConfig(): void {
|
|
81
|
+
webFetchConfig = { allowedDomains: [] };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function isHostAllowed(host: string): boolean {
|
|
85
|
+
if (webFetchConfig.allowedDomains.length === 0) return true;
|
|
86
|
+
const lower = host.toLowerCase();
|
|
87
|
+
return webFetchConfig.allowedDomains.some(
|
|
88
|
+
(entry) => lower === entry || lower.endsWith(`.${entry}`),
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ─── WebFetch ─────────────────────────────────────────────────────────
|
|
93
|
+
|
|
94
|
+
const webFetchSchema = z.object({
|
|
95
|
+
url: z.string().min(1),
|
|
96
|
+
prompt: z.string().optional(),
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
export type RawFetch = (req: Request) => Promise<Response>;
|
|
100
|
+
let rawFetch: RawFetch = (req) => globalThis.fetch(req);
|
|
101
|
+
export function _setRawFetch(fn: RawFetch | undefined): void {
|
|
102
|
+
rawFetch = fn ?? ((req) => globalThis.fetch(req));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function readBodyCapped(res: Response): Promise<string> {
|
|
106
|
+
if (res.body === null) return "";
|
|
107
|
+
const reader = res.body.getReader();
|
|
108
|
+
const chunks: Uint8Array[] = [];
|
|
109
|
+
let total = 0;
|
|
110
|
+
try {
|
|
111
|
+
while (true) {
|
|
112
|
+
const { done, value } = await reader.read();
|
|
113
|
+
if (done) break;
|
|
114
|
+
total += value.byteLength;
|
|
115
|
+
if (total > MAX_BODY_BYTES) {
|
|
116
|
+
try {
|
|
117
|
+
await reader.cancel();
|
|
118
|
+
} catch {
|
|
119
|
+
/* ignore */
|
|
120
|
+
}
|
|
121
|
+
throw new WebFetchPermissionError(
|
|
122
|
+
`response body exceeded ${MAX_BODY_BYTES} bytes — aborted`,
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
chunks.push(value);
|
|
126
|
+
}
|
|
127
|
+
} finally {
|
|
128
|
+
try {
|
|
129
|
+
reader.releaseLock();
|
|
130
|
+
} catch {
|
|
131
|
+
/* ignore */
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const merged = new Uint8Array(total);
|
|
135
|
+
let offset = 0;
|
|
136
|
+
for (const chunk of chunks) {
|
|
137
|
+
merged.set(chunk, offset);
|
|
138
|
+
offset += chunk.byteLength;
|
|
139
|
+
}
|
|
140
|
+
return new TextDecoder("utf-8", { fatal: false }).decode(merged);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function performWebFetch(initialUrl: URL, signal: AbortSignal): Promise<Response> {
|
|
144
|
+
let currentUrl = initialUrl;
|
|
145
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
146
|
+
if (currentUrl.protocol !== "http:" && currentUrl.protocol !== "https:") {
|
|
147
|
+
throw new WebFetchPermissionError(
|
|
148
|
+
`WebFetch denied: scheme "${currentUrl.protocol}" — only http/https allowed`,
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
if (!isHostAllowed(currentUrl.hostname)) {
|
|
152
|
+
throw new WebFetchPermissionError(
|
|
153
|
+
`WebFetch denied: host "${currentUrl.hostname}" is not in allowed_domains`,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const res = await rawFetch(
|
|
158
|
+
new Request(currentUrl.toString(), {
|
|
159
|
+
method: "GET",
|
|
160
|
+
redirect: "manual",
|
|
161
|
+
signal,
|
|
162
|
+
headers: { "user-agent": "crewhaus-tool-web/0.1" },
|
|
163
|
+
}),
|
|
164
|
+
);
|
|
165
|
+
if (res.status >= 300 && res.status < 400 && res.headers.has("location")) {
|
|
166
|
+
const loc = res.headers.get("location") ?? "";
|
|
167
|
+
let next: URL;
|
|
168
|
+
try {
|
|
169
|
+
next = new URL(loc, currentUrl);
|
|
170
|
+
} catch {
|
|
171
|
+
throw new WebFetchPermissionError(`invalid redirect target "${loc}"`);
|
|
172
|
+
}
|
|
173
|
+
currentUrl = next;
|
|
174
|
+
try {
|
|
175
|
+
await res.body?.cancel();
|
|
176
|
+
} catch {
|
|
177
|
+
/* ignore */
|
|
178
|
+
}
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
return res;
|
|
182
|
+
}
|
|
183
|
+
throw new WebFetchPermissionError(`too many redirects (>${MAX_REDIRECTS})`);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Convert HTML into compact markdown via cheerio + turndown. Strips
|
|
188
|
+
* script/style/noscript before conversion so non-content noise doesn't
|
|
189
|
+
* blow out the result.
|
|
190
|
+
*/
|
|
191
|
+
export function htmlToMarkdown(html: string): string {
|
|
192
|
+
const $ = loadHtml(html);
|
|
193
|
+
$("script, style, noscript, link, iframe, svg").remove();
|
|
194
|
+
// Prefer <main> / <article> when present so we don't drag headers/footers in.
|
|
195
|
+
let bodyHtml = $("article").first().html();
|
|
196
|
+
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = $("main").first().html();
|
|
197
|
+
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = $("body").html();
|
|
198
|
+
if (bodyHtml === null || bodyHtml === undefined) bodyHtml = html;
|
|
199
|
+
const turndown = new TurndownService({
|
|
200
|
+
headingStyle: "atx",
|
|
201
|
+
codeBlockStyle: "fenced",
|
|
202
|
+
bulletListMarker: "-",
|
|
203
|
+
});
|
|
204
|
+
return turndown.turndown(bodyHtml).trim();
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export const webFetch: RegisteredTool = buildTool({
|
|
208
|
+
name: "WebFetch",
|
|
209
|
+
description:
|
|
210
|
+
"Fetch an http(s) URL and return its content. HTML is converted to compact markdown; plain text and JSON pass through. ≤5 MB body, ≤5 redirects, 30 s timeout. Optional allow-list via WebFetch.allowed_domains in the spec's tool_config block.",
|
|
211
|
+
inputSchema: webFetchSchema,
|
|
212
|
+
readOnly: true,
|
|
213
|
+
concurrencySafe: true,
|
|
214
|
+
// Pillar 3 sink-side: URL parameters can encode exfiltrated data ("Safe URL"
|
|
215
|
+
// pattern from OpenAI's 2026-05 prompt-injection paper).
|
|
216
|
+
scope: "external",
|
|
217
|
+
execute: async (input, ctx) => {
|
|
218
|
+
let url: URL;
|
|
219
|
+
try {
|
|
220
|
+
url = new URL(input.url);
|
|
221
|
+
} catch {
|
|
222
|
+
throw new WebFetchPermissionError(`invalid URL "${input.url}"`);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const ctrl = new AbortController();
|
|
226
|
+
const timer = setTimeout(() => ctrl.abort(new Error("fetch timeout")), DEFAULT_TIMEOUT_MS);
|
|
227
|
+
if (ctx?.signal !== undefined) {
|
|
228
|
+
if (ctx.signal.aborted) ctrl.abort(ctx.signal.reason);
|
|
229
|
+
else
|
|
230
|
+
ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
|
|
231
|
+
once: true,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
let res: Response;
|
|
235
|
+
try {
|
|
236
|
+
res = await performWebFetch(url, ctrl.signal);
|
|
237
|
+
} finally {
|
|
238
|
+
clearTimeout(timer);
|
|
239
|
+
}
|
|
240
|
+
const body = await readBodyCapped(res);
|
|
241
|
+
const ct = (res.headers.get("content-type") ?? "").toLowerCase();
|
|
242
|
+
let content: string;
|
|
243
|
+
if (ct.includes("text/html") || ct.includes("application/xhtml")) {
|
|
244
|
+
content = htmlToMarkdown(body);
|
|
245
|
+
} else if (
|
|
246
|
+
ct.includes("text/plain") ||
|
|
247
|
+
ct.includes("application/json") ||
|
|
248
|
+
ct === "" ||
|
|
249
|
+
ct.startsWith("text/")
|
|
250
|
+
) {
|
|
251
|
+
content = body;
|
|
252
|
+
} else {
|
|
253
|
+
content = `[non-textual content: ${ct}, ${body.length} bytes — not displayed]`;
|
|
254
|
+
}
|
|
255
|
+
const header = `URL: ${url.toString()}\nStatus: ${res.status}\nContent-Type: ${ct}\n\n`;
|
|
256
|
+
if (input.prompt !== undefined && input.prompt !== "") {
|
|
257
|
+
return `${header}[user prompt: ${input.prompt}]\n\n${content}`;
|
|
258
|
+
}
|
|
259
|
+
return `${header}${content}`;
|
|
260
|
+
},
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
// ─── WebSearch ─────────────────────────────────────────────────────────
|
|
264
|
+
|
|
265
|
+
const webSearchSchema = z.object({
|
|
266
|
+
query: z.string().min(1),
|
|
267
|
+
allowed_domains: z.array(z.string()).optional(),
|
|
268
|
+
blocked_domains: z.array(z.string()).optional(),
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
type SearchHit = {
|
|
272
|
+
readonly title: string;
|
|
273
|
+
readonly url: string;
|
|
274
|
+
readonly snippet: string;
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Provider dispatch. Anthropic server-side `web_search` lands when Section
|
|
279
|
+
* 17 (multi-provider model layer) introduces `model.features.web_search`.
|
|
280
|
+
* Until then, dispatch to a third-party provider via env. When neither
|
|
281
|
+
* env var is set, return a clean refusal so the model handles it
|
|
282
|
+
* gracefully.
|
|
283
|
+
*/
|
|
284
|
+
async function dispatchSearch(
|
|
285
|
+
query: string,
|
|
286
|
+
allowed: readonly string[] | undefined,
|
|
287
|
+
blocked: readonly string[] | undefined,
|
|
288
|
+
signal: AbortSignal,
|
|
289
|
+
): Promise<readonly SearchHit[] | string> {
|
|
290
|
+
const provider = (process.env["CREWHAUS_SEARCH_PROVIDER"] ?? "").toLowerCase();
|
|
291
|
+
const apiKey = process.env["CREWHAUS_SEARCH_API_KEY"] ?? "";
|
|
292
|
+
if (provider === "" || apiKey === "") {
|
|
293
|
+
return "WebSearch unavailable: set CREWHAUS_SEARCH_PROVIDER (brave|tavily) and CREWHAUS_SEARCH_API_KEY in the environment.";
|
|
294
|
+
}
|
|
295
|
+
if (provider === "brave") return braveSearch(query, apiKey, allowed, blocked, signal);
|
|
296
|
+
if (provider === "tavily") return tavilySearch(query, apiKey, allowed, blocked, signal);
|
|
297
|
+
return `WebSearch unavailable: unknown provider "${provider}" — supported: brave, tavily.`;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
async function braveSearch(
|
|
301
|
+
query: string,
|
|
302
|
+
apiKey: string,
|
|
303
|
+
allowed: readonly string[] | undefined,
|
|
304
|
+
blocked: readonly string[] | undefined,
|
|
305
|
+
signal: AbortSignal,
|
|
306
|
+
): Promise<readonly SearchHit[]> {
|
|
307
|
+
const params = new URLSearchParams({ q: query, count: "10" });
|
|
308
|
+
// Brave uses a single `result_filter` and per-request goggles; treat
|
|
309
|
+
// allowed/blocked as advisory hints rather than binding constraints
|
|
310
|
+
// (the provider doesn't enforce them server-side).
|
|
311
|
+
const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
|
|
312
|
+
const res = await rawFetch(
|
|
313
|
+
new Request(url, {
|
|
314
|
+
headers: { accept: "application/json", "x-subscription-token": apiKey },
|
|
315
|
+
signal,
|
|
316
|
+
}),
|
|
317
|
+
);
|
|
318
|
+
if (!res.ok) {
|
|
319
|
+
throw new WebFetchPermissionError(`Brave search failed: HTTP ${res.status}`);
|
|
320
|
+
}
|
|
321
|
+
const json = (await res.json()) as { web?: { results?: BraveResult[] } };
|
|
322
|
+
const raw = json.web?.results ?? [];
|
|
323
|
+
return raw
|
|
324
|
+
.map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.description ?? "" }))
|
|
325
|
+
.filter((h) => filterByDomains(h.url, allowed, blocked));
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
interface BraveResult {
|
|
329
|
+
readonly title?: string;
|
|
330
|
+
readonly url?: string;
|
|
331
|
+
readonly description?: string;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
async function tavilySearch(
|
|
335
|
+
query: string,
|
|
336
|
+
apiKey: string,
|
|
337
|
+
allowed: readonly string[] | undefined,
|
|
338
|
+
blocked: readonly string[] | undefined,
|
|
339
|
+
signal: AbortSignal,
|
|
340
|
+
): Promise<readonly SearchHit[]> {
|
|
341
|
+
const res = await rawFetch(
|
|
342
|
+
new Request("https://api.tavily.com/search", {
|
|
343
|
+
method: "POST",
|
|
344
|
+
headers: { "content-type": "application/json" },
|
|
345
|
+
body: JSON.stringify({
|
|
346
|
+
api_key: apiKey,
|
|
347
|
+
query,
|
|
348
|
+
max_results: 10,
|
|
349
|
+
...(allowed !== undefined ? { include_domains: allowed } : {}),
|
|
350
|
+
...(blocked !== undefined ? { exclude_domains: blocked } : {}),
|
|
351
|
+
}),
|
|
352
|
+
signal,
|
|
353
|
+
}),
|
|
354
|
+
);
|
|
355
|
+
if (!res.ok) {
|
|
356
|
+
throw new WebFetchPermissionError(`Tavily search failed: HTTP ${res.status}`);
|
|
357
|
+
}
|
|
358
|
+
const json = (await res.json()) as { results?: TavilyResult[] };
|
|
359
|
+
const raw = json.results ?? [];
|
|
360
|
+
return raw
|
|
361
|
+
.map((r) => ({ title: r.title ?? "", url: r.url ?? "", snippet: r.content ?? "" }))
|
|
362
|
+
.filter((h) => filterByDomains(h.url, allowed, blocked));
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
interface TavilyResult {
|
|
366
|
+
readonly title?: string;
|
|
367
|
+
readonly url?: string;
|
|
368
|
+
readonly content?: string;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function filterByDomains(
|
|
372
|
+
hitUrl: string,
|
|
373
|
+
allowed: readonly string[] | undefined,
|
|
374
|
+
blocked: readonly string[] | undefined,
|
|
375
|
+
): boolean {
|
|
376
|
+
let host: string;
|
|
377
|
+
try {
|
|
378
|
+
host = new URL(hitUrl).hostname.toLowerCase();
|
|
379
|
+
} catch {
|
|
380
|
+
return false;
|
|
381
|
+
}
|
|
382
|
+
if (blocked?.some((b) => host === b.toLowerCase() || host.endsWith(`.${b.toLowerCase()}`))) {
|
|
383
|
+
return false;
|
|
384
|
+
}
|
|
385
|
+
if (allowed && allowed.length > 0) {
|
|
386
|
+
return allowed.some((a) => host === a.toLowerCase() || host.endsWith(`.${a.toLowerCase()}`));
|
|
387
|
+
}
|
|
388
|
+
return true;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function formatHits(hits: readonly SearchHit[]): string {
|
|
392
|
+
if (hits.length === 0) return "[no results]";
|
|
393
|
+
return hits.map((h, i) => `${i + 1}. ${h.title}\n ${h.url}\n ${h.snippet}`).join("\n\n");
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
export const webSearch: RegisteredTool = buildTool({
|
|
397
|
+
name: "WebSearch",
|
|
398
|
+
description:
|
|
399
|
+
"Search the web for up-to-date information. Provider chosen via the CREWHAUS_SEARCH_PROVIDER env var (brave or tavily); CREWHAUS_SEARCH_API_KEY supplies the credential. Returns a numbered list of {title, url, snippet}. Optional allowed_domains / blocked_domains filter the results.",
|
|
400
|
+
inputSchema: webSearchSchema,
|
|
401
|
+
readOnly: true,
|
|
402
|
+
concurrencySafe: true,
|
|
403
|
+
// Pillar 3 sink-side: search-query parameter is an exfiltration vector.
|
|
404
|
+
scope: "external",
|
|
405
|
+
execute: async (input, ctx) => {
|
|
406
|
+
const ctrl = new AbortController();
|
|
407
|
+
const timer = setTimeout(() => ctrl.abort(new Error("search timeout")), DEFAULT_TIMEOUT_MS);
|
|
408
|
+
if (ctx?.signal !== undefined) {
|
|
409
|
+
if (ctx.signal.aborted) ctrl.abort(ctx.signal.reason);
|
|
410
|
+
else
|
|
411
|
+
ctx.signal.addEventListener("abort", () => ctrl.abort(ctx.signal?.reason), {
|
|
412
|
+
once: true,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
try {
|
|
416
|
+
const result = await dispatchSearch(
|
|
417
|
+
input.query,
|
|
418
|
+
input.allowed_domains,
|
|
419
|
+
input.blocked_domains,
|
|
420
|
+
ctrl.signal,
|
|
421
|
+
);
|
|
422
|
+
if (typeof result === "string") return result;
|
|
423
|
+
return formatHits(result);
|
|
424
|
+
} finally {
|
|
425
|
+
clearTimeout(timer);
|
|
426
|
+
}
|
|
427
|
+
},
|
|
428
|
+
});
|