@crewhaus/tool-web 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crewhaus/tool-web",
3
- "version": "0.1.1",
3
+ "version": "0.1.2",
4
4
  "type": "module",
5
5
  "description": "WebFetch (URL → markdown via cheerio + turndown) and WebSearch (provider dispatch)",
6
6
  "main": "src/index.ts",
@@ -12,24 +12,24 @@
12
12
  "test": "bun test src"
13
13
  },
14
14
  "dependencies": {
15
- "@crewhaus/errors": "0.1.1",
16
- "@crewhaus/tool-builder": "0.1.1",
17
- "@crewhaus/tool-catalog": "0.1.1",
15
+ "@crewhaus/errors": "0.1.2",
16
+ "@crewhaus/tool-builder": "0.1.2",
17
+ "@crewhaus/tool-catalog": "0.1.2",
18
18
  "cheerio": "^1.0.0",
19
19
  "turndown": "^7.2.0",
20
20
  "zod": "^3.23.8"
21
21
  },
22
22
  "devDependencies": {
23
- "@crewhaus/tool-executor": "0.1.1",
24
- "@crewhaus/tool-permission-matcher": "0.1.1",
25
- "@crewhaus/tool-validate": "0.1.1",
23
+ "@crewhaus/tool-executor": "0.1.2",
24
+ "@crewhaus/tool-permission-matcher": "0.1.2",
25
+ "@crewhaus/tool-validate": "0.1.2",
26
26
  "@types/turndown": "^5.0.5"
27
27
  },
28
28
  "license": "Apache-2.0",
29
29
  "author": {
30
30
  "name": "Max Meier",
31
- "email": "max@studiomax.io",
32
- "url": "https://studiomax.io"
31
+ "email": "max@crewhaus.ai",
32
+ "url": "https://crewhaus.ai"
33
33
  },
34
34
  "repository": {
35
35
  "type": "git",
@@ -41,12 +41,7 @@
41
41
  "url": "https://github.com/crewhaus/factory/issues"
42
42
  },
43
43
  "publishConfig": {
44
- "access": "restricted"
45
- },
46
- "files": [
47
- "src",
48
- "README.md",
49
- "LICENSE",
50
- "NOTICE"
51
- ]
44
+ "access": "public"
45
+ },
46
+ "files": ["src", "README.md", "LICENSE", "NOTICE"]
52
47
  }
package/src/index.test.ts CHANGED
@@ -1,8 +1,10 @@
1
- import { afterEach, beforeEach, describe, expect, test } from "bun:test";
1
+ import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
2
2
  import {
3
3
  WebFetchPermissionError,
4
4
  _resetWebFetchConfig,
5
+ _setDnsLookup,
5
6
  _setRawFetch,
7
+ getWebFetchConfig,
6
8
  htmlToMarkdown,
7
9
  registerWebFetchConfig,
8
10
  webFetch,
@@ -27,6 +29,9 @@ const ORIGINAL_ENV: Record<string, string | undefined> = {};
27
29
  beforeEach(() => {
28
30
  _resetWebFetchConfig();
29
31
  _setRawFetch(undefined);
32
+ // Keep the SSRF DNS backstop offline + deterministic: every host resolves
33
+ // to a public IP by default. Individual tests override to exercise rebinding.
34
+ _setDnsLookup(async () => ({ address: "93.184.216.34", family: 4 }));
30
35
  for (const key of ["CREWHAUS_SEARCH_PROVIDER", "CREWHAUS_SEARCH_API_KEY"]) {
31
36
  ORIGINAL_ENV[key] = process.env[key];
32
37
  delete process.env[key];
@@ -36,6 +41,7 @@ beforeEach(() => {
36
41
  afterEach(() => {
37
42
  _resetWebFetchConfig();
38
43
  _setRawFetch(undefined);
44
+ _setDnsLookup(undefined);
39
45
  for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
40
46
  if (value === undefined) delete process.env[key];
41
47
  else process.env[key] = value;
@@ -165,6 +171,130 @@ describe("T8 — WebFetch scheme + allow-list", () => {
165
171
  });
166
172
  });
167
173
 
174
+ describe("WebFetch — SSRF guard (#141)", () => {
175
+ test("rejects the cloud-metadata IP even with an empty allow-list", async () => {
176
+ _setRawFetch(async () => {
177
+ throw new Error("must not fetch");
178
+ });
179
+ await expect(
180
+ webFetch.execute({
181
+ url: "http://169.254.169.254/latest/meta-data/iam/security-credentials/",
182
+ }),
183
+ ).rejects.toBeInstanceOf(WebFetchPermissionError);
184
+ });
185
+
186
+ test("rejects loopback localhost", async () => {
187
+ _setRawFetch(async () => {
188
+ throw new Error("must not fetch");
189
+ });
190
+ await expect(webFetch.execute({ url: "http://localhost:6379/" })).rejects.toBeInstanceOf(
191
+ WebFetchPermissionError,
192
+ );
193
+ });
194
+
195
+ test("rejects an RFC1918 IP literal", async () => {
196
+ _setRawFetch(async () => {
197
+ throw new Error("must not fetch");
198
+ });
199
+ await expect(webFetch.execute({ url: "http://10.0.0.5/" })).rejects.toBeInstanceOf(
200
+ WebFetchPermissionError,
201
+ );
202
+ });
203
+
204
+ test("rejects the IPv6 loopback [::1]", async () => {
205
+ _setRawFetch(async () => {
206
+ throw new Error("must not fetch");
207
+ });
208
+ await expect(webFetch.execute({ url: "http://[::1]:8080/" })).rejects.toBeInstanceOf(
209
+ WebFetchPermissionError,
210
+ );
211
+ });
212
+
213
+ test("rejects DNS rebinding: a public host that resolves to 127.0.0.1", async () => {
214
+ _setDnsLookup(async () => ({ address: "127.0.0.1", family: 4 }));
215
+ _setRawFetch(async () => {
216
+ throw new Error("must not fetch");
217
+ });
218
+ await expect(
219
+ webFetch.execute({ url: "https://totally-legit.example/" }),
220
+ ).rejects.toBeInstanceOf(WebFetchPermissionError);
221
+ });
222
+
223
+ test("guard fires even when the internal host is explicitly allow-listed", async () => {
224
+ registerWebFetchConfig({ allowed_domains: ["169.254.169.254"] });
225
+ _setRawFetch(async () => {
226
+ throw new Error("must not fetch");
227
+ });
228
+ await expect(webFetch.execute({ url: "http://169.254.169.254/" })).rejects.toBeInstanceOf(
229
+ WebFetchPermissionError,
230
+ );
231
+ });
232
+
233
+ test("blocks a redirect that lands on an internal host (per-hop)", async () => {
234
+ let hops = 0;
235
+ _setRawFetch(async () => {
236
+ hops++;
237
+ return new Response(null, {
238
+ status: 302,
239
+ headers: { location: "http://169.254.169.254/" },
240
+ });
241
+ });
242
+ await expect(webFetch.execute({ url: "https://example.com/" })).rejects.toBeInstanceOf(
243
+ WebFetchPermissionError,
244
+ );
245
+ expect(hops).toBe(1);
246
+ });
247
+
248
+ test("still allows a normal public host (guard is not over-broad)", async () => {
249
+ _setRawFetch(
250
+ async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
251
+ );
252
+ await expect(webFetch.execute({ url: "https://example.com/" })).resolves.toBeDefined();
253
+ });
254
+
255
+ // `new URL` compresses [::ffff:127.0.0.1] to [::ffff:7f00:1]; the old
256
+ // dotted-only regex missed the hex form. Now classified via the embedded v4.
257
+ test("rejects the hex IPv4-mapped IPv6 loopback [::ffff:7f00:1]", async () => {
258
+ _setRawFetch(async () => {
259
+ throw new Error("must not fetch");
260
+ });
261
+ await expect(webFetch.execute({ url: "http://[::ffff:127.0.0.1]/" })).rejects.toBeInstanceOf(
262
+ WebFetchPermissionError,
263
+ );
264
+ });
265
+
266
+ test("rejects an octal-encoded loopback literal", async () => {
267
+ _setRawFetch(async () => {
268
+ throw new Error("must not fetch");
269
+ });
270
+ await expect(webFetch.execute({ url: "http://0177.0.0.1/" })).rejects.toBeInstanceOf(
271
+ WebFetchPermissionError,
272
+ );
273
+ });
274
+
275
+ // The actual rebinding defense: the socket is pinned to the IP we vetted.
276
+ test("pins the connection to the resolved IP (closes the rebinding TOCTOU)", async () => {
277
+ _setDnsLookup(async () => ({ address: "93.184.216.34", family: 4 }));
278
+ let seenPin: string | undefined;
279
+ _setRawFetch(async (_req, pinnedIp) => {
280
+ seenPin = pinnedIp;
281
+ return new Response("ok", { status: 200, headers: { "content-type": "text/plain" } });
282
+ });
283
+ await webFetch.execute({ url: "https://innocent.example/" });
284
+ expect(seenPin).toBe("93.184.216.34");
285
+ });
286
+
287
+ test("pins a public IP-literal host to the literal itself", async () => {
288
+ let seenPin: string | undefined;
289
+ _setRawFetch(async (_req, pinnedIp) => {
290
+ seenPin = pinnedIp;
291
+ return new Response("ok", { status: 200, headers: { "content-type": "text/plain" } });
292
+ });
293
+ await webFetch.execute({ url: "https://93.184.216.34/" });
294
+ expect(seenPin).toBe("93.184.216.34");
295
+ });
296
+ });
297
+
168
298
  describe("T8 — WebFetch redirects", () => {
169
299
  test("follows up to 5 redirects then aborts", async () => {
170
300
  let hops = 0;
@@ -315,4 +445,330 @@ describe("WebSearch — env-driven dispatch", () => {
315
445
  expect(result).toContain("good");
316
446
  expect(result).not.toContain("evil.com");
317
447
  });
448
+
449
+ test("allowed_domains keeps only matching hosts (exact + subdomain)", async () => {
450
+ process.env["CREWHAUS_SEARCH_PROVIDER"] = "brave";
451
+ process.env["CREWHAUS_SEARCH_API_KEY"] = "x";
452
+ _setRawFetch(
453
+ async () =>
454
+ new Response(
455
+ JSON.stringify({
456
+ web: {
457
+ results: [
458
+ { title: "keep-exact", url: "https://example.com/a", description: "" },
459
+ { title: "keep-sub", url: "https://docs.example.com/b", description: "" },
460
+ { title: "drop-other", url: "https://other.org/c", description: "" },
461
+ { title: "drop-bad-url", url: "::not a url::", description: "" },
462
+ ],
463
+ },
464
+ }),
465
+ { status: 200, headers: { "content-type": "application/json" } },
466
+ ),
467
+ );
468
+ const result = await webSearch.execute({
469
+ query: "x",
470
+ allowed_domains: ["Example.com"],
471
+ });
472
+ if (typeof result !== "string") throw new Error("expected string result");
473
+ expect(result).toContain("keep-exact");
474
+ expect(result).toContain("keep-sub");
475
+ expect(result).not.toContain("drop-other");
476
+ expect(result).not.toContain("drop-bad-url");
477
+ });
478
+
479
+ test("when every hit is filtered out, formatHits returns the empty marker", async () => {
480
+ process.env["CREWHAUS_SEARCH_PROVIDER"] = "brave";
481
+ process.env["CREWHAUS_SEARCH_API_KEY"] = "x";
482
+ _setRawFetch(
483
+ async () =>
484
+ new Response(
485
+ JSON.stringify({
486
+ web: { results: [{ title: "gone", url: "https://nope.org/x", description: "" }] },
487
+ }),
488
+ { status: 200, headers: { "content-type": "application/json" } },
489
+ ),
490
+ );
491
+ const result = await webSearch.execute({ query: "x", allowed_domains: ["example.com"] });
492
+ expect(result).toBe("[no results]");
493
+ });
494
+ });
495
+
496
+ describe("WebFetch — content-type branches", () => {
497
+ test("application/json passes through unchanged", async () => {
498
+ _setRawFetch(
499
+ async () =>
500
+ new Response('{"ok":true}', {
501
+ status: 200,
502
+ headers: { "content-type": "application/json" },
503
+ }),
504
+ );
505
+ const result = await webFetch.execute({ url: "https://example.com/data.json" });
506
+ if (typeof result !== "string") throw new Error("expected string result");
507
+ expect(result).toContain('{"ok":true}');
508
+ expect(result).toContain("Content-Type: application/json");
509
+ });
510
+
511
+ test("application/xhtml+xml is rendered as markdown", async () => {
512
+ _setRawFetch(
513
+ async () =>
514
+ new Response("<h1>Xhtml Title</h1><p>body</p>", {
515
+ status: 200,
516
+ headers: { "content-type": "application/xhtml+xml" },
517
+ }),
518
+ );
519
+ const result = await webFetch.execute({ url: "https://example.com/page.xhtml" });
520
+ if (typeof result !== "string") throw new Error("expected string result");
521
+ expect(result).toContain("# Xhtml Title");
522
+ });
523
+
524
+ test("a missing content-type (empty) is treated as text and passed through", async () => {
525
+ // Construct a Response and strip its content-type so `ct === ""` fires.
526
+ _setRawFetch(async () => {
527
+ const res = new Response("raw bytes here", { status: 200 });
528
+ res.headers.delete("content-type");
529
+ return res;
530
+ });
531
+ const result = await webFetch.execute({ url: "https://example.com/blob" });
532
+ if (typeof result !== "string") throw new Error("expected string result");
533
+ expect(result).toContain("raw bytes here");
534
+ });
535
+
536
+ test("an unknown binary content-type yields the non-textual placeholder", async () => {
537
+ _setRawFetch(
538
+ async () =>
539
+ new Response("PKzipbytes", {
540
+ status: 200,
541
+ headers: { "content-type": "application/zip" },
542
+ }),
543
+ );
544
+ const result = await webFetch.execute({ url: "https://example.com/archive.zip" });
545
+ if (typeof result !== "string") throw new Error("expected string result");
546
+ expect(result).toContain("[non-textual content: application/zip");
547
+ expect(result).toContain("not displayed]");
548
+ });
549
+
550
+ test("an empty (null-body) response yields an empty body string", async () => {
551
+ // 204 responses have res.body === null — exercises the early return in
552
+ // readBodyCapped.
553
+ _setRawFetch(async () => new Response(null, { status: 204 }));
554
+ const result = await webFetch.execute({ url: "https://example.com/empty" });
555
+ if (typeof result !== "string") throw new Error("expected string result");
556
+ expect(result).toContain("Status: 204");
557
+ });
558
+ });
559
+
560
+ describe("WebFetch — invalid URL + redirect edge cases", () => {
561
+ test("a malformed URL throws WebFetchPermissionError before any fetch", async () => {
562
+ _setRawFetch(async () => {
563
+ throw new Error("must not fetch");
564
+ });
565
+ await expect(webFetch.execute({ url: "not a url" })).rejects.toBeInstanceOf(
566
+ WebFetchPermissionError,
567
+ );
568
+ });
569
+
570
+ test("a redirect with an unparseable Location header is rejected", async () => {
571
+ _setRawFetch(
572
+ async () =>
573
+ new Response(null, {
574
+ status: 302,
575
+ // A valid HTTP header value that `new URL(loc, base)` still rejects
576
+ // (incomplete IPv6 literal) drives the redirect-parse catch branch.
577
+ headers: { location: "http://[" },
578
+ }),
579
+ );
580
+ await expect(webFetch.execute({ url: "https://example.com/" })).rejects.toBeInstanceOf(
581
+ WebFetchPermissionError,
582
+ );
583
+ });
584
+
585
+ test("a relative redirect Location is resolved against the current URL", async () => {
586
+ let hop = 0;
587
+ _setRawFetch(async (req) => {
588
+ hop++;
589
+ if (hop === 1) {
590
+ return new Response(null, { status: 301, headers: { location: "/landing" } });
591
+ }
592
+ expect(req.url).toBe("https://example.com/landing");
593
+ return new Response("arrived", { status: 200, headers: { "content-type": "text/plain" } });
594
+ });
595
+ const result = await webFetch.execute({ url: "https://example.com/start" });
596
+ if (typeof result !== "string") throw new Error("expected string result");
597
+ expect(result).toContain("arrived");
598
+ expect(hop).toBe(2);
599
+ });
600
+
601
+ test("a 3xx without a Location header is treated as a terminal response", async () => {
602
+ _setRawFetch(
603
+ async () =>
604
+ new Response("moved but no location", {
605
+ status: 304,
606
+ headers: { "content-type": "text/plain" },
607
+ }),
608
+ );
609
+ const result = await webFetch.execute({ url: "https://example.com/" });
610
+ if (typeof result !== "string") throw new Error("expected string result");
611
+ expect(result).toContain("Status: 304");
612
+ expect(result).toContain("moved but no location");
613
+ });
614
+ });
615
+
616
+ describe("getWebFetchConfig", () => {
617
+ test("reflects the most recently registered allow-list (lower-cased)", () => {
618
+ expect(getWebFetchConfig().allowedDomains).toEqual([]);
619
+ registerWebFetchConfig({ allowed_domains: ["Example.COM", "Foo.org"] });
620
+ expect(getWebFetchConfig().allowedDomains).toEqual(["example.com", "foo.org"]);
621
+ });
622
+
623
+ test("registerWebFetchConfig honours the camelCase alias", () => {
624
+ registerWebFetchConfig({ allowedDomains: ["Bar.NET"] });
625
+ expect(getWebFetchConfig().allowedDomains).toEqual(["bar.net"]);
626
+ });
627
+ });
628
+
629
+ describe("WebFetch — ctx.signal cancellation wiring", () => {
630
+ test("an already-aborted ctx.signal still completes against a mocked fetch", async () => {
631
+ // Drives the `ctx.signal.aborted === true` branch: ctrl.abort(reason) runs,
632
+ // but our stub rawFetch ignores the signal and returns a Response.
633
+ _setRawFetch(
634
+ async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
635
+ );
636
+ const result = await webFetch.execute(
637
+ { url: "https://example.com/" },
638
+ { signal: AbortSignal.abort(new Error("pre-aborted")) },
639
+ );
640
+ if (typeof result !== "string") throw new Error("expected string result");
641
+ expect(result).toContain("ok");
642
+ });
643
+
644
+ test("a later ctx.signal abort fires the once-listener without affecting the result", async () => {
645
+ _setRawFetch(
646
+ async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
647
+ );
648
+ const ctrl = new AbortController();
649
+ const result = await webFetch.execute({ url: "https://example.com/" }, { signal: ctrl.signal });
650
+ // Abort AFTER the fetch resolved — fires the registered "abort" listener
651
+ // (its callback body runs) on the already-settled inner controller.
652
+ ctrl.abort(new Error("late"));
653
+ if (typeof result !== "string") throw new Error("expected string result");
654
+ expect(result).toContain("ok");
655
+ });
656
+
657
+ test("the fetch-timeout setTimeout callback aborts the controller when fired", async () => {
658
+ // Capture the timeout callback instead of waiting 30s, then invoke it by
659
+ // hand so its body (ctrl.abort(new Error("fetch timeout"))) is exercised
660
+ // deterministically — no real timer, no leaked handle.
661
+ const captured: Array<() => void> = [];
662
+ const setSpy = spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void) => {
663
+ captured.push(fn);
664
+ return 0 as unknown as ReturnType<typeof setTimeout>;
665
+ }) as typeof setTimeout);
666
+ const clearSpy = spyOn(globalThis, "clearTimeout").mockImplementation(
667
+ (() => {}) as typeof clearTimeout,
668
+ );
669
+ try {
670
+ _setRawFetch(
671
+ async () => new Response("ok", { status: 200, headers: { "content-type": "text/plain" } }),
672
+ );
673
+ const result = await webFetch.execute({ url: "https://example.com/" });
674
+ if (typeof result !== "string") throw new Error("expected string result");
675
+ expect(result).toContain("ok");
676
+ expect(captured.length).toBeGreaterThanOrEqual(1);
677
+ // Invoke the captured timeout callback — covers the abort closure body.
678
+ for (const fn of captured) fn();
679
+ } finally {
680
+ setSpy.mockRestore();
681
+ clearSpy.mockRestore();
682
+ }
683
+ });
684
+ });
685
+
686
+ describe("WebSearch — ctx.signal cancellation wiring", () => {
687
+ test("an already-aborted ctx.signal still returns the refusal (env unset)", async () => {
688
+ const result = await webSearch.execute(
689
+ { query: "anything" },
690
+ { signal: AbortSignal.abort(new Error("pre-aborted")) },
691
+ );
692
+ if (typeof result !== "string") throw new Error("expected string result");
693
+ expect(result).toContain("WebSearch unavailable");
694
+ });
695
+
696
+ test("a later ctx.signal abort fires the once-listener", async () => {
697
+ const ctrl = new AbortController();
698
+ const result = await webSearch.execute({ query: "anything" }, { signal: ctrl.signal });
699
+ ctrl.abort(new Error("late"));
700
+ if (typeof result !== "string") throw new Error("expected string result");
701
+ expect(result).toContain("WebSearch unavailable");
702
+ });
703
+
704
+ test("the search-timeout setTimeout callback aborts the controller when fired", async () => {
705
+ const captured: Array<() => void> = [];
706
+ const setSpy = spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void) => {
707
+ captured.push(fn);
708
+ return 0 as unknown as ReturnType<typeof setTimeout>;
709
+ }) as typeof setTimeout);
710
+ const clearSpy = spyOn(globalThis, "clearTimeout").mockImplementation(
711
+ (() => {}) as typeof clearTimeout,
712
+ );
713
+ try {
714
+ const result = await webSearch.execute({ query: "anything" });
715
+ if (typeof result !== "string") throw new Error("expected string result");
716
+ expect(result).toContain("WebSearch unavailable");
717
+ for (const fn of captured) fn();
718
+ } finally {
719
+ setSpy.mockRestore();
720
+ clearSpy.mockRestore();
721
+ }
722
+ });
723
+ });
724
+
725
+ describe("WebFetch — production default fetch/dns wrappers", () => {
726
+ test("defaultRawFetch delegates to globalThis.fetch when not stubbed", async () => {
727
+ // Reset rawFetch to its production default, then mock the global fetch so
728
+ // the default wrapper's body (globalThis.fetch(req)) runs with no network.
729
+ const fetchSpy = spyOn(globalThis, "fetch").mockImplementation(
730
+ (async () =>
731
+ new Response("from global fetch", {
732
+ status: 200,
733
+ headers: { "content-type": "text/plain" },
734
+ })) as unknown as typeof globalThis.fetch,
735
+ );
736
+ try {
737
+ _setRawFetch(undefined); // rawFetch === defaultRawFetch now
738
+ const result = await webFetch.execute({ url: "https://example.com/" });
739
+ if (typeof result !== "string") throw new Error("expected string result");
740
+ expect(result).toContain("from global fetch");
741
+ expect(fetchSpy).toHaveBeenCalled();
742
+ } finally {
743
+ fetchSpy.mockRestore();
744
+ _setRawFetch(undefined);
745
+ }
746
+ });
747
+
748
+ test("defaultDnsLookup delegates to node:dns/promises lookup when not stubbed", async () => {
749
+ // Mock the dns module so the production resolver wrapper body runs offline.
750
+ const lookupMock = mock(async () => ({ address: "203.0.113.7", family: 4 }));
751
+ await mock.module("node:dns/promises", () => ({ lookup: lookupMock }));
752
+ const fetchSpy = spyOn(globalThis, "fetch").mockImplementation(
753
+ (async () =>
754
+ new Response("ok", {
755
+ status: 200,
756
+ headers: { "content-type": "text/plain" },
757
+ })) as unknown as typeof globalThis.fetch,
758
+ );
759
+ try {
760
+ _setDnsLookup(undefined); // dnsLookupFn === defaultDnsLookup now
761
+ _setRawFetch(undefined);
762
+ const result = await webFetch.execute({ url: "https://public.example/" });
763
+ if (typeof result !== "string") throw new Error("expected string result");
764
+ expect(result).toContain("ok");
765
+ expect(lookupMock).toHaveBeenCalled();
766
+ } finally {
767
+ fetchSpy.mockRestore();
768
+ // Restore the real module so later suites/files are unaffected.
769
+ mock.restore();
770
+ _setDnsLookup(undefined);
771
+ _setRawFetch(undefined);
772
+ }
773
+ });
318
774
  });
package/src/index.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { lookup as dnsLookup } from "node:dns/promises";
1
2
  import { CrewhausError } from "@crewhaus/errors";
2
3
  import { buildTool } from "@crewhaus/tool-builder";
3
4
  import type { RegisteredTool } from "@crewhaus/tool-catalog";
@@ -12,6 +13,9 @@ import { z } from "zod";
12
13
  * - Validates http/https scheme.
13
14
  * - Optional allow-list via `getWebFetchConfig().allowedDomains`. When
14
15
  * non-empty, host must equal an entry or be a subdomain of one.
16
+ * - SSRF guard (independent of the allow-list, re-checked on every
17
+ * redirect hop): rejects loopback, link-local / 169.254.0.0/16,
18
+ * RFC1918, CGNAT, *.local / *.localhost, and DNS-rebinding targets.
15
19
  * - Manual redirect handling, max 5; allow-list re-checked at every hop.
16
20
  * - 30 s default timeout (`AbortController`, honours `ctx.signal`).
17
21
  * - 5 MB response body cap.
@@ -89,6 +93,160 @@ function isHostAllowed(host: string): boolean {
89
93
  );
90
94
  }
91
95
 
96
+ // ─── SSRF guard ─────────────────────────────────────────────────────────
97
+ // Defence-in-depth, INDEPENDENT of the allow-list. WebFetch is reachable
98
+ // with model-/prompt-influenced URLs; without this guard an empty allow-list
99
+ // (the documented default) lets an attacker pivot to cloud-metadata
100
+ // (169.254.169.254), loopback, or RFC1918 services. Mirrors tool-fetch's
101
+ // `assertNotSsrf`, including the DNS-resolution backstop for rebinding.
102
+
103
+ /**
104
+ * DNS resolver injection point used by tests. Production callers leave it at
105
+ * `dnsLookup` from `node:dns/promises`; tests stub it to assert the rebinding
106
+ * path or to keep the suite offline.
107
+ */
108
+ export type DnsLookupFn = (
109
+ host: string,
110
+ ) => Promise<{ readonly address: string; readonly family: number }>;
111
+ // Production default — a single named function shared by both the initial
112
+ // binding and the `_setDnsLookup(undefined)` reset path, so there is one
113
+ // resolver definition to reason about (and to cover) rather than two
114
+ // duplicate closures.
115
+ const defaultDnsLookup: DnsLookupFn = (host) => dnsLookup(host, { verbatim: false });
116
+ let dnsLookupFn: DnsLookupFn = defaultDnsLookup;
117
+ export function _setDnsLookup(fn: DnsLookupFn | undefined): void {
118
+ dnsLookupFn = fn ?? defaultDnsLookup;
119
+ }
120
+
121
+ // Normalise octal/hex/integer IPv4 literals (e.g. 0177.0.0.1, 0x7f000001,
122
+ // 2130706433) to dotted-decimal so isPrivateIp can classify them — browsers
123
+ // and many HTTP stacks accept these forms, which would otherwise bypass a
124
+ // naive dotted-only check. Mirrors @crewhaus/tool-fetch / crawler.
125
+ function normalizeIpv4(raw: string): string | null {
126
+ const trimmed = raw.trim();
127
+ if (trimmed === "") return null;
128
+ const parseComponent = (s: string): number | null => {
129
+ if (s === "") return null;
130
+ let value: number;
131
+ if (/^0[xX][0-9a-fA-F]+$/.test(s)) value = Number.parseInt(s.slice(2), 16);
132
+ else if (/^0[0-7]+$/.test(s)) value = Number.parseInt(s, 8);
133
+ else if (/^[0-9]+$/.test(s)) value = Number.parseInt(s, 10);
134
+ else return null;
135
+ return Number.isNaN(value) ? null : value;
136
+ };
137
+ const segments = trimmed.split(".");
138
+ if (segments.length > 4) return null;
139
+ const components: number[] = [];
140
+ for (const seg of segments) {
141
+ const value = parseComponent(seg);
142
+ if (value === null || value < 0) return null;
143
+ components.push(value);
144
+ }
145
+ const n = components.length;
146
+ const octets = [0, 0, 0, 0];
147
+ for (let i = 0; i < n - 1; i++) {
148
+ const c = components[i] as number;
149
+ if (c > 255) return null;
150
+ octets[i] = c;
151
+ }
152
+ const last = components[n - 1] as number;
153
+ const maxLast = 2 ** (8 * (4 - (n - 1)));
154
+ if (last >= maxLast) return null;
155
+ let rest = last;
156
+ for (let i = 3; i >= n - 1; i--) {
157
+ octets[i] = rest & 0xff;
158
+ rest = Math.floor(rest / 256);
159
+ }
160
+ return octets.join(".");
161
+ }
162
+
163
+ function isPrivateIp(addr: string): boolean {
164
+ const ip = addr.replace(/^\[/, "").replace(/\]$/, "");
165
+ const lowerIp = ip.toLowerCase();
166
+
167
+ if (lowerIp === "::" || lowerIp === "::1") return true; // IPv6 unspecified / loopback
168
+ if (lowerIp.startsWith("fe80:")) return true; // IPv6 link-local
169
+ if (lowerIp.startsWith("fc") || lowerIp.startsWith("fd")) return true; // IPv6 unique-local
170
+
171
+ // IPv4-mapped IPv6 — both dotted (::ffff:127.0.0.1) AND hex-compressed
172
+ // (::ffff:7f00:1, which `new URL` normalizes [::ffff:127.0.0.1] into).
173
+ const mapped = lowerIp.match(/^::ffff:(.+)$/);
174
+ if (mapped) {
175
+ const tail = mapped[1] as string;
176
+ if (tail.includes(".")) return isPrivateIp(tail);
177
+ const hexGroups = tail.split(":");
178
+ if (hexGroups.length === 2 && hexGroups.every((g) => /^[0-9a-f]{1,4}$/.test(g))) {
179
+ const hi = Number.parseInt(hexGroups[0] as string, 16);
180
+ const lo = Number.parseInt(hexGroups[1] as string, 16);
181
+ return isPrivateIp([(hi >> 8) & 0xff, hi & 0xff, (lo >> 8) & 0xff, lo & 0xff].join("."));
182
+ }
183
+ return false;
184
+ }
185
+
186
+ const normalized = normalizeIpv4(ip);
187
+ if (normalized === null) return false;
188
+ const parts = normalized.split(".").map((p) => Number.parseInt(p, 10));
189
+ if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {
190
+ return false;
191
+ }
192
+ const [a, b] = parts as [number, number, number, number];
193
+ if (a === 127) return true; // 127.0.0.0/8 loopback
194
+ if (a === 10) return true; // 10.0.0.0/8 RFC1918
195
+ if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local + cloud metadata
196
+ if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 RFC1918
197
+ if (a === 192 && b === 168) return true; // 192.168.0.0/16 RFC1918
198
+ if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGNAT (RFC6598)
199
+ if (a === 192 && b === 0 && parts[2] === 0) return true; // 192.0.0.0/24 (RFC6890)
200
+ if (a === 198 && (b === 18 || b === 19)) return true; // 198.18.0.0/15 (RFC2544)
201
+ if (a === 0) return true; // 0.0.0.0/8
202
+ return false;
203
+ }
204
+
205
+ /**
206
+ * Reject any host that is — or resolves to — a private/loopback/link-local
207
+ * address. Applied at the initial URL and at every redirect hop, regardless
208
+ * of the allow-list. Fails closed if the name cannot be resolved.
209
+ *
210
+ * Returns the validated IP so the caller can PIN the connection to that exact
211
+ * address — resolving here and letting the default `fetch` re-resolve at
212
+ * connect time is a DNS-rebinding TOCTOU (CWE-367): a hostile resolver can
213
+ * answer public for this check and private for the socket. For an IP-literal
214
+ * host the pinned value is the (normalized) literal itself.
215
+ */
216
+ async function assertNotSsrf(hostname: string): Promise<string> {
217
+ const lower = hostname.toLowerCase();
218
+ if (lower === "localhost" || lower.endsWith(".localhost")) {
219
+ throw new WebFetchPermissionError(`WebFetch denied: host "${hostname}" resolves to loopback`);
220
+ }
221
+ if (lower.endsWith(".local")) {
222
+ throw new WebFetchPermissionError(`WebFetch denied: mDNS host "${hostname}" is not allowed`);
223
+ }
224
+ const unbracketed = lower.replace(/^\[/, "").replace(/\]$/, "");
225
+ if (isPrivateIp(unbracketed)) {
226
+ throw new WebFetchPermissionError(
227
+ `WebFetch denied: host "${hostname}" is a private/loopback IP`,
228
+ );
229
+ }
230
+ // An IP literal is its own pinned target — nothing to resolve or rebind.
231
+ const literal = normalizeIpv4(unbracketed) ?? (unbracketed.includes(":") ? unbracketed : null);
232
+ if (literal !== null) return literal;
233
+ // Resolve so a public-looking name pointing at an internal IP (DNS
234
+ // rebinding) is still caught — and return the address so the caller dials it.
235
+ let resolved: { readonly address: string; readonly family: number };
236
+ try {
237
+ resolved = await dnsLookupFn(lower);
238
+ } catch (err) {
239
+ const msg = err instanceof Error ? err.message : String(err);
240
+ throw new WebFetchPermissionError(`WebFetch denied: cannot resolve "${hostname}": ${msg}`);
241
+ }
242
+ if (isPrivateIp(resolved.address)) {
243
+ throw new WebFetchPermissionError(
244
+ `WebFetch denied: host "${hostname}" resolves to private IP ${resolved.address}`,
245
+ );
246
+ }
247
+ return resolved.address;
248
+ }
249
+
92
250
  // ─── WebFetch ─────────────────────────────────────────────────────────
93
251
 
94
252
  const webFetchSchema = z.object({
@@ -96,10 +254,49 @@ const webFetchSchema = z.object({
96
254
  prompt: z.string().optional(),
97
255
  });
98
256
 
99
- export type RawFetch = (req: Request) => Promise<Response>;
100
- let rawFetch: RawFetch = (req) => globalThis.fetch(req);
257
+ /**
258
+ * `pinnedIp` is the address `assertNotSsrf` validated for the request's host
259
+ * (empty string ⇒ no pin, e.g. fixed trusted API hosts). The production
260
+ * fetcher dials that exact IP, preserving the Host header and TLS SNI, so the
261
+ * socket can't be rebound to a private address between the SSRF check and
262
+ * connect. Test stubs may ignore the argument.
263
+ */
264
+ export type RawFetch = (req: Request, pinnedIp: string) => Promise<Response>;
265
+
266
+ // Dial `pinnedIp` directly while keeping the real host for the Host header and
267
+ // TLS SNI. Mirrors @crewhaus/tool-fetch.
268
+ function pinnedFetch(req: Request, pinnedIp: string): Promise<Response> {
269
+ const original = new URL(req.url);
270
+ const host = original.hostname;
271
+ const hostUnbracketed = host.replace(/^\[/, "").replace(/\]$/, "");
272
+ if (pinnedIp === "" || hostUnbracketed === pinnedIp) {
273
+ return globalThis.fetch(req);
274
+ }
275
+ const hostForUrl = pinnedIp.includes(":") ? `[${pinnedIp}]` : pinnedIp;
276
+ const pinnedUrl = new URL(original.toString());
277
+ pinnedUrl.hostname = hostForUrl;
278
+ const headers = new Headers(req.headers);
279
+ headers.set("host", original.port === "" ? host : `${host}:${original.port}`);
280
+ const init: RequestInit & { tls?: { serverName: string } } = {
281
+ method: req.method,
282
+ headers,
283
+ redirect: "manual",
284
+ signal: req.signal,
285
+ tls: { serverName: host },
286
+ };
287
+ if (req.method !== "GET" && req.method !== "HEAD") {
288
+ init.body = req.body;
289
+ (init as { duplex?: string }).duplex = "half";
290
+ }
291
+ return globalThis.fetch(pinnedUrl.toString(), init);
292
+ }
293
+
294
+ // Production default — one named fetcher shared by the initial binding and the
295
+ // `_setRawFetch(undefined)` reset path (see defaultDnsLookup for rationale).
296
+ const defaultRawFetch: RawFetch = pinnedFetch;
297
+ let rawFetch: RawFetch = defaultRawFetch;
101
298
  export function _setRawFetch(fn: RawFetch | undefined): void {
102
- rawFetch = fn ?? ((req) => globalThis.fetch(req));
299
+ rawFetch = fn ?? defaultRawFetch;
103
300
  }
104
301
 
105
302
  async function readBodyCapped(res: Response): Promise<string> {
@@ -148,6 +345,10 @@ async function performWebFetch(initialUrl: URL, signal: AbortSignal): Promise<Re
148
345
  `WebFetch denied: scheme "${currentUrl.protocol}" — only http/https allowed`,
149
346
  );
150
347
  }
348
+ // SSRF guard — independent of the allow-list, enforced on every hop.
349
+ // The returned IP pins the socket so a rebinding resolver can't swap in a
350
+ // private address between this check and connect.
351
+ const pinnedIp = await assertNotSsrf(currentUrl.hostname);
151
352
  if (!isHostAllowed(currentUrl.hostname)) {
152
353
  throw new WebFetchPermissionError(
153
354
  `WebFetch denied: host "${currentUrl.hostname}" is not in allowed_domains`,
@@ -161,6 +362,7 @@ async function performWebFetch(initialUrl: URL, signal: AbortSignal): Promise<Re
161
362
  signal,
162
363
  headers: { "user-agent": "crewhaus-tool-web/0.1" },
163
364
  }),
365
+ pinnedIp,
164
366
  );
165
367
  if (res.status >= 300 && res.status < 400 && res.headers.has("location")) {
166
368
  const loc = res.headers.get("location") ?? "";
@@ -214,6 +416,8 @@ export const webFetch: RegisteredTool = buildTool({
214
416
  // Pillar 3 sink-side: URL parameters can encode exfiltrated data ("Safe URL"
215
417
  // pattern from OpenAI's 2026-05 prompt-injection paper).
216
418
  scope: "external",
419
+ // FR-002 — declare the io-capability fact (HTTP fetch) for the audit.
420
+ ioCapability: "network",
217
421
  execute: async (input, ctx) => {
218
422
  let url: URL;
219
423
  try {
@@ -309,11 +513,13 @@ async function braveSearch(
309
513
  // allowed/blocked as advisory hints rather than binding constraints
310
514
  // (the provider doesn't enforce them server-side).
311
515
  const url = `https://api.search.brave.com/res/v1/web/search?${params.toString()}`;
516
+ // Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
312
517
  const res = await rawFetch(
313
518
  new Request(url, {
314
519
  headers: { accept: "application/json", "x-subscription-token": apiKey },
315
520
  signal,
316
521
  }),
522
+ "",
317
523
  );
318
524
  if (!res.ok) {
319
525
  throw new WebFetchPermissionError(`Brave search failed: HTTP ${res.status}`);
@@ -351,6 +557,8 @@ async function tavilySearch(
351
557
  }),
352
558
  signal,
353
559
  }),
560
+ // Fixed, trusted API host — not model-controlled, so no SSRF pin needed.
561
+ "",
354
562
  );
355
563
  if (!res.ok) {
356
564
  throw new WebFetchPermissionError(`Tavily search failed: HTTP ${res.status}`);
@@ -402,6 +610,8 @@ export const webSearch: RegisteredTool = buildTool({
402
610
  concurrencySafe: true,
403
611
  // Pillar 3 sink-side: search-query parameter is an exfiltration vector.
404
612
  scope: "external",
613
+ // FR-002 — declare the io-capability fact (remote search API) for the audit.
614
+ ioCapability: "network",
405
615
  execute: async (input, ctx) => {
406
616
  const ctrl = new AbortController();
407
617
  const timer = setTimeout(() => ctrl.abort(new Error("search timeout")), DEFAULT_TIMEOUT_MS);