@startsimpli/integrations 0.2.34 → 0.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -4
- package/src/__tests__/corpus-folders.test.tsx +727 -0
- package/src/__tests__/corpus-pages.test.tsx +4 -4
- package/src/__tests__/ingest.test.ts +127 -0
- package/src/__tests__/safe-fetch-pinning.test.ts +198 -0
- package/src/__tests__/safe-fetch.test.ts +186 -0
- package/src/components/FolderTree.tsx +425 -0
- package/src/components/index.ts +6 -0
- package/src/components/pages/LibraryPage.tsx +333 -35
- package/src/corpus/client.ts +88 -4
- package/src/corpus/entities.ts +75 -0
- package/src/corpus/fixtures.ts +263 -1
- package/src/corpus/index.ts +8 -1
- package/src/corpus/types.ts +93 -0
- package/src/ingest/index.ts +126 -0
- package/src/safe-fetch/index.ts +387 -0
|
@@ -161,11 +161,11 @@ describe('LibraryPage — filtering and paging are SERVER round trips', () => {
|
|
|
161
161
|
});
|
|
162
162
|
render(<LibraryPage api={api} scope={SCOPE} pageSize={2} />);
|
|
163
163
|
|
|
164
|
-
await screen.findByText(/page 1 of
|
|
164
|
+
await screen.findByText(/page 1 of \d+/);
|
|
165
165
|
await user.click(screen.getByRole('button', { name: 'Next' }));
|
|
166
166
|
|
|
167
167
|
await waitFor(() => expect(queries.some((query) => query.page === 2)).toBe(true));
|
|
168
|
-
expect(await screen.findByText(/page 2 of
|
|
168
|
+
expect(await screen.findByText(/page 2 of \d+/)).toBeInTheDocument();
|
|
169
169
|
});
|
|
170
170
|
|
|
171
171
|
it('returns to page 1 when a filter changes', async () => {
|
|
@@ -178,9 +178,9 @@ describe('LibraryPage — filtering and paging are SERVER round trips', () => {
|
|
|
178
178
|
});
|
|
179
179
|
render(<LibraryPage api={api} scope={SCOPE} pageSize={2} />);
|
|
180
180
|
|
|
181
|
-
await screen.findByText(/page 1 of
|
|
181
|
+
await screen.findByText(/page 1 of \d+/);
|
|
182
182
|
await user.click(screen.getByRole('button', { name: 'Next' }));
|
|
183
|
-
await screen.findByText(/page 2 of
|
|
183
|
+
await screen.findByText(/page 2 of \d+/);
|
|
184
184
|
|
|
185
185
|
await user.selectOptions(screen.getByLabelText('Source'), 'dropbox');
|
|
186
186
|
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// @vitest-environment node
|
|
2
|
+
/**
|
|
3
|
+
* Server-side ingest helpers (crochet_patterns-llmj, design D1/PK2): readable
|
|
4
|
+
* text from a fetched page, and the text layer of a stored PDF.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, expect, it } from 'vitest';
|
|
7
|
+
|
|
8
|
+
import { extractPdfText, looksScanned, readableText, SCANNED_CHARS_PER_PAGE } from '../ingest';
|
|
9
|
+
|
|
10
|
+
describe('readableText', () => {
|
|
11
|
+
const article = (rounds: number) =>
|
|
12
|
+
'<html><head><title>Amigurumi Bunny | Hook Blog</title><style>.ad{color:red}</style></head><body>' +
|
|
13
|
+
'<header><nav><a href="/">Home</a> <a href="/shop">Shop</a></nav></header>' +
|
|
14
|
+
'<aside class="sidebar">Subscribe to our newsletter for weekly deals</aside>' +
|
|
15
|
+
'<article><h1>Amigurumi Bunny</h1><p>A small bunny worked in the round with worsted yarn and a 3.5 mm hook.</p><ul>' +
|
|
16
|
+
Array.from({ length: rounds }, (_, i) => `<li>Rnd ${i + 1}: sc in each st around, working in a spiral without joining (${6 * (i + 1)})</li>`).join('') +
|
|
17
|
+
'</ul><p>Fasten off and stuff firmly.</p></article>' +
|
|
18
|
+
'<footer>Copyright 2026 Hook Blog</footer><script>trackEverything()</script></body></html>';
|
|
19
|
+
|
|
20
|
+
it('keeps the article, one round per line, and drops scripts, styles, nav and footer', () => {
|
|
21
|
+
const { title, text } = readableText(article(12), 'https://blog.test/bunny');
|
|
22
|
+
expect(title).toContain('Amigurumi Bunny');
|
|
23
|
+
const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
|
|
24
|
+
expect(lines).toContain('Rnd 1: sc in each st around, working in a spiral without joining (6)');
|
|
25
|
+
expect(lines).toContain('Rnd 12: sc in each st around, working in a spiral without joining (72)');
|
|
26
|
+
expect(text).toContain('Fasten off and stuff firmly.');
|
|
27
|
+
for (const junk of ['trackEverything', 'color:red', 'Copyright 2026', 'newsletter', 'Shop']) {
|
|
28
|
+
expect(text).not.toContain(junk);
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it('falls back to the page body when the page is too short for an article, still one round per line', () => {
|
|
33
|
+
const { text } = readableText(
|
|
34
|
+
'<html><head><title>Tiny</title><script>x()</script></head><body><p>Rnd 1: 6 sc in MR<br>Rnd 2: inc x6 (12)</p><div>Rnd 3: sc around (12)</div></body></html>',
|
|
35
|
+
);
|
|
36
|
+
const lines = text.split('\n').map((l) => l.trim()).filter(Boolean);
|
|
37
|
+
expect(lines).toEqual(['Rnd 1: 6 sc in MR', 'Rnd 2: inc x6 (12)', 'Rnd 3: sc around (12)']);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('reads a bare fragment', () => {
|
|
41
|
+
expect(readableText('Rnd 1: 6 sc<br>Rnd 2: inc').text).toBe('Rnd 1: 6 sc\nRnd 2: inc');
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('collapses inline whitespace and decodes entities', () => {
|
|
45
|
+
const { text } = readableText('<body><p>Rnd 1: 6 sc\n in <b>MR</b> & join</p></body>');
|
|
46
|
+
expect(text).toBe('Rnd 1: 6 sc in MR & join');
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/** A minimal, valid PDF with one Helvetica text line per entry of `pages[i]`. */
|
|
51
|
+
function makePdf(pages: string[][]): Uint8Array {
|
|
52
|
+
const esc = (s: string) => s.replace(/[\\()]/g, (c) => `\\${c}`);
|
|
53
|
+
const objs: string[] = [];
|
|
54
|
+
const n = pages.length;
|
|
55
|
+
// 1 catalog, 2 pages, 3 font, then (page, content) pairs
|
|
56
|
+
objs.push('<< /Type /Catalog /Pages 2 0 R >>');
|
|
57
|
+
objs.push(`<< /Type /Pages /Count ${n} /Kids [${pages.map((_, i) => `${4 + 2 * i} 0 R`).join(' ')}] >>`);
|
|
58
|
+
objs.push('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>');
|
|
59
|
+
pages.forEach((lines, i) => {
|
|
60
|
+
const stream = lines.length
|
|
61
|
+
? `BT /F1 11 Tf 14 TL 72 720 Td ${lines.map((l) => `(${esc(l)}) Tj T*`).join(' ')} ET`
|
|
62
|
+
: '';
|
|
63
|
+
objs.push(
|
|
64
|
+
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + 2 * i} 0 R >>`,
|
|
65
|
+
);
|
|
66
|
+
objs.push(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
|
|
67
|
+
});
|
|
68
|
+
let out = '%PDF-1.4\n';
|
|
69
|
+
const offsets: number[] = [];
|
|
70
|
+
objs.forEach((body, i) => {
|
|
71
|
+
offsets.push(out.length);
|
|
72
|
+
out += `${i + 1} 0 obj\n${body}\nendobj\n`;
|
|
73
|
+
});
|
|
74
|
+
const xref = out.length;
|
|
75
|
+
out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n`;
|
|
76
|
+
for (const o of offsets) out += `${String(o).padStart(10, '0')} 00000 n \n`;
|
|
77
|
+
out += `trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
|
|
78
|
+
return new TextEncoder().encode(out);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const ROUNDS = Array.from({ length: 12 }, (_, i) => `Rnd ${i + 1}: sc in each st around (${6 * (i + 1)})`);
|
|
82
|
+
|
|
83
|
+
describe('extractPdfText', () => {
|
|
84
|
+
it('returns the text layer page by page', async () => {
|
|
85
|
+
const bytes = makePdf([ROUNDS.slice(0, 6), ROUNDS.slice(6)]);
|
|
86
|
+
const r = await extractPdfText(bytes);
|
|
87
|
+
expect(r.pages).toHaveLength(2);
|
|
88
|
+
expect(r.pages[0]).toContain('Rnd 1: sc in each st around (6)');
|
|
89
|
+
expect(r.pages[0]).toContain('Rnd 6: sc in each st around (36)');
|
|
90
|
+
expect(r.pages[0]).not.toContain('Rnd 7:');
|
|
91
|
+
expect(r.pages[1]).toContain('Rnd 12: sc in each st around (72)');
|
|
92
|
+
expect(r.totalChars).toBe(r.pages[0]!.length + r.pages[1]!.length);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('a page with a full text layer does not look scanned', async () => {
|
|
96
|
+
const r = await extractPdfText(makePdf([ROUNDS]));
|
|
97
|
+
expect(r.totalChars).toBeGreaterThanOrEqual(SCANNED_CHARS_PER_PAGE);
|
|
98
|
+
expect(looksScanned(r)).toBe(false);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('does not detach or alter the caller’s buffer (the stored original)', async () => {
|
|
102
|
+
const bytes = makePdf([ROUNDS]);
|
|
103
|
+
const copy = bytes.slice();
|
|
104
|
+
await extractPdfText(bytes);
|
|
105
|
+
expect(bytes.byteLength).toBe(copy.byteLength);
|
|
106
|
+
expect(Buffer.from(bytes).equals(Buffer.from(copy))).toBe(true);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('a PDF with no text layer reads as empty and looks scanned', async () => {
|
|
110
|
+
const r = await extractPdfText(makePdf([[], []]));
|
|
111
|
+
expect(r.pages).toEqual(['', '']);
|
|
112
|
+
expect(r.totalChars).toBe(0);
|
|
113
|
+
expect(looksScanned(r)).toBe(true);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('rejects bytes that are not a PDF', async () => {
|
|
117
|
+
await expect(extractPdfText(new TextEncoder().encode('<html>not a pdf</html>'))).rejects.toThrow();
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
describe('looksScanned', () => {
|
|
122
|
+
it(`is true below ${SCANNED_CHARS_PER_PAGE} chars per page on average, false at or above`, () => {
|
|
123
|
+
expect(looksScanned({ pages: [], totalChars: 0 })).toBe(true);
|
|
124
|
+
expect(looksScanned({ pages: ['a', 'b'], totalChars: 2 * SCANNED_CHARS_PER_PAGE - 1 })).toBe(true);
|
|
125
|
+
expect(looksScanned({ pages: ['a', 'b'], totalChars: 2 * SCANNED_CHARS_PER_PAGE })).toBe(false);
|
|
126
|
+
});
|
|
127
|
+
});
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
// @vitest-environment node
|
|
2
|
+
/**
|
|
3
|
+
* safeFetch DNS pinning (crochet_patterns-yfj8 #1, design D5) and the address
|
|
4
|
+
* blocklist. The connection must go to the address that was validated: a
|
|
5
|
+
* record that answers public at pre-flight and private at connect is refused at
|
|
6
|
+
* connect. Uses a loopback http server only — no external network.
|
|
7
|
+
*/
|
|
8
|
+
import { createServer, type Server } from 'node:http';
|
|
9
|
+
import type { AddressInfo } from 'node:net';
|
|
10
|
+
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
|
11
|
+
import { Agent, fetch as undiciFetch } from 'undici';
|
|
12
|
+
|
|
13
|
+
import { isBlockedAddress, pinnedLookup, safeFetch, SafeFetchError, type ResolvedAddress } from '../safe-fetch';
|
|
14
|
+
|
|
15
|
+
const PUBLIC = '93.184.216.34';
|
|
16
|
+
const asAddrs = (...addrs: string[]): ResolvedAddress[] =>
|
|
17
|
+
addrs.map((address) => ({ address, family: address.includes(':') ? 6 : 4 }));
|
|
18
|
+
|
|
19
|
+
describe('isBlockedAddress', () => {
|
|
20
|
+
it.each([
|
|
21
|
+
'127.0.0.1',
|
|
22
|
+
'127.255.0.9',
|
|
23
|
+
'0.0.0.0',
|
|
24
|
+
'0.1.2.3',
|
|
25
|
+
'10.9.8.7',
|
|
26
|
+
'100.64.0.1', // CGNAT
|
|
27
|
+
'100.127.255.255',
|
|
28
|
+
'169.254.169.254',
|
|
29
|
+
'172.16.0.1',
|
|
30
|
+
'172.31.255.255',
|
|
31
|
+
'192.0.0.8',
|
|
32
|
+
'192.168.0.1',
|
|
33
|
+
'198.18.0.1', // benchmarking
|
|
34
|
+
'198.19.255.255',
|
|
35
|
+
'224.0.0.1', // multicast
|
|
36
|
+
'239.255.255.250',
|
|
37
|
+
'240.0.0.1', // reserved
|
|
38
|
+
'255.255.255.255',
|
|
39
|
+
'::',
|
|
40
|
+
'::1',
|
|
41
|
+
'[::1]',
|
|
42
|
+
'fc00::1',
|
|
43
|
+
'fd12:3456::1',
|
|
44
|
+
'fe80::1',
|
|
45
|
+
'fe80::1%en0',
|
|
46
|
+
'fec0::1',
|
|
47
|
+
'ff02::1',
|
|
48
|
+
'::ffff:169.254.169.254',
|
|
49
|
+
'::ffff:a9fe:a9fe', // the form WHATWG URL rewrites the above to
|
|
50
|
+
'::ffff:127.0.0.1',
|
|
51
|
+
'::127.0.0.1', // v4-compatible
|
|
52
|
+
'64:ff9b::a9fe:a9fe', // NAT64 of the metadata address
|
|
53
|
+
'64:ff9b::10.0.0.1',
|
|
54
|
+
'64:ff9b:1::1', // local-use NAT64
|
|
55
|
+
'2002:7f00:0001::1', // 6to4 of 127.0.0.1
|
|
56
|
+
'2002:c0a8:0101::', // 6to4 of 192.168.1.1
|
|
57
|
+
'not-an-ip',
|
|
58
|
+
'',
|
|
59
|
+
])('blocks %s', (addr) => {
|
|
60
|
+
expect(isBlockedAddress(addr)).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it.each([
|
|
64
|
+
PUBLIC,
|
|
65
|
+
'8.8.8.8',
|
|
66
|
+
'1.1.1.1',
|
|
67
|
+
'172.32.0.1',
|
|
68
|
+
'172.15.255.255',
|
|
69
|
+
'100.128.0.1',
|
|
70
|
+
'100.63.255.255',
|
|
71
|
+
'198.20.0.1',
|
|
72
|
+
'223.255.255.255',
|
|
73
|
+
'2606:4700:4700::1111',
|
|
74
|
+
'2001:4860:4860::8888',
|
|
75
|
+
'::ffff:8.8.8.8',
|
|
76
|
+
'64:ff9b::808:808', // NAT64 of 8.8.8.8
|
|
77
|
+
'2002:0808:0808::1', // 6to4 of 8.8.8.8
|
|
78
|
+
])('allows %s', (addr) => {
|
|
79
|
+
expect(isBlockedAddress(addr)).toBe(false);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe('safeFetch url hygiene', () => {
|
|
84
|
+
it.each(['https://user:pass@blog.test/', 'https://user@blog.test/'])('refuses credentials in %s', async (url) => {
|
|
85
|
+
const fetch = vi.fn();
|
|
86
|
+
const err = await safeFetch(url, { fetch, lookup: async () => asAddrs(PUBLIC) }).catch((e: unknown) => e);
|
|
87
|
+
expect(err).toBeInstanceOf(SafeFetchError);
|
|
88
|
+
expect((err as SafeFetchError).code).toBe('BAD_URL');
|
|
89
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it.each([
|
|
93
|
+
['decimal', 'http://2130706433/'],
|
|
94
|
+
['hex', 'http://0x7f.0.0.1/'],
|
|
95
|
+
['NAT64 metadata literal', 'http://[64:ff9b::169.254.169.254]/'],
|
|
96
|
+
['6to4 loopback literal', 'http://[2002:7f00:1::1]/'],
|
|
97
|
+
])('refuses the %s form of an internal IP literal without resolving it', async (_label, url) => {
|
|
98
|
+
const fetch = vi.fn();
|
|
99
|
+
const lookup = vi.fn(async () => asAddrs(PUBLIC));
|
|
100
|
+
const err = await safeFetch(url, { fetch, lookup }).catch((e: unknown) => e);
|
|
101
|
+
expect((err as SafeFetchError).code).toBe('BLOCKED_ADDRESS');
|
|
102
|
+
expect(lookup).not.toHaveBeenCalled();
|
|
103
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
function callLookup(fn: ReturnType<typeof pinnedLookup>, host: string, options: object) {
|
|
108
|
+
return new Promise<{ err: NodeJS.ErrnoException | null; address: unknown; family?: number }>((resolve) => {
|
|
109
|
+
fn(host, options as never, (err, address, family) => resolve({ err, address, family }));
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
describe('pinnedLookup', () => {
|
|
114
|
+
it('hands back exactly the validated answer in the {all:true} form, resolving once', async () => {
|
|
115
|
+
const resolve = vi.fn(async () => asAddrs(PUBLIC, '2606:4700:4700::1111'));
|
|
116
|
+
const r = await callLookup(pinnedLookup(resolve), 'blog.test', { all: true });
|
|
117
|
+
expect(r.err).toBeNull();
|
|
118
|
+
expect(r.address).toEqual([
|
|
119
|
+
{ address: PUBLIC, family: 4 },
|
|
120
|
+
{ address: '2606:4700:4700::1111', family: 6 },
|
|
121
|
+
]);
|
|
122
|
+
expect(resolve).toHaveBeenCalledTimes(1);
|
|
123
|
+
expect(resolve).toHaveBeenCalledWith('blog.test');
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('uses the (address, family) form without all, and honours options.family', async () => {
|
|
127
|
+
const resolve = async () => asAddrs('2606:4700:4700::1111', PUBLIC);
|
|
128
|
+
const plain = await callLookup(pinnedLookup(resolve), 'blog.test', {});
|
|
129
|
+
expect(plain).toMatchObject({ err: null, address: '2606:4700:4700::1111', family: 6 });
|
|
130
|
+
const v4 = await callLookup(pinnedLookup(resolve), 'blog.test', { family: 4 });
|
|
131
|
+
expect(v4).toMatchObject({ err: null, address: PUBLIC, family: 4 });
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it('refuses the whole answer when any address is blocked', async () => {
|
|
135
|
+
const r = await callLookup(pinnedLookup(async () => asAddrs(PUBLIC, '169.254.169.254')), 'mixed.test', {
|
|
136
|
+
all: true,
|
|
137
|
+
});
|
|
138
|
+
expect(r.err).toBeInstanceOf(SafeFetchError);
|
|
139
|
+
expect((r.err as unknown as SafeFetchError).code).toBe('BLOCKED_ADDRESS');
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
describe('safeFetch connects only to the address it validated (no injected fetch)', () => {
|
|
144
|
+
let server: Server;
|
|
145
|
+
let port: number;
|
|
146
|
+
let connections = 0;
|
|
147
|
+
|
|
148
|
+
beforeAll(async () => {
|
|
149
|
+
server = createServer((_req, res) => {
|
|
150
|
+
res.writeHead(200, { 'content-type': 'text/html' });
|
|
151
|
+
res.end('<html><body>internal</body></html>');
|
|
152
|
+
});
|
|
153
|
+
server.on('connection', () => {
|
|
154
|
+
connections++;
|
|
155
|
+
});
|
|
156
|
+
await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
|
|
157
|
+
port = (server.address() as AddressInfo).port;
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
afterAll(async () => {
|
|
161
|
+
await new Promise<void>((r) => server.close(() => r()));
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it('control: an undici Agent connect.lookup decides where the socket goes', async () => {
|
|
165
|
+
// Proves the loopback server is reachable through a custom lookup, so the
|
|
166
|
+
// zero-connection assertion below is caused by pinning, not by an unreachable host.
|
|
167
|
+
const before = connections;
|
|
168
|
+
const agent = new Agent({
|
|
169
|
+
connect: {
|
|
170
|
+
lookup: (_h, opts, cb) =>
|
|
171
|
+
(opts as { all?: boolean }).all
|
|
172
|
+
? (cb as (e: null, a: Array<{ address: string; family: number }>) => void)(null, [
|
|
173
|
+
{ address: '127.0.0.1', family: 4 },
|
|
174
|
+
])
|
|
175
|
+
: cb(null, '127.0.0.1', 4),
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
try {
|
|
179
|
+
const res = await undiciFetch(`http://rebind.test:${port}/`, { dispatcher: agent });
|
|
180
|
+
expect(await res.text()).toContain('internal');
|
|
181
|
+
} finally {
|
|
182
|
+
await agent.close();
|
|
183
|
+
}
|
|
184
|
+
expect(connections).toBe(before + 1);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('a record that answers public at pre-flight and private at connect is refused at connect', async () => {
|
|
188
|
+
const before = connections;
|
|
189
|
+
const answers = [asAddrs(PUBLIC), asAddrs('127.0.0.1')];
|
|
190
|
+
const lookup = vi.fn(async () => answers.shift() ?? asAddrs('127.0.0.1'));
|
|
191
|
+
const err = await safeFetch(`http://rebind.test:${port}/`, { lookup, timeoutMs: 5000 }).catch((e: unknown) => e);
|
|
192
|
+
expect(err).toBeInstanceOf(SafeFetchError);
|
|
193
|
+
expect((err as SafeFetchError).code).toBe('BLOCKED_ADDRESS');
|
|
194
|
+
// Once for the pre-flight, once inside connect.lookup — the socket used the second, checked answer.
|
|
195
|
+
expect(lookup).toHaveBeenCalledTimes(2);
|
|
196
|
+
expect(connections).toBe(before);
|
|
197
|
+
});
|
|
198
|
+
});
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// @vitest-environment node
|
|
2
|
+
/**
|
|
3
|
+
* safeFetch SSRF guards (crochet_patterns-llmj — "paste a link to a pattern").
|
|
4
|
+
* DNS and fetch are injected fakes: no real network.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
7
|
+
|
|
8
|
+
import { safeFetch, SafeFetchError, type ResolvedAddress } from '../safe-fetch';
|
|
9
|
+
|
|
10
|
+
const PUBLIC = '93.184.216.34';
|
|
11
|
+
|
|
12
|
+
function fakeDns(map: Record<string, string[]> = {}) {
|
|
13
|
+
return vi.fn(async (host: string): Promise<ResolvedAddress[]> => {
|
|
14
|
+
const addrs = map[host] ?? [PUBLIC];
|
|
15
|
+
return addrs.map((address) => ({ address, family: address.includes(':') ? 6 : 4 }));
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const html = (body = '<html><body>Rnd 1: 6 sc in magic ring</body></html>', type = 'text/html; charset=utf-8') =>
|
|
20
|
+
new Response(body, { status: 200, headers: { 'content-type': type } });
|
|
21
|
+
const redirect = (location: string) => new Response(null, { status: 302, headers: { location } });
|
|
22
|
+
|
|
23
|
+
function fakeFetch(...responses: Array<Response | ((init?: RequestInit) => Promise<Response>)>) {
|
|
24
|
+
return vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
|
|
25
|
+
const next = responses.shift();
|
|
26
|
+
if (!next) throw new Error('fake fetch: no more responses');
|
|
27
|
+
return typeof next === 'function' ? next(init) : next;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function refusal(p: Promise<unknown>): Promise<SafeFetchError> {
|
|
32
|
+
const err = await p.then(
|
|
33
|
+
() => undefined,
|
|
34
|
+
(e: unknown) => e,
|
|
35
|
+
);
|
|
36
|
+
expect(err).toBeInstanceOf(SafeFetchError);
|
|
37
|
+
return err as SafeFetchError;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe('safeFetch refuses internal addresses', () => {
|
|
41
|
+
it.each([
|
|
42
|
+
['loopback v4', 'http://127.0.0.1/pattern'],
|
|
43
|
+
['loopback v6', 'http://[::1]/pattern'],
|
|
44
|
+
['cloud metadata', 'http://169.254.169.254/latest/meta-data/'],
|
|
45
|
+
['10/8', 'http://10.0.0.5/'],
|
|
46
|
+
['172.16/12 low end', 'http://172.16.0.1/'],
|
|
47
|
+
['172.16/12 high end', 'http://172.31.255.255/'],
|
|
48
|
+
['192.168/16', 'http://192.168.1.1/'],
|
|
49
|
+
['unspecified', 'http://0.0.0.0/'],
|
|
50
|
+
['v4-mapped v6 metadata', 'http://[::ffff:169.254.169.254]/'],
|
|
51
|
+
])('%s (%s) as an IP literal, without fetching', async (_label, url) => {
|
|
52
|
+
const fetch = fakeFetch(html());
|
|
53
|
+
const err = await refusal(safeFetch(url, { fetch, lookup: fakeDns() }));
|
|
54
|
+
expect(err.code).toBe('BLOCKED_ADDRESS');
|
|
55
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('a DNS name that resolves to a private IP', async () => {
|
|
59
|
+
const fetch = fakeFetch(html());
|
|
60
|
+
const lookup = fakeDns({ 'patterns.evil.test': ['10.1.2.3'] });
|
|
61
|
+
const err = await refusal(safeFetch('https://patterns.evil.test/bunny', { fetch, lookup }));
|
|
62
|
+
expect(err.code).toBe('BLOCKED_ADDRESS');
|
|
63
|
+
expect(lookup).toHaveBeenCalledWith('patterns.evil.test');
|
|
64
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('a DNS name where ANY of its addresses is private', async () => {
|
|
68
|
+
const fetch = fakeFetch(html());
|
|
69
|
+
const lookup = fakeDns({ 'mixed.test': [PUBLIC, '192.168.0.9'] });
|
|
70
|
+
const err = await refusal(safeFetch('https://mixed.test/', { fetch, lookup }));
|
|
71
|
+
expect(err.code).toBe('BLOCKED_ADDRESS');
|
|
72
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('a redirect hop to a host that resolves privately', async () => {
|
|
76
|
+
const fetch = fakeFetch(redirect('http://internal.test/admin'), html());
|
|
77
|
+
const lookup = fakeDns({ 'internal.test': ['192.168.0.10'] });
|
|
78
|
+
const err = await refusal(safeFetch('https://blog.test/bunny', { fetch, lookup }));
|
|
79
|
+
expect(err.code).toBe('BLOCKED_ADDRESS');
|
|
80
|
+
expect(fetch).toHaveBeenCalledTimes(1);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('a redirect hop straight to the metadata IP', async () => {
|
|
84
|
+
const fetch = fakeFetch(redirect('http://169.254.169.254/latest/meta-data/'), html());
|
|
85
|
+
const err = await refusal(safeFetch('https://blog.test/bunny', { fetch, lookup: fakeDns() }));
|
|
86
|
+
expect(err.code).toBe('BLOCKED_ADDRESS');
|
|
87
|
+
expect(fetch).toHaveBeenCalledTimes(1);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('a non-http scheme', async () => {
|
|
91
|
+
const fetch = fakeFetch(html());
|
|
92
|
+
const err = await refusal(safeFetch('file:///etc/passwd', { fetch, lookup: fakeDns() }));
|
|
93
|
+
expect(err.code).toBe('BAD_URL');
|
|
94
|
+
expect(fetch).not.toHaveBeenCalled();
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
describe('safeFetch fetches public pages', () => {
|
|
99
|
+
it('returns the body of a public html page, fetched with redirect: manual', async () => {
|
|
100
|
+
const fetch = fakeFetch(html());
|
|
101
|
+
const res = await safeFetch('https://blog.test/bunny', { fetch, lookup: fakeDns() });
|
|
102
|
+
expect(res.status).toBe(200);
|
|
103
|
+
expect(res.contentType).toBe('text/html');
|
|
104
|
+
expect(new TextDecoder().decode(res.body)).toContain('6 sc in magic ring');
|
|
105
|
+
expect(fetch.mock.calls[0]![1]?.redirect).toBe('manual');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('allows 172.32.0.1 (just outside 172.16/12)', async () => {
|
|
109
|
+
const fetch = fakeFetch(html());
|
|
110
|
+
await expect(safeFetch('http://172.32.0.1/', { fetch, lookup: fakeDns() })).resolves.toMatchObject({ status: 200 });
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('follows 3 redirects (relative Location resolved), re-checking every hop with redirect: manual', async () => {
|
|
114
|
+
const fetch = fakeFetch(redirect('https://a.test/1'), redirect('/2'), redirect('https://c.test/3'), html());
|
|
115
|
+
const lookup = fakeDns();
|
|
116
|
+
const res = await safeFetch('https://start.test/', { fetch, lookup });
|
|
117
|
+
expect(res.finalUrl).toBe('https://c.test/3');
|
|
118
|
+
expect(fetch).toHaveBeenCalledTimes(4);
|
|
119
|
+
expect(String(fetch.mock.calls[2]![0])).toBe('https://a.test/2');
|
|
120
|
+
for (const call of fetch.mock.calls) expect(call[1]?.redirect).toBe('manual');
|
|
121
|
+
expect(lookup.mock.calls.map((c) => c[0])).toEqual(expect.arrayContaining(['start.test', 'a.test', 'c.test']));
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('refuses a 4th redirect', async () => {
|
|
125
|
+
const fetch = fakeFetch(redirect('/1'), redirect('/2'), redirect('/3'), redirect('/4'), html());
|
|
126
|
+
const err = await refusal(safeFetch('https://start.test/', { fetch, lookup: fakeDns() }));
|
|
127
|
+
expect(err.code).toBe('TOO_MANY_REDIRECTS');
|
|
128
|
+
expect(fetch).toHaveBeenCalledTimes(4);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
describe('safeFetch content limits', () => {
|
|
133
|
+
it('accepts application/pdf', async () => {
|
|
134
|
+
const fetch = fakeFetch(new Response(new Uint8Array([0x25, 0x50, 0x44, 0x46]), { headers: { 'content-type': 'application/pdf' } }));
|
|
135
|
+
const res = await safeFetch('https://blog.test/bunny.pdf', { fetch, lookup: fakeDns() });
|
|
136
|
+
expect(res.contentType).toBe('application/pdf');
|
|
137
|
+
expect(res.body.byteLength).toBe(4);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it.each(['application/json', 'image/png', 'application/octet-stream'])('refuses %s', async (type) => {
|
|
141
|
+
const fetch = fakeFetch(html('{}', type));
|
|
142
|
+
const err = await refusal(safeFetch('https://blog.test/x', { fetch, lookup: fakeDns() }));
|
|
143
|
+
expect(err.code).toBe('UNSUPPORTED_TYPE');
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it('refuses a response with no content type', async () => {
|
|
147
|
+
const fetch = fakeFetch(new Response(new Uint8Array([1, 2, 3]), { status: 200 }));
|
|
148
|
+
const err = await refusal(safeFetch('https://blog.test/x', { fetch, lookup: fakeDns() }));
|
|
149
|
+
expect(err.code).toBe('UNSUPPORTED_TYPE');
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('refuses a declared content-length over maxBytes', async () => {
|
|
153
|
+
const fetch = fakeFetch(
|
|
154
|
+
new Response('x'.repeat(10), { headers: { 'content-type': 'text/html', 'content-length': '5000000' } }),
|
|
155
|
+
);
|
|
156
|
+
const err = await refusal(safeFetch('https://blog.test/x', { fetch, lookup: fakeDns(), maxBytes: 1000 }));
|
|
157
|
+
expect(err.code).toBe('TOO_LARGE');
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it('aborts a streamed body (no content-length) once it passes maxBytes', async () => {
|
|
161
|
+
let pulled = 0;
|
|
162
|
+
const stream = new ReadableStream<Uint8Array>({
|
|
163
|
+
pull(c) {
|
|
164
|
+
pulled++;
|
|
165
|
+
if (pulled > 1000) return c.close();
|
|
166
|
+
c.enqueue(new Uint8Array(512));
|
|
167
|
+
},
|
|
168
|
+
});
|
|
169
|
+
const fetch = fakeFetch(new Response(stream, { headers: { 'content-type': 'text/html' } }));
|
|
170
|
+
const err = await refusal(safeFetch('https://blog.test/x', { fetch, lookup: fakeDns(), maxBytes: 2048 }));
|
|
171
|
+
expect(err.code).toBe('TOO_LARGE');
|
|
172
|
+
expect(pulled).toBeLessThan(1000);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('aborts on timeout', async () => {
|
|
176
|
+
const hang = (init?: RequestInit) =>
|
|
177
|
+
new Promise<Response>((_resolve, reject) => {
|
|
178
|
+
init?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
|
|
179
|
+
});
|
|
180
|
+
const fetch = fakeFetch(hang);
|
|
181
|
+
const started = Date.now();
|
|
182
|
+
const err = await refusal(safeFetch('https://slow.test/', { fetch, lookup: fakeDns(), timeoutMs: 50 }));
|
|
183
|
+
expect(err.code).toBe('TIMEOUT');
|
|
184
|
+
expect(Date.now() - started).toBeLessThan(2000);
|
|
185
|
+
});
|
|
186
|
+
});
|