@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/net/__tests__/client.test.ts

@@ -0,0 +1,273 @@
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { isWebResearchError } from '../../contract/errors'
+import { createHttpClient } from '../client'
+
+/**
+ * Real sockets, not a stubbed transport.
+ *
+ * The behaviour worth pinning here lives below `fetch`: redirect re-vetting,
+ * what happens to a caller's credentials on a hop, and whether a stalled body
+ * read can ever be interrupted. A double of the transport would assert the
+ * double. The SSRF guard blocks loopback by design, so these tests use the
+ * `allowPrivateHosts` allowlist to name the one host they serve from.
+ */
+
+type Handler = (req: IncomingMessage, res: ServerResponse) => void
+
+const LOOPBACK = ['127.0.0.1']
+
+function listen(handler: Handler): Promise<{ server: Server; origin: string; port: number }> {
+  return new Promise((resolve) => {
+    const server = createServer(handler)
+    server.listen(0, '127.0.0.1', () => {
+      const { port } = server.address() as AddressInfo
+      resolve({ server, origin: `http://127.0.0.1:${port}`, port })
+    })
+  })
+}
+
+function close(server: Server): Promise<void> {
+  return new Promise((resolve) => {
+    server.closeAllConnections?.()
+    server.close(() => resolve())
+  })
+}
+
+const client = (overrides = {}) => createHttpClient({ allowPrivateHosts: LOOPBACK, ...overrides })
+
+describe('http client against a real server', () => {
+  const servers: Server[] = []
+  afterEach(async () => {
+    await Promise.all(servers.splice(0).map(close))
+  })
+  const track = <T extends { server: Server }>(started: T): T => {
+    servers.push(started.server)
+    return started
+  }
+
+  it('refuses a private host that is not on the allowlist', async () => {
+    const { origin } = track(await listen((_req, res) => res.end('ok')))
+
+    await expect(createHttpClient().request(origin)).rejects.toMatchObject({ code: 'ssrf_blocked' })
+  })
+
+  it('reads a body and reports the final url', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(200, { 'content-type': 'text/html' })
+        res.end('<html><body>hello</body></html>')
+      }),
+    )
+
+    const response = await client().request(`${origin}/page`)
+
+    expect(response.status).toBe(200)
+    expect(response.body).toContain('hello')
+    expect(response.url).toBe(`${origin}/page`)
+  })
+
+  it('keeps caller headers on a same-origin redirect', async () => {
+    const seen: Array<string | undefined> = []
+    const { origin } = track(
+      await listen((req, res) => {
+        seen.push(req.headers.authorization)
+        if (req.url === '/start') {
+          res.writeHead(302, { location: '/end' })
+          res.end()
+          return
+        }
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.end('done')
+      }),
+    )
+
+    await client().request(`${origin}/start`, { headers: { authorization: 'Bearer secret' } })
+
+    expect(seen).toEqual(['Bearer secret', 'Bearer secret'])
+  })
+
+  it('drops caller headers when a redirect changes origin', async () => {
+    let received: string | undefined = 'not-called'
+    const target = track(
+      await listen((req, res) => {
+        received = req.headers.authorization
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.end('done')
+      }),
+    )
+    const { origin } = track(
+      await listen((_req, res) => {
+        // A different port is a different origin, which is all the rule keys on.
+        res.writeHead(302, { location: `${target.origin}/landed` })
+        res.end()
+      }),
+    )
+
+    const response = await client().request(`${origin}/start`, {
+      headers: { authorization: 'Bearer secret' },
+    })
+
+    // The whole point: a vendor key must not be replayed to wherever a 302 points.
+    expect(received).toBeUndefined()
+    expect(response.body).toBe('done')
+  })
+
+  it('re-vets every redirect hop against the SSRF guard', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' })
+        res.end()
+      }),
+    )
+
+    // The allowlist named 127.0.0.1 only, so the metadata endpoint is still refused.
+    await expect(client().request(`${origin}/start`)).rejects.toMatchObject({ code: 'ssrf_blocked' })
+  })
+
+  it('stops after the redirect limit instead of looping', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(302, { location: '/again' })
+        res.end()
+      }),
+    )
+
+    await expect(client({ maxRedirects: 2 }).request(`${origin}/start`)).rejects.toMatchObject({
+      code: 'too_many_redirects',
+    })
+  })
+
+  it('caps the body and reports truncation', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.end('x'.repeat(100_000))
+      }),
+    )
+
+    const response = await client().request(origin, { maxBytes: 1024 })
+
+    expect(response.truncated).toBe(true)
+    expect(response.body.length).toBeLessThanOrEqual(1024)
+  })
+
+  it('refuses to decode a content type the caller did not accept', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(200, { 'content-type': 'image/png' })
+        res.end('binary')
+      }),
+    )
+
+    await expect(client().request(origin, { accept: ['text/html'] })).rejects.toMatchObject({
+      code: 'unsupported_content_type',
+    })
+  })
+
+  it('reports a 429 as blocked and carries retry-after', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(429, { 'retry-after': '7' })
+        res.end('slow down')
+      }),
+    )
+
+    const error = await client({ maxRetries: 0 }).request(origin).catch((caught: unknown) => caught)
+
+    expect(isWebResearchError(error) && error.code).toBe('blocked')
+    expect(isWebResearchError(error) && error.retryAfterMs).toBe(7000)
+  })
+
+  it('retries a 5xx and succeeds on a later attempt', async () => {
+    let attempts = 0
+    const { origin } = track(
+      await listen((_req, res) => {
+        attempts += 1
+        if (attempts < 3) {
+          res.writeHead(503)
+          res.end('nope')
+          return
+        }
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.end('recovered')
+      }),
+    )
+
+    const response = await client({ sleep: async () => {} }).request(origin)
+
+    expect(attempts).toBe(3)
+    expect(response.body).toBe('recovered')
+  })
+
+  it('does not retry a 404', async () => {
+    let attempts = 0
+    const { origin } = track(
+      await listen((_req, res) => {
+        attempts += 1
+        res.writeHead(404)
+        res.end('gone')
+      }),
+    )
+
+    await expect(client({ sleep: async () => {} }).request(origin)).rejects.toMatchObject({
+      code: 'bad_response',
+    })
+    expect(attempts).toBe(1)
+  })
+
+  it('gives up once one call has used its whole budget', async () => {
+    let hops = 0
+    const { origin } = track(
+      await listen((_req, res) => {
+        hops += 1
+        res.writeHead(302, { location: '/again' })
+        res.end()
+      }),
+    )
+
+    // Each hop is fast, but the budget is what bounds the call — a chain of
+    // redirects and retries could otherwise run for minutes inside an adapter
+    // budget of seconds.
+    let clock = 0
+    await expect(
+      client({ maxRedirects: 50, maxTotalMs: 100, now: () => (clock += 60) }).request(`${origin}/start`),
+    ).rejects.toMatchObject({ code: 'timeout' })
+    expect(hops).toBeLessThan(50)
+  })
+
+  it('times out a body that never finishes', async () => {
+    const { origin } = track(
+      await listen((_req, res) => {
+        // Headers land, then the server stalls forever. Before the read timeout
+        // existed this pinned a socket and an adapter promise for the process
+        // lifetime: the engine walked away on its deadline and cancelled nothing.
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.write('partial')
+      }),
+    )
+
+    const error = await client({ timeoutMs: 300 }).request(origin).catch((caught: unknown) => caught)
+
+    expect(isWebResearchError(error) && error.code).toBe('timeout')
+  }, 10_000)
+
+  it('aborts a body read that is already in flight', async () => {
+    const controller = new AbortController()
+    const { origin } = track(
+      await listen((_req, res) => {
+        res.writeHead(200, { 'content-type': 'text/plain' })
+        res.write('partial')
+        setTimeout(() => controller.abort(), 50).unref?.()
+      }),
+    )
+
+    const error = await client({ timeoutMs: 30_000 })
+      .request(origin, { signal: controller.signal })
+      .catch((caught: unknown) => caught)
+
+    // The abort listener used to be removed once headers arrived, which left the
+    // read uninterruptible by any signal or deadline.
+    expect(isWebResearchError(error) && error.code).toBe('aborted')
+  }, 10_000)
+})

package/src/net/__tests__/ssrf.test.ts

@@ -0,0 +1,165 @@
+import { assertPublicUrl, isLoopbackHostname, isPrivateAddress, type LookupFn } from '../ssrf'
+import { isWebResearchError } from '../../contract/errors'
+
+const publicLookup: LookupFn = async () => [{ address: '93.184.216.34', family: 4 }]
+const privateLookup: LookupFn = async () => [{ address: '10.0.0.5', family: 4 }]
+const failingLookup: LookupFn = async () => {
+  throw new Error('ENOTFOUND')
+}
+
+describe('isPrivateAddress', () => {
+  it.each([
+    '127.0.0.1',
+    '10.1.2.3',
+    '172.16.0.1',
+    '192.168.1.1',
+    '169.254.169.254',
+    '100.64.0.1',
+    '0.0.0.0',
+    '224.0.0.1',
+    '240.0.0.1',
+    '198.18.0.1',
+  ])('blocks IPv4 %s', (address) => {
+    expect(isPrivateAddress(address)).toBe(true)
+  })
+
+  it.each(['93.184.216.34', '8.8.8.8', '1.1.1.1'])('allows public IPv4 %s', (address) => {
+    expect(isPrivateAddress(address)).toBe(false)
+  })
+
+  it.each(['::1', '::', 'fc00::1', 'fd12:3456::1', 'fe80::1', 'ff02::1'])(
+    'blocks IPv6 %s',
+    (address) => {
+      expect(isPrivateAddress(address)).toBe(true)
+    },
+  )
+
+  it('allows a public IPv6 address', () => {
+    expect(isPrivateAddress('2606:2800:220:1:248:1893:25c8:1946')).toBe(false)
+  })
+
+  // Regression: `new URL('http://[::ffff:127.0.0.1]/').hostname` is '::ffff:7f00:1',
+  // so a guard matching only the dotted-quad spelling lets loopback through.
+  it.each(['::ffff:7f00:1', '::ffff:127.0.0.1', '::ffff:a9fe:a9fe', '::ffff:169.254.169.254'])(
+    'blocks IPv4-mapped IPv6 %s in either spelling',
+    (address) => {
+      expect(isPrivateAddress(address)).toBe(true)
+    },
+  )
+
+  it('blocks NAT64-embedded private IPv4', () => {
+    expect(isPrivateAddress('64:ff9b::a00:1')).toBe(true)
+  })
+
+  it('treats an unparseable address as unsafe', () => {
+    expect(isPrivateAddress('not-an-ip')).toBe(true)
+    expect(isPrivateAddress('12345::zz')).toBe(true)
+  })
+})
+
+describe('isLoopbackHostname', () => {
+  it.each(['localhost', 'LOCALHOST', 'api.localhost'])('matches %s', (host) => {
+    expect(isLoopbackHostname(host)).toBe(true)
+  })
+
+  it('does not match a lookalike', () => {
+    expect(isLoopbackHostname('notlocalhost.com')).toBe(false)
+  })
+})
+
+describe('assertPublicUrl', () => {
+  it('returns the vetted addresses so the caller can pin them', async () => {
+    const vetted = await assertPublicUrl('https://example.com/a', 'test', { lookup: publicLookup })
+    expect(vetted.addresses).toEqual(['93.184.216.34'])
+    expect(vetted.url.href).toBe('https://example.com/a')
+  })
+
+  it.each(['file:///etc/passwd', 'javascript:alert(1)', 'ftp://example.com'])(
+    'rejects scheme in %s',
+    async (url) => {
+      await expect(assertPublicUrl(url, 'test', { lookup: publicLookup })).rejects.toThrow()
+    },
+  )
+
+  it('rejects credentials in the URL', async () => {
+    await expect(
+      assertPublicUrl('https://user:pass@example.com/', 'test', { lookup: publicLookup }),
+    ).rejects.toThrow(/Credentials/)
+  })
+
+  it('rejects a loopback hostname before consulting DNS', async () => {
+    const lookup = jest.fn<ReturnType<LookupFn>, [string]>()
+    await expect(assertPublicUrl('http://localhost:3000/', 'test', { lookup })).rejects.toThrow()
+    expect(lookup).not.toHaveBeenCalled()
+  })
+
+  it('rejects a host resolving to a private address', async () => {
+    await expect(
+      assertPublicUrl('https://internal.example.com/', 'test', { lookup: privateLookup }),
+    ).rejects.toThrow(/non-public address/)
+  })
+
+  it('rejects a bracketed IPv4-mapped loopback literal', async () => {
+    await expect(
+      assertPublicUrl('http://[::ffff:127.0.0.1]/', 'test', { lookup: publicLookup }),
+    ).rejects.toThrow(/non-public address/)
+  })
+
+  it('fails closed on DNS failure by default, because it cannot pin without addresses', async () => {
+    await expect(
+      assertPublicUrl('https://example.com/', 'test', { lookup: failingLookup }),
+    ).rejects.toThrow(/Could not resolve/)
+  })
+
+  it('can fail open for callers that do not pin', async () => {
+    const vetted = await assertPublicUrl('https://example.com/', 'test', {
+      lookup: failingLookup,
+      failClosed: false,
+    })
+    expect(vetted.addresses).toEqual([])
+  })
+
+  it('throws a branded error that survives realm boundaries', async () => {
+    const error = await assertPublicUrl('http://127.0.0.1/', 'test').catch((caught: unknown) => caught)
+    expect(isWebResearchError(error)).toBe(true)
+  })
+
+  describe('allowPrivateHosts', () => {
+    it('lets a named host resolve privately', async () => {
+      const vetted = await assertPublicUrl('http://searxng:8080/search', 'test', {
+        lookup: async () => [{ address: '10.1.2.3', family: 4 }],
+        allowPrivateHosts: ['searxng'],
+      })
+      expect(vetted.addresses).toEqual(['10.1.2.3'])
+    })
+
+    it('matches on a dot boundary, never a bare suffix', async () => {
+      const options = {
+        lookup: async () => [{ address: '10.1.2.3', family: 4 }],
+        allowPrivateHosts: ['internal.example.com'],
+      }
+      await expect(
+        assertPublicUrl('http://search.internal.example.com/', 'test', options),
+      ).resolves.toBeDefined()
+      await expect(
+        assertPublicUrl('http://notinternal.example.com/', 'test', options),
+      ).rejects.toMatchObject({ code: 'ssrf_blocked' })
+    })
+
+    it('still refuses a host nobody named', async () => {
+      await expect(
+        assertPublicUrl('http://169.254.169.254/latest/meta-data/', 'test', {
+          allowPrivateHosts: ['searxng'],
+        }),
+      ).rejects.toMatchObject({ code: 'ssrf_blocked' })
+    })
+
+    it('unblocks loopback only for the named host', async () => {
+      await expect(assertPublicUrl('http://127.0.0.1/', 'test', { allowPrivateHosts: ['127.0.0.1'] }))
+        .resolves.toBeDefined()
+      await expect(
+        assertPublicUrl('http://localhost/', 'test', { allowPrivateHosts: ['127.0.0.1'] }),
+      ).rejects.toMatchObject({ code: 'ssrf_blocked' })
+    })
+  })
+})