@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/net/client.ts

@@ -0,0 +1,434 @@
+import { Agent as HttpAgent, request as sendHttp, type IncomingMessage } from 'node:http'
+import { Agent as HttpsAgent, request as sendHttps } from 'node:https'
+import { isIP, type LookupFunction } from 'node:net'
+import type { Readable } from 'node:stream'
+import { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'
+import { WebResearchError, errorMessage, isWebResearchError } from '../contract/errors'
+import type { HttpClient, HttpRequestInit, HttpResponse } from '../contract/http'
+import { assertPublicUrl, type LookupFn } from './ssrf'
+
+const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (compatible; OpenMercatoResearch/1.0; +https://github.com/open-mercato/open-mercato)'
+const DEFAULT_MAX_BYTES = 512 * 1024
+const DEFAULT_TIMEOUT_MS = 10_000
+const MAX_REDIRECTS = 5
+const MAX_RETRIES = 2
+const MAX_BACKOFF_MS = 5_000
+/**
+ * Hard ceiling on a single `sleep()`. Backoff is already clamped to
+ * `MAX_BACKOFF_MS`, but the politeness delay is derived from configuration and
+ * a `Retry-After` header is remote input, so the timer itself is capped too —
+ * no caller can park a request for an unbounded stretch.
+ */
+const MAX_SLEEP_MS = 60_000
+/**
+ * Ceiling on one `request()` call, redirects and retries included.
+ *
+ * `timeoutMs` bounds a single attempt, so without this a call could legitimately
+ * run for redirects x retries x timeout plus backoff — minutes, inside an adapter
+ * budget of seconds. The adapter's signal would eventually cut it, but only after
+ * the work was already spent.
+ */
+const DEFAULT_TOTAL_TIMEOUT_MS = 30_000
+
+export type HttpClientOptions = {
+  readonly userAgent?: string
+  readonly acceptLanguage?: string
+  readonly maxBytes?: number
+  readonly timeoutMs?: number
+  readonly maxRedirects?: number
+  readonly maxRetries?: number
+  /** Ceiling on one call including every redirect hop and retry. */
+  readonly maxTotalMs?: number
+  /** Minimum gap between requests to the same host. */
+  readonly politenessMs?: number
+  readonly lookup?: LookupFn
+  /**
+   * Hosts allowed to resolve to a non-public address. Empty by default; see
+   * `assertPublicUrl` for why this is an allowlist and not a boolean.
+   */
+  readonly allowPrivateHosts?: readonly string[]
+  readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
+  readonly now?: () => number
+}
+
+/**
+ * Resolves a hostname to exactly the addresses the SSRF guard already vetted.
+ * Without this the kernel re-resolves at connect time, reopening the rebinding
+ * window between check and connect. Only name resolution is overridden, so SNI
+ * and certificate validation still see the real hostname.
+ */
+export function createPinnedLookup(addresses: readonly string[]): LookupFunction {
+  const entries = addresses.map((address) => ({ address, family: isIP(address) }))
+  return (hostname, options, callback) => {
+    if (entries.length === 0) {
+      callback(new Error(`No pinned address for ${hostname}`), '', 0)
+      return
+    }
+    const requestedFamily = typeof options === 'number' ? options : (options?.family ?? 0)
+    const matching = requestedFamily
+      ? entries.filter((entry) => entry.family === requestedFamily)
+      : entries
+    const selected = matching.length > 0 ? matching : entries
+    if (typeof options === 'object' && options?.all) {
+      callback(null, selected)
+      return
+    }
+    callback(null, selected[0].address, selected[0].family)
+  }
+}
+
+function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
+  // Explicit finite check + relational clamp rather than `Math.min`: a NaN
+  // duration must not reach `setTimeout`, and the upper bound has to be
+  // provable at the call site.
+  if (!Number.isFinite(ms) || ms <= 0) return Promise.resolve()
+  const bounded = ms > MAX_SLEEP_MS ? MAX_SLEEP_MS : ms
+  return new Promise((resolve) => {
+    const timer = setTimeout(finish, bounded)
+    function finish(): void {
+      clearTimeout(timer)
+      signal?.removeEventListener('abort', finish)
+      resolve()
+    }
+    signal?.addEventListener('abort', finish, { once: true })
+  })
+}
+
+function decompress(response: IncomingMessage): Readable {
+  const encoding = String(response.headers['content-encoding'] ?? '').toLowerCase()
+  if (encoding === 'gzip' || encoding === 'x-gzip') return response.pipe(createGunzip())
+  if (encoding === 'deflate') return response.pipe(createInflate())
+  if (encoding === 'br') return response.pipe(createBrotliDecompress())
+  return response
+}
+
+/**
+ * Reads at most `maxBytes`, decoding incrementally so a cap landing inside a
+ * multi-byte sequence does not produce a replacement character at the tail.
+ */
+async function readCapped(
+  stream: Readable,
+  maxBytes: number,
+): Promise<{ text: string; truncated: boolean; bytes: number }> {
+  const decoder = new TextDecoder('utf-8')
+  let text = ''
+  let bytes = 0
+  let truncated = false
+  try {
+    for await (const chunk of stream) {
+      const buffer = chunk as Buffer
+      if (bytes >= maxBytes) {
+        truncated = true
+        break
+      }
+      const remaining = maxBytes - bytes
+      if (buffer.byteLength > remaining) {
+        text += decoder.decode(buffer.subarray(0, remaining), { stream: true })
+        bytes = maxBytes
+        truncated = true
+        break
+      }
+      text += decoder.decode(buffer, { stream: true })
+      bytes += buffer.byteLength
+    }
+  } finally {
+    stream.destroy()
+  }
+  text += decoder.decode()
+  return { text, truncated, bytes }
+}
+
+type SentRequest = {
+  readonly response: IncomingMessage
+  readonly dispose: () => void
+  /**
+   * Why we tore the exchange down, if we did.
+   *
+   * Destroying a request does not deliver our reason to the response stream —
+   * Node surfaces its own socket error there — so a read cut short by our own
+   * timeout or abort would otherwise be reported as a generic network fault,
+   * and an adapter would call a deadline breach a retriable error.
+   */
+  readonly terminalError: () => WebResearchError | null
+}
+
+type SendOptions = {
+  readonly method: string
+  readonly headers: Record<string, string>
+  readonly body?: string
+  readonly timeoutMs: number
+  readonly signal?: AbortSignal
+}
+
+function send(target: URL, addresses: readonly string[], options: SendOptions): Promise<SentRequest> {
+  return new Promise((resolve, reject) => {
+    const secure = target.protocol === 'https:'
+    const agent = secure
+      ? new HttpsAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) })
+      : new HttpAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) })
+    const transport = secure ? sendHttps : sendHttp
+    const request = transport(
+      {
+        protocol: target.protocol,
+        hostname: target.hostname,
+        port: target.port || (secure ? 443 : 80),
+        path: `${target.pathname}${target.search}`,
+        method: options.method,
+        headers: options.headers,
+        agent,
+      },
+      (response) => {
+        clearTimeout(headerTimer)
+        resolve({ response, dispose, terminalError: () => terminal })
+      },
+    )
+
+    let terminal: WebResearchError | null = null
+    const fail = (error: WebResearchError): void => {
+      terminal = error
+      request.destroy(error)
+    }
+
+    /**
+     * Teardown for the whole exchange, headers *and* body. The abort listener and
+     * the socket idle timer deliberately stay armed past the response callback:
+     * releasing them there leaves a body read that no signal and no deadline can
+     * interrupt, so a server that trickles bytes forever pins a socket and an
+     * adapter promise for the life of the process.
+     */
+    const dispose = (): void => {
+      clearTimeout(headerTimer)
+      request.setTimeout(0)
+      options.signal?.removeEventListener('abort', onAbort)
+      request.destroy()
+      agent.destroy()
+    }
+
+    const headerTimer = setTimeout(() => {
+      fail(new WebResearchError('timeout', `Request to ${target.href} timed out`))
+    }, options.timeoutMs)
+
+    request.setTimeout(options.timeoutMs, () => {
+      fail(new WebResearchError('timeout', `Request to ${target.href} stalled`))
+    })
+
+    function onAbort(): void {
+      fail(new WebResearchError('aborted', `Request to ${target.href} was aborted`))
+    }
+    options.signal?.addEventListener('abort', onAbort, { once: true })
+
+    request.on('error', (error) => {
+      dispose()
+      reject(
+        terminal ??
+          (isWebResearchError(error)
+            ? error
+            : new WebResearchError('network', `Request to ${target.href} failed: ${errorMessage(error)}`, {
+                cause: error,
+              })),
+      )
+    })
+
+    if (options.body !== undefined) request.write(options.body)
+    request.end()
+  })
+}
+
+function headerMap(response: IncomingMessage): Map<string, string> {
+  const headers = new Map<string, string>()
+  for (const [key, value] of Object.entries(response.headers)) {
+    if (value === undefined) continue
+    headers.set(key.toLowerCase(), Array.isArray(value) ? value.join(', ') : value)
+  }
+  return headers
+}
+
+function retryAfterMs(headers: Map<string, string>): number | undefined {
+  const raw = headers.get('retry-after')
+  if (!raw) return undefined
+  const seconds = Number.parseInt(raw.trim(), 10)
+  if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000
+  const date = Date.parse(raw)
+  if (Number.isFinite(date)) return Math.max(0, date - Date.now())
+  return undefined
+}
+
+function contentTypeOf(headers: Map<string, string>): string | null {
+  const raw = headers.get('content-type')
+  if (!raw) return null
+  return raw.split(';')[0].trim().toLowerCase()
+}
+
+function isAcceptable(contentType: string | null, accept: readonly string[] | undefined): boolean {
+  if (!accept || accept.length === 0) return true
+  if (contentType === null) return true
+  return accept.some((prefix) => contentType.startsWith(prefix))
+}
+
+function statusError(status: number, url: string, headers: Map<string, string>): WebResearchError {
+  if (status === 429) {
+    return new WebResearchError('blocked', `Rate limited by ${url} (HTTP 429)`, {
+      status,
+      retryAfterMs: retryAfterMs(headers),
+    })
+  }
+  if (status === 403 || status === 401) {
+    return new WebResearchError('blocked', `Access denied by ${url} (HTTP ${status})`, { status })
+  }
+  return new WebResearchError('bad_response', `Request to ${url} failed with HTTP ${status}`, { status })
+}
+
+function isRetriable(error: WebResearchError): boolean {
+  if (error.code === 'blocked') return error.status === 429
+  if (error.code === 'network') return true
+  return error.code === 'bad_response' && (error.status ?? 0) >= 500
+}
+
+/**
+ * The single hardened egress path: SSRF-vetted, DNS-pinned per redirect hop,
+ * manually redirected so every hop is re-vetted, byte-capped, content-type
+ * gated, and polite to each host.
+ */
+export function createHttpClient(options: HttpClientOptions = {}): HttpClient {
+  const userAgent = options.userAgent ?? DEFAULT_USER_AGENT
+  const acceptLanguage = options.acceptLanguage ?? 'en-US,en;q=0.9'
+  const defaultMaxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
+  const defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  const maxRedirects = options.maxRedirects ?? MAX_REDIRECTS
+  const maxRetries = options.maxRetries ?? MAX_RETRIES
+  const maxTotalMs = options.maxTotalMs ?? DEFAULT_TOTAL_TIMEOUT_MS
+  const politenessMs = options.politenessMs ?? 0
+  const sleep = options.sleep ?? defaultSleep
+  const now = options.now ?? Date.now
+  const nextAllowedAt = new Map<string, number>()
+
+  async function waitForHost(host: string, signal?: AbortSignal): Promise<void> {
+    if (politenessMs <= 0) return
+    const earliest = nextAllowedAt.get(host) ?? 0
+    const current = now()
+    const delay = earliest - current
+    nextAllowedAt.set(host, Math.max(current, earliest) + politenessMs)
+    if (delay > 0) await sleep(delay, signal)
+  }
+
+  async function attempt(
+    target: URL,
+    addresses: readonly string[],
+    init: HttpRequestInit,
+    carryCallerHeaders: boolean,
+  ) {
+    const headers: Record<string, string> = {
+      'user-agent': userAgent,
+      'accept-language': acceptLanguage,
+      'accept-encoding': 'gzip, deflate, br',
+      accept: init.accept?.length ? `${init.accept.join(', ')}, */*;q=0.1` : '*/*',
+      ...(init.body === undefined ? {} : { 'content-type': 'application/json' }),
+      ...(carryCallerHeaders ? (init.headers ?? {}) : {}),
+    }
+    return send(target, addresses, {
+      method: init.method ?? 'GET',
+      headers,
+      ...(init.body === undefined ? {} : { body: init.body }),
+      timeoutMs: init.timeoutMs ?? defaultTimeoutMs,
+      ...(init.signal ? { signal: init.signal } : {}),
+    })
+  }
+
+  async function request(rawUrl: string, init: HttpRequestInit = {}): Promise<HttpResponse> {
+    const maxBytes = init.maxBytes ?? defaultMaxBytes
+    const expiresAt = now() + maxTotalMs
+    const outOfTime = (): boolean => now() >= expiresAt
+    let currentUrl = rawUrl
+    // Caller headers carry vendor credentials (`authorization`, an API-key header).
+    // A redirect to another origin must not replay them, or a compromised or
+    // merely sloppy endpoint can harvest a tenant's key with a single 302.
+    let sameOrigin = true
+
+    for (let hop = 0; hop <= maxRedirects; hop += 1) {
+      if (outOfTime()) {
+        throw new WebResearchError('timeout', `Request to ${rawUrl} exceeded its ${maxTotalMs}ms budget`)
+      }
+      const vetted = await assertPublicUrl(currentUrl, 'web research fetch', {
+        ...(options.lookup ? { lookup: options.lookup } : {}),
+        ...(options.allowPrivateHosts ? { allowPrivateHosts: options.allowPrivateHosts } : {}),
+      })
+      await waitForHost(vetted.url.host, init.signal)
+
+      let sent: SentRequest | null = null
+      let lastError: WebResearchError | null = null
+      for (let retry = 0; retry <= maxRetries; retry += 1) {
+        try {
+          sent = await attempt(vetted.url, vetted.addresses, init, sameOrigin)
+          const status = sent.response.statusCode ?? 0
+          if (status >= 400) {
+            const error = statusError(status, vetted.url.href, headerMap(sent.response))
+            sent.response.resume()
+            sent.dispose()
+            sent = null
+            throw error
+          }
+          break
+        } catch (error) {
+          // Branded rather than `instanceof`: this package typechecks as `src`
+          // and runs as `dist`, so the two class identities can differ.
+          const typed = isWebResearchError(error)
+            ? error
+            : new WebResearchError('network', errorMessage(error), { cause: error })
+          lastError = typed
+          if (typed.code === 'aborted' || retry === maxRetries || !isRetriable(typed)) throw typed
+          // Retrying past the budget only delays the failure the caller already earned.
+          if (outOfTime()) throw typed
+          const backoff = Math.min(typed.retryAfterMs ?? 250 * 2 ** retry, MAX_BACKOFF_MS)
+          await sleep(backoff, init.signal)
+        }
+      }
+      if (!sent) throw lastError ?? new WebResearchError('network', `Request to ${currentUrl} failed`)
+
+      const headers = headerMap(sent.response)
+      const status = sent.response.statusCode ?? 0
+
+      if (status >= 300 && status < 400) {
+        const location = headers.get('location')
+        sent.response.resume()
+        sent.dispose()
+        if (!location) {
+          throw new WebResearchError('bad_response', `Redirect from ${currentUrl} without a Location header`)
+        }
+        const next = new URL(location, vetted.url)
+        if (next.origin !== vetted.url.origin) sameOrigin = false
+        currentUrl = next.toString()
+        continue
+      }
+
+      const contentType = contentTypeOf(headers)
+      if (!isAcceptable(contentType, init.accept)) {
+        sent.response.resume()
+        sent.dispose()
+        throw new WebResearchError(
+          'unsupported_content_type',
+          `Refusing to decode ${contentType ?? 'unknown'} from ${vetted.url.href} as text`,
+        )
+      }
+
+      try {
+        const { text, truncated } = await readCapped(decompress(sent.response), maxBytes)
+        return { url: vetted.url.href, status, headers, contentType, body: text, truncated }
+      } catch (error) {
+        const terminal = sent.terminalError()
+        if (terminal) throw terminal
+        throw isWebResearchError(error)
+          ? error
+          : new WebResearchError('network', `Reading ${vetted.url.href} failed: ${errorMessage(error)}`, {
+              cause: error,
+            })
+      } finally {
+        sent.dispose()
+      }
+    }
+
+    throw new WebResearchError('too_many_redirects', `Exceeded ${maxRedirects} redirects from ${rawUrl}`)
+  }
+
+  return { request }
+}

package/src/net/ssrf.ts

@@ -0,0 +1,235 @@
+import { lookup as dnsLookup } from 'node:dns/promises'
+import { isIP } from 'node:net'
+import { WebResearchError } from '../contract/errors'
+
+export type LookupFn = (hostname: string) => Promise<Array<{ address: string; family: number }>>
+
+const defaultLookup: LookupFn = async (hostname) => {
+  const records = await dnsLookup(hostname, { all: true })
+  return records.map((record) => ({ address: record.address, family: record.family }))
+}
+
+function ipv4ToInt(address: string): number | null {
+  const parts = address.split('.')
+  if (parts.length !== 4) return null
+  let value = 0
+  for (const part of parts) {
+    if (!/^\d{1,3}$/.test(part)) return null
+    const octet = Number(part)
+    if (octet > 255) return null
+    value = value * 256 + octet
+  }
+  return value >>> 0
+}
+
+function isPrivateIpv4(address: string): boolean {
+  const value = ipv4ToInt(address)
+  if (value === null) return true
+  const inRange = (base: string, maskBits: number): boolean => {
+    const baseValue = ipv4ToInt(base)
+    if (baseValue === null) return false
+    const mask = maskBits === 0 ? 0 : (0xffffffff << (32 - maskBits)) >>> 0
+    return (value & mask) === (baseValue & mask)
+  }
+  return (
+    inRange('0.0.0.0', 8) ||
+    inRange('10.0.0.0', 8) ||
+    inRange('100.64.0.0', 10) ||
+    inRange('127.0.0.0', 8) ||
+    inRange('169.254.0.0', 16) ||
+    inRange('172.16.0.0', 12) ||
+    inRange('192.0.0.0', 24) ||
+    inRange('192.168.0.0', 16) ||
+    inRange('198.18.0.0', 15) ||
+    inRange('224.0.0.0', 4) ||
+    inRange('240.0.0.0', 4)
+  )
+}
+
+function normalizeIpv6(address: string): string {
+  const zoneIndex = address.indexOf('%')
+  return (zoneIndex === -1 ? address : address.slice(0, zoneIndex)).toLowerCase()
+}
+
+function parseHextets(segment: string): number[] | null {
+  if (segment === '') return []
+  const parsed: number[] = []
+  for (const part of segment.split(':')) {
+    if (!/^[0-9a-f]{1,4}$/.test(part)) return null
+    parsed.push(Number.parseInt(part, 16))
+  }
+  return parsed
+}
+
+/** Expands any IPv6 spelling (`::`, embedded dotted-quad) to its eight hextets. */
+function expandIpv6(raw: string): number[] | null {
+  let address = raw
+  const dotted = /:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(address)
+  if (dotted) {
+    const value = ipv4ToInt(dotted[1])
+    if (value === null) return null
+    const high = ((value >>> 16) & 0xffff).toString(16)
+    const low = (value & 0xffff).toString(16)
+    address = `${address.slice(0, dotted.index)}:${high}:${low}`
+  }
+  const halves = address.split('::')
+  if (halves.length > 2) return null
+  const head = parseHextets(halves[0] ?? '')
+  if (head === null) return null
+  if (halves.length === 1) return head.length === 8 ? head : null
+  const tail = parseHextets(halves[1] ?? '')
+  if (tail === null) return null
+  const missing = 8 - head.length - tail.length
+  if (missing < 0) return null
+  return [...head, ...Array<number>(missing).fill(0), ...tail]
+}
+
+function hextetsToIpv4(high: number, low: number): string {
+  return `${high >>> 8}.${high & 0xff}.${low >>> 8}.${low & 0xff}`
+}
+
+/**
+ * Recovers the IPv4 address tunnelled inside an IPv6 one. This is load-bearing:
+ * `new URL('http://[::ffff:127.0.0.1]/').hostname` is `::ffff:7f00:1`, so a guard
+ * that only pattern-matches the dotted-quad spelling never fires and loopback —
+ * and the cloud metadata endpoint — walk straight through.
+ */
+function embeddedIpv4(hextets: readonly number[]): string | null {
+  const [first, second, third, fourth, fifth, sixth, seventh, eighth] = hextets
+  const zeroPrefix = first === 0 && second === 0 && third === 0 && fourth === 0 && fifth === 0
+  if (zeroPrefix && (sixth === 0xffff || sixth === 0)) return hextetsToIpv4(seventh, eighth)
+  const nat64 =
+    first === 0x0064 && second === 0xff9b && third === 0 && fourth === 0 && fifth === 0 && sixth === 0
+  if (nat64) return hextetsToIpv4(seventh, eighth)
+  return null
+}
+
+function isPrivateIpv6(address: string): boolean {
+  const normalized = normalizeIpv6(address)
+  if (normalized === '::1' || normalized === '::') return true
+  const hextets = expandIpv6(normalized)
+  if (hextets === null) return true
+  const embedded = embeddedIpv4(hextets)
+  if (embedded !== null) return isPrivateIpv4(embedded)
+  const first = hextets[0]
+  if ((first & 0xfe00) === 0xfc00) return true
+  if ((first & 0xffc0) === 0xfe80) return true
+  if ((first & 0xff00) === 0xff00) return true
+  return false
+}
+
+export function isPrivateAddress(address: string): boolean {
+  const family = isIP(address)
+  if (family === 4) return isPrivateIpv4(address)
+  if (family === 6) return isPrivateIpv6(address)
+  return true
+}
+
+/** `localhost` and anything under it never reach DNS. */
+export function isLoopbackHostname(hostname: string): boolean {
+  const host = hostname.toLowerCase()
+  return host === 'localhost' || host.endsWith('.localhost')
+}
+
+export type AssertPublicUrlOptions = {
+  readonly lookup?: LookupFn
+  /**
+   * Whether an unresolvable host is a hard failure. Defaults to `true` — unlike
+   * reference implementations that fail open here, we pin the connection to the
+   * addresses this check returns, so no addresses means no pinning, and letting
+   * the request proceed would silently drop a control rather than merely fail.
+   */
+  readonly failClosed?: boolean
+  /**
+   * Hosts permitted to resolve to a non-public address.
+   *
+   * An allowlist rather than a blanket "allow private", because the two are not
+   * remotely equivalent: a boolean opens every internal address to whatever URL
+   * a model or a search result happens to produce, while this only unblocks
+   * destinations an operator named. It exists because a service the operator
+   * runs themselves — a SearXNG instance on the container network, a self-hosted
+   * scraper — is otherwise unreachable by design.
+   *
+   * Matching is exact host or a dot-boundary suffix, so `internal.example.com`
+   * covers `search.internal.example.com` but never `notinternal.example.com`.
+   */
+  readonly allowPrivateHosts?: readonly string[]
+}
+
+/** Exact host or a dot-boundary suffix; never a bare substring. */
+export function isPrivateHostAllowed(hostname: string, allowlist: readonly string[] | undefined): boolean {
+  if (!allowlist || allowlist.length === 0) return false
+  const host = hostname.toLowerCase()
+  return allowlist.some((entry) => {
+    const allowed = entry.trim().toLowerCase()
+    if (allowed.length === 0) return false
+    return host === allowed || host.endsWith(`.${allowed}`)
+  })
+}
+
+export type VettedUrl = {
+  readonly url: URL
+  /** Vetted addresses to pin the connection to; empty only when failClosed is off. */
+  readonly addresses: readonly string[]
+}
+
+export async function assertPublicUrl(
+  rawUrl: string,
+  label = 'request',
+  options: AssertPublicUrlOptions = {},
+): Promise<VettedUrl> {
+  let parsed: URL
+  try {
+    parsed = new URL(rawUrl)
+  } catch {
+    throw new WebResearchError('ssrf_blocked', `Invalid URL for ${label}`)
+  }
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    throw new WebResearchError('ssrf_blocked', `Blocked URL scheme for ${label}: ${parsed.protocol}`)
+  }
+  if (parsed.username || parsed.password) {
+    throw new WebResearchError('ssrf_blocked', `Credentials in URL are not allowed for ${label}`)
+  }
+
+  const hostname = parsed.hostname.replace(/^\[|\]$/g, '')
+  const privateAllowed = isPrivateHostAllowed(hostname, options.allowPrivateHosts)
+
+  if (!privateAllowed && isLoopbackHostname(hostname)) {
+    throw new WebResearchError('ssrf_blocked', `Blocked loopback host for ${label}: ${hostname}`)
+  }
+
+  if (isIP(hostname)) {
+    if (!privateAllowed && isPrivateAddress(hostname)) {
+      throw new WebResearchError('ssrf_blocked', `Blocked non-public address for ${label}: ${hostname}`)
+    }
+    return { url: parsed, addresses: [hostname] }
+  }
+
+  const lookup = options.lookup ?? defaultLookup
+  const failClosed = options.failClosed ?? true
+  let records: Array<{ address: string; family: number }>
+  try {
+    records = await lookup(hostname)
+  } catch {
+    if (failClosed) {
+      throw new WebResearchError('ssrf_blocked', `Could not resolve host for ${label}: ${hostname}`)
+    }
+    return { url: parsed, addresses: [] }
+  }
+  if (records.length === 0) {
+    if (failClosed) {
+      throw new WebResearchError('ssrf_blocked', `Host did not resolve for ${label}: ${hostname}`)
+    }
+    return { url: parsed, addresses: [] }
+  }
+  for (const { address } of records) {
+    if (!privateAllowed && isPrivateAddress(address)) {
+      throw new WebResearchError(
+        'ssrf_blocked',
+        `Host ${hostname} resolves to non-public address ${address} for ${label}`,
+      )
+    }
+  }
+  return { url: parsed, addresses: records.map((record) => record.address) }
+}

package/src/net/url.ts

@@ -0,0 +1,13 @@
+/**
+ * Trims every trailing `/` from a base URL.
+ *
+ * Written as a scan rather than `replace(/\/+$/, '')` on purpose: the regex
+ * form backtracks quadratically on a configured base URL that ends in a long
+ * run of slashes, which CodeQL flags as a polynomial ReDoS. This walks the
+ * string once from the end.
+ */
+export function stripTrailingSlashes(value: string): string {
+  let end = value.length
+  while (end > 0 && value.charCodeAt(end - 1) === 0x2f) end -= 1
+  return value.slice(0, end)
+}