@open-mercato/web-research-browser 0.9.1-develop.7547.1.ade91a3f6b → 0.9.1-develop.7552.1.4acf199f7a

package/src/egressProxy.ts

@@ -0,0 +1,345 @@
+import {
+  createServer,
+  request as sendHttp,
+  type ClientRequest,
+  type IncomingHttpHeaders,
+  type IncomingMessage,
+  type OutgoingHttpHeaders,
+  type ServerResponse,
+} from 'node:http'
+import { connect as netConnect, type Socket } from 'node:net'
+import type { Duplex } from 'node:stream'
+import { assertPublicUrl, createPinnedLookup, type LookupFn, type VettedUrl } from '@open-mercato/web-research'
+
+export type EgressDialTarget = {
+  readonly host: string
+  readonly port: number
+  /** The addresses `assertPublicUrl` vetted for this connection; the only ones a dial may use. */
+  readonly addresses: readonly string[]
+}
+
+export type EgressConnect = (target: EgressDialTarget) => Socket
+
+export type EgressProxyOptions = {
+  readonly lookup?: LookupFn
+  readonly connect?: EgressConnect
+  readonly vetTimeoutMs?: number
+  readonly dialTimeoutMs?: number
+  readonly idleTimeoutMs?: number
+}
+
+export type EgressProxy = {
+  readonly url: string
+  close(): Promise<void>
+}
+
+const EGRESS_VET_TIMEOUT_MS = 10_000
+const EGRESS_DIAL_TIMEOUT_MS = 10_000
+const EGRESS_IDLE_TIMEOUT_MS = 120_000
+
+const AUTHORITY_PATTERN = /^(\[[0-9A-Fa-f:.]+\]|[A-Za-z0-9._-]+):(\d{1,5})$/
+
+const HOP_BY_HOP_HEADERS = new Set([
+  'connection',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'proxy-connection',
+  'te',
+  'trailer',
+  'transfer-encoding',
+  'upgrade',
+  'host',
+])
+
+const STATUS_TEXT: Record<number, string> = {
+  400: 'Bad Request',
+  403: 'Forbidden',
+  502: 'Bad Gateway',
+  504: 'Gateway Timeout',
+}
+
+/** Dials only the vetted addresses: the hostname is never handed to DNS. */
+export const pinnedConnect: EgressConnect = ({ host, port, addresses }) =>
+  netConnect({ host, port, lookup: createPinnedLookup(addresses), autoSelectFamily: true })
+
+function parseConnectAuthority(raw: string): { host: string; port: number } | null {
+  const match = AUTHORITY_PATTERN.exec(raw)
+  if (!match) return null
+  const port = Number(match[2])
+  if (!Number.isInteger(port) || port < 1 || port > 65535) return null
+  return { host: match[1], port }
+}
+
+function parseAbsoluteForm(raw: string): URL | null {
+  if (!/^http:\/\//i.test(raw)) return null
+  try {
+    const url = new URL(raw)
+    if (url.protocol !== 'http:' || url.hostname === '') return null
+    return url
+  } catch {
+    return null
+  }
+}
+
+function forwardableHeaders(headers: IncomingHttpHeaders): OutgoingHttpHeaders {
+  const listed = new Set(
+    String(headers.connection ?? '')
+      .split(',')
+      .map((token) => token.trim().toLowerCase())
+      .filter((token) => token.length > 0),
+  )
+  const result: OutgoingHttpHeaders = {}
+  for (const [name, value] of Object.entries(headers)) {
+    if (value === undefined) continue
+    const key = name.toLowerCase()
+    if (HOP_BY_HOP_HEADERS.has(key) || key.startsWith('proxy-') || listed.has(key)) continue
+    result[key] = value
+  }
+  return result
+}
+
+function dialHost(vetted: VettedUrl): string {
+  return vetted.url.hostname.replace(/^\[|\]$/g, '')
+}
+
+function refuseSocket(socket: Duplex, status: number): void {
+  if (socket.destroyed || !socket.writable) {
+    socket.destroy()
+    return
+  }
+  socket.end(`HTTP/1.1 ${status} ${STATUS_TEXT[status] ?? 'Error'}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`, () =>
+    socket.destroy(),
+  )
+}
+
+function refuseResponse(res: ServerResponse, status: number, reason: string): void {
+  if (res.headersSent || res.destroyed) {
+    res.destroy()
+    return
+  }
+  res.writeHead(status, { 'content-type': 'text/plain; charset=utf-8', connection: 'close' })
+  res.end(`om-web-research egress proxy: ${reason}\n`)
+}
+
+/**
+ * Loopback HTTP proxy that every connection of the sidecar's Chromium goes through.
+ *
+ * Each CONNECT tunnel and each absolute-form request is vetted with
+ * `assertPublicUrl` on its own, then dialled only to the addresses that check
+ * returned, so redirect hops, WebSockets, workers and a host whose DNS answer
+ * changes between connections are all held to the same rule. Nothing is cached.
+ */
+export async function startEgressProxy(options: EgressProxyOptions = {}): Promise<EgressProxy> {
+  const connect = options.connect ?? pinnedConnect
+  const vetTimeoutMs = options.vetTimeoutMs ?? EGRESS_VET_TIMEOUT_MS
+  const dialTimeoutMs = options.dialTimeoutMs ?? EGRESS_DIAL_TIMEOUT_MS
+  const idleTimeoutMs = options.idleTimeoutMs ?? EGRESS_IDLE_TIMEOUT_MS
+  const sockets = new Set<Duplex>()
+  let closed = false
+
+  const track = (socket: Duplex): void => {
+    sockets.add(socket)
+    socket.once('close', () => sockets.delete(socket))
+  }
+
+  const vet = (rawUrl: string): Promise<VettedUrl | 'blocked' | 'timeout'> =>
+    new Promise((resolve) => {
+      const timer = setTimeout(() => resolve('timeout'), vetTimeoutMs)
+      assertPublicUrl(rawUrl, 'browser egress', {
+        failClosed: true,
+        ...(options.lookup ? { lookup: options.lookup } : {}),
+      }).then(
+        (vetted) => {
+          clearTimeout(timer)
+          resolve(vetted)
+        },
+        () => {
+          clearTimeout(timer)
+          resolve('blocked')
+        },
+      )
+    })
+
+  const dial = (target: EgressDialTarget): Promise<Socket> =>
+    new Promise((resolve, reject) => {
+      let socket: Socket
+      try {
+        socket = connect(target)
+      } catch (error) {
+        reject(error)
+        return
+      }
+      track(socket)
+      socket.on('error', () => socket.destroy())
+      const timer = setTimeout(() => {
+        socket.destroy()
+        reject(new Error(`dial to ${target.host}:${target.port} timed out`))
+      }, dialTimeoutMs)
+      socket.once('close', () => {
+        clearTimeout(timer)
+        reject(new Error(`dial to ${target.host}:${target.port} failed`))
+      })
+      socket.once('connect', () => {
+        clearTimeout(timer)
+        if (closed) {
+          socket.destroy()
+          return
+        }
+        resolve(socket)
+      })
+    })
+
+  const handleConnect = async (req: IncomingMessage, client: Duplex, head: Buffer): Promise<void> => {
+    track(client)
+    client.on('error', () => client.destroy())
+    let clientLeft = false
+    const markLeft = (): void => {
+      clientLeft = true
+    }
+    client.once('end', markLeft)
+    client.once('close', markLeft)
+    const authority = parseConnectAuthority(req.url ?? '')
+    if (!authority) return refuseSocket(client, 400)
+    const vetted = await vet(`http://${authority.host}:${authority.port}/`)
+    if (vetted === 'timeout') return refuseSocket(client, 504)
+    if (vetted === 'blocked') return refuseSocket(client, 403)
+    if (closed || clientLeft) {
+      client.destroy()
+      return
+    }
+    let upstream: Socket
+    try {
+      upstream = await dial({ host: dialHost(vetted), port: authority.port, addresses: vetted.addresses })
+    } catch {
+      return refuseSocket(client, 502)
+    }
+    if (clientLeft) {
+      upstream.destroy()
+      client.destroy()
+      return
+    }
+    upstream.setTimeout(idleTimeoutMs, () => {
+      upstream.destroy()
+      client.destroy()
+    })
+    upstream.once('close', () => client.destroy())
+    client.once('close', () => upstream.destroy())
+    client.write('HTTP/1.1 200 Connection Established\r\n\r\n')
+    if (head.length > 0) upstream.write(head)
+    client.pipe(upstream)
+    upstream.pipe(client)
+  }
+
+  const handleRequest = async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
+    req.on('error', () => res.destroy())
+    res.on('error', () => res.destroy())
+    const target = parseAbsoluteForm(req.url ?? '')
+    if (!target) return refuseResponse(res, 400, 'only absolute-form http:// requests are proxied')
+    if (req.headers.upgrade !== undefined) return refuseResponse(res, 400, 'upgrade requests must use CONNECT')
+    const vetted = await vet(target.href)
+    if (vetted === 'timeout') return refuseResponse(res, 504, 'destination could not be vetted in time')
+    if (vetted === 'blocked') return refuseResponse(res, 403, 'destination is not a public address')
+    if (closed || res.destroyed) return
+    let upstreamSocket: Socket
+    try {
+      upstreamSocket = await dial({
+        host: dialHost(vetted),
+        port: Number(vetted.url.port || 80),
+        addresses: vetted.addresses,
+      })
+    } catch {
+      return refuseResponse(res, 502, 'destination unreachable')
+    }
+    if (res.destroyed) {
+      upstreamSocket.destroy()
+      return
+    }
+    upstreamSocket.setTimeout(idleTimeoutMs, () => upstreamSocket.destroy())
+
+    const headers = forwardableHeaders(req.headers)
+    headers.host = vetted.url.host
+    headers.connection = 'close'
+    const forwardResponse = (upstreamResponse: IncomingMessage): void => {
+      upstreamResponse.on('error', () => res.destroy())
+      const status = upstreamResponse.statusCode ?? 0
+      if (status < 200 || status > 999) {
+        upstreamResponse.destroy()
+        refuseResponse(res, 502, 'upstream sent an invalid status')
+        return
+      }
+      try {
+        res.writeHead(status, forwardableHeaders(upstreamResponse.headers))
+      } catch {
+        upstreamResponse.destroy()
+        res.destroy()
+        return
+      }
+      upstreamResponse.pipe(res)
+    }
+    let upstreamRequest: ClientRequest
+    try {
+      upstreamRequest = sendHttp(
+        {
+          method: req.method,
+          path: `${vetted.url.pathname}${vetted.url.search}`,
+          headers,
+          setHost: false,
+          createConnection: () => upstreamSocket,
+        },
+        forwardResponse,
+      )
+    } catch {
+      upstreamSocket.destroy()
+      return refuseResponse(res, 502, 'upstream request could not be built')
+    }
+    upstreamRequest.on('error', () => refuseResponse(res, 502, 'upstream failed'))
+    upstreamRequest.on('upgrade', (_response, socket) => {
+      socket.destroy()
+      refuseResponse(res, 502, 'upstream attempted a protocol upgrade')
+    })
+    res.once('close', () => upstreamRequest.destroy())
+    req.pipe(upstreamRequest)
+  }
+
+  const server = createServer()
+  server.timeout = idleTimeoutMs
+  server.on('connection', (socket) => track(socket))
+  server.on('request', (req, res) => {
+    void handleRequest(req, res).catch(() => refuseResponse(res, 502, 'proxy failure'))
+  })
+  server.on('connect', (req, socket, head) => {
+    void handleConnect(req, socket, head).catch(() => socket.destroy())
+  })
+  server.on('upgrade', (_req, socket) => {
+    socket.on('error', () => socket.destroy())
+    refuseSocket(socket, 400)
+  })
+  server.on('clientError', (_error, socket) => {
+    refuseSocket(socket, 400)
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    server.once('error', reject)
+    server.listen(0, '127.0.0.1', () => {
+      server.removeListener('error', reject)
+      resolve()
+    })
+  })
+  server.on('error', () => undefined)
+  const address = server.address()
+  if (!address || typeof address === 'string') {
+    server.close()
+    throw new Error('[internal] egress proxy did not bind a TCP port')
+  }
+
+  return {
+    url: `http://127.0.0.1:${address.port}`,
+    close: () =>
+      new Promise<void>((resolve) => {
+        closed = true
+        server.close(() => resolve())
+        for (const socket of sockets) socket.destroy()
+      }),
+  }
+}

package/src/playwright.ts

@@ -1,5 +1,5 @@
 import { spawn } from 'node:child_process'
-import { assertPublicUrl } from '@open-mercato/web-research'
+import { assertPublicUrl, type LookupFn } from '@open-mercato/web-research'
 import { SidecarError } from './protocol'
 
 /**
@@ -35,8 +35,14 @@ export type BrowserHandle = {
   close(): Promise<void>
 }
 
+export type LaunchOptions = {
+  headless?: boolean
+  args?: string[]
+  proxy?: { server: string; bypass?: string }
+}
+
 export type BrowserType = {
-  launch(options?: { headless?: boolean; args?: string[] }): Promise<BrowserHandle>
+  launch(options?: LaunchOptions): Promise<BrowserHandle>
 }
 
 function isModuleNotFound(message: string): boolean {
@@ -115,7 +121,7 @@ const defaultInstallRunner: InstallRunner = async (onProgress) =>
  */
 export async function launchChromium(
   browserType: BrowserType,
-  options: { headless?: boolean; args?: string[] },
+  options: LaunchOptions,
   deps: { install?: InstallRunner; onProgress?: (line: string) => void } = {},
 ): Promise<BrowserHandle> {
   try {
@@ -150,25 +156,34 @@ export async function launchChromium(
 
 const MAX_VERDICT_CACHE = 2048
 
+export type NavigationGuardOptions = {
+  /** @deprecated Every routed request is filtered now; this option is ignored. */
+  filterSubresources?: boolean
+  warn?: (message: string) => void
+  lookup?: LookupFn
+}
+
 /**
- * Blocks private-origin navigations for the life of the context.
+ * Aborts every routed request whose URL is not public, navigations and
+ * subresources alike, for the life of the context.
  *
- * The parent and the sidecar both vet the URL before a `goto`, but a page reached
- * through a legitimate public navigation can redirect or script-navigate itself
- * to a metadata endpoint, and those hops never pass through the RPC layer.
+ * This is an early-abort layer, not the egress boundary: `context.route()` never
+ * sees redirect hops, WebSockets, service workers or UDP, and it vets the name
+ * with `node:dns` while Chromium dials whatever its own resolver returns. The
+ * boundary is the pinned egress proxy every connection is forced through (see
+ * `egressProxy.ts`). Navigation verdicts are never cached; subresource verdicts
+ * are cached per host for the life of the context.
  *
- * Fails closed: Chromium resolves names with its own resolver, so a hostname
- * `node:dns` cannot vet must not be allowed through on the assumption it is dead.
+ * Fails closed: a context without request interception is refused, and a URL
+ * that cannot be parsed is aborted.
  */
 export async function installNavigationGuard(
   context: ContextHandle,
-  options: { filterSubresources?: boolean; warn?: (message: string) => void } = {},
+  options: NavigationGuardOptions = {},
 ): Promise<void> {
   if (typeof context.route !== 'function') {
-    options.warn?.('context.route() unavailable — in-page navigation SSRF guard NOT installed')
-    return
+    throw new SidecarError('context.route() is unavailable, so the request guard cannot be installed', 'init')
   }
-  const filterSubresources = options.filterSubresources ?? false
   const verdicts = new Map<string, boolean>()
   const remember = (host: string, blocked: boolean): void => {
     if (verdicts.size >= MAX_VERDICT_CACHE) {
@@ -181,14 +196,12 @@ export async function installNavigationGuard(
   await context.route('**/*', async (route) => {
     const request = route.request()
     const isNavigation = request.isNavigationRequest()
-    if (!isNavigation && !filterSubresources) return route.continue()
-
     const url = request.url()
     let host: string
     try {
       host = new URL(url).host
     } catch {
-      return isNavigation ? route.abort('blockedbyclient') : route.continue()
+      return route.abort('blockedbyclient')
     }
 
     const cached = isNavigation ? undefined : verdicts.get(host)
@@ -198,6 +211,7 @@ export async function installNavigationGuard(
     try {
       await assertPublicUrl(url, isNavigation ? 'browser navigation' : 'browser subresource', {
         failClosed: true,
+        ...(options.lookup ? { lookup: options.lookup } : {}),
       })
       remember(host, false)
     } catch {
@@ -208,6 +222,14 @@ export async function installNavigationGuard(
   })
 }
 
+/**
+ * Denies Chromium its own DNS, so any path that bypasses the egress proxy by
+ * hostname fails instead of resolving. The proxy address is a literal, so it is
+ * unaffected. Only meaningful alongside the egress proxy, so it is kept out of
+ * {@link LAUNCH_ARGS}.
+ */
+export const RESOLVER_LOCKDOWN_ARGS = ['--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1']
+
 /**
  * Launch flags aimed at not announcing automation. Playwright's defaults set
  * `navigator.webdriver` and a headless UA, which the SERP endpoints this adapter
@@ -215,12 +237,15 @@ export async function installNavigationGuard(
  *
  * `--disable-dev-shm-usage` is a sizing workaround, not a security control: it
  * makes Chromium write shared memory to /tmp instead of a container's small
- * /dev/shm. The sandbox is deliberately NOT disabled here — see
+ * /dev/shm. The WebRTC policy and `--disable-quic` keep UDP from leaving outside
+ * the egress proxy. The sandbox is deliberately NOT disabled here — see
  * {@link resolveLaunchArgs}.
  */
 export const LAUNCH_ARGS = [
   '--disable-blink-features=AutomationControlled',
   '--disable-dev-shm-usage',
+  '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+  '--disable-quic',
 ]
 
 export const SANDBOX_OPT_OUT_ENV = 'OM_WEB_RESEARCH_BROWSER_NO_SANDBOX'