@open-mercato/web-research-browser 0.9.1-develop.7548.1.ee60458c45 → 0.9.1-develop.7553.1.8dbab05b5a

package/src/__tests__/launchWiring.test.ts

@@ -0,0 +1,222 @@
+import { connect as netConnect } from 'node:net'
+import { createState, dispatch, teardown, type SidecarState } from '../dispatch'
+import * as egressProxyModule from '../egressProxy'
+import {
+  LAUNCH_ARGS,
+  RESOLVER_LOCKDOWN_ARGS,
+  type BrowserHandle,
+  type BrowserType,
+  type ContextHandle,
+  type LaunchOptions,
+  type RouteHandle,
+} from '../playwright'
+
+const FILTER_ENV = 'OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES'
+const PUBLIC_PAGE = 'https://93.184.216.34/'
+
+type FakeBrowserType = BrowserType & { launches: LaunchOptions[] }
+
+function fakeBrowser(): BrowserHandle {
+  const page = {
+    goto: async () => undefined,
+    content: async () => '<html></html>',
+    url: () => PUBLIC_PAGE,
+    close: async () => undefined,
+  }
+  const context: ContextHandle = {
+    newPage: async () => page,
+    close: async () => undefined,
+    route: async (_pattern: string, _handler: (route: RouteHandle) => Promise<void> | void) => undefined,
+  }
+  return { newContext: async () => context, close: async () => undefined }
+}
+
+function fakeBrowserType(launch?: (options: LaunchOptions) => Promise<BrowserHandle>): FakeBrowserType {
+  const type: FakeBrowserType = {
+    launches: [],
+    async launch(options = {}) {
+      type.launches.push(options)
+      if (launch) return launch(options)
+      await new Promise((resolve) => setTimeout(resolve, 5))
+      return fakeBrowser()
+    },
+  }
+  return type
+}
+
+function render(state: SidecarState, leaseId: string) {
+  return dispatch(state, { id: `render-${leaseId}`, method: 'render', params: { url: PUBLIC_PAGE, leaseId } })
+}
+
+function proxyAccepts(url: string): Promise<boolean> {
+  const { port } = new URL(url)
+  return new Promise((resolve) => {
+    const probe = netConnect({ host: '127.0.0.1', port: Number(port) })
+    probe.once('connect', () => {
+      probe.destroy()
+      resolve(true)
+    })
+    probe.once('error', () => resolve(false))
+  })
+}
+
+describe('browser launch wiring (#7218)', () => {
+  const savedFilter = process.env[FILTER_ENV]
+  const states: SidecarState[] = []
+
+  const stateWith = (overrides: Partial<SidecarState>): SidecarState => {
+    const state = createState({ warn: () => undefined, ...overrides })
+    states.push(state)
+    return state
+  }
+
+  beforeEach(() => {
+    delete process.env[FILTER_ENV]
+  })
+
+  afterEach(async () => {
+    jest.restoreAllMocks()
+    for (const state of states.splice(0)) await teardown(state)
+  })
+
+  afterAll(() => {
+    if (savedFilter === undefined) delete process.env[FILTER_ENV]
+    else process.env[FILTER_ENV] = savedFilter
+  })
+
+  it('launches Chromium through a live egress proxy with loopback not bypassed and UDP restricted', async () => {
+    const browserType = fakeBrowserType()
+    const state = stateWith({ loadBrowserType: async () => browserType })
+
+    expect((await render(state, 'lease-1')).ok).toBe(true)
+
+    expect(browserType.launches).toHaveLength(1)
+    const options = browserType.launches[0]
+    expect(options?.proxy).toEqual({ server: state.egressProxy?.url, bypass: '<-loopback>' })
+    expect(options?.proxy?.server).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/)
+    expect(await proxyAccepts(String(options?.proxy?.server))).toBe(true)
+    expect(options?.args).toEqual(
+      expect.arrayContaining([
+        '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+        '--disable-quic',
+        ...RESOLVER_LOCKDOWN_ARGS,
+      ]),
+    )
+    expect(LAUNCH_ARGS).not.toEqual(expect.arrayContaining(RESOLVER_LOCKDOWN_ARGS))
+  })
+
+  it('launches again on the next render after a failed launch', async () => {
+    let attempts = 0
+    const browserType = fakeBrowserType(async () => {
+      attempts += 1
+      if (attempts === 1) throw new Error('Target page crashed')
+      return fakeBrowser()
+    })
+    const state = stateWith({ loadBrowserType: async () => browserType })
+
+    expect((await render(state, 'lease-1')).ok).toBe(false)
+    expect((await render(state, 'lease-2')).ok).toBe(true)
+
+    expect(browserType.launches).toHaveLength(2)
+    expect(await proxyAccepts(String(state.egressProxy?.url))).toBe(true)
+  })
+
+  it('closes the proxy it tore down, not one a concurrent render started meanwhile', async () => {
+    let releaseClose: () => void = () => undefined
+    const closing = new Promise<void>((resolve) => {
+      releaseClose = resolve
+    })
+    let launches = 0
+    const browserType = fakeBrowserType(async () => {
+      launches += 1
+      const browser = fakeBrowser()
+      if (launches === 1) return { ...browser, close: () => closing }
+      return browser
+    })
+    const state = stateWith({ loadBrowserType: async () => browserType })
+    await render(state, 'lease-1')
+    const firstProxy = String(state.egressProxy?.url)
+
+    const tearingDown = teardown(state)
+    await new Promise((resolve) => setTimeout(resolve, 5))
+    expect((await render(state, 'lease-2')).ok).toBe(true)
+    const secondProxy = String(state.egressProxy?.url)
+    releaseClose()
+    await tearingDown
+
+    expect(secondProxy).not.toBe(firstProxy)
+    expect(await proxyAccepts(firstProxy)).toBe(false)
+    expect(await proxyAccepts(secondProxy)).toBe(true)
+  })
+
+  it('does not launch when the egress proxy cannot start', async () => {
+    jest.spyOn(egressProxyModule, 'startEgressProxy').mockRejectedValueOnce(new Error('EADDRNOTAVAIL'))
+    const browserType = fakeBrowserType()
+    const state = stateWith({ loadBrowserType: async () => browserType })
+
+    const reply = await render(state, 'lease-1')
+
+    expect(reply.ok).toBe(false)
+    expect(browserType.launches).toHaveLength(0)
+    expect(state.browser).toBeNull()
+  })
+
+  it('closes the egress proxy when the launch fails', async () => {
+    const browserType = fakeBrowserType(async () => {
+      throw new Error('Target page crashed')
+    })
+    const state = stateWith({ loadBrowserType: async () => browserType })
+
+    const reply = await render(state, 'lease-1')
+
+    expect(reply.ok).toBe(false)
+    const server = browserType.launches[0]?.proxy?.server
+    expect(server).toBeDefined()
+    expect(await proxyAccepts(String(server))).toBe(false)
+    expect(state.egressProxy ?? null).toBeNull()
+  })
+
+  it('closes the egress proxy on teardown', async () => {
+    const state = stateWith({ loadBrowserType: async () => fakeBrowserType() })
+    await render(state, 'lease-1')
+    const server = String(state.egressProxy?.url)
+    expect(await proxyAccepts(server)).toBe(true)
+
+    await teardown(state)
+
+    expect(await proxyAccepts(server)).toBe(false)
+    expect(state.egressProxy).toBeNull()
+  })
+
+  it('launches once when two leases open concurrently', async () => {
+    const browserType = fakeBrowserType()
+    const state = stateWith({ loadBrowserType: async () => browserType })
+
+    const replies = await Promise.all([render(state, 'lease-a'), render(state, 'lease-b')])
+
+    expect(replies.map((reply) => reply.ok)).toEqual([true, true])
+    expect(browserType.launches).toHaveLength(1)
+  })
+
+  it.each(['0', '1'])('warns that the filter env var set to %s is ignored', async (value) => {
+    process.env[FILTER_ENV] = value
+    const warnings: string[] = []
+    const state = stateWith({ loadBrowserType: async () => fakeBrowserType(), warn: (line) => warnings.push(line) })
+
+    await render(state, 'lease-1')
+    await render(state, 'lease-2')
+
+    const notices = warnings.filter((line) => line.includes(FILTER_ENV))
+    expect(notices).toHaveLength(1)
+    expect(notices[0]).toContain('always on')
+  })
+
+  it('does not warn about the filter env var when it is unset', async () => {
+    const warnings: string[] = []
+    const state = stateWith({ loadBrowserType: async () => fakeBrowserType(), warn: (line) => warnings.push(line) })
+
+    await render(state, 'lease-1')
+
+    expect(warnings.filter((line) => line.includes(FILTER_ENV))).toEqual([])
+  })
+})

package/src/__tests__/navigationGuard.test.ts

@@ -0,0 +1,200 @@
+import type { LookupFn } from '@open-mercato/web-research'
+import { createState, dispatch } from '../dispatch'
+import { installNavigationGuard, type BrowserHandle, type ContextHandle, type RouteHandle } from '../playwright'
+import { SidecarError } from '../protocol'
+
+type RouteHandler = (route: RouteHandle) => Promise<void> | void
+
+type GuardedContext = ContextHandle & {
+  handler(): RouteHandler | undefined
+}
+
+const FILTER_ENV = 'OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES'
+const PUBLIC_PAGE = 'https://93.184.216.34/'
+const METADATA = 'http://169.254.169.254/latest/meta-data/'
+const LOOPBACK = 'http://127.0.0.1:5432/'
+
+function fakeContext(): GuardedContext {
+  let routeHandler: RouteHandler | undefined
+  return {
+    newPage: async () => ({}),
+    close: async () => undefined,
+    route: async (_pattern: string, handler: RouteHandler) => {
+      routeHandler = handler
+    },
+    handler: () => routeHandler,
+  }
+}
+
+async function settle(handler: RouteHandler | undefined, url: string, isNavigation: boolean): Promise<string> {
+  if (!handler) throw new Error('[internal] no route handler was installed')
+  let outcome = 'pending'
+  await handler({
+    request: () => ({ url: () => url, isNavigationRequest: () => isNavigation }),
+    abort: async () => {
+      outcome = 'abort'
+    },
+    continue: async () => {
+      outcome = 'continue'
+    },
+  })
+  return outcome
+}
+
+type RenderFixture = {
+  browser: BrowserHandle
+  context: GuardedContext
+  contextOptions: Array<Record<string, unknown> | undefined>
+  subresourceOutcomes: string[]
+  contentReads: () => number
+}
+
+function renderFixture(finalUrl: string = PUBLIC_PAGE): RenderFixture {
+  const contextOptions: Array<Record<string, unknown> | undefined> = []
+  const context = fakeContext()
+  const subresourceOutcomes: string[] = []
+  let reads = 0
+  const page = {
+    goto: async () => {
+      subresourceOutcomes.push(await settle(context.handler(), METADATA, false))
+    },
+    content: async () => {
+      reads += 1
+      return '<html><body>page</body></html>'
+    },
+    url: () => finalUrl,
+    close: async () => undefined,
+  }
+  context.newPage = async () => page
+  const browser: BrowserHandle = {
+    newContext: async (options) => {
+      contextOptions.push(options)
+      return context
+    },
+    close: async () => undefined,
+  }
+  return { browser, context, contextOptions, subresourceOutcomes, contentReads: () => reads }
+}
+
+describe('browser request guard (#7218)', () => {
+  const savedFilter = process.env[FILTER_ENV]
+
+  beforeEach(() => {
+    delete process.env[FILTER_ENV]
+  })
+
+  afterAll(() => {
+    if (savedFilter === undefined) delete process.env[FILTER_ENV]
+    else process.env[FILTER_ENV] = savedFilter
+  })
+
+  it('still blocks a private navigation and allows a public one', async () => {
+    const context = fakeContext()
+    await installNavigationGuard(context)
+
+    expect(await settle(context.handler(), METADATA, true)).toBe('abort')
+    expect(await settle(context.handler(), PUBLIC_PAGE, true)).toBe('continue')
+  })
+
+  it('blocks private subresources with default options', async () => {
+    const context = fakeContext()
+    await installNavigationGuard(context)
+
+    expect(await settle(context.handler(), METADATA, false)).toBe('abort')
+    expect(await settle(context.handler(), LOOPBACK, false)).toBe('abort')
+    expect(await settle(context.handler(), 'https://93.184.216.34/app.js', false)).toBe('continue')
+  })
+
+  it('ignores the deprecated filterSubresources: false option', async () => {
+    const context = fakeContext()
+    await installNavigationGuard(context, { filterSubresources: false })
+
+    expect(await settle(context.handler(), METADATA, false)).toBe('abort')
+  })
+
+  it('blocks a private subresource of a rendered page in the default sidecar configuration', async () => {
+    const fixture = renderFixture()
+    const state = createState({ browser: fixture.browser, warn: () => undefined })
+
+    const reply = await dispatch(state, {
+      id: 'render-1',
+      method: 'render',
+      params: { url: PUBLIC_PAGE, leaseId: 'lease-1' },
+    })
+
+    expect(reply.ok).toBe(true)
+    expect(fixture.subresourceOutcomes).toEqual(['abort'])
+    expect(fixture.contextOptions[0]).toMatchObject({ serviceWorkers: 'block' })
+  })
+
+  it.each(['0', '1'])('ignores the filter env var set to %s and still filters subresources', async (value) => {
+    process.env[FILTER_ENV] = value
+    const fixture = renderFixture()
+    const state = createState({ browser: fixture.browser, warn: () => undefined })
+
+    const reply = await dispatch(state, {
+      id: 'render-env',
+      method: 'render',
+      params: { url: PUBLIC_PAGE, leaseId: 'lease-env' },
+    })
+
+    expect(reply.ok).toBe(true)
+    expect(fixture.subresourceOutcomes).toEqual(['abort'])
+  })
+
+  it('refuses a render whose page ended on a non-public URL and returns no html', async () => {
+    const fixture = renderFixture('http://127.0.0.1:8080/admin')
+    const state = createState({ browser: fixture.browser, warn: () => undefined })
+
+    const reply = await dispatch(state, {
+      id: 'render-final',
+      method: 'render',
+      params: { url: PUBLIC_PAGE, leaseId: 'lease-final' },
+    })
+
+    expect(reply.ok).toBe(false)
+    if (reply.ok) return
+    expect(reply.error.kind).toBe('navigation')
+    expect(JSON.stringify(reply)).not.toContain('<html')
+    expect(fixture.contentReads()).toBe(0)
+  })
+
+  it('aborts a request whose URL cannot be parsed instead of letting it through', async () => {
+    const context = fakeContext()
+    await installNavigationGuard(context)
+
+    expect(await settle(context.handler(), 'http://[::1', false)).toBe('abort')
+    expect(await settle(context.handler(), 'http://[::1', true)).toBe('abort')
+  })
+
+  it('fails closed when request interception cannot be installed', async () => {
+    const context: ContextHandle = { newPage: async () => ({}), close: async () => undefined }
+
+    const error = await installNavigationGuard(context, { warn: () => undefined }).catch((caught: unknown) => caught)
+
+    expect(error).toBeInstanceOf(SidecarError)
+    expect((error as SidecarError).kind).toBe('init')
+  })
+
+  it('caches a blocked subresource verdict even if the name later resolves publicly', async () => {
+    const answers = ['10.0.0.7', '93.184.216.34']
+    const lookup: LookupFn = jest.fn(async () => [{ address: answers.shift() ?? '93.184.216.34', family: 4 }])
+    const context = fakeContext()
+    await installNavigationGuard(context, { lookup })
+
+    expect(await settle(context.handler(), 'https://flip.test/a.js', false)).toBe('abort')
+    expect(await settle(context.handler(), 'https://flip.test/b.js', false)).toBe('abort')
+    expect(lookup).toHaveBeenCalledTimes(1)
+  })
+
+  it('re-vets navigations instead of caching them', async () => {
+    const answers = ['93.184.216.34', '10.0.0.7']
+    const lookup: LookupFn = jest.fn(async () => [{ address: answers.shift() ?? '10.0.0.7', family: 4 }])
+    const context = fakeContext()
+    await installNavigationGuard(context, { lookup })
+
+    expect(await settle(context.handler(), 'https://flip.test/', true)).toBe('continue')
+    expect(await settle(context.handler(), 'https://flip.test/', true)).toBe('abort')
+    expect(lookup).toHaveBeenCalledTimes(2)
+  })
+})

package/src/dispatch.ts

@@ -1,6 +1,8 @@
-import { assertPublicUrl } from '@open-mercato/web-research'
+import { assertPublicUrl, type LookupFn } from '@open-mercato/web-research'
+import { startEgressProxy, type EgressConnect, type EgressProxy } from './egressProxy'
 import {
   DEFAULT_USER_AGENT,
+  RESOLVER_LOCKDOWN_ARGS,
   isSandboxDisabled,
   resolveLaunchArgs,
   importChromium,
@@ -8,6 +10,7 @@ import {
   launchChromium,
   type InstallRunner,
   type BrowserHandle,
+  type BrowserType,
   type ContextHandle,
   type PageHandle,
 } from './playwright'
@@ -27,14 +30,25 @@ export const LEASE_IDLE_TIMEOUT_MS = 5 * 60_000
 export type SidecarState = {
   browser: BrowserHandle | null
   leases: Map<string, Lease>
+  /** @deprecated Subresource filtering is always on; this field is ignored. */
   filterSubresources: boolean
   warn: (message: string) => void
   /** Injectable so a lease-expiry test does not have to wait five minutes. */
   now: () => number
   /** Injectable so tests never shell out to `npx playwright install`. */
   installRunner?: InstallRunner
+  /** Injectable DNS for the URL checks and the egress proxy, so tests never resolve real names. */
+  lookup?: LookupFn
+  /** Injectable dial for the egress proxy, so tests can stand in for public hosts. */
+  connect?: EgressConnect
+  /** Injectable so tests never import Playwright. */
+  loadBrowserType?: () => Promise<BrowserType>
+  egressProxy?: EgressProxy | null
+  launching?: Promise<BrowserHandle> | null
 }
 
+const FILTER_SUBRESOURCES_ENV = 'OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES'
+
 export const DEFAULT_GOTO_TIMEOUT_MS = 30_000
 const MAX_GOTO_TIMEOUT_MS = 120_000
 
@@ -42,16 +56,19 @@ export function createState(overrides: Partial<SidecarState> = {}): SidecarState
   return {
     browser: null,
     leases: new Map(),
-    filterSubresources: process.env.OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES === '1',
+    filterSubresources: true,
     warn: (message) => process.stderr.write(`om-web-research-sidecar: ${message}\n`),
     now: () => Date.now(),
     ...overrides,
   }
 }
 
-async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
-  if (state.browser) return state.browser
-  const chromium = await importChromium()
+function lookupOption(state: SidecarState): { lookup?: LookupFn } {
+  return state.lookup ? { lookup: state.lookup } : {}
+}
+
+async function launchBrowser(state: SidecarState): Promise<BrowserHandle> {
+  const browserType = await (state.loadBrowserType ?? importChromium)()
   if (isSandboxDisabled()) {
     // Said once per browser, not once per render, but said: this process renders
     // pages chosen by a search engine or a model with the one mitigation
@@ -60,12 +77,45 @@ async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
       'SECURITY: Chromium sandbox is disabled by OM_WEB_RESEARCH_BROWSER_NO_SANDBOX=1 — a renderer compromise reaches this host',
     )
   }
-  state.browser = await launchChromium(
-    chromium,
-    { headless: true, args: resolveLaunchArgs() },
-    { onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },
-  )
-  return state.browser
+  if (process.env[FILTER_SUBRESOURCES_ENV] !== undefined) {
+    state.warn(`subresource filtering is now always on; ${FILTER_SUBRESOURCES_ENV} is ignored`)
+  }
+  const proxy = await startEgressProxy({
+    ...lookupOption(state),
+    ...(state.connect ? { connect: state.connect } : {}),
+  })
+  let browser: BrowserHandle
+  try {
+    browser = await launchChromium(
+      browserType,
+      {
+        headless: true,
+        args: [...resolveLaunchArgs(), ...RESOLVER_LOCKDOWN_ARGS],
+        proxy: { server: proxy.url, bypass: '<-loopback>' },
+      },
+      { onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },
+    )
+  } catch (error) {
+    await proxy.close()
+    throw error
+  }
+  state.egressProxy = proxy
+  state.browser = browser
+  return browser
+}
+
+/**
+ * Single-flight: two leases opening at once share one launch, so neither a
+ * browser nor its egress proxy is orphaned outside `teardown`.
+ */
+async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
+  if (state.browser) return state.browser
+  if (!state.launching) {
+    state.launching = launchBrowser(state).finally(() => {
+      state.launching = null
+    })
+  }
+  return state.launching
 }
 
 /**
@@ -81,8 +131,8 @@ async function openLease(state: SidecarState, leaseId: string, userAgent: string
     return existing
   }
   const browser = await ensureBrowser(state)
-  const context = await browser.newContext({ userAgent, locale: 'en-US' })
-  await installNavigationGuard(context, { filterSubresources: state.filterSubresources, warn: state.warn })
+  const context = await browser.newContext({ userAgent, locale: 'en-US', serviceWorkers: 'block' })
+  await installNavigationGuard(context, { warn: state.warn, ...lookupOption(state) })
   const page = (await context.newPage()) as unknown as PageHandle
   const lease: Lease = { context, page, touchedAt: state.now() }
   state.leases.set(leaseId, lease)
@@ -111,9 +161,12 @@ async function closeLease(state: SidecarState, leaseId: string): Promise<void> {
 }
 
 export async function teardown(state: SidecarState): Promise<void> {
+  if (state.launching) await state.launching.catch(() => null)
   for (const leaseId of [...state.leases.keys()]) await closeLease(state, leaseId)
   const browser = state.browser
+  const proxy = state.egressProxy
   state.browser = null
+  state.egressProxy = null
   if (browser) {
     try {
       await browser.close()
@@ -121,6 +174,7 @@ export async function teardown(state: SidecarState): Promise<void> {
       // Best effort; the parent escalates to SIGKILL if we hang.
     }
   }
+  if (proxy) await proxy.close()
 }
 
 function stringParam(request: SidecarRequest, key: string): string {
@@ -142,7 +196,7 @@ async function dispatchInner(state: SidecarState, request: SidecarRequest): Prom
       // Re-checked here, in a distinct process from the caller, and BEFORE
       // Playwright is imported — a blocked URL never launches a browser.
       try {
-        await assertPublicUrl(url, 'browser render', { failClosed: true })
+        await assertPublicUrl(url, 'browser render', { failClosed: true, ...lookupOption(state) })
       } catch (error) {
         return {
           id: request.id,
@@ -168,8 +222,18 @@ async function dispatchInner(state: SidecarState, request: SidecarRequest): Prom
           error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
         }
       }
+      const finalUrl = lease.page.url()
+      try {
+        await assertPublicUrl(finalUrl, 'browser final URL', { failClosed: true, ...lookupOption(state) })
+      } catch (error) {
+        return {
+          id: request.id,
+          ok: false,
+          error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
+        }
+      }
       const html = await lease.page.content()
-      return { id: request.id, ok: true, result: { url: lease.page.url(), html } }
+      return { id: request.id, ok: true, result: { url: finalUrl, html } }
     }
 
     case 'release':