@open-mercato/web-research-browser 0.7.1-develop.7194.1.ab4fc81f82

package/src/dispatch.ts

@@ -0,0 +1,205 @@
+import { assertPublicUrl } from '@open-mercato/web-research'
+import {
+  DEFAULT_USER_AGENT,
+  isSandboxDisabled,
+  resolveLaunchArgs,
+  importChromium,
+  installNavigationGuard,
+  launchChromium,
+  type InstallRunner,
+  type BrowserHandle,
+  type ContextHandle,
+  type PageHandle,
+} from './playwright'
+import { SidecarError, type SidecarReply, type SidecarRequest } from './protocol'
+
+export type Lease = { context: ContextHandle; page: PageHandle; touchedAt: number }
+
+/**
+ * How long an untouched lease may live.
+ *
+ * `release` normally closes a context, but the caller can die between `render`
+ * and `release` — a killed parent, a dropped socket — and every orphan is a live
+ * BrowserContext holding memory for as long as the sidecar runs.
+ */
+export const LEASE_IDLE_TIMEOUT_MS = 5 * 60_000
+
+export type SidecarState = {
+  browser: BrowserHandle | null
+  leases: Map<string, Lease>
+  filterSubresources: boolean
+  warn: (message: string) => void
+  /** Injectable so a lease-expiry test does not have to wait five minutes. */
+  now: () => number
+  /** Injectable so tests never shell out to `npx playwright install`. */
+  installRunner?: InstallRunner
+}
+
+export const DEFAULT_GOTO_TIMEOUT_MS = 30_000
+const MAX_GOTO_TIMEOUT_MS = 120_000
+
+export function createState(overrides: Partial<SidecarState> = {}): SidecarState {
+  return {
+    browser: null,
+    leases: new Map(),
+    filterSubresources: process.env.OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES === '1',
+    warn: (message) => process.stderr.write(`om-web-research-sidecar: ${message}\n`),
+    now: () => Date.now(),
+    ...overrides,
+  }
+}
+
+async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
+  if (state.browser) return state.browser
+  const chromium = await importChromium()
+  if (isSandboxDisabled()) {
+    // Said once per browser, not once per render, but said: this process renders
+    // pages chosen by a search engine or a model with the one mitigation
+    // Chromium offers against a hostile renderer turned off.
+    state.warn(
+      'SECURITY: Chromium sandbox is disabled by OM_WEB_RESEARCH_BROWSER_NO_SANDBOX=1 — a renderer compromise reaches this host',
+    )
+  }
+  state.browser = await launchChromium(
+    chromium,
+    { headless: true, args: resolveLaunchArgs() },
+    { onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },
+  )
+  return state.browser
+}
+
+/**
+ * One BrowserContext per lease. Contexts are Playwright's real isolation boundary
+ * (cookies, storage, cache) and cost ~10ms, so two concurrent searches never share
+ * a cookie jar the way a single shared page would.
+ */
+async function openLease(state: SidecarState, leaseId: string, userAgent: string): Promise<Lease> {
+  await reapIdleLeases(state)
+  const existing = state.leases.get(leaseId)
+  if (existing) {
+    existing.touchedAt = state.now()
+    return existing
+  }
+  const browser = await ensureBrowser(state)
+  const context = await browser.newContext({ userAgent, locale: 'en-US' })
+  await installNavigationGuard(context, { filterSubresources: state.filterSubresources, warn: state.warn })
+  const page = (await context.newPage()) as unknown as PageHandle
+  const lease: Lease = { context, page, touchedAt: state.now() }
+  state.leases.set(leaseId, lease)
+  return lease
+}
+
+/** Closes leases nobody released. Swept on open, so an idle sidecar stays idle. */
+export async function reapIdleLeases(state: SidecarState): Promise<void> {
+  const cutoff = state.now() - LEASE_IDLE_TIMEOUT_MS
+  for (const [leaseId, lease] of [...state.leases]) {
+    if (lease.touchedAt > cutoff) continue
+    state.warn(`reaping lease ${leaseId} idle for over ${LEASE_IDLE_TIMEOUT_MS}ms`)
+    await closeLease(state, leaseId)
+  }
+}
+
+async function closeLease(state: SidecarState, leaseId: string): Promise<void> {
+  const lease = state.leases.get(leaseId)
+  if (!lease) return
+  state.leases.delete(leaseId)
+  try {
+    await lease.context.close()
+  } catch {
+    // A context that already died takes nothing with it.
+  }
+}
+
+export async function teardown(state: SidecarState): Promise<void> {
+  for (const leaseId of [...state.leases.keys()]) await closeLease(state, leaseId)
+  const browser = state.browser
+  state.browser = null
+  if (browser) {
+    try {
+      await browser.close()
+    } catch {
+      // Best effort; the parent escalates to SIGKILL if we hang.
+    }
+  }
+}
+
+function stringParam(request: SidecarRequest, key: string): string {
+  const value = request.params?.[key]
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new SidecarError(`missing "${key}" parameter`, 'runtime')
+  }
+  return value
+}
+
+async function dispatchInner(state: SidecarState, request: SidecarRequest): Promise<SidecarReply> {
+  switch (request.method) {
+    case 'ping':
+      return { id: request.id, ok: true, result: { ready: true } }
+
+    case 'render': {
+      const url = stringParam(request, 'url')
+      const leaseId = stringParam(request, 'leaseId')
+      // Re-checked here, in a distinct process from the caller, and BEFORE
+      // Playwright is imported — a blocked URL never launches a browser.
+      try {
+        await assertPublicUrl(url, 'browser render', { failClosed: true })
+      } catch (error) {
+        return {
+          id: request.id,
+          ok: false,
+          error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
+        }
+      }
+
+      const userAgent = typeof request.params?.userAgent === 'string' ? request.params.userAgent : DEFAULT_USER_AGENT
+      const timeout = Math.min(
+        typeof request.params?.timeoutMs === 'number' ? request.params.timeoutMs : DEFAULT_GOTO_TIMEOUT_MS,
+        MAX_GOTO_TIMEOUT_MS,
+      )
+      const waitUntil = typeof request.params?.waitUntil === 'string' ? request.params.waitUntil : 'domcontentloaded'
+
+      const lease = await openLease(state, leaseId, userAgent)
+      try {
+        await lease.page.goto(url, { waitUntil, timeout })
+      } catch (error) {
+        return {
+          id: request.id,
+          ok: false,
+          error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
+        }
+      }
+      const html = await lease.page.content()
+      return { id: request.id, ok: true, result: { url: lease.page.url(), html } }
+    }
+
+    case 'release':
+      await closeLease(state, stringParam(request, 'leaseId'))
+      return { id: request.id, ok: true }
+
+    case 'close':
+      await teardown(state)
+      return { id: request.id, ok: true }
+
+    default:
+      return {
+        id: request.id,
+        ok: false,
+        error: { message: `unknown method: ${request.method}`, kind: 'runtime' },
+      }
+  }
+}
+
+export async function dispatch(state: SidecarState, request: SidecarRequest): Promise<SidecarReply> {
+  try {
+    return await dispatchInner(state, request)
+  } catch (error) {
+    return {
+      id: request.id,
+      ok: false,
+      error: {
+        message: error instanceof Error ? error.message : String(error),
+        kind: error instanceof SidecarError ? error.kind : 'unknown',
+      },
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export {
+  browserAdapterModule,
+  browserOptionsSchema,
+  createBrowserAdapter,
+  type BrowserOptions,
+} from './adapter'
+export { Sidecar, createSidecar, type SidecarOptions, type SidecarStream, type SpawnFn } from './client'
+export { SidecarError, type SidecarErrorKind, type SidecarReply, type SidecarRequest } from './protocol'
+
+export { browserAdapterModule as default } from './adapter'

package/src/playwright.ts

@@ -0,0 +1,265 @@
+import { spawn } from 'node:child_process'
+import { assertPublicUrl } from '@open-mercato/web-research'
+import { SidecarError } from './protocol'
+
+/**
+ * Structural projections of the Playwright surface we touch.
+ *
+ * Playwright is a real dependency of this package — installing the adapter gets
+ * you the tooling it needs — but it is still reached through a dynamic import
+ * inside the sidecar, so the app and MCP processes never load it. The types stay
+ * structural for the same reason: nothing here should pull the SDK into a bundle
+ * that only ever spawns the child.
+ */
+export type PageHandle = {
+  goto(url: string, options?: { waitUntil?: string; timeout?: number }): Promise<unknown>
+  content(): Promise<string>
+  url(): string
+  close(): Promise<void>
+}
+
+export type ContextHandle = {
+  newPage(): Promise<unknown>
+  close(): Promise<void>
+  route?(pattern: string, handler: (route: RouteHandle) => Promise<void> | void): Promise<void>
+}
+
+export type RouteHandle = {
+  request(): { url(): string; isNavigationRequest(): boolean }
+  abort(reason?: string): Promise<void>
+  continue(): Promise<void>
+}
+
+export type BrowserHandle = {
+  newContext(options?: Record<string, unknown>): Promise<ContextHandle>
+  close(): Promise<void>
+}
+
+export type BrowserType = {
+  launch(options?: { headless?: boolean; args?: string[] }): Promise<BrowserHandle>
+}
+
+function isModuleNotFound(message: string): boolean {
+  return (
+    /cannot find (package|module) ['"]?playwright/i.test(message) ||
+    /ERR_MODULE_NOT_FOUND/i.test(message) ||
+    /failed to resolve ['"]?playwright/i.test(message)
+  )
+}
+
+export async function importChromium(): Promise<BrowserType> {
+  try {
+    const playwright = (await import('playwright')) as unknown as { chromium: BrowserType }
+    return playwright.chromium
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    // Playwright is a dependency of this package, so a module-not-found here
+    // means a broken install (hoisting, a pruned image layer), not a missing opt-in.
+    throw new SidecarError(
+      `Playwright could not be loaded even though it is a dependency of this package — reinstall dependencies. Underlying: ${message}`,
+      isModuleNotFound(message) ? 'needs-install' : 'init',
+    )
+  }
+}
+
+/**
+ * Playwright's launch error when the npm package is installed but the browser
+ * binary was never downloaded. The wording has been stable across 1.x, and it is
+ * the only signal Playwright gives for this case.
+ */
+function isMissingBrowserError(error: unknown): boolean {
+  return error instanceof Error && /Executable doesn'?t exist at/i.test(error.message)
+}
+
+/** Generous, but bounded — an unbounded install hangs the sidecar forever. */
+const INSTALL_TIMEOUT_MS = 10 * 60_000
+
+export type InstallRunner = (onProgress: (line: string) => void) => Promise<void>
+
+const defaultInstallRunner: InstallRunner = async (onProgress) =>
+  new Promise((resolve, reject) => {
+    const child = spawn('npx', ['playwright', 'install', 'chromium'], { stdio: ['ignore', 'pipe', 'pipe'] })
+    let tail = ''
+    const collect = (chunk: Buffer): void => {
+      tail = `${tail}${chunk.toString('utf8')}`.slice(-4000)
+    }
+    child.stdout?.on('data', collect)
+    child.stderr?.on('data', collect)
+
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL')
+      reject(new Error(`\`npx playwright install chromium\` exceeded ${INSTALL_TIMEOUT_MS}ms`))
+    }, INSTALL_TIMEOUT_MS)
+
+    child.once('error', (error) => {
+      clearTimeout(timer)
+      reject(error)
+    })
+    child.once('close', (code) => {
+      clearTimeout(timer)
+      if (code === 0) {
+        onProgress('chromium install complete')
+        resolve()
+        return
+      }
+      reject(new Error(`\`npx playwright install chromium\` failed (exit ${code}): ${tail.trim()}`))
+    })
+  })
+
+/**
+ * Launches Chromium, downloading the binary once if it is missing.
+ *
+ * Installing the npm package does not fetch the ~150MB browser, so an operator
+ * who enables this tier would otherwise hit a launch failure with no obvious
+ * remedy. One retry only — a second failure is a real fault, not a cold cache.
+ */
+export async function launchChromium(
+  browserType: BrowserType,
+  options: { headless?: boolean; args?: string[] },
+  deps: { install?: InstallRunner; onProgress?: (line: string) => void } = {},
+): Promise<BrowserHandle> {
+  try {
+    return await browserType.launch(options)
+  } catch (error) {
+    // Deliberately not retried without the sandbox. Falling back silently would
+    // mean the security posture depends on whether a container happened to be
+    // configured, with nobody told which one they got.
+    if (isSandboxFailure(error)) {
+      throw new SidecarError(
+        `Chromium could not start its sandbox, and this adapter renders untrusted pages so it will not run without one. ` +
+          `Fix the container (run as a non-root user, or grant SYS_ADMIN, or enable user namespaces), ` +
+          `or accept the risk explicitly with ${SANDBOX_OPT_OUT_ENV}=1. Underlying: ${error instanceof Error ? error.message : String(error)}`,
+        'init',
+      )
+    }
+    if (!isMissingBrowserError(error)) throw error
+    const onProgress = deps.onProgress ?? (() => {})
+    onProgress('chromium binary missing, running `npx playwright install chromium` (one-time, ~150MB)')
+    try {
+      await (deps.install ?? defaultInstallRunner)(onProgress)
+    } catch (installError) {
+      const detail = installError instanceof Error ? installError.message : String(installError)
+      throw new SidecarError(
+        `Playwright browser auto-install failed: ${detail}. Run \`npx playwright install chromium\` manually.`,
+        'init',
+      )
+    }
+    return await browserType.launch(options)
+  }
+}
+
+const MAX_VERDICT_CACHE = 2048
+
+/**
+ * Blocks private-origin navigations for the life of the context.
+ *
+ * The parent and the sidecar both vet the URL before a `goto`, but a page reached
+ * through a legitimate public navigation can redirect or script-navigate itself
+ * to a metadata endpoint, and those hops never pass through the RPC layer.
+ *
+ * Fails closed: Chromium resolves names with its own resolver, so a hostname
+ * `node:dns` cannot vet must not be allowed through on the assumption it is dead.
+ */
+export async function installNavigationGuard(
+  context: ContextHandle,
+  options: { filterSubresources?: boolean; warn?: (message: string) => void } = {},
+): Promise<void> {
+  if (typeof context.route !== 'function') {
+    options.warn?.('context.route() unavailable — in-page navigation SSRF guard NOT installed')
+    return
+  }
+  const filterSubresources = options.filterSubresources ?? false
+  const verdicts = new Map<string, boolean>()
+  const remember = (host: string, blocked: boolean): void => {
+    if (verdicts.size >= MAX_VERDICT_CACHE) {
+      const oldest = verdicts.keys().next().value
+      if (oldest !== undefined) verdicts.delete(oldest)
+    }
+    verdicts.set(host, blocked)
+  }
+
+  await context.route('**/*', async (route) => {
+    const request = route.request()
+    const isNavigation = request.isNavigationRequest()
+    if (!isNavigation && !filterSubresources) return route.continue()
+
+    const url = request.url()
+    let host: string
+    try {
+      host = new URL(url).host
+    } catch {
+      return isNavigation ? route.abort('blockedbyclient') : route.continue()
+    }
+
+    const cached = isNavigation ? undefined : verdicts.get(host)
+    if (cached === true) return route.abort('blockedbyclient')
+    if (cached === false) return route.continue()
+
+    try {
+      await assertPublicUrl(url, isNavigation ? 'browser navigation' : 'browser subresource', {
+        failClosed: true,
+      })
+      remember(host, false)
+    } catch {
+      remember(host, true)
+      return route.abort('blockedbyclient')
+    }
+    return route.continue()
+  })
+}
+
+/**
+ * Launch flags aimed at not announcing automation. Playwright's defaults set
+ * `navigator.webdriver` and a headless UA, which the SERP endpoints this adapter
+ * exists to reach specifically look for.
+ *
+ * `--disable-dev-shm-usage` is a sizing workaround, not a security control: it
+ * makes Chromium write shared memory to /tmp instead of a container's small
+ * /dev/shm. The sandbox is deliberately NOT disabled here — see
+ * {@link resolveLaunchArgs}.
+ */
+export const LAUNCH_ARGS = [
+  '--disable-blink-features=AutomationControlled',
+  '--disable-dev-shm-usage',
+]
+
+export const SANDBOX_OPT_OUT_ENV = 'OM_WEB_RESEARCH_BROWSER_NO_SANDBOX'
+
+/**
+ * Launch flags for this process, sandbox included unless an operator opted out.
+ *
+ * This adapter exists to render pages chosen by a search engine or a model, so
+ * the renderer is by definition parsing hostile input. `--no-sandbox` turns a
+ * renderer compromise into code running as whatever user the app runs as, which
+ * is the one mitigation Chromium provides for exactly this threat.
+ *
+ * It used to be on unconditionally because the sandbox needs privileges a
+ * root-in-Docker container does not grant by default. That is a container
+ * problem with container fixes (run as a non-root user, or grant `SYS_ADMIN`, or
+ * enable user namespaces); silently shipping every deployment the insecure
+ * variant to spare those is the wrong trade. The opt-out stays available for
+ * environments that genuinely cannot, and it is loud.
+ */
+export function resolveLaunchArgs(env: NodeJS.ProcessEnv = process.env): string[] {
+  return env[SANDBOX_OPT_OUT_ENV] === '1' ? [...LAUNCH_ARGS, '--no-sandbox'] : [...LAUNCH_ARGS]
+}
+
+export function isSandboxDisabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  return env[SANDBOX_OPT_OUT_ENV] === '1'
+}
+
+/**
+ * Chromium's refusal when it cannot set up its sandbox. The wording differs
+ * between the root check and the namespace failure, so both shapes are matched.
+ */
+export function isSandboxFailure(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+  return (
+    /running as root without --no-sandbox/i.test(error.message) ||
+    /No usable sandbox/i.test(error.message) ||
+    /clone helper|namespace sandbox|SUID sandbox/i.test(error.message)
+  )
+}
+
+export const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'

package/src/protocol.ts

@@ -0,0 +1,51 @@
+export type SidecarErrorKind = 'init' | 'needs-install' | 'navigation' | 'runtime' | 'timeout' | 'unknown'
+
+export type SidecarRequest = {
+  readonly id: string
+  readonly method: string
+  readonly params?: Record<string, unknown>
+}
+
+export type SidecarReply =
+  | { readonly id: string; readonly ok: true; readonly result?: unknown }
+  | {
+      readonly id: string
+      readonly ok: false
+      readonly error: { readonly message: string; readonly kind: SidecarErrorKind }
+    }
+
+export class SidecarError extends Error {
+  readonly kind: SidecarErrorKind
+
+  constructor(message: string, kind: SidecarErrorKind = 'runtime') {
+    super(message)
+    this.name = 'SidecarError'
+    this.kind = kind
+  }
+}
+
+/**
+ * Line-delimited JSON, one object per line, correlated by `id`. Deliberately not
+ * JSON-RPC 2.0: the surface is four methods over a pipe we own both ends of, and
+ * an `ok` discriminant is easier to narrow than presence-of-`error`.
+ */
+export function encodeLine(value: SidecarRequest | SidecarReply): string {
+  return `${JSON.stringify(value)}\n`
+}
+
+export function decodeLine(line: string): unknown {
+  try {
+    return JSON.parse(line)
+  } catch {
+    return null
+  }
+}
+
+export function isReply(value: unknown): value is SidecarReply {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    typeof (value as { id?: unknown }).id === 'string' &&
+    typeof (value as { ok?: unknown }).ok === 'boolean'
+  )
+}

package/src/sidecar.ts

@@ -0,0 +1,108 @@
+/**
+ * Playwright sidecar entry point.
+ *
+ * Runs as a child process so Playwright never enters the app or MCP bundle and a
+ * crashed browser cannot take the server down. Speaks line-delimited JSON on
+ * stdio; every request is serialized per lease so a `goto` cannot race a
+ * `release` on the same context.
+ */
+
+import { createState, dispatch, teardown, type SidecarState } from './dispatch'
+import { decodeLine, encodeLine, type SidecarReply, type SidecarRequest } from './protocol'
+
+const state: SidecarState = createState()
+
+function write(reply: SidecarReply): void {
+  try {
+    process.stdout.write(encodeLine(reply))
+  } catch {
+    // stdout is gone; the parent will time the call out and reap us.
+  }
+}
+
+/** Per-lease chains: work on different contexts proceeds concurrently. */
+const chains = new Map<string, Promise<void>>()
+
+function chainKey(request: SidecarRequest): string {
+  const leaseId = request.params?.leaseId
+  return typeof leaseId === 'string' ? leaseId : '__global__'
+}
+
+export function enqueue(line: string): void {
+  const parsed = decodeLine(line)
+  if (typeof parsed !== 'object' || parsed === null) return
+  const request = parsed as SidecarRequest
+  if (typeof request.id !== 'string' || typeof request.method !== 'string') return
+
+  const key = chainKey(request)
+  const previous = chains.get(key) ?? Promise.resolve()
+  const next: Promise<void> = previous
+    .then(async () => {
+      write(await dispatch(state, request))
+    })
+    .catch((error: unknown) => {
+      write({
+        id: request.id,
+        ok: false,
+        error: { message: error instanceof Error ? error.message : String(error), kind: 'unknown' },
+      })
+    })
+    .finally(() => {
+      // Every render takes a fresh lease id, so without this the map grows one
+      // settled promise per request for the life of the process. Only the tail is
+      // dropped; a later request on the same lease owns the entry by then.
+      if (chains.get(key) === next) chains.delete(key)
+    })
+  chains.set(key, next)
+}
+
+async function shutdownAndExit(code: number): Promise<void> {
+  try {
+    await teardown(state)
+  } catch {
+    // Exiting regardless.
+  }
+  process.exit(code)
+}
+
+/**
+ * A SIGKILL'd parent produces no stdin EOF, so without this poll a Chromium can
+ * outlive the server that spawned it and leak memory until the box is restarted.
+ */
+function startParentWatchdog(): void {
+  const parentPid = process.ppid
+  if (!parentPid || parentPid <= 1) return
+  const interval = setInterval(() => {
+    try {
+      process.kill(parentPid, 0)
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ESRCH') {
+        clearInterval(interval)
+        void shutdownAndExit(0)
+      }
+      // EPERM means it exists but we may not signal it — still alive.
+    }
+  }, 2000)
+  interval.unref?.()
+}
+
+export function main(): void {
+  let buffer = ''
+  process.stdin.setEncoding('utf8')
+  process.stdin.on('data', (chunk: string) => {
+    buffer += chunk
+    let newline = buffer.indexOf('\n')
+    while (newline !== -1) {
+      const line = buffer.slice(0, newline)
+      buffer = buffer.slice(newline + 1)
+      if (line.trim()) enqueue(line)
+      newline = buffer.indexOf('\n')
+    }
+  })
+  process.stdin.on('end', () => void shutdownAndExit(0))
+  process.stdout.on('error', () => void shutdownAndExit(0))
+  startParentWatchdog()
+}
+
+// Guarded so tests can import the module without starting the stdio loop.
+if (process.env.OM_WEB_RESEARCH_SIDECAR_TEST !== '1') main()

package/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/__tests__/**"]
+}

package/watch.mjs

@@ -0,0 +1,7 @@
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { watch } from '../../scripts/watch.mjs'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+watch(__dirname)