@open-mercato/web-research-browser 0.7.1-develop.7194.1.ab4fc81f82

package/src/__tests__/install.test.ts

@@ -0,0 +1,72 @@
+import { launchChromium, type BrowserHandle, type BrowserType } from '../playwright'
+
+const MISSING_BINARY = "Executable doesn't exist at /root/.cache/ms-playwright/chromium-1140/chrome-linux/chrome"
+
+const fakeBrowser = { newContext: async () => ({}) as never, close: async () => {} } as BrowserHandle
+
+function browserThatFails(times: number, message: string): BrowserType & { launches: number } {
+  const type = {
+    launches: 0,
+    async launch() {
+      type.launches += 1
+      if (type.launches <= times) throw new Error(message)
+      return fakeBrowser
+    },
+  }
+  return type as BrowserType & { launches: number }
+}
+
+describe('launchChromium', () => {
+  it('launches directly when the binary is present', async () => {
+    const type = browserThatFails(0, MISSING_BINARY)
+    const install = jest.fn(async () => {})
+
+    await expect(launchChromium(type, { headless: true }, { install })).resolves.toBe(fakeBrowser)
+    expect(install).not.toHaveBeenCalled()
+    expect(type.launches).toBe(1)
+  })
+
+  // Installing the npm package does not fetch the ~150MB browser, so an operator
+  // who enables the tier would otherwise hit a dead-end launch failure.
+  it('downloads the binary once and retries the launch', async () => {
+    const type = browserThatFails(1, MISSING_BINARY)
+    const install = jest.fn(async () => {})
+    const progress: string[] = []
+
+    await expect(
+      launchChromium(type, { headless: true }, { install, onProgress: (line) => progress.push(line) }),
+    ).resolves.toBe(fakeBrowser)
+
+    expect(install).toHaveBeenCalledTimes(1)
+    expect(type.launches).toBe(2)
+    expect(progress.join(' ')).toContain('one-time')
+  })
+
+  it('does not retry a launch failure that is not a missing binary', async () => {
+    const type = browserThatFails(1, 'Target page crashed')
+    const install = jest.fn(async () => {})
+
+    await expect(launchChromium(type, { headless: true }, { install })).rejects.toThrow('Target page crashed')
+    expect(install).not.toHaveBeenCalled()
+  })
+
+  it('reports an actionable error when the download itself fails', async () => {
+    const type = browserThatFails(1, MISSING_BINARY)
+    const install = jest.fn(async () => {
+      throw new Error('ENOTFOUND registry.npmjs.org')
+    })
+
+    await expect(launchChromium(type, { headless: true }, { install })).rejects.toThrow(
+      /auto-install failed.*ENOTFOUND.*manually/is,
+    )
+  })
+
+  it('gives up after one retry rather than looping', async () => {
+    const type = browserThatFails(5, MISSING_BINARY)
+    const install = jest.fn(async () => {})
+
+    await expect(launchChromium(type, { headless: true }, { install })).rejects.toThrow(/Executable doesn't exist/)
+    expect(install).toHaveBeenCalledTimes(1)
+    expect(type.launches).toBe(2)
+  })
+})

package/src/__tests__/leases.test.ts

@@ -0,0 +1,50 @@
+import { LEASE_IDLE_TIMEOUT_MS, createState, reapIdleLeases, type Lease } from '../dispatch'
+
+function fakeLease(touchedAt: number, onClose: () => void): Lease {
+  return {
+    touchedAt,
+    context: {
+      newPage: async () => ({}),
+      close: async () => {
+        onClose()
+      },
+    },
+    page: {
+      goto: async () => undefined,
+      content: async () => '',
+      url: () => 'https://example.com',
+      close: async () => undefined,
+    },
+  } as unknown as Lease
+}
+
+describe('lease reaping', () => {
+  it('closes a lease nobody released', async () => {
+    // `release` normally closes the context, but a caller can die between render
+    // and release — and every orphan is a live BrowserContext holding memory for
+    // as long as the sidecar runs.
+    let closed = 0
+    const clock = { value: 1_000_000 }
+    const state = createState({ now: () => clock.value, warn: () => {} })
+    state.leases.set('orphan', fakeLease(clock.value, () => (closed += 1)))
+
+    clock.value += LEASE_IDLE_TIMEOUT_MS + 1
+    await reapIdleLeases(state)
+
+    expect(closed).toBe(1)
+    expect(state.leases.size).toBe(0)
+  })
+
+  it('leaves a lease that is still in use alone', async () => {
+    let closed = 0
+    const clock = { value: 1_000_000 }
+    const state = createState({ now: () => clock.value, warn: () => {} })
+    state.leases.set('active', fakeLease(clock.value, () => (closed += 1)))
+
+    clock.value += LEASE_IDLE_TIMEOUT_MS - 1_000
+    await reapIdleLeases(state)
+
+    expect(closed).toBe(0)
+    expect(state.leases.size).toBe(1)
+  })
+})

package/src/__tests__/sandbox.test.ts

@@ -0,0 +1,67 @@
+import {
+  LAUNCH_ARGS,
+  SANDBOX_OPT_OUT_ENV,
+  isSandboxDisabled,
+  isSandboxFailure,
+  launchChromium,
+  resolveLaunchArgs,
+  type BrowserType,
+} from '../playwright'
+import { SidecarError } from '../protocol'
+
+describe('Chromium sandbox posture', () => {
+  it('keeps the sandbox on by default', () => {
+    // This adapter renders pages chosen by a search engine or a model, so the
+    // renderer parses hostile input by definition. `--no-sandbox` turns a
+    // renderer compromise into code running as the app's own user.
+    expect(LAUNCH_ARGS).not.toContain('--no-sandbox')
+    expect(resolveLaunchArgs({})).not.toContain('--no-sandbox')
+    expect(isSandboxDisabled({})).toBe(false)
+  })
+
+  it('disables it only on an explicit opt-out', () => {
+    const env = { [SANDBOX_OPT_OUT_ENV]: '1' } as NodeJS.ProcessEnv
+    expect(resolveLaunchArgs(env)).toContain('--no-sandbox')
+    expect(isSandboxDisabled(env)).toBe(true)
+    // Anything other than the exact opt-out keeps the sandbox.
+    expect(resolveLaunchArgs({ [SANDBOX_OPT_OUT_ENV]: 'true' } as NodeJS.ProcessEnv)).not.toContain(
+      '--no-sandbox',
+    )
+  })
+
+  it('recognises Chromium refusing to run unsandboxed', () => {
+    expect(isSandboxFailure(new Error('Running as root without --no-sandbox is not supported.'))).toBe(true)
+    expect(isSandboxFailure(new Error('No usable sandbox! Update your kernel'))).toBe(true)
+    expect(isSandboxFailure(new Error('Executable doesn\'t exist at /root/.cache/ms-playwright'))).toBe(false)
+  })
+
+  it('refuses to fall back to an unsandboxed browser on its own', async () => {
+    const browserType = {
+      launch: async () => {
+        throw new Error('Running as root without --no-sandbox is not supported.')
+      },
+    } as unknown as BrowserType
+
+    const error = await launchChromium(browserType, { headless: true }).catch((caught: unknown) => caught)
+
+    // A silent retry without the sandbox would make the security posture depend
+    // on container configuration with nobody told which one they got.
+    expect(error).toBeInstanceOf(SidecarError)
+    expect(String((error as Error).message)).toContain(SANDBOX_OPT_OUT_ENV)
+  })
+
+  it('still auto-installs a missing browser binary', async () => {
+    let launches = 0
+    const browserType = {
+      launch: async () => {
+        launches += 1
+        if (launches === 1) throw new Error("Executable doesn't exist at /ms-playwright/chromium/chrome")
+        return { newContext: async () => ({}), close: async () => {} }
+      },
+    } as unknown as BrowserType
+
+    await launchChromium(browserType, { headless: true }, { install: async () => {} })
+
+    expect(launches).toBe(2)
+  })
+})

package/src/adapter.ts

@@ -0,0 +1,196 @@
+import { randomUUID } from 'node:crypto'
+import {
+  CONTRACT_VERSION,
+  READY,
+  blocked,
+  classifyPage,
+  defineAdapterModule,
+  extractMainContent,
+  extractTitle,
+  htmlToText,
+  notReady,
+  searchOk,
+  type FetchOutcome,
+  type SearchAdapter,
+  type SearchOutcome,
+} from '@open-mercato/web-research'
+import { SERP_PROFILES, extractSerpResults, type SerpEngineId } from '@open-mercato/web-research-serp'
+import { z } from 'zod'
+import { createSidecar, type Sidecar, type SidecarOptions } from './client'
+import { SidecarError } from './protocol'
+
+export const browserOptionsSchema = z.object({
+  engines: z.array(z.enum(['ddg-html', 'ddg-lite'])).min(1).optional(),
+  userAgent: z.string().min(1).optional(),
+  timeoutMs: z.number().int().min(1000).max(120_000).optional(),
+  sidecar: z.custom<SidecarOptions>((value) => typeof value === 'object' && value !== null).optional(),
+})
+
+export type BrowserOptions = z.infer<typeof browserOptionsSchema>
+
+const DEFAULT_ENGINES: readonly SerpEngineId[] = ['ddg-html']
+const DEFAULT_TIMEOUT_MS = 30_000
+
+type RenderResult = { url: string; html: string }
+
+function isRenderResult(value: unknown): value is RenderResult {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    typeof (value as { html?: unknown }).html === 'string' &&
+    typeof (value as { url?: unknown }).url === 'string'
+  )
+}
+
+export function createBrowserAdapter(options: BrowserOptions): SearchAdapter {
+  const engines = options.engines ?? DEFAULT_ENGINES
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  let sidecar: Sidecar | null = null
+  // Remembered so `readiness()` stops promising a tier that cannot start. It is
+  // sync and I/O-free by contract, so it cannot probe — but it can report what
+  // the last attempt already proved, instead of letting the scheduler pay the
+  // escalation cost again to rediscover it.
+  let startupFailure: string | null = null
+
+  const client = (): Sidecar => {
+    sidecar ??= createSidecar(options.sidecar ?? {})
+    return sidecar
+  }
+
+  /**
+   * Every render takes its own lease, so concurrent work never shares a cookie jar.
+   *
+   * The budget is the smaller of the configured timeout and whatever the engine
+   * has left: a render that outlives `deadlineAt` keeps a browser context open on
+   * a page nobody is waiting for any more.
+   */
+  const render = async (url: string, deadlineAt: number, signal: AbortSignal): Promise<RenderResult> => {
+    const leaseId = randomUUID()
+    const budget = Math.max(1_000, Math.min(timeoutMs, deadlineAt - Date.now()))
+    const active = client()
+    try {
+      const result = await active.call(
+        'render',
+        { url, leaseId, timeoutMs: budget, ...(options.userAgent ? { userAgent: options.userAgent } : {}) },
+        signal,
+      )
+      if (!isRenderResult(result)) throw new Error('sidecar returned a malformed render result')
+      startupFailure = null
+      return result
+    } catch (error) {
+      if (error instanceof SidecarError && (error.kind === 'init' || error.kind === 'needs-install')) {
+        startupFailure = error.message
+      }
+      throw error
+    } finally {
+      await active.call('release', { leaseId }).catch(() => undefined)
+    }
+  }
+
+  return {
+    id: 'browser',
+    kind: 'browser',
+    capabilities: {
+      search: true,
+      fetch: true,
+      snippets: true,
+      content: true,
+      freshness: true,
+      siteFilter: true,
+      // Roughly two orders of magnitude more expensive than an HTTP read, which
+      // is why the engine only reaches for this after a source reports blocked.
+      cost: 'metered',
+    },
+    // Participation is governed solely by the policy row. The scheduler already
+    // keeps `kind: 'browser'` out of normal waves, so an enabled browser adapter
+    // is reached only for escalation — a second per-adapter switch just made the
+    // UI show two toggles meaning different things.
+    /** `heavy` — spawns and pings the sidecar process. */
+    probeCost: 'heavy' as const,
+    readiness: () => (startupFailure === null ? READY : notReady(startupFailure)),
+
+    async search(request, context): Promise<SearchOutcome> {
+      let lastReason = 'no engine produced results'
+
+      for (const engine of engines) {
+        if (context.signal.aborted) break
+        const profile = SERP_PROFILES[engine]
+        context.report('started', `rendering ${profile.label} in a browser`)
+        try {
+          const rendered = await render(profile.buildUrl(request), context.deadlineAt, context.signal)
+          const results = extractSerpResults(rendered.html, profile.selector, profile.baseUrl, request.limit)
+          if (results.length > 0) {
+            context.report('ok', `${profile.label} returned ${results.length} result(s)`, {
+              resultCount: results.length,
+            })
+            return searchOk(results)
+          }
+          const verdict = classifyPage({
+            status: 200,
+            contentType: 'text/html',
+            html: rendered.html,
+            text: '',
+          })
+          lastReason =
+            verdict.verdict === 'blocked'
+              ? `${profile.label} served a challenge page even in a browser (${verdict.reason ?? 'unknown'})`
+              : `${profile.label} rendered but produced no parseable results`
+        } catch (error) {
+          lastReason = error instanceof Error ? error.message : String(error)
+        }
+        context.report('blocked', lastReason)
+      }
+
+      return blocked(lastReason)
+    },
+
+    async fetch(request, context): Promise<FetchOutcome> {
+      try {
+        const rendered = await render(request.url, context.deadlineAt, context.signal)
+        const text = extractMainContent(rendered.html) || htmlToText(rendered.html)
+        return {
+          status: 'ok',
+          page: {
+            url: rendered.url,
+            title: extractTitle(rendered.html),
+            text,
+            contentType: 'text/html',
+            status: 200,
+            truncated: false,
+            renderedWith: 'browser',
+          },
+        }
+      } catch (error) {
+        return {
+          status: 'error',
+          reason: error instanceof Error ? error.message : String(error),
+          retriable: false,
+        }
+      }
+    },
+
+    async healthCheck() {
+      try {
+        await client().call('ping', {})
+        startupFailure = null
+        return { ok: true }
+      } catch (error) {
+        return { ok: false, detail: error instanceof Error ? error.message : String(error) }
+      }
+    },
+
+    async dispose() {
+      const active = sidecar
+      sidecar = null
+      if (active) await active.close()
+    },
+  }
+}
+
+export const browserAdapterModule = defineAdapterModule({
+  id: 'browser',
+  kind: 'browser',
+  contractVersion: CONTRACT_VERSION,
+  optionsSchema: browserOptionsSchema,
+  createAdapter: createBrowserAdapter,
+})

package/src/client.ts

@@ -0,0 +1,245 @@
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { SidecarError, decodeLine, encodeLine, isReply, type SidecarReply } from './protocol'
+
+export type SidecarStream = {
+  stdin: { write(chunk: string): unknown; on?(event: 'error', handler: (error: NodeJS.ErrnoException) => void): unknown }
+  stdout: { setEncoding?(encoding: string): unknown; on(event: 'data', handler: (chunk: string) => void): unknown }
+  stderr: { setEncoding?(encoding: string): unknown; on(event: 'data', handler: (chunk: string) => void): unknown }
+  once(event: 'exit', handler: (code: number | null) => void): unknown
+  kill(signal?: NodeJS.Signals): unknown
+}
+
+export type SpawnFn = (scriptPath: string) => SidecarStream
+
+export type SidecarOptions = {
+  readonly sidecarPath?: string
+  readonly spawnFn?: SpawnFn
+  readonly callTimeoutMs?: number
+  readonly closeTimeoutMs?: number
+  readonly maxPending?: number
+}
+
+const DEFAULT_CALL_TIMEOUT_MS = 150_000
+/**
+ * Deliberately short and separate from the call timeout: shutdown must not block
+ * for minutes on a wedged child before SIGTERM is even attempted.
+ */
+const DEFAULT_CLOSE_TIMEOUT_MS = 2_000
+const DEFAULT_MAX_PENDING = 128
+const SIGKILL_GRACE_MS = 2_000
+/** A full-page screenshot or a large DOM fits comfortably; a runaway does not. */
+const MAX_STDOUT_BUFFER = 64 * 1024 * 1024
+const STDERR_RING = 24
+
+function defaultSidecarPath(): string {
+  return path.join(path.dirname(fileURLToPath(import.meta.url)), 'sidecar.js')
+}
+
+function defaultSpawn(scriptPath: string): SidecarStream {
+  const child: ChildProcessWithoutNullStreams = spawn(process.execPath, [scriptPath], {
+    stdio: ['pipe', 'pipe', 'pipe'],
+  })
+  return child as unknown as SidecarStream
+}
+
+type Pending = {
+  resolve: (value: unknown) => void
+  reject: (error: Error) => void
+  timer: NodeJS.Timeout
+}
+
+export class Sidecar {
+  private child: SidecarStream | null = null
+  private startError: Error | null = null
+  private buffer = ''
+  private recentStderr: string[] = []
+  private readonly pending = new Map<string, Pending>()
+
+  constructor(
+    private readonly sidecarPath: string,
+    private readonly spawnFn: SpawnFn,
+    private readonly callTimeoutMs: number,
+    private readonly closeTimeoutMs: number,
+    private readonly maxPending: number,
+  ) {}
+
+  private ensure(): void {
+    if (this.child) return
+    if (this.startError) throw this.startError
+    try {
+      this.child = this.spawnFn(this.sidecarPath)
+    } catch (error) {
+      this.startError = error instanceof Error ? error : new Error(String(error))
+      throw this.startError
+    }
+    // Reset per-process state on every spawn. A child that died mid-line would
+    // otherwise leave a JSON fragment that corrupts the next spawn's first
+    // reply — which then parses as garbage and hangs that call until timeout.
+    this.buffer = ''
+    this.recentStderr = []
+
+    this.child.stdout.setEncoding?.('utf8')
+    this.child.stdout.on('data', (chunk: string) => {
+      this.buffer += chunk
+      let newline = this.buffer.indexOf('\n')
+      while (newline !== -1) {
+        const line = this.buffer.slice(0, newline)
+        this.buffer = this.buffer.slice(newline + 1)
+        if (line.trim()) this.handleLine(line)
+        newline = this.buffer.indexOf('\n')
+      }
+      if (this.buffer.length > MAX_STDOUT_BUFFER) {
+        this.recentStderr.push(`dropped ${this.buffer.length} bytes of un-delimited sidecar output`)
+        this.buffer = ''
+      }
+    })
+
+    // A dead child leaves its stdin pipe broken, and Node reports that as an
+    // asynchronous 'error' event (EPIPE) rather than as a throw from write() —
+    // so the try/catch around the write in `call` never sees it. Unhandled, a
+    // stream 'error' escalates past the adapter and the call throws instead of
+    // resolving to an outcome. Recording it keeps the event handled; the 'exit'
+    // handler below is what rejects the calls still in flight.
+    this.child.stdin.on?.('error', (error: NodeJS.ErrnoException) => {
+      this.recentStderr.push(`sidecar stdin ${error.code ?? 'error'}: ${error.message}`)
+      if (this.recentStderr.length > STDERR_RING) this.recentStderr.shift()
+    })
+
+    this.child.stderr.setEncoding?.('utf8')
+    this.child.stderr.on('data', (chunk: string) => {
+      for (const line of chunk.split('\n')) {
+        if (!line.trim()) continue
+        this.recentStderr.push(line)
+        if (this.recentStderr.length > STDERR_RING) this.recentStderr.shift()
+      }
+    })
+
+    this.child.once('exit', (code) => {
+      const tail = this.recentStderr.slice(-8).join('\n').trim()
+      const error = new SidecarError(
+        `browser sidecar exited unexpectedly (code=${code ?? 'null'})${tail ? `:\n${tail}` : ''}`,
+        'init',
+      )
+      for (const [, entry] of this.pending) {
+        clearTimeout(entry.timer)
+        entry.reject(error)
+      }
+      this.pending.clear()
+      this.child = null
+    })
+  }
+
+  private handleLine(line: string): void {
+    const parsed = decodeLine(line)
+    if (!isReply(parsed)) return
+    const entry = this.pending.get(parsed.id)
+    if (!entry) return
+    this.pending.delete(parsed.id)
+    clearTimeout(entry.timer)
+    if (parsed.ok) {
+      entry.resolve(parsed.result)
+      return
+    }
+    entry.reject(new SidecarError(parsed.error.message, parsed.error.kind))
+  }
+
+  async call(method: string, params: Record<string, unknown> = {}, signal?: AbortSignal): Promise<unknown> {
+    this.ensure()
+    const child = this.child
+    if (!child) throw new SidecarError('browser sidecar is not running', 'init')
+    if (this.pending.size >= this.maxPending) {
+      throw new SidecarError(`browser sidecar busy (${this.pending.size} in flight)`, 'runtime')
+    }
+
+    const id = randomUUID()
+    return new Promise<unknown>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id)
+        signal?.removeEventListener('abort', onAbort)
+        reject(new SidecarError(`browser sidecar call "${method}" timed out`, 'timeout'))
+      }, this.callTimeoutMs)
+      timer.unref?.()
+
+      // Abort rejects THIS call only. The sidecar is shared, so killing it would
+      // take healthy concurrent leases down with it; a late reply is ignored
+      // because its id is no longer pending.
+      const onAbort = (): void => {
+        if (!this.pending.delete(id)) return
+        clearTimeout(timer)
+        reject(new SidecarError(`browser sidecar call "${method}" was aborted`, 'timeout'))
+      }
+      signal?.addEventListener('abort', onAbort, { once: true })
+
+      this.pending.set(id, {
+        resolve: (value) => {
+          signal?.removeEventListener('abort', onAbort)
+          resolve(value)
+        },
+        reject: (error) => {
+          signal?.removeEventListener('abort', onAbort)
+          reject(error)
+        },
+        timer,
+      })
+
+      try {
+        child.stdin.write(encodeLine({ id, method, params }))
+      } catch (error) {
+        this.pending.delete(id)
+        clearTimeout(timer)
+        signal?.removeEventListener('abort', onAbort)
+        reject(error instanceof Error ? error : new Error(String(error)))
+      }
+    })
+  }
+
+  async close(): Promise<void> {
+    const child = this.child
+    if (!child) return
+    try {
+      const closeSignal = AbortSignal.timeout(this.closeTimeoutMs)
+      await this.call('close', {}, closeSignal)
+    } catch {
+      // Wedged or already gone; escalate below.
+    }
+    await new Promise<void>((resolve) => {
+      let settled = false
+      const done = (): void => {
+        if (settled) return
+        settled = true
+        clearTimeout(timer)
+        resolve()
+      }
+      child.once('exit', done)
+      // Armed before the kill so a synchronous exit still finds the timer set.
+      const timer = setTimeout(() => {
+        try {
+          child.kill('SIGKILL')
+        } catch {
+          // Already gone.
+        }
+        done()
+      }, SIGKILL_GRACE_MS)
+      timer.unref?.()
+      try {
+        child.kill('SIGTERM')
+      } catch {
+        done()
+      }
+    })
+    this.child = null
+  }
+}
+
+export function createSidecar(options: SidecarOptions = {}): Sidecar {
+  return new Sidecar(
+    options.sidecarPath ?? defaultSidecarPath(),
+    options.spawnFn ?? defaultSpawn,
+    options.callTimeoutMs ?? DEFAULT_CALL_TIMEOUT_MS,
+    options.closeTimeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS,
+    options.maxPending ?? DEFAULT_MAX_PENDING,
+  )
+}