@tangle-network/agent-bench 0.1.0

package/src/benchmarks/types.ts

@@ -0,0 +1,58 @@
+/**
+ * BenchmarkAdapter — the seam every external benchmark implements so the
+ * agent-runtime loop can be A/B'd (blind vs steering vs steering+memory) against
+ * a REAL, DETERMINISTIC judge. The worker produces an `artifact` (a patch, a
+ * command transcript, a final state); the benchmark's own harness scores it.
+ *
+ * The point: no self-authored judge, no invented score noise. The number comes
+ * from the benchmark's published evaluation harness.
+ */
+
+import type { OutputAdapter } from '@tangle-network/agent-runtime/loops'
+
+export interface BenchTask {
+  /** Stable benchmark instance id. */
+  id: string
+  /** The task statement handed to the worker agent. */
+  prompt: string
+  split?: string
+  /** Benchmark-specific fields the worker/judge need (repo, base_commit, gold, …). */
+  metadata?: Record<string, unknown>
+}
+
+export interface BenchScore {
+  /** Did the deterministic judge pass (tests resolved / state correct)? */
+  resolved: boolean
+  /** 0..1 — 1 = fully resolved; partial credit where the harness supports it. */
+  score: number
+  detail?: string
+}
+
+export interface LoadOptions {
+  limit?: number
+  split?: string
+  ids?: string[]
+}
+
+export interface BenchmarkAdapter {
+  readonly name: string
+  /** Throw with actionable guidance when the harness/judge isn't installed/runnable. */
+  preflight(): Promise<void>
+  loadTasks(opts?: LoadOptions): Promise<BenchTask[]>
+  /** DETERMINISTIC judge: score the worker's produced artifact for a task. */
+  judge(task: BenchTask, artifact: string): Promise<BenchScore>
+  /** Gold/oracle artifact — lets us self-verify the judge before spending model tokens. */
+  goldArtifact(task: BenchTask): Promise<string | undefined>
+  /** How to extract the judged artifact from a run's event stream. Optional —
+   *  defaults to the agent's final answer text (the research/QA case). SWE sets
+   *  it to a patch parser. This is `benchmark = adapter` owning its deliverable,
+   *  so the gate runner (`runGate` / `runBenchmark`) needs no
+   *  per-benchmark branching. */
+  output?: OutputAdapter<string>
+  /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the
+   *  worker (e.g. AppWorld's interactive ReAct episode runs inside the engine,
+   *  not as a chat completion) — the experiment uses this instead of the
+   *  BACKEND-selected client; the steer still flows through the per-round prompt.
+   *  Typed loosely to avoid a runtime import cycle; the harness casts it. */
+  leafClient?: (cfg: { model: string; routerBaseUrl: string; routerKey: string }) => unknown
+}

package/src/browser/adapters/bad-design-audit.ts

@@ -0,0 +1,153 @@
+/**
+ * UiReviewerAdapter for browser-agent-driver's `bad design-audit --json` — the
+ * reviewer that carries the DETERMINISTIC floor (axe a11y + WCAG contrast) the
+ * panel verdict is derived from. One of N pluggable reviewers behind the neutral
+ * UiReviewerAdapter contract; its subjective LLM findings are surfaced, its
+ * MEASURED floor is what `judgeUiFloor` attests.
+ *
+ * `badDesignAuditToReviewRuns` is a pure, unit-testable mapper kept separate from
+ * the spawn so the report-shape mapping is exercised without a browser/LLM.
+ */
+
+import { spawn } from 'node:child_process'
+import { mkdtemp, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type {
+  DesignMeasurements,
+  UiFinding,
+  UiReviewRun,
+  UiReviewTarget,
+  UiReviewerAdapter,
+} from '../ui-reviewer'
+
+export interface BadDesignAuditConfig {
+  /** Path to the bad CLI entry (default: $BAD_CLI or ~/code/browser-agent-driver/dist/cli.js). */
+  badCli?: string
+  /** Design-audit profile (saas/defi/…); forwarded as --profile when set. */
+  profile?: string
+  model?: string
+  /** OpenAI-compatible endpoint — point at our router. */
+  baseUrl?: string
+  apiKey?: string
+  /** Pages to crawl from the start URL (bad --pages). */
+  pages?: number
+  timeoutMs?: number
+}
+
+interface BadFinding {
+  category?: string
+  severity?: 'critical' | 'major' | 'minor'
+  title?: string
+  description?: string
+  recommendation?: string
+  selector?: string
+}
+interface BadMeasurements {
+  contrast?: { aaFailures?: unknown[]; totalChecked?: number; summary?: { aaPassRate?: number } }
+  a11y?: { violations?: Array<{ id?: string; impact?: string }> }
+}
+interface BadPage {
+  route?: string
+  url?: string
+  findings?: BadFinding[]
+  measurements?: BadMeasurements
+  score?: number
+  healthScore?: number
+}
+/** Wire shape of `bad design-audit --json` (the report.json the CLI writes). */
+export interface BadDesignReport {
+  pages?: BadPage[]
+  summary?: { healthScore?: number; avgScore?: number }
+}
+
+/** Pure mapper: `bad design-audit --json` report → neutral UiReviewRun[] (one per
+ *  page). The measured floor (axe + contrast) becomes DesignMeasurements; the LLM
+ *  findings become UiFinding[]; healthScore is recorded as selfReportedScore only. */
+export function badDesignAuditToReviewRuns(report: BadDesignReport): UiReviewRun[] {
+  const pages = report.pages ?? []
+  return pages.map((p, i) => {
+    const route = p.route ?? p.url ?? `page-${i}`
+    const findings: UiFinding[] = (p.findings ?? []).map((f) => ({
+      reviewerId: 'bad-design-audit',
+      lens: f.category ?? 'design',
+      severity: f.severity ?? 'minor',
+      route,
+      title: f.title ?? '(finding)',
+      observation: f.description ?? f.title ?? '',
+      suggestedFix: f.recommendation,
+      selector: f.selector,
+    }))
+    const m = p.measurements
+    let measurements: DesignMeasurements | undefined
+    if (m?.contrast || m?.a11y) {
+      const failures = m.contrast?.aaFailures?.length ?? 0
+      const total = m.contrast?.totalChecked ?? 0
+      measurements = {
+        a11yViolations: (m.a11y?.violations ?? []).map((v) => ({ id: v.id ?? 'unknown', impact: v.impact })),
+        contrastAaPassRate: m.contrast?.summary?.aaPassRate ?? (total > 0 ? 1 - failures / total : 1),
+        contrastAaFailures: failures,
+        contrastTotalChecked: total,
+      }
+    }
+    return {
+      reviewerId: 'bad-design-audit',
+      route,
+      findings,
+      measurements,
+      selfReportedScore: p.healthScore ?? p.score,
+    }
+  })
+}
+
+function resolveCli(cfg: BadDesignAuditConfig): string {
+  return cfg.badCli ?? process.env.BAD_CLI ?? join(process.env.HOME ?? '', 'code/browser-agent-driver/dist/cli.js')
+}
+
+/** A UiReviewerAdapter backed by `bad design-audit`. Drives a real browser audit
+ *  (LLM findings + measured axe/contrast floor); the panel judge attests the floor. */
+export function badDesignAuditReviewer(cfg: BadDesignAuditConfig = {}): UiReviewerAdapter {
+  const cli = resolveCli(cfg)
+  const timeoutMs = cfg.timeoutMs ?? 600_000
+  return {
+    id: 'bad-design-audit',
+    async review(target: UiReviewTarget): Promise<UiReviewRun[]> {
+      const dir = await mkdtemp(join(tmpdir(), 'bad-design-'))
+      const args = [cli, 'design-audit', '--url', target.url, '--json', '--output', dir]
+      if (cfg.profile) args.push('--profile', cfg.profile)
+      if (cfg.pages) args.push('--pages', String(cfg.pages))
+      if (cfg.model) args.push('--model', cfg.model)
+      if (cfg.baseUrl) args.push('--base-url', cfg.baseUrl)
+      if (cfg.apiKey) args.push('--api-key', cfg.apiKey)
+      if (target.storageState) args.push('--storage-state', target.storageState)
+      try {
+        await runChild('node', args, dir, timeoutMs)
+        const report = JSON.parse(await readFile(join(dir, 'report.json'), 'utf8')) as BadDesignReport
+        return badDesignAuditToReviewRuns(report)
+      } finally {
+        await rm(dir, { recursive: true, force: true }).catch(() => {})
+      }
+    },
+  }
+}
+
+function runChild(cmd: string, args: string[], cwd: string, timeoutMs: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] })
+    let err = ''
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL')
+      reject(new Error(`bad design-audit timed out after ${timeoutMs}ms`))
+    }, timeoutMs)
+    child.stderr.on('data', (d) => (err += d.toString()))
+    child.on('error', (e) => {
+      clearTimeout(timer)
+      reject(new Error(`bad design-audit failed to spawn: ${e.message}`))
+    })
+    child.on('close', (code) => {
+      clearTimeout(timer)
+      if (code !== 0) reject(new Error(`bad design-audit exited ${code}: ${err.slice(-300)}`))
+      else resolve()
+    })
+  })
+}

package/src/browser/adapters/bad-design-audit.verify.ts

@@ -0,0 +1,85 @@
+/**
+ * Deterministic self-check for the bad-design-audit reviewer mapper —
+ * `tsx src/browser/adapters/bad-design-audit.verify.ts`.
+ *
+ * Proves the `bad design-audit --json` report maps to the neutral UiReviewRun with
+ * the DETERMINISTIC floor intact, and that the panel verdict is then driven by that
+ * measured floor — not the audit's self-reported healthScore.
+ */
+
+import { judgeUiFloor } from '../ui-reviewer'
+import { badDesignAuditToReviewRuns, type BadDesignReport } from './bad-design-audit'
+
+let failures = 0
+function check(name: string, cond: boolean, detail = '') {
+  if (cond) console.log(`  ✓ ${name}`)
+  else {
+    failures += 1
+    console.log(`  ✗ ${name} ${detail}`)
+  }
+}
+
+// A bad design-audit --json report shaped exactly like the CLI writes: one clean
+// page + one page whose MEASURED floor is blocking, while it self-reports a high
+// healthScore (the dishonesty the floor verdict must override).
+const report: BadDesignReport = {
+  summary: { avgScore: 7, healthScore: 80 },
+  pages: [
+    {
+      route: 'home',
+      url: 'https://shop.test/',
+      score: 8,
+      healthScore: 88,
+      findings: [
+        { category: 'hierarchy', severity: 'major', title: 'Weak CTA hierarchy', description: 'primary + secondary look identical', recommendation: 'differentiate weight', selector: 'button.cta' },
+      ],
+      measurements: {
+        contrast: { aaFailures: [{}, {}], totalChecked: 100, summary: { aaPassRate: 0.98 } },
+        a11y: { violations: [{ id: 'image-alt', impact: 'moderate' }] },
+      },
+    },
+    {
+      route: 'checkout',
+      url: 'https://shop.test/checkout',
+      score: 4,
+      healthScore: 95, // self-reports GREAT — but the measured floor is blocking
+      findings: [],
+      measurements: {
+        contrast: { aaFailures: new Array(40).fill({}), totalChecked: 100, summary: { aaPassRate: 0.6 } },
+        a11y: {
+          violations: [
+            { id: 'label', impact: 'critical' },
+            { id: 'button-name', impact: 'critical' },
+            { id: 'aria-required', impact: 'critical' },
+            { id: 'link-name', impact: 'serious' },
+            { id: 'color-contrast', impact: 'serious' },
+          ],
+        },
+      },
+    },
+  ],
+}
+
+console.log('bad-design-audit reviewer mapper verify:')
+
+const runs = badDesignAuditToReviewRuns(report)
+check('maps one UiReviewRun per page', runs.length === 2, `got ${runs.length}`)
+check('carries the deterministic floor (axe + contrast) on each page', runs.every((r) => r.measurements != null))
+const home = runs.find((r) => r.route === 'home')!
+check('home contrast pass rate mapped', home.measurements?.contrastAaPassRate === 0.98)
+check('home finding mapped to a UiFinding (lens=category, severity preserved, selector kept)', home.findings.length === 1 && home.findings[0]?.lens === 'hierarchy' && home.findings[0]?.severity === 'major' && home.findings[0]?.selector === 'button.cta')
+check('self-reported healthScore recorded but NOT the verdict', home.selfReportedScore === 88)
+
+const verdict = judgeUiFloor(runs)
+check('panel verdict is BLOCKING — the measured checkout floor (3 critical a11y + 40% contrast fail) overrides the 95 self-report', verdict.resolved === false, verdict.detail)
+check('blocking score is low', verdict.score < 0.6, verdict.detail)
+
+// A reviewer with NO measurements (e.g. a findings-only audit) contributes no floor.
+const noFloor = badDesignAuditToReviewRuns({ pages: [{ route: 'x', findings: [], score: 5 }] } satisfies BadDesignReport)
+check('a page without measurements yields no floor (subjective-only run)', noFloor[0]?.measurements === undefined)
+
+if (failures > 0) {
+  console.log(`\n❌ ${failures} check(s) failed`)
+  process.exit(1)
+}
+console.log('\n✅ bad-design-audit reviewer verified — floor mapped from --json, verdict driven by the measured floor not the self-report')

package/src/browser/adapters/bad.ts

@@ -0,0 +1,165 @@
+/**
+ * Reference BrowserAgentAdapter for @tangle-network/browser-agent-driver (the `bad`
+ * CLI). One of N pluggable drivers behind the neutral harness — it conforms to the
+ * SAME BrowserAgentAdapter contract as a browser-use runner or any other agent.
+ *
+ * `bad run --cases <file> --sink <dir>` emits a structured report.json; we map its
+ * `agentResult` to the neutral BrowserRun and let `judgeBrowserRun` derive the
+ * DETERMINISTIC verdict. We record `bad`'s own success/goalVerification only as
+ * `selfReportedSuccess` — it never decides the outcome (attestation, not trust).
+ *
+ * `badReportToRun` is a pure, unit-testable mapper kept separate from the spawn so
+ * the report-shape mapping can be exercised without launching a browser.
+ */
+
+import { spawn } from 'node:child_process'
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { BrowserAgentAdapter, BrowserRun, BrowserStep, BrowserTask } from '../agent-adapter'
+
+export interface BadAdapterConfig {
+  /** Path to the bad CLI entry (default: $BAD_CLI or ~/code/browser-agent-driver/dist/cli.js). */
+  badCli?: string
+  model?: string
+  provider?: string
+  /** OpenAI-compatible endpoint — point at our router. */
+  baseUrl?: string
+  apiKey?: string
+  maxTurns?: number
+  timeoutMs?: number
+  /** Capture a per-turn screenshot into each turn's state.screenshot (bad
+   *  full-evidence mode). Off by default — opt in for the navigation film. */
+  captureScreenshots?: boolean
+}
+
+interface BadReport {
+  results?: Array<{
+    testCase?: { id?: string }
+    // Real token/cost telemetry (bad >=0.33 reports these per result) — forwarded to
+    // BrowserRun so a real run never reads as a {0,0} backend-integrity stub.
+    estimatedCostUsd?: number
+    inputTokens?: number
+    outputTokens?: number
+    agentResult?: {
+      success?: boolean
+      result?: string
+      turns?: Array<{
+        turn?: number
+        // state.screenshot is INLINE base64 (no data: prefix) — the durable frame
+        // source for the film, since the adapter's temp sink is reaped after the run.
+        state?: { url?: string; title?: string; snapshot?: string; screenshot?: string }
+        action?: { action?: string; selector?: string; text?: string; result?: string }
+      }>
+    }
+  }>
+}
+
+/** Pure mapper: bad's report.json → the neutral BrowserRun for taskId. The final
+ *  turn's state.url is the observable final URL; its snapshot is the final DOM the
+ *  deterministic dom-* checks read. */
+export function badReportToRun(report: BadReport, taskId: string): BrowserRun {
+  const res = report.results?.find((r) => r.testCase?.id === taskId) ?? report.results?.[0]
+  const ar = res?.agentResult ?? {}
+  const turns = Array.isArray(ar.turns) ? ar.turns : []
+  const steps: BrowserStep[] = turns.map((t, i) => ({
+    index: typeof t.turn === 'number' ? t.turn : i + 1,
+    url: t.state?.url ?? '',
+    action: t.action?.action ?? 'decide',
+    target: t.action?.selector ?? (t.action?.text ? `"${t.action.text}"` : undefined),
+    // The frame for the film: bad's per-turn base64 JPEG, normalized to a data: URI.
+    screenshot: t.state?.screenshot ? `data:image/jpeg;base64,${t.state.screenshot}` : undefined,
+  }))
+  const last = turns[turns.length - 1]
+  const input = typeof res?.inputTokens === 'number' ? res.inputTokens : undefined
+  const output = typeof res?.outputTokens === 'number' ? res.outputTokens : undefined
+  return {
+    taskId,
+    steps,
+    finalUrl: last?.state?.url ?? '',
+    finalDom: typeof last?.state?.snapshot === 'string' ? last.state.snapshot : undefined,
+    answer: typeof ar.result === 'string' ? ar.result : undefined,
+    selfReportedSuccess: typeof ar.success === 'boolean' ? ar.success : undefined,
+    usage: input !== undefined || output !== undefined ? { input: input ?? 0, output: output ?? 0 } : undefined,
+    costUsd: typeof res?.estimatedCostUsd === 'number' ? res.estimatedCostUsd : undefined,
+    driverId: 'bad',
+  }
+}
+
+function resolveCli(cfg: BadAdapterConfig): string {
+  return cfg.badCli ?? process.env.BAD_CLI ?? join(process.env.HOME ?? '', 'code/browser-agent-driver/dist/cli.js')
+}
+
+/** A BrowserAgentAdapter backed by the `bad` CLI. Runs one task through a real
+ *  browser; the harness judge attests the outcome. */
+export function badBrowserAdapter(cfg: BadAdapterConfig = {}): BrowserAgentAdapter {
+  const cli = resolveCli(cfg)
+  const timeoutMs = cfg.timeoutMs ?? 300_000
+  return {
+    id: 'bad',
+    async run(task: BrowserTask): Promise<BrowserRun> {
+      const dir = await mkdtemp(join(tmpdir(), 'bad-adapter-'))
+      const casesPath = join(dir, 'cases.json')
+      const sink = join(dir, 'sink')
+      const cases = [
+        {
+          id: task.id,
+          name: task.id,
+          goal: task.goal,
+          url: task.startUrl,
+          maxTurns: task.maxSteps ?? cfg.maxTurns ?? 20,
+        },
+      ]
+      await writeFile(casesPath, JSON.stringify(cases))
+      const args = [
+        cli,
+        'run',
+        '--cases',
+        casesPath,
+        '--sink',
+        sink,
+        '--provider',
+        cfg.provider ?? 'openai',
+        '--model',
+        cfg.model ?? 'deepseek-v4-flash',
+        '--max-turns',
+        String(task.maxSteps ?? cfg.maxTurns ?? 20),
+        '--headless',
+      ]
+      // full-evidence + every-turn capture populates each turn's state.screenshot,
+      // which badReportToRun lifts into BrowserStep.screenshot for the film.
+      if (cfg.captureScreenshots) args.push('--mode', 'full-evidence', '--screenshot-interval', '1')
+      if (cfg.baseUrl) args.push('--base-url', cfg.baseUrl)
+      if (cfg.apiKey) args.push('--api-key', cfg.apiKey)
+      if (task.storageState) args.push('--storage-state', task.storageState)
+      try {
+        await runChild('node', args, dir, timeoutMs)
+        const report = JSON.parse(await readFile(join(sink, 'report.json'), 'utf8')) as BadReport
+        return badReportToRun(report, task.id)
+      } finally {
+        await rm(dir, { recursive: true, force: true }).catch(() => {})
+      }
+    },
+  }
+}
+
+function runChild(cmd: string, args: string[], cwd: string, timeoutMs: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] })
+    let err = ''
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL')
+      reject(new Error(`bad adapter timed out after ${timeoutMs}ms`))
+    }, timeoutMs)
+    child.stderr.on('data', (d) => (err += d.toString()))
+    child.on('error', (e) => {
+      clearTimeout(timer)
+      reject(new Error(`bad adapter failed to spawn: ${e.message}`))
+    })
+    child.on('close', (code) => {
+      clearTimeout(timer)
+      if (code !== 0) reject(new Error(`bad exited ${code}: ${err.slice(-300)}`))
+      else resolve()
+    })
+  })
+}

package/src/browser/agent-adapter.ts

@@ -0,0 +1,145 @@
+/**
+ * Driver-agnostic browser-agent harness — the GENERAL seam.
+ *
+ * The benchmark owns the task and the verdict; the browser AGENT is pluggable.
+ * Any driver — @tangle-network/browser-agent-driver, browser-use (Python),
+ * a hosted API, a future tool — conforms to ONE neutral contract: given a goal +
+ * start URL, produce a normalized trajectory and a final observable state. We do
+ * NOT shape this around any one driver's SDK; it is the least-common-denominator
+ * a browser agent can emit, crossed with what a deterministic judge needs.
+ *
+ * Attestability is the whole point. We NEVER trust the driver's own `success`
+ * flag or its LLM goal-verification (that is the unverifiable self-report the
+ * benchmark exists to replace). Our judge re-derives the outcome from DETERMINISTIC
+ * criteria (url / DOM state / element checks) against the run's final state. A
+ * driver's published number is a hypothesis our harness attests or refutes.
+ *
+ * Language-agnostic by a process boundary: a driver that isn't TypeScript (e.g.
+ * browser-use) is wrapped by `processBrowserAdapter`, which hands the task as JSON
+ * on argv/stdin and reads a BrowserRun back as JSON on stdout. No SDK coupling, no
+ * overfit — adding a driver is a thin runner that speaks this JSON, nothing more.
+ */
+
+/** A deterministic, driver-independent success check the JUDGE applies to the
+ *  run's final state — never the agent's self-assessment. Mirrors the kinds a
+ *  benchmark harness actually ships (WebArena program-state, Mind2Web element). */
+export type SuccessSpec =
+  | { type: 'url-contains'; value: string }
+  | { type: 'url-matches'; value: string } // regex on finalUrl
+  | { type: 'final-answer-matches'; value: string } // regex on the agent's answer (extraction tasks)
+  | { type: 'dom-contains'; value: string } // substring of the final DOM/text snapshot
+  | { type: 'dom-selector-text'; selector: string; value: string } // element text equals/contains
+  | { type: 'program'; id: string } // a benchmark-provided programmatic check keyed by id
+
+/** A task handed to ANY browser agent. `success` is the benchmark's deterministic
+ *  verdict spec — evaluated by us, not the driver. */
+export interface BrowserTask {
+  id: string
+  goal: string
+  startUrl: string
+  maxSteps?: number
+  /** Path to a Playwright storageState for authenticated tasks (optional). */
+  storageState?: string
+  /** Deterministic success criteria the judge applies to the produced run. */
+  success: SuccessSpec[]
+  /** Benchmark-specific extras the judge may need (gold answer, program inputs). */
+  metadata?: Record<string, unknown>
+}
+
+/** One normalized step, common to every driver. */
+export interface BrowserStep {
+  index: number
+  /** URL at this step. */
+  url: string
+  /** A normalized action label (e.g. 'click', 'type', 'navigate', 'scroll'). */
+  action: string
+  /** Human/agent-readable target or value (selector, text typed, link). */
+  target?: string
+  /** Agent's reasoning for this step, if the driver exposes it. */
+  reasoning?: string
+  /** Path to a per-step screenshot on disk (for the run-capsule film). */
+  screenshotPath?: string
+  /** Inline base64 `data:` URI of the per-step frame, when the driver carries the
+   *  image in its report rather than on disk (e.g. `bad`'s state.screenshot, whose
+   *  sink is reaped post-run). Preferred over `screenshotPath` by the film converter. */
+  screenshot?: string
+  /** Target element box at action time, if known — for replay cursor overlays. */
+  actionBounds?: { x: number; y: number; width: number; height: number }
+}
+
+/** The normalized result ANY driver produces — the judge scores this, not the
+ *  driver's opinion of itself. */
+export interface BrowserRun {
+  taskId: string
+  steps: BrowserStep[]
+  /** URL the agent ended on. */
+  finalUrl: string
+  /** Final visible text / DOM snapshot for dom-* deterministic checks. */
+  finalDom?: string
+  /** The agent's final free-text answer (extraction tasks). */
+  answer?: string
+  /** The driver's OWN success claim — recorded for comparison, NEVER the verdict. */
+  selfReportedSuccess?: boolean
+  /** Real token usage + cost if the driver reports it (backend-integrity). */
+  usage?: { input: number; output: number }
+  costUsd?: number
+  /** Which driver produced this run (for compareDrivers attribution). */
+  driverId: string
+}
+
+/** The general seam every browser driver implements (directly in TS, or via the
+ *  process wrapper below). `id` is the attribution key for head-to-head. */
+export interface BrowserAgentAdapter {
+  readonly id: string
+  run(task: BrowserTask): Promise<BrowserRun>
+}
+
+/** Optional per-benchmark programmatic checks, keyed by SuccessSpec {type:'program', id}. */
+export type ProgramChecks = Record<string, (run: BrowserRun, task: BrowserTask) => boolean>
+
+/** The DETERMINISTIC, attestable verdict: evaluate the benchmark's SuccessSpec list
+ *  against the run's final observable state. Independent of selfReportedSuccess.
+ *  resolved = ALL criteria pass; score = fraction passed (a gradient for the optimizer). */
+export function judgeBrowserRun(
+  task: BrowserTask,
+  run: BrowserRun,
+  programs: ProgramChecks = {},
+): { resolved: boolean; score: number; detail: string } {
+  const checks = task.success
+  if (checks.length === 0) throw new Error(`browser task ${task.id} has no success criteria — cannot attest`)
+  const results = checks.map((c) => ({ c, ok: evalCriterion(c, run, task, programs) }))
+  const passed = results.filter((r) => r.ok).length
+  const resolved = passed === checks.length
+  return {
+    resolved,
+    score: passed / checks.length,
+    detail: JSON.stringify({
+      passed,
+      total: checks.length,
+      selfReported: run.selfReportedSuccess,
+      finalUrl: run.finalUrl,
+      perCriterion: results.map((r) => ({ type: r.c.type, ok: r.ok })),
+    }),
+  }
+}
+
+function evalCriterion(c: SuccessSpec, run: BrowserRun, task: BrowserTask, programs: ProgramChecks): boolean {
+  switch (c.type) {
+    case 'url-contains':
+      return run.finalUrl.includes(c.value)
+    case 'url-matches':
+      return new RegExp(c.value).test(run.finalUrl)
+    case 'final-answer-matches':
+      return run.answer != null && new RegExp(c.value, 'i').test(run.answer)
+    case 'dom-contains':
+      return run.finalDom != null && run.finalDom.includes(c.value)
+    case 'dom-selector-text':
+      // Deterministic only with a structured snapshot; require finalDom to carry it.
+      return run.finalDom != null && run.finalDom.includes(c.value)
+    case 'program': {
+      const fn = programs[c.id]
+      if (!fn) throw new Error(`browser task ${task.id}: no program check registered for id '${c.id}'`)
+      return fn(run, task)
+    }
+  }
+}