@tangle-network/agent-bench 0.1.0

package/src/browser/verify.ts

@@ -0,0 +1,96 @@
+/**
+ * Deterministic self-check for the driver-agnostic browser harness — run with
+ * `tsx src/browser/verify.ts` (bench convention: a verify script, not a live run).
+ *
+ * The regression it defends: the verdict is ATTESTABLE — derived from the run's
+ * final observable state, NEVER the driver's self-reported success. A driver can
+ * claim success=true; if the deterministic criteria fail, resolved MUST be false.
+ */
+
+import { judgeBrowserRun, type BrowserTask } from './agent-adapter'
+import { badReportToRun } from './adapters/bad'
+
+let failures = 0
+function check(name: string, cond: boolean, detail = '') {
+  if (cond) {
+    console.log(`  ✓ ${name}`)
+  } else {
+    failures += 1
+    console.log(`  ✗ ${name} ${detail}`)
+  }
+}
+
+// A bad report.json shaped exactly like the real one, with the OpenSCAD page reached.
+const successReport = {
+  results: [
+    {
+      testCase: { id: 'wiki-openscad' },
+      agentResult: {
+        success: true,
+        result: "The 'OpenSCAD' article is open on Wikipedia.",
+        turns: [
+          { turn: 1, state: { url: 'https://en.wikipedia.org/wiki/Main_Page', snapshot: 'search box' }, action: { action: 'type', text: 'OpenSCAD' } },
+          { turn: 2, state: { url: 'https://en.wikipedia.org/wiki/Main_Page' }, action: { action: 'click', selector: '@result' } },
+          { turn: 3, state: { url: 'https://en.wikipedia.org/wiki/OpenSCAD', snapshot: 'OpenSCAD - Wikipedia. Constructive Solid Geometry modeller.' }, action: { action: 'complete' } },
+        ],
+      },
+    },
+  ],
+}
+
+// The SAME driver claiming success, but the browser never left Main_Page.
+const liarReport = {
+  results: [
+    {
+      testCase: { id: 'wiki-openscad' },
+      agentResult: {
+        success: true, // self-reported — must be IGNORED
+        result: 'Done!',
+        turns: [{ turn: 1, state: { url: 'https://en.wikipedia.org/wiki/Main_Page', snapshot: 'main page' }, action: { action: 'complete' } }],
+      },
+    },
+  ],
+}
+
+const task: BrowserTask = {
+  id: 'wiki-openscad',
+  goal: 'Open the OpenSCAD article',
+  startUrl: 'https://en.wikipedia.org',
+  success: [{ type: 'url-contains', value: 'OpenSCAD' }],
+}
+
+console.log('browser harness verify:')
+
+const okRun = badReportToRun(successReport, 'wiki-openscad')
+check('mapper extracts final url from last turn', okRun.finalUrl === 'https://en.wikipedia.org/wiki/OpenSCAD', okRun.finalUrl)
+check('mapper records all steps', okRun.steps.length === 3)
+check('mapper carries selfReportedSuccess', okRun.selfReportedSuccess === true)
+const okV = judgeBrowserRun(task, okRun)
+check('genuine success attests resolved=true score=1', okV.resolved === true && okV.score === 1, okV.detail)
+
+const liarRun = badReportToRun(liarReport, 'wiki-openscad')
+const liarV = judgeBrowserRun(task, liarRun)
+check('SELF-REPORTED success is IGNORED — failed criteria → resolved=false', liarV.resolved === false && liarV.score === 0, liarV.detail)
+
+// Partial credit + dom check: 2 criteria, only url passes (snapshot lacks the phrase).
+const twoCrit: BrowserTask = {
+  ...task,
+  success: [{ type: 'url-contains', value: 'OpenSCAD' }, { type: 'dom-contains', value: 'Marching Cubes' }],
+}
+const partialV = judgeBrowserRun(twoCrit, okRun)
+check('partial credit: 1 of 2 criteria → score=0.5 resolved=false', partialV.score === 0.5 && partialV.resolved === false, partialV.detail)
+
+// Fail loud: no criteria is a harness error, not a silent pass.
+let threw = false
+try {
+  judgeBrowserRun({ ...task, success: [] }, okRun)
+} catch {
+  threw = true
+}
+check('empty success spec throws (fail-loud, never silent-attest)', threw)
+
+if (failures > 0) {
+  console.log(`\n❌ ${failures} check(s) failed`)
+  process.exit(1)
+}
+console.log('\n✅ browser harness verified — verdict is attestable, self-report ignored')

package/src/clbench-codebase-gate.mts

@@ -0,0 +1,314 @@
+/**
+ * CL-Bench (Continual) Codebase Adaptation — verifier-grounded selector gate.
+ *
+ * Of CL-Bench's six domains, codebase_adaptation is the ONLY deployable checker:
+ * its scorer applies the instance's provided `test_patch` and runs pytest in the
+ * instance's Docker image, keying off the exit code (the rest grade against realized
+ * outcomes the agent never has = oracles). That makes it the clean analogue of the
+ * HumanEval gate — an INDEPENDENT deployable check (tests ≠ the answer), unlike the
+ * CL-bench Context gate where the rubric judge IS the metric.
+ *
+ * Each instance is SWE-bench format (repo @ base_commit + a GitHub issue). The worker
+ * is a real sandbox rollout (opencode clones the repo, fixes the source, writes a diff
+ * to a file we read back — a large diff truncates in the chat stream). We score each
+ * candidate patch with CL-Bench's OWN scorer via `clbench_codebase_judge.py` (run in
+ * CL-Bench's venv) — verified to self-check (gold patch passes, empty fails). Two
+ * paired arms over the same instances:
+ *   random@K  — K identical-issue rollouts (the compute control)
+ *   diverse@K — K rollouts, the i-th with a strategy lens prepended (composeStrategies)
+ * verifierGroundedSelect picks by pytest-pass; we report blind / random@k / diverse@k /
+ * oracle@k with paired-bootstrap CIs, and write a corpus RunRecord/task the existing
+ * `corpus-replay --selector=verifier` + `corpus-report` consume unchanged. Fail loud:
+ * an infra-errored rollout is excluded (infraError), never scored 0.
+ *
+ *   dotenvx run -f … -- env N=4 K=3 WORKER_MODEL=deepseek-chat CONCURRENCY=2 \
+ *     CLBENCH_DIR=/tmp/clbench-continual CORPUS=/tmp/clbench-codebase.jsonl \
+ *     tsx src/clbench-codebase-gate.mts
+ */
+
+import { execFile } from 'node:child_process'
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { acquireSandbox } from '@tangle-network/agent-runtime/loops'
+import { Sandbox } from '@tangle-network/sandbox'
+import { composeStrategies } from './directives'
+import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
+import { verifierGroundedSelect } from './selector'
+import { type PairedLift, pairedLift, pool } from './stats.mts'
+
+const execFileAsync = promisify(execFile)
+const PATCH_PATH = '/tmp/solution.patch'
+const randomSuffix = () => Math.random().toString(36).slice(2, 10)
+
+function must(name: string): string {
+  const v = process.env[name]
+  if (!v) throw new Error(`env ${name} is required`)
+  return v
+}
+
+interface Instance {
+  instanceId: string
+  repo: string
+  baseCommit: string
+  problemStatement: string
+}
+
+/** Load CL-Bench codebase_adaptation instances (SWE-bench format) from the cloned
+ *  repo's final-dataset.jsonl. Fail loud on a malformed/short record. */
+function loadInstances(clbenchDir: string, limit: number, offset: number): Instance[] {
+  const path = join(clbenchDir, 'data/codebase_adaptation/final-dataset.jsonl')
+  const text = readFileSync(path, 'utf8')
+  const out: Instance[] = []
+  for (const line of text.split('\n')) {
+    if (line.trim() === '') continue
+    const d = JSON.parse(line) as { instance_id?: string; repo?: string; base_commit?: string; problem_statement?: string }
+    if (!d.instance_id || !d.repo || !d.base_commit || !d.problem_statement) {
+      throw new Error(`malformed codebase_adaptation record: ${line.slice(0, 120)}`)
+    }
+    out.push({ instanceId: d.instance_id, repo: d.repo, baseCommit: d.base_commit, problemStatement: d.problem_statement })
+  }
+  if (out.length === 0) throw new Error('codebase_adaptation parsed to 0 instances')
+  if (offset >= out.length) throw new Error(`OFFSET ${offset} >= dataset size ${out.length}`)
+  return out.slice(offset, offset + limit)
+}
+
+function rolloutPrompt(inst: Instance, lens: string | undefined): string {
+  return [
+    lens ? `${lens}\n` : '',
+    `Clone https://github.com/${inst.repo} into /work, then \`cd /work && git checkout ${inst.baseCommit}\`.`,
+    '',
+    'Resolve this GitHub issue by editing the SOURCE only (never the tests — the evaluation re-runs its own hidden tests on a fresh clone):',
+    '',
+    inst.problemStatement,
+    '',
+    'Work iteratively: install the package editable (`pip install -e .`), reproduce the issue, implement the fix, and re-run the existing tests until they pass.',
+    `When done, from /work run EXACTLY:`,
+    `  git add -A && git diff --cached -- . ':(exclude)*/test*' > ${PATCH_PATH}`,
+    `Then stop. The patch file is the only deliverable — do NOT paste the diff in your reply.`,
+  ].filter((s) => s !== '').join('\n')
+}
+
+interface ShotCfg {
+  sandboxBaseUrl: string
+  routerBaseUrl: string
+  routerKey: string
+  model: string
+  /** in-box opencode provider. `openai-compat` (default) is the generic passthrough —
+   *  it does NOT validate the model against opencode's registry, so router-served cheap
+   *  models (deepseek-chat, moonshotai/kimi-k2.6, glm) work; `openai`/`anthropic` only
+   *  accept their registered model names (e.g. gpt-4.1). */
+  provider: string
+  timeoutMs: number
+}
+
+interface Shot {
+  patch: string
+  /** rollout completed and produced a (possibly empty) patch; false ⇒ infra error (excluded). */
+  ran: boolean
+  detail?: string
+}
+
+/** One fault-isolated sandbox rollout → a patch (read from the box FS). ANY rollout
+ *  error becomes a recorded infra failure (ran=false), never a throw that kills the pool. */
+async function runRollout(inst: Instance, lens: string | undefined, cfg: ShotCfg): Promise<Shot> {
+  const client = new Sandbox({ baseUrl: cfg.sandboxBaseUrl, apiKey: cfg.routerKey })
+  let box: Awaited<ReturnType<typeof acquireSandbox>> | undefined
+  try {
+    box = await acquireSandbox(client, {
+      name: `clbench-cb-${inst.instanceId}-${randomSuffix()}`.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(0, 60),
+      environment: 'universal',
+      // backend.model pins provider/model/baseUrl only — in-box model auth is the
+      // box-provisioned OPENCODE_MODEL_API_KEY (foreign keys are 403'd at egress).
+      backend: { type: 'opencode', model: { provider: cfg.provider, model: cfg.model, baseUrl: cfg.routerBaseUrl } },
+    })
+    const signal = cfg.timeoutMs > 0 ? AbortSignal.timeout(cfg.timeoutMs) : undefined
+    for await (const _ev of box.streamPrompt(rolloutPrompt(inst, lens), signal ? { signal } : {})) {
+      // drain; the deliverable is the patch FILE, not the stream
+    }
+    let patch = ''
+    try {
+      patch = await box.fs.read(PATCH_PATH)
+    } catch {
+      patch = '' // missing patch file ⇒ the agent produced nothing (a real empty, ran=true)
+    }
+    return { patch, ran: true }
+  } catch (err) {
+    return { patch: '', ran: false, detail: `rollout error: ${(err instanceof Error ? err.message : String(err)).slice(0, 200)}` }
+  } finally {
+    try {
+      if (box) await box.delete()
+    } catch {
+      // staging reaps on expiry
+    }
+  }
+}
+
+/** Score one candidate patch with CL-Bench's deployable pytest checker via the bridge
+ *  (run in CL-Bench's venv, cwd = its repo root so `src.tasks...` resolves). Returns
+ *  pass (0/1) or null on an infra/judge failure (excluded, never scored 0). */
+async function judgePatch(inst: Instance, patch: string, clbenchDir: string): Promise<number | null> {
+  if (patch.trim() === '') return 0 // empty patch is a legitimate fail, not an infra error
+  const dir = mkdtempSync(join(tmpdir(), 'clbench-cb-'))
+  const patchFile = join(dir, 'candidate.patch')
+  writeFileSync(patchFile, patch)
+  try {
+    const { stdout } = await execFileAsync(
+      join(clbenchDir, '.venv/bin/python'),
+      [
+        join(process.cwd(), 'scripts/clbench_codebase_judge.py'),
+        '--dataset',
+        join(clbenchDir, 'data/codebase_adaptation/final-dataset.jsonl'),
+        '--instance-id',
+        inst.instanceId,
+        '--patch-file',
+        patchFile,
+      ],
+      { cwd: clbenchDir, maxBuffer: 8 * 1024 * 1024, timeout: 600_000 },
+    )
+    const last = stdout.trim().split('\n').at(-1) ?? '{}'
+    const verdict = JSON.parse(last) as { success?: boolean }
+    return verdict.success ? 1 : 0
+  } catch (err) {
+    console.error(`  judge infra error ${inst.instanceId}: ${(err instanceof Error ? err.message : String(err)).slice(0, 160)}`)
+    return null
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+const pct = (x: number) => `${(x * 100).toFixed(1)}%`
+const pp = (x: number) => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp`
+
+async function main(): Promise<void> {
+  const n = Number(process.env.N ?? 4)
+  const k = Number(process.env.K ?? 3)
+  const offset = Number(process.env.OFFSET ?? 0)
+  const model = process.env.WORKER_MODEL ?? 'deepseek-chat'
+  // openai-compat = generic passthrough so cheap router models resolve in-box (see ShotCfg).
+  const provider = process.env.WORKER_PROVIDER ?? 'openai-compat'
+  const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
+  const routerKey = must('TANGLE_API_KEY')
+  const sandboxBaseUrl = process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools'
+  const clbenchDir = process.env.CLBENCH_DIR ?? '/tmp/clbench-continual'
+  const rolloutConc = Number(process.env.CONCURRENCY ?? 3)
+  const judgeConc = Number(process.env.JUDGE_CONCURRENCY ?? 2)
+  const timeoutMs = process.env.SHOT_TIMEOUT_MS ? Number(process.env.SHOT_TIMEOUT_MS) : 900_000
+  const corpusPath = process.env.CORPUS ?? '/tmp/clbench-codebase.jsonl'
+  if (!Number.isInteger(n) || n < 1) throw new Error(`N must be a positive integer, got ${process.env.N}`)
+  if (!Number.isInteger(k) || k < 1) throw new Error(`K must be a positive integer, got ${process.env.K}`)
+
+  const cfg: ShotCfg = { sandboxBaseUrl, routerBaseUrl, routerKey, model, provider, timeoutMs }
+  console.log(`=== CL-Bench Codebase Adaptation selector gate · N=${n} K=${k} offset=${offset} model=${provider}/${model} ===`)
+  console.log(`  sandbox=${sandboxBaseUrl}  judge=CL-Bench pytest-in-Docker (deployable)  clbench=${clbenchDir}`)
+
+  const instances = loadInstances(clbenchDir, n, offset)
+  console.log(`loaded ${instances.length} instance(s): ${instances.map((i) => i.instanceId).join(', ')}`)
+
+  type Unit = { instIdx: number; arm: 'random' | 'diverse'; shot: number; lens: string | undefined }
+  const units: Unit[] = []
+  for (let ii = 0; ii < instances.length; ii += 1) {
+    const lenses = composeStrategies('', k) // lens prefixes only ('' base ⇒ "<lens>\n\n")
+    for (let s = 0; s < k; s += 1) {
+      units.push({ instIdx: ii, arm: 'random', shot: s, lens: undefined })
+      units.push({ instIdx: ii, arm: 'diverse', shot: s, lens: (lenses[s] as string).trim() })
+    }
+  }
+  console.log(`\n▶ phase 1: ${units.length} rollouts (${instances.length}×${k}×2 arms) via sandbox, conc=${rolloutConc}`)
+  const shots = await pool(units, rolloutConc, async (u) => {
+    const inst = instances[u.instIdx] as Instance
+    const s = await runRollout(inst, u.lens, cfg)
+    console.log(`  rollout ${inst.instanceId} ${u.arm}#${u.shot}: ${s.ran ? `patch ${s.patch.length}B` : `INFRA (${s.detail})`}`)
+    return s
+  })
+
+  console.log(`\n▶ phase 2: judging ${shots.length} patches with the deployable checker, conc=${judgeConc}`)
+  const passes = await pool(units, judgeConc, async (u, i) => {
+    const shot = shots[i] as Shot
+    if (!shot.ran) return null // infra error ⇒ excluded
+    const inst = instances[u.instIdx] as Instance
+    const p = await judgePatch(inst, shot.patch, clbenchDir)
+    console.log(`  judge ${inst.instanceId} ${u.arm}#${u.shot}: ${p === null ? 'INFRA' : p ? 'PASS' : 'fail'}`)
+    return p
+  })
+
+  // Regroup; an attempt with a null pass (infra) is dropped from its arm.
+  const byInst = instances.map(() => ({ random: [] as (number | null)[], diverse: [] as (number | null)[], rPatch: [] as string[], dPatch: [] as string[] }))
+  units.forEach((u, i) => {
+    const grp = byInst[u.instIdx] as { random: (number | null)[]; diverse: (number | null)[]; rPatch: string[]; dPatch: string[] }
+    const pass = passes[i] as number | null
+    const patch = (shots[i] as Shot).patch
+    if (u.arm === 'random') { grp.random[u.shot] = pass; grp.rPatch[u.shot] = patch } else { grp.diverse[u.shot] = pass; grp.dPatch[u.shot] = patch }
+  })
+
+  // Per-instance {0,1} outcomes; instances with no valid attempt in BOTH arms are excluded.
+  const blind: number[] = []
+  const randomAtK: number[] = []
+  const diverseAtK: number[] = []
+  const oracleAtK: number[] = []
+  let excluded = 0
+  for (const grp of byInst) {
+    const rValid = grp.random.filter((p): p is number => p !== null && p !== undefined)
+    const dValid = grp.diverse.filter((p): p is number => p !== null && p !== undefined)
+    if (rValid.length === 0 || dValid.length === 0) { excluded += 1; continue }
+    blind.push(rValid[0] as number)
+    randomAtK.push(rValid[verifierGroundedSelect(rValid)] as number)
+    diverseAtK.push(dValid[verifierGroundedSelect(dValid)] as number)
+    oracleAtK.push(dValid.some((p) => p > 0) ? 1 : 0)
+  }
+  const rate = (xs: number[]) => (xs.length === 0 ? 0 : xs.reduce((s, x) => s + x, 0) / xs.length)
+
+  console.log(`\n${'='.repeat(78)}`)
+  console.log(`RESULTS · CL-Bench Codebase Adaptation · n=${blind.length} scored (${excluded} excluded) · k=${k} · model=${model}`)
+  console.log('='.repeat(78))
+  console.log(`  blind pass@1               ${pct(rate(blind))}`)
+  console.log(`  random@k (verifier-pick)   ${pct(rate(randomAtK))}`)
+  console.log(`  diverse@k (verifier-pick)  ${pct(rate(diverseAtK))}`)
+  console.log(`  oracle@k (diverse, any)    ${pct(rate(oracleAtK))}`)
+  if (blind.length >= 2) {
+    const row = (label: string, l: PairedLift) =>
+      console.log(`  ${label.padEnd(34)} ${pp(l.point).padStart(7)}   CI [${pp(l.low)}, ${pp(l.high)}]   (paired ${l.pairs}, discordant ${l.discordant})`)
+    console.log(`\n  PAIRED LIFTS (95% bootstrap CI):`)
+    row('random@k − blind (compute)', pairedLift(blind, randomAtK))
+    row('diverse@k − random@k (verifier)', pairedLift(randomAtK, diverseAtK))
+    row('diverse@k − blind (total)', pairedLift(blind, diverseAtK))
+  } else {
+    console.log('\n  (n<2 scored — paired CIs need ≥2; this is a plumbing smoke, not a signal)')
+  }
+
+  // Corpus: random@k arm, ranked by the deployable pytest verifier.
+  for (let ii = 0; ii < instances.length; ii += 1) {
+    const inst = instances[ii] as Instance
+    const grp = byInst[ii] as { random: (number | null)[]; rPatch: string[] }
+    const attempts: AttemptRecord[] = grp.random.map((p, round) => ({
+      round,
+      prompt: 'clbench-codebase-rollout',
+      output: (grp.rPatch[round] ?? '').slice(0, 4000),
+      ...(p === null ? { error: 'infra' } : { valid: p > 0, score: p }),
+      eventCount: 1,
+      eventTypes: { 'sandbox.stream': 1 },
+      traceTail: (grp.rPatch[round] ?? '').slice(-600),
+    }))
+    const validPasses = grp.random.filter((p): p is number => p !== null && p !== undefined)
+    const record = buildRunRecordFromAttempts(attempts, {
+      benchmark: 'clbench-codebase',
+      instanceId: inst.instanceId,
+      condition: `random@${k}`,
+      model,
+      // blindResolved reads off the FIRST non-null score (== full pass), not
+      // attempts[0].valid — a partial-credit first shot is valid but not "blind
+      // resolved". Pass it explicitly so the helper preserves the exact value.
+      blindResolved: validPasses[0] === 1,
+      resolved: validPasses.some((p) => p > 0),
+      infraError: validPasses.length === 0,
+    })
+    await appendRunRecord(corpusPath, record)
+  }
+  console.log(`\n=== wrote ${instances.length} task(s) → ${corpusPath} · gate: tsx src/corpus-replay.mts ${corpusPath} --selector=verifier ===`)
+}
+
+main().catch((err) => {
+  console.error(`clbench-codebase-gate: ${err instanceof Error ? err.message : String(err)}`)
+  process.exit(1)
+})