@tangle-network/agent-bench 0.1.0

package/src/atom-mcp-e2e.mts

@@ -0,0 +1,223 @@
+/**
+ * THE WHOLE REAL THING, end to end. No mock, no stub.
+ *
+ * An opencode SUPERVISOR (via the cli-bridge) mounts the coordination MCP over a live Scope and
+ * carries the real `supervise` SKILL.md (a file in its cwd's skill dir — loaded natively by the
+ * harness, not stapled into a prompt). It authors a worker profile and calls spawn_worker.
+ * Each WORKER is an opencode coding session in its OWN cwd that edits files and is graded by a
+ * REAL test it must pass (the deployable check = `valid`). The supervisor settles only on a
+ * delivered worker. Real models (bridge non-Claude), real check, real driver↔worker transcripts.
+ *
+ *   ROUTER_BASE=http://127.0.0.1:3355/v1 TANGLE_API_KEY=<bridge-bearer> \
+ *   WORKER_MODEL=opencode/zai-coding-plan/glm-5-turbo npx tsx bench/src/atom-mcp-e2e.mts
+ */
+
+import { execFileSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import {
+  type Agent,
+  type AgentProfile,
+  type AgentSpec,
+  contentAddress,
+  createExecutorRegistry,
+  createSupervisor,
+  type Executor,
+  type ExecutorResult,
+  gitWorkspace,
+  InMemoryResultBlobStore,
+  InMemorySpawnJournal,
+  runInWorkspace,
+  type Scope,
+  type Workspace,
+} from '../../src/runtime/index'
+import { asAuthoredProfile } from '../../src/runtime/supervise/authoring'
+import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp'
+
+const BRIDGE = (process.env.ROUTER_BASE ?? 'http://127.0.0.1:3355/v1').replace(/\/$/, '')
+const BEARER = process.env.TANGLE_API_KEY ?? ''
+const MODEL = process.env.WORKER_MODEL ?? 'opencode/zai-coding-plan/glm-5-turbo'
+const REPO = join(dirname(fileURLToPath(import.meta.url)), '..', '..')
+const SKILL_MD = readFileSync(join(REPO, 'skills', 'supervise', 'SKILL.md'), 'utf8')
+
+const TASK = 'In solution.py, implement add(a, b) so it returns the sum a + b and test_solution.py passes.'
+
+/** Seed a bare git repo with the failing task — the SHARED workspace ref every worker clones. */
+function seedWorkspaceRepo(): string {
+  const git = (args: string[], cwd?: string): void => {
+    execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'user.email=t@t', '-c', 'user.name=t', ...args], {
+      cwd,
+      stdio: 'pipe',
+    })
+  }
+  const bare = `${mkdtempSync(join(tmpdir(), 'e2e-ws-'))}.git`
+  git(['init', '--bare', '-b', 'main', bare])
+  const seed = mkdtempSync(join(tmpdir(), 'e2e-seed-'))
+  git(['clone', bare, seed])
+  writeFileSync(join(seed, 'solution.py'), 'def add(a, b):\n    raise NotImplementedError\n')
+  writeFileSync(
+    join(seed, 'test_solution.py'),
+    'from solution import add\nassert add(2, 3) == 5\nassert add(-1, 1) == 0\nassert add(0, 0) == 0\nprint("PASS")\n',
+  )
+  git(['add', '-A'], seed)
+  git(['commit', '-m', 'task'], seed)
+  git(['push', 'origin', 'main'], seed)
+  rmSync(seed, { recursive: true, force: true })
+  return bare
+}
+
+/** The deployable check: run the test in the worker's cwd. Exit 0 = delivered. No LLM judge. */
+function checkPasses(cwd: string): boolean {
+  try {
+    execFileSync('python3', ['test_solution.py'], { cwd, stdio: 'pipe', timeout: 30_000 })
+    return true
+  } catch {
+    return false
+  }
+}
+
+async function bridgeChat(opts: {
+  messages: Array<{ role: string; content: string }>
+  cwd?: string
+  mcpUrl?: string
+}): Promise<string> {
+  const r = await fetch(`${BRIDGE}/chat/completions`, {
+    method: 'POST',
+    headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' },
+    body: JSON.stringify({
+      model: MODEL,
+      messages: opts.messages,
+      ...(opts.cwd ? { cwd: opts.cwd } : {}),
+      ...(opts.mcpUrl ? { mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } } } : {}),
+    }),
+  })
+  if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})`
+  const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> }
+  return j.choices?.[0]?.message?.content ?? ''
+}
+
+const transcripts: Array<{ who: string; said: string; delivered?: boolean }> = []
+
+/** A WORKER = a real opencode coding session in a clone of the SHARED workspace, graded by the
+ *  real test; its delivery is committed back so the next worker builds on it (not isolated). */
+function makeWorker(rawProfile: unknown, ws: Workspace, n: number): Agent<unknown, unknown> {
+  const p = asAuthoredProfile(rawProfile)
+  const name = p?.name ?? `worker-${n}`
+  let artifact: ExecutorResult<unknown> | undefined
+  const inner: Executor<unknown> = {
+    runtime: 'router',
+    async execute() {
+      const sys = p?.systemPrompt ?? TASK
+      const run = await runInWorkspace(
+        ws,
+        async (cwd) => {
+          const said = await bridgeChat({
+            messages: [
+              {
+                role: 'user',
+                content: `${sys}\n\nYou are working in the current directory (it already holds prior workers' committed progress). Edit the files so that running \`python3 test_solution.py\` prints PASS. Do it now.`,
+              },
+            ],
+            cwd,
+          })
+          const valid = checkPasses(cwd)
+          transcripts.push({ who: name, said: said.slice(0, 300), delivered: valid })
+          return { valid, value: said.slice(0, 120), message: `${name}: ${valid ? 'delivered' : 'wip'}` }
+        },
+        { tmpPrefix: 'e2e-worker-', commitOnInvalid: true },
+      )
+      const delivered = run.valid
+      artifact = {
+        outRef: contentAddress(`${name}:${delivered}`),
+        out: { worker: name, delivered, rev: run.commit?.ok ? run.commit.rev : undefined, profileSystemPrompt: sys.slice(0, 120) },
+        verdict: { valid: delivered, score: delivered ? 1 : 0 },
+        spent: { iterations: 1, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 },
+      }
+      return artifact
+    },
+    teardown: () => Promise.resolve({ destroyed: true }),
+    resultArtifact: () => {
+      if (!artifact) throw new Error('worker resultArtifact before execute')
+      return artifact
+    },
+  }
+  const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: inner }
+  return { name, act: async () => '', executorSpec: spec } as Agent<unknown, unknown> & { executorSpec: AgentSpec }
+}
+
+async function main(): Promise<void> {
+  console.log(`atom-mcp-e2e: model=${MODEL}  (real boxes, real MCP, real test, shared workspace)`)
+  const bareRef = seedWorkspaceRepo()
+  const ws = gitWorkspace({ ref: bareRef })
+  const blobs = new InMemoryResultBlobStore()
+  let n = 0
+
+  const root: Agent<unknown, unknown> = {
+    name: 'supervisor',
+    async act(_t, scope: Scope<unknown>) {
+      const mcp = await serveCoordinationMcp({
+        scope,
+        blobs,
+        makeWorkerAgent: (raw) => makeWorker(raw, ws, n++),
+        perWorker: { maxIterations: 2, maxTokens: 200_000 },
+      })
+      // The supervisor's cwd carries the REAL skill file (opencode loads it from the cwd skill dirs).
+      const supCwd = mkdtempSync(join(tmpdir(), 'e2e-sup-'))
+      for (const d of ['.opencode/skills/supervise', '.claude/skills/supervise']) {
+        mkdirSync(join(supCwd, d), { recursive: true })
+        writeFileSync(join(supCwd, d, 'SKILL.md'), SKILL_MD)
+      }
+      try {
+        console.error(`[e2e] coordination MCP at ${mcp.url}; supervisor cwd=${supCwd}`)
+        const said = await bridgeChat({
+          messages: [
+            {
+              role: 'user',
+              content: `${TASK}\n\nYou are a SUPERVISOR. You have the "supervise" skill and a "coordination" MCP with tools spawn_worker, await_event, stop. Do NOT write code yourself. Author a worker profile (a JSON object with name + a rich systemPrompt telling the worker exactly what to implement) and call spawn_worker with it, then await_event, and stop once a worker delivered (valid:true).`,
+            },
+          ],
+          cwd: supCwd,
+          mcpUrl: mcp.url,
+        })
+        transcripts.push({ who: 'supervisor', said: said.slice(0, 400) })
+        const settled = mcp.settled()
+        const delivered = settled.filter((w) => w.status === 'done' && w.valid === true)
+        console.error(`[e2e] supervisor spawned ${settled.length} worker(s), ${delivered.length} delivered`)
+        return delivered[0]?.outRef ? await blobs.get(delivered[0].outRef) : undefined
+      } finally {
+        await mcp.close()
+        rmSync(supCwd, { recursive: true, force: true })
+      }
+    },
+  }
+
+  const result = await createSupervisor<unknown, unknown>().run(root, TASK, {
+    budget: { maxIterations: 100, maxTokens: 2_000_000 },
+    runId: 'e2e',
+    journal: new InMemorySpawnJournal(),
+    blobs,
+    executors: createExecutorRegistry(),
+    maxDepth: 4,
+    now: () => Date.now(),
+  })
+  rmSync(bareRef, { recursive: true, force: true })
+
+  console.log('\n── transcripts (real driver↔worker) ──')
+  for (const t of transcripts) {
+    console.log(`\n[${t.who}${t.delivered === undefined ? '' : t.delivered ? ' · DELIVERED' : ' · failed'}]`)
+    console.log(`  ${t.said.replace(/\n/g, '\n  ')}`)
+  }
+  console.log('\n── verdict ──')
+  console.log(
+    result.kind === 'winner'
+      ? `✅ REAL E2E DELIVERED — supervisor (via MCP + skill) drove an in-box worker that coded + passed the real test. out=${JSON.stringify(result.out)}`
+      : `❌ no delivery (result=${result.kind}) — see transcripts above`,
+  )
+}
+
+main().catch((e) => {
+  console.error(e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/benchmarks/_harness.ts

@@ -0,0 +1,206 @@
+/**
+ * Shared code-bench harness. The "stage the artifact → run the benchmark's own
+ * evaluator in an external process (mkdtemp / execFile / .venv python / Docker)
+ * → read its JSON report → { resolved, score }" spine that swe-bench,
+ * terminal-bench, commit0, programbench and aec-bench all need. Factored out so
+ * the Docker/venv/report-reading logic lives in ONE place instead of being
+ * copy-pasted per adapter.
+ *
+ * It owns NO benchmark policy: each adapter passes the argv for its evaluator
+ * and a `parseReport` that maps that evaluator's report JSON → a BenchScore. The
+ * harness owns process spawning, temp-dir lifecycle, large-buffer/timeout config
+ * and fail-loud diagnostics.
+ *
+ * Fail-loud: an absent .venv / harness / Docker daemon THROWS from `preflight`
+ * (the adapter passes the import line + the exact fix). A staged run that exits
+ * nonzero throws with the captured stderr — never a fabricated score.
+ */
+
+import { execFile, spawn } from 'node:child_process'
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+import type { BenchScore } from './types'
+
+const execFileAsync = promisify(execFile)
+
+/** Repo root for the bench package (…/bench), so `.venv` and `fixtures` resolve. */
+export const benchRoot = fileURLToPath(new URL('../..', import.meta.url))
+/** The bench venv interpreter every python-backed evaluator runs through. */
+export const venvPython = join(benchRoot, '.venv', 'bin', 'python')
+
+/** Interpreter for a NAMED isolated venv (e.g. `.venv-commit0`). Benches whose pip
+ *  deps conflict with the shared `.venv` (commit0 downgrades pydantic/sqlalchemy)
+ *  get their own venv and pass its python explicitly — keeping the shared one clean. */
+export const venvPythonAt = (venvDir: string): string => join(benchRoot, venvDir, 'bin', 'python')
+/** Report/transcript reads are large; 256 MiB matches the SWE harness budget. */
+export const bigBuffer = 1024 * 1024 * 256
+
+/** Path to a named executable inside the bench venv (e.g. `venvBin('tb')`). */
+export function venvBin(name: string): string {
+  return join(benchRoot, '.venv', 'bin', name)
+}
+
+/**
+ * Run the bench venv python with an inline script (`-c`); return stdout. Throws
+ * (with stderr) on a nonzero exit — the loaders rely on this to fail loud rather
+ * than parse a partial dump.
+ */
+export async function runVenvPython(
+  script: string,
+  args: string[] = [],
+  timeoutMs = 0,
+  python: string = venvPython,
+): Promise<string> {
+  const { stdout } = await execFileAsync(python, ['-c', script, ...args], {
+    maxBuffer: bigBuffer,
+    timeout: timeoutMs,
+  })
+  return stdout
+}
+
+/**
+ * Preflight a python-backed harness: import the module(s) and (optionally) ping
+ * Docker, all inside the bench venv. On failure THROWS the captured error joined
+ * to the adapter's `fix` guidance — the contract every code-bench preflight wants.
+ */
+export async function preflightVenvImports(opts: {
+  /** Module names to `import` (e.g. ['swebench']); '' entries are ignored. */
+  modules: string[]
+  /** Also `docker.from_env().ping()` — true for Docker-backed evaluators. */
+  requireDocker?: boolean
+  /** Actionable remediation appended to the thrown message. */
+  fix: string
+  /** Override the interpreter (e.g. an isolated `.venv-commit0`). Default: shared `.venv`. */
+  python?: string
+}): Promise<void> {
+  const imports = opts.modules.filter((m) => m.length > 0)
+  const lines = [...imports.map((m) => `import ${m}`)]
+  if (opts.requireDocker) lines.push('import docker', 'docker.from_env().ping()')
+  lines.push("print('ok')")
+  try {
+    await runVenvPython(lines.join('\n'), [], 0, opts.python ?? venvPython)
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err)
+    throw new Error(`${msg}\n${opts.fix}`)
+  }
+}
+
+/**
+ * Run a bench-local python driver script (in the bench venv) while piping
+ * `input` to its stdin, returning stdout. The driver's verdict JSON is its LAST
+ * stdout line; callers parse that and inspect an `error` field (fail loud).
+ *
+ * Uses spawn + an explicit `stdin.end(input)` rather than promisify(execFile)'s
+ * `input` option, because that option is NOT honored by async execFile — stdin
+ * is left open and a driver that does `sys.stdin.read()` blocks forever. The
+ * artifact-piping judges (commit0, appworld) MUST go through this.
+ */
+export function runVenvScriptStdin(
+  scriptPath: string,
+  args: string[],
+  input: string,
+  opts: { cwd?: string; timeoutMs?: number; python?: string } = {},
+): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    const child = spawn(opts.python ?? venvPython, [scriptPath, ...args], {
+      cwd: opts.cwd ?? benchRoot,
+      ...(opts.timeoutMs ? { timeout: opts.timeoutMs } : {}),
+    })
+    let stdout = ''
+    let stderr = ''
+    let bytes = 0
+    child.stdout.on('data', (c: Buffer) => {
+      bytes += c.length
+      if (bytes <= bigBuffer) stdout += c.toString('utf8')
+    })
+    child.stderr.on('data', (c: Buffer) => {
+      stderr += c.toString('utf8')
+    })
+    child.on('error', (err) => reject(err))
+    child.on('close', (code) => {
+      if (code === 0) resolve(stdout)
+      else reject(new Error((stderr || stdout || `exit ${code}`).slice(0, 1500)))
+    })
+    child.stdin.end(input)
+  })
+}
+
+export interface StagedRunSpec {
+  /** mkdtemp prefix, e.g. 'swebench-' / 'commit0-'. */
+  tmpPrefix: string
+  /**
+   * Write the artifact + any harness inputs into the temp dir. Returns nothing;
+   * `argv`/`cwd` consume `dir` to point the evaluator at what was written.
+   */
+  stage(dir: string): Promise<void>
+  /** The external evaluator to spawn. `bin` defaults to the bench venv python. */
+  bin?: string
+  /** argv for the evaluator, computed from the temp `dir`. */
+  argv(dir: string): string[]
+  /** Working directory for the evaluator. Defaults to the temp `dir`. */
+  cwd?(dir: string): string
+  /** Hard timeout for the evaluator (ms); 0 = none. */
+  timeoutMs?: number
+  /**
+   * Read the evaluator's report(s) out of `dir` and map to a BenchScore. Throws
+   * if the expected report is absent/malformed (fail loud — no default score).
+   */
+  parseReport(dir: string): Promise<BenchScore>
+  /** Keep the temp dir on disk (debugging). Default false → always cleaned up. */
+  keepTmp?: boolean
+}
+
+/**
+ * The shared judge body: mkdtemp → stage → spawn evaluator → parseReport →
+ * cleanup. The evaluator's stdout/stderr is surfaced on failure; the temp dir is
+ * always removed in `finally` unless `keepTmp`.
+ */
+export async function runStagedJudge(spec: StagedRunSpec): Promise<BenchScore> {
+  const dir = await mkdtemp(join(tmpdir(), spec.tmpPrefix))
+  try {
+    await spec.stage(dir)
+    const bin = spec.bin ?? venvPython
+    try {
+      await execFileAsync(bin, spec.argv(dir), {
+        cwd: spec.cwd ? spec.cwd(dir) : dir,
+        maxBuffer: bigBuffer,
+        ...(spec.timeoutMs ? { timeout: spec.timeoutMs } : {}),
+      })
+    } catch (err) {
+      const e = err as { stderr?: string; stdout?: string; message?: string }
+      const detail = (e.stderr || e.stdout || e.message || String(err)).slice(0, 2000)
+      throw new Error(`${spec.tmpPrefix.replace(/-$/, '')} evaluator failed (${bin} ${spec.argv(dir).join(' ')}):\n${detail}`)
+    }
+    return await spec.parseReport(dir)
+  } finally {
+    if (!spec.keepTmp) await rm(dir, { recursive: true, force: true }).catch(() => {})
+  }
+}
+
+/** Read + JSON.parse a report file from a staged run; throws with the path on failure. */
+export async function readJsonReport<T>(path: string): Promise<T> {
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf8')
+  } catch (err) {
+    throw new Error(`expected report not written: ${path} (${err instanceof Error ? err.message : err})`)
+  }
+  try {
+    return JSON.parse(raw) as T
+  } catch (err) {
+    throw new Error(`report not valid JSON at ${path}: ${err instanceof Error ? err.message : err}`)
+  }
+}
+
+/** Write a UTF-8 file into a staged dir (artifact / preds.json / attempt.sh / …). */
+export async function stageFile(path: string, content: string): Promise<void> {
+  await writeFile(path, content)
+}
+
+/** Sanitize an instance id into a filesystem/run-id-safe token. */
+export function safeRunId(prefix: string, id: string): string {
+  return `${prefix}-${id}`.replace(/[^a-zA-Z0-9_.-]/g, '_')
+}

package/src/benchmarks/aec-bench.test.mts

@@ -0,0 +1,53 @@
+/**
+ * Offline AEC-Bench adapter test. AEC's judge runs the task's own verify.py with
+ * python3 (stdlib only, no Docker, no pip), so the FULL judge is exercised here:
+ * gold → score 1, empty → score 0, with per-field partial credit in detail. Run:
+ *   AEC_FIXTURES=1 npx tsx --test src/benchmarks/aec-bench.test.mts
+ */
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { createAecBenchAdapter } from './aec-bench'
+
+process.env.AEC_FIXTURES = '1'
+
+const id = 'electrical/catenary-sag'
+
+test('loadTasks (fixtures) yields self-contained tasks with verifier metadata', async () => {
+  const a = createAecBenchAdapter()
+  const tasks = await a.loadTasks({ ids: [id] })
+  assert.equal(tasks.length, 1)
+  const t = tasks[0]
+  assert.equal(t.id, id)
+  assert.equal(t.split, 'electrical')
+  assert.ok(t.prompt.length > 0, 'prompt is the instruction.md')
+  const md = t.metadata as Record<string, unknown>
+  assert.equal(typeof md.verifyPy, 'string')
+  assert.ok((md.verifyPy as string).length > 0)
+  assert.equal(typeof md.goldenPassMd, 'string')
+})
+
+test('loadTasks limit slices the fixture set', async () => {
+  const a = createAecBenchAdapter()
+  const tasks = await a.loadTasks({ limit: 1 })
+  assert.equal(tasks.length, 1)
+})
+
+test('judge: REAL verify.py — gold resolves with full per-field credit', async () => {
+  const a = createAecBenchAdapter()
+  const [t] = await a.loadTasks({ ids: [id] })
+  const gold = await a.goldArtifact(t)
+  assert.equal(typeof gold, 'string')
+  const score = await a.judge(t, gold as string)
+  assert.equal(score.resolved, true)
+  assert.equal(score.score, 1)
+  const detail = JSON.parse(score.detail as string) as { fields: Record<string, number> }
+  for (const v of Object.values(detail.fields)) assert.equal(v, 1)
+})
+
+test('judge: REAL verify.py — empty artifact fails closed to 0 (no fabricated score)', async () => {
+  const a = createAecBenchAdapter()
+  const [t] = await a.loadTasks({ ids: [id] })
+  const score = await a.judge(t, '')
+  assert.equal(score.resolved, false)
+  assert.equal(score.score, 0)
+})