@tangle-network/agent-bench 0.13.7 → 0.13.13

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.13.13
+
+Require Knowledge 17.1.6 and Eval `>=0.191.0 <0.194.0` through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.12
+
+Accept Eval 0.185 and 0.186 and require Knowledge 17.1.2 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.11
+
+Accept Eval 0.184 and require Knowledge 17.1.1 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.10
+
+Accept Sandbox SDK 0.47 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.9
+
+Accept agent-interface 2.11.0 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.8
+
+Accept Sandbox SDK 0.46 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
 ## 0.13.7
 
 Support Eval 0.183 and require Knowledge 17.1 through the shared dependency catalog.

package/HARNESS.md

@@ -28,6 +28,8 @@ Use these labels literally. Do not promote one level into another in prose.
 
 A localization score, output-shape check, LLM quality judge, or toy deterministic reward can be useful for development. None is a substitute for the benchmark's outcome evaluator.
 
+The [September 24 public benchmark proof](evidence/public-agent-benchmarks-20260924/README.md) retains three officially graded agent tasks, traces, and validity limits across Terminal-Bench and SWE-bench Verified.
+
 ## Supported commands
 
 ### Package and integration contracts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.13.7",
+  "version": "0.13.13",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.183.0 <0.184.0",
-    "@tangle-network/agent-interface": "^2.10.0",
-    "@tangle-network/agent-knowledge": "^17.1.0",
-    "@tangle-network/sandbox": ">=0.36.4 <0.46.0",
-    "@tangle-network/agent-runtime": "^0.249.1"
+    "@tangle-network/agent-eval": ">=0.191.0 <0.194.0",
+    "@tangle-network/agent-interface": "^2.11.0",
+    "@tangle-network/agent-knowledge": "^17.1.6",
+    "@tangle-network/sandbox": ">=0.36.4 <0.48.0",
+    "@tangle-network/agent-runtime": "^0.277.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/swe-bench-env.test.ts

@@ -47,6 +47,11 @@ const makeRelativeSymlinkRepo = (): { root: string; source: string; destination:
     `core.hooksPath=${hooksDir}`,
     '-c',
     'commit.gpgsign=false',
+    // A commit starts `git maintenance run --auto` in the background, which creates and removes
+    // .git/objects/maintenance.lock while the test copies the checkout. The copy then fails with
+    // ENOENT on a file that vanished: 5 of 240 copies on Node 22.23 and 24.11, 0 of 400 without it.
+    '-c',
+    'maintenance.auto=false',
     'commit',
     '--quiet',
     '-m',

package/src/swe-local-proof.mts

@@ -16,7 +16,7 @@
  *     node_modules/.bin/tsx bench/src/swe-local-proof.mts
  */
 import { execFile } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
@@ -40,12 +40,24 @@ async function main(): Promise<void> {
   // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool + the run-aware prompt. Default OFF ⇒
   // the read/edit-only baseline (glm-5.2 7/12) unchanged.
   const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+  const artifactDir = process.env.RUN_ARTIFACT_DIR
+  if (artifactDir) mkdirSync(artifactDir, { recursive: true })
 
   console.log(`═══ SWE-bench Verified — SEE-able LOCAL proof (no tangle sandbox) ═══`)
   console.log(`model=${model}  ids=${ids.join(',')}  innerTurns=${innerTurns}  maxTokens=${maxTokens}  budget=${budget}  runTool=${enableRun}`)
   console.log(`router=${routerBaseUrl}`)
 
-  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun })
+  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, {
+    ids,
+    enableRun,
+    ...(artifactDir ? {
+      adapterOptions: {
+        captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({
+          destination: join(artifactDir, taskId, `judge-${attemptSequence}`),
+        }),
+      },
+    } : {}),
+  })
   const workerProfile = withBenchProfile(
     {
       name: 'swe-local-proof-worker',
@@ -85,10 +97,27 @@ async function main(): Promise<void> {
   const captured = new Map<string, Rec>()
   const judged = new Map<string, BenchScore>()
   const toolStats = new Map<string, { list: number; read: number; edit_ok: number; edit_fail: number; run: number; run_err: number }>()
+  let activeTaskId = ''
+  const recordTool = (name: string, args: unknown, result: unknown, error?: string) => {
+    if (!artifactDir || !activeTaskId) return
+    const taskDir = join(artifactDir, activeTaskId)
+    mkdirSync(taskDir, { recursive: true })
+    appendFileSync(join(taskDir, 'tools.jsonl'), JSON.stringify({
+      at: new Date().toISOString(), name, args,
+      ...(error ? { error } : { result }),
+    }) + '\n')
+  }
   const proxy: AgenticSurface = {
     ...environment,
     async call(handle, name, args) {
-      const res = await environment.call(handle, name, args)
+      let res: string
+      try {
+        res = await environment.call(handle, name, args)
+      } catch (error) {
+        recordTool(name, args, undefined, error instanceof Error ? error.message : String(error))
+        throw error
+      }
+      recordTool(name, args, res)
       // Count tool usage per workspace so we can SEE whether the agent ever edited, and whether
       // edits succeeded or bounced off old_string matching. handle.id keys the workspace.
       const st = toolStats.get(handle.id) ?? { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 }
@@ -161,6 +190,7 @@ async function main(): Promise<void> {
 
   let anyResolved = 0
   for (const task of taskList) {
+    activeTaskId = task.id
     const t0 = Date.now()
     const r = await runAgentic({
       surface: proxy,
@@ -171,6 +201,13 @@ async function main(): Promise<void> {
       workerProfile,
       analystProfile,
       budget,
+      ...(artifactDir ? { hooks: { onEvent(event: unknown) {
+        const taskDir = join(artifactDir, activeTaskId)
+        mkdirSync(taskDir, { recursive: true })
+        appendFileSync(join(taskDir, 'runtime-events.jsonl'), JSON.stringify(event, (_key, value) =>
+          value instanceof Error ? { name: value.name, message: value.message, stack: value.stack } : value,
+        ) + '\n')
+      } } } : {}),
     })
     const rec = captured.get(task.id)
     const st = toolStats.get(task.id)
@@ -186,6 +223,38 @@ async function main(): Promise<void> {
     console.log(`  patch APPLIED (git apply --check on clean base): ${rec?.applied ? 'YES' : 'NO'}${rec?.applyErr ? ` (${rec.applyErr})` : ''}`)
     console.log(`  swebench judge RESOLVED: ${resolved ? '1' : '0'}`)
     if (rec?.score?.detail) console.log(`  judge report: ${rec.score.detail.slice(0, 400)}`)
+    if (artifactDir) {
+      const taskDir = join(artifactDir, task.id)
+      mkdirSync(taskDir, { recursive: true })
+      writeFileSync(join(taskDir, 'patch.diff'), rec?.patch ?? '')
+      writeFileSync(join(taskDir, 'summary.json'), JSON.stringify({
+        benchmark: 'SWE-bench Verified',
+        taskId: task.id,
+        modelRequested: model,
+        prompt: task.userPrompt,
+        turnsLimit: innerTurns,
+        maxTokens,
+        budget,
+        enableRun,
+        shots: r.shots,
+        completions: r.completions,
+        tokens: r.tokens,
+        tokensKnown: r.tokensKnown,
+        billedCostUsd: null,
+        reportedUsd: r.usd,
+        reportedUsdKnown: r.usdKnown,
+        progression: r.progression,
+        runScore: r.score,
+        runResolved: r.resolved,
+        wallMs: Date.now() - t0,
+        toolStats: st ?? null,
+        patchBytes,
+        patchLines,
+        files,
+        patchApplies: rec?.applied ?? false,
+        score: rec?.score ?? null,
+      }, null, 2) + '\n')
+    }
   }
   console.log(`\n>>> resolved ${anyResolved}/${taskList.length}`)
 }

package/src/terminal-compare.ts

@@ -58,7 +58,7 @@ const PROFILE = agentProfileSchema.parse({
     systemPrompt:
       'Solve the Terminal-Bench task completely in the provided container and verify the result before finishing.',
   },
-  permission: {
+  permissions: {
     edit: 'allow',
     bash: 'allow',
     webfetch: 'allow',
@@ -360,14 +360,9 @@ async function captureRunRecord(
   await appendRunRecord(CORPUS, record)
 }
 
-/**
- * Serially `docker compose build` each task image before the concurrent fan-out.
- * tb builds per-task images on first use; building two cold images concurrently
- * contends on the shared docker build backend and can return nonzero. Warming the
- * cache serially makes the parallel rounds hit a warm cache and never race.
- * Dataset-agnostic: derives the compose path from the tb cache layout
- * (<cache>/<name>/<version>/<task>/docker-compose.yaml).
- */
+/** Warm each task image serially before the concurrent fan-out. Terminal-Bench
+ * supplies compose variables only inside its own runner, so build the task's
+ * Dockerfile directly and let its later compose build reuse the cached layers. */
 async function prebuildImages(taskIds: string[]): Promise<void> {
   const [name, version] = DATASET.split('==')
   if (!name || !version) {
@@ -376,18 +371,19 @@ async function prebuildImages(taskIds: string[]): Promise<void> {
   }
   const cacheRoot = join(homedir(), '.cache', 'terminal-bench', name, version)
   for (const taskId of taskIds) {
-    const composePath = join(cacheRoot, taskId, 'docker-compose.yaml')
+    const taskDir = join(cacheRoot, taskId)
+    const dockerfile = join(taskDir, 'Dockerfile')
     try {
-      await stat(composePath)
+      await stat(dockerfile)
     } catch {
-      console.log(`  prebuild: no compose at ${composePath}; tb will materialize ${taskId} on first run`)
+      console.log(`  prebuild: no Dockerfile at ${dockerfile}; tb will materialize ${taskId} on first run`)
       continue
     }
     process.stdout.write(`  prebuild: ${taskId} … `)
     await new Promise<void>((resolve, reject) => {
       execFile(
         'docker',
-        ['compose', '-p', `tbprebuild-${taskId}`, '-f', composePath, 'build'],
+        ['build', '--tag', `tbprebuild-${taskId}`.toLowerCase().replace(/[^a-z0-9_.-]/g, '-'), '--file', dockerfile, taskDir],
         { maxBuffer: 1024 * 1024 * 256, timeout: ROUND_CAP_MS },
         (err) => (err ? reject(new Error(`prebuild ${taskId} failed: ${err.message}`)) : resolve()),
       )

package/tb_agents/opencode_router_agent.py

@@ -202,7 +202,7 @@ class OpenCodeRouterAgent(OpenCodeAgent):
             "$schema": "https://opencode.ai/config.json",
             # Headless benchmark runs cannot answer interactive permission prompts.
             # Keep this identical for raw and supervisor arms.
-            "permission": self._profile.get("permission", {}),
+            "permission": self._profile.get("permissions", {}),
             "provider": {
                 self._provider: {
                     "npm": "@ai-sdk/openai-compatible",