@tangle-network/agent-bench 0.13.11 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 0.14.0
+
+Require Eval 0.201 through 0.203, Interface 2.15, Knowledge 18, and Sandbox 0.58.4 through the shared catalog.
+Consume Runtime 0.287 through the workspace dependency.
+
+## 0.13.13
+
+Require Knowledge 17.1.6 and Eval `>=0.191.0 <0.194.0` through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
+## 0.13.12
+
+Accept Eval 0.185 and 0.186 and require Knowledge 17.1.2 through the shared dependency catalog.
+Benchmark behavior is unchanged.
+
 ## 0.13.11
 
 Accept Eval 0.184 and require Knowledge 17.1.1 through the shared dependency catalog.

package/HARNESS.md

@@ -28,6 +28,8 @@ Use these labels literally. Do not promote one level into another in prose.
 
 A localization score, output-shape check, LLM quality judge, or toy deterministic reward can be useful for development. None is a substitute for the benchmark's outcome evaluator.
 
+The [September 24 public benchmark proof](evidence/public-agent-benchmarks-20260924/README.md) retains three officially graded agent tasks, traces, and validity limits across Terminal-Bench and SWE-bench Verified.
+
 ## Supported commands
 
 ### Package and integration contracts
@@ -116,9 +118,9 @@ cd bench
 pnpm tsx src/swe-self-improve.mts
 ```
 
-This fixture uses `runStrategyEvolution` with SWE-bench tasks and a frozen holdout.
-It does not exercise `improve`, and it deletes its temporary run directory on exit.
-It therefore cannot provide retained improvement or lineage evidence.
+This fixture uses `runStrategyEvolution` with SWE-bench tasks split into train, selection and sealed test slices.
+It does not exercise `improve`.
+It keeps its run directory, whose search ledger is the checkpoint and the lineage record of one search.
 Use `examples/improve` for the maintained offline API fixture.
 Use the consuming labs for registered learning campaigns with retained execution and comparison evidence.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.13.11",
+  "version": "0.14.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.183.0 <0.185.0",
-    "@tangle-network/agent-interface": "^2.11.0",
-    "@tangle-network/agent-knowledge": "^17.1.1",
-    "@tangle-network/sandbox": ">=0.36.4 <0.48.0",
-    "@tangle-network/agent-runtime": "^0.256.0"
+    "@tangle-network/agent-eval": ">=0.201.0 <0.204.0",
+    "@tangle-network/agent-interface": "^2.15.0",
+    "@tangle-network/agent-knowledge": "^18.0.0",
+    "@tangle-network/sandbox": ">=0.58.4 <0.59.0",
+    "@tangle-network/agent-runtime": "^0.287.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/selector.ts

@@ -1,5 +1,5 @@
 /**
- * Deployable, non-oracle selector (docs/roadmap-rsi.md Phase 1).
+ * Deployable, non-oracle selector.
  *
  * best-of-N only pays if you can pick the good attempt WITHOUT the judge. Today
  * the loop's winner is judge-selected (verdict.score) — an oracle upper bound, not

package/src/swe-bench-env.ts

@@ -277,8 +277,9 @@ async function resolveInstanceImage(ws: Ws, expected?: SweImageIdentity): Promis
 }
 
 /** Build the SWE-bench Environment + a DISJOINT-slice task supplier over the Verified split. The
- *  supplier keys tasks by dataset offset so `runStrategyEvolution`'s train [0,trainN) and holdout
- *  [trainN+off,…) never overlap. Verified is loaded once; instances carry their repo/base_commit. */
+ *  supplier keys tasks by dataset offset, so consecutive slices (train, selection and test for
+ *  `runStrategyEvolution`) never overlap. Verified is loaded once; instances carry their
+ *  repo/base_commit. */
 export async function createSweBenchEnvironment(
   poolN = 80,
   opts: {

package/src/swe-local-proof.mts

@@ -16,7 +16,7 @@
  *     node_modules/.bin/tsx bench/src/swe-local-proof.mts
  */
 import { execFile } from 'node:child_process'
-import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
@@ -40,12 +40,24 @@ async function main(): Promise<void> {
   // WITH-TOOLS arm: RUN_TOOL=1 exposes the jailed `run` tool + the run-aware prompt. Default OFF ⇒
   // the read/edit-only baseline (glm-5.2 7/12) unchanged.
   const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase())
+  const artifactDir = process.env.RUN_ARTIFACT_DIR
+  if (artifactDir) mkdirSync(artifactDir, { recursive: true })
 
   console.log(`═══ SWE-bench Verified — SEE-able LOCAL proof (no tangle sandbox) ═══`)
   console.log(`model=${model}  ids=${ids.join(',')}  innerTurns=${innerTurns}  maxTokens=${maxTokens}  budget=${budget}  runTool=${enableRun}`)
   console.log(`router=${routerBaseUrl}`)
 
-  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun })
+  const { environment, tasks, adapter } = await createSweBenchEnvironment(ids.length, {
+    ids,
+    enableRun,
+    ...(artifactDir ? {
+      adapterOptions: {
+        captureEvaluatorArtifacts: ({ taskId, attemptSequence }) => ({
+          destination: join(artifactDir, taskId, `judge-${attemptSequence}`),
+        }),
+      },
+    } : {}),
+  })
   const workerProfile = withBenchProfile(
     {
       name: 'swe-local-proof-worker',
@@ -85,10 +97,27 @@ async function main(): Promise<void> {
   const captured = new Map<string, Rec>()
   const judged = new Map<string, BenchScore>()
   const toolStats = new Map<string, { list: number; read: number; edit_ok: number; edit_fail: number; run: number; run_err: number }>()
+  let activeTaskId = ''
+  const recordTool = (name: string, args: unknown, result: unknown, error?: string) => {
+    if (!artifactDir || !activeTaskId) return
+    const taskDir = join(artifactDir, activeTaskId)
+    mkdirSync(taskDir, { recursive: true })
+    appendFileSync(join(taskDir, 'tools.jsonl'), JSON.stringify({
+      at: new Date().toISOString(), name, args,
+      ...(error ? { error } : { result }),
+    }) + '\n')
+  }
   const proxy: AgenticSurface = {
     ...environment,
     async call(handle, name, args) {
-      const res = await environment.call(handle, name, args)
+      let res: string
+      try {
+        res = await environment.call(handle, name, args)
+      } catch (error) {
+        recordTool(name, args, undefined, error instanceof Error ? error.message : String(error))
+        throw error
+      }
+      recordTool(name, args, res)
       // Count tool usage per workspace so we can SEE whether the agent ever edited, and whether
       // edits succeeded or bounced off old_string matching. handle.id keys the workspace.
       const st = toolStats.get(handle.id) ?? { list: 0, read: 0, edit_ok: 0, edit_fail: 0, run: 0, run_err: 0 }
@@ -161,6 +190,7 @@ async function main(): Promise<void> {
 
   let anyResolved = 0
   for (const task of taskList) {
+    activeTaskId = task.id
     const t0 = Date.now()
     const r = await runAgentic({
       surface: proxy,
@@ -171,6 +201,13 @@ async function main(): Promise<void> {
       workerProfile,
       analystProfile,
       budget,
+      ...(artifactDir ? { hooks: { onEvent(event: unknown) {
+        const taskDir = join(artifactDir, activeTaskId)
+        mkdirSync(taskDir, { recursive: true })
+        appendFileSync(join(taskDir, 'runtime-events.jsonl'), JSON.stringify(event, (_key, value) =>
+          value instanceof Error ? { name: value.name, message: value.message, stack: value.stack } : value,
+        ) + '\n')
+      } } } : {}),
     })
     const rec = captured.get(task.id)
     const st = toolStats.get(task.id)
@@ -186,6 +223,38 @@ async function main(): Promise<void> {
     console.log(`  patch APPLIED (git apply --check on clean base): ${rec?.applied ? 'YES' : 'NO'}${rec?.applyErr ? ` (${rec.applyErr})` : ''}`)
     console.log(`  swebench judge RESOLVED: ${resolved ? '1' : '0'}`)
     if (rec?.score?.detail) console.log(`  judge report: ${rec.score.detail.slice(0, 400)}`)
+    if (artifactDir) {
+      const taskDir = join(artifactDir, task.id)
+      mkdirSync(taskDir, { recursive: true })
+      writeFileSync(join(taskDir, 'patch.diff'), rec?.patch ?? '')
+      writeFileSync(join(taskDir, 'summary.json'), JSON.stringify({
+        benchmark: 'SWE-bench Verified',
+        taskId: task.id,
+        modelRequested: model,
+        prompt: task.userPrompt,
+        turnsLimit: innerTurns,
+        maxTokens,
+        budget,
+        enableRun,
+        shots: r.shots,
+        completions: r.completions,
+        tokens: r.tokens,
+        tokensKnown: r.tokensKnown,
+        billedCostUsd: null,
+        reportedUsd: r.usd,
+        reportedUsdKnown: r.usdKnown,
+        progression: r.progression,
+        runScore: r.score,
+        runResolved: r.resolved,
+        wallMs: Date.now() - t0,
+        toolStats: st ?? null,
+        patchBytes,
+        patchLines,
+        files,
+        patchApplies: rec?.applied ?? false,
+        score: rec?.score ?? null,
+      }, null, 2) + '\n')
+    }
   }
   console.log(`\n>>> resolved ${anyResolved}/${taskList.length}`)
 }

package/src/swe-self-improve.mts

@@ -1,20 +1,22 @@
 /**
- * SWE-bench self-improvement — the PROPER, no-cheating run: a frontier worker over the SWE-bench
- * `Environment`, with `runStrategyEvolution` enforcing the train→freeze→holdout split (the substrate
- * draws a disjoint holdout slice and gates once — adaptive reuse is impossible). CONTAMINATION CAVEAT
- * applies (public fixes may be memorized) — reported, never claimed clean.
+ * SWE-bench self-improvement: a frontier worker over the SWE-bench `Environment`, with
+ * `runStrategyEvolution` searching strategies on disjoint train, selection and test slices. The
+ * author reads train results only, strategies are ranked on the private selection slice, and the
+ * claim runs once on the sealed test slice. CONTAMINATION CAVEAT applies (public fixes may be
+ * memorized) — reported, never claimed clean.
  *
  *   CALIBRATE first (cost gate):  TANGLE_API_KEY=… CALIBRATE=1 N=3 tsx bench/src/swe-self-improve.mts
- *   Full run:                     TANGLE_API_KEY=… TRAIN_N=6 HOLDOUT_N=8 GENERATIONS=2 tsx bench/src/swe-self-improve.mts
+ *   Full run:                     TANGLE_API_KEY=… TRAIN_N=4 SELECTION_N=12 TEST_N=12 tsx bench/src/swe-self-improve.mts
+ *
+ * The run directory is kept (`OUT_DIR`, default `.swe-run`): its ledger is the checkpoint, so the
+ * same command continues an interrupted search.
  */
-import { mkdtempSync, rmSync } from 'node:fs'
 import { join } from 'node:path'
-import type { AgentProfile } from '@tangle-network/agent-interface'
+import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface'
 import {
   refine,
   runAgentic,
   runStrategyEvolution,
-  sample,
   strategyAuthorSystemPrompt,
 } from '@tangle-network/agent-runtime/kernel'
 import { createSweBenchEnvironment } from './swe-bench-env'
@@ -72,46 +74,52 @@ async function main(): Promise<void> {
     return
   }
 
-  const report = await (async () => {
-    const outDir = mkdtempSync(join(process.cwd(), '.swe-run-'))
-    try {
-      return await runStrategyEvolution({
-        environment,
-        tasks,
-        trainN: Number(process.env.TRAIN_N ?? 6),
-        holdoutN: Number(process.env.HOLDOUT_N ?? 8),
-        worker: { routerBaseUrl, routerKey, workerProfile },
-        author: {
-          profile: authorProfile(authorModel, 'swe-strategy-author'),
-          executor: { backend: 'router', routerBaseUrl, routerKey },
-          fallbackProfile: authorProfile(
-            process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash',
-            'swe-strategy-author-fallback',
-          ),
-        },
-        baselines: [sample, refine],
-        budget: Number(process.env.BUDGET ?? 2),
-        generations: Number(process.env.GENERATIONS ?? 2),
-        populationSize: Number(process.env.POP ?? 2),
-        outDir,
-      })
-    } finally {
-      rmSync(outDir, { recursive: true, force: true })
-    }
-  })()
+  const trainN = Number(process.env.TRAIN_N ?? 4)
+  const selectionN = Number(process.env.SELECTION_N ?? 12)
+  const testN = Number(process.env.TEST_N ?? 12)
+  const all = await tasks(0, trainN + selectionN + testN)
+  const report = await runStrategyEvolution({
+    environment,
+    train: all.slice(0, trainN),
+    selection: all.slice(trainN, trainN + selectionN),
+    test: all.slice(trainN + selectionN),
+    claim: {
+      use: 'comparison',
+      population: { id: 'swe-bench-verified', description: 'SWE-bench Verified instances' },
+      samplingFrame: 'consecutive SWE-bench Verified instances from the loaded pool',
+      independentUnit: 'id',
+      generalization: 'new-units',
+      minimumEffect: Number(process.env.MIN_EFFECT ?? 0.15),
+    },
+    executionRef: canonicalCandidateDigest({
+      environment: 'bench/src/swe-bench-env.ts',
+      harness: 'swebench-docker',
+      innerTurns,
+    }),
+    worker: { routerBaseUrl, routerKey, workerProfile },
+    author: {
+      profile: authorProfile(authorModel, 'swe-strategy-author'),
+      executor: { backend: 'router', routerBaseUrl, routerKey },
+      fallbackProfile: authorProfile(
+        process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash',
+        'swe-strategy-author-fallback',
+      ),
+    },
+    root: refine,
+    budget: Number(process.env.BUDGET ?? 2),
+    maxExpansions: Number(process.env.EXPANSIONS ?? 4),
+    outDir: join(process.cwd(), process.env.OUT_DIR ?? '.swe-run'),
+  })
 
-  const v = report.verdict
-  console.log('\n═══ SWE-bench SELF-IMPROVEMENT — certified on a FROZEN holdout (CONTAMINATION-flagged) ═══')
-  console.log(`worker=${workerModel}  author=${authorModel}`)
-  console.log(`gen0 champion:   ${report.gen0Champion.name}`)
-  console.log(`final champion:  ${report.finalChampion.name}`)
-  console.log(`PROMOTED:        ${v.promoted}  (${v.reason})`)
-  console.log(`held-out lift:   mean ${v.lift.mean.toFixed(3)}  95% CI [${v.lift.low.toFixed(3)}, ${v.lift.high.toFixed(3)}]  n=${v.n}`)
-  console.log(
-    v.promoted
-      ? '\n>>> The search taught the agent a strategy that resolves MORE real bugs it never trained on, beyond luck. (Report the contamination caveat: public fixes may be memorized.)'
-      : '\n>>> No promotion: the evolved strategy did not beat gen0 on the fresh holdout beyond noise (honest null).',
-  )
+  const shipped = report.claim.finalists.find((f) => f.nodeId === report.claim.selected)?.test
+  console.log('\n═══ SWE-bench SELF-IMPROVEMENT — claimed on a SEALED test slice (CONTAMINATION-flagged) ═══')
+  console.log(`worker=${workerModel}  author=${authorModel}  ledger=${report.ledger}`)
+  console.log(`strategies:  ${report.strategies.map((s) => `${s.name} (${s.status})`).join(', ')}`)
+  console.log(`selected:    ${report.selected.name}`)
+  console.log(`DECISION:    ${report.decision}  (${report.reason})`)
+  if (shipped) {
+    console.log(`test lift:   ${shipped.delta.toFixed(3)}  [${shipped.interval[0].toFixed(3)}, ${shipped.interval[1].toFixed(3)}]  n=${shipped.pairs}`)
+  }
 }
 
 main().catch((e) => {

package/src/terminal-compare.ts

@@ -58,7 +58,7 @@ const PROFILE = agentProfileSchema.parse({
     systemPrompt:
       'Solve the Terminal-Bench task completely in the provided container and verify the result before finishing.',
   },
-  permission: {
+  permissions: {
     edit: 'allow',
     bash: 'allow',
     webfetch: 'allow',
@@ -360,14 +360,9 @@ async function captureRunRecord(
   await appendRunRecord(CORPUS, record)
 }
 
-/**
- * Serially `docker compose build` each task image before the concurrent fan-out.
- * tb builds per-task images on first use; building two cold images concurrently
- * contends on the shared docker build backend and can return nonzero. Warming the
- * cache serially makes the parallel rounds hit a warm cache and never race.
- * Dataset-agnostic: derives the compose path from the tb cache layout
- * (<cache>/<name>/<version>/<task>/docker-compose.yaml).
- */
+/** Warm each task image serially before the concurrent fan-out. Terminal-Bench
+ * supplies compose variables only inside its own runner, so build the task's
+ * Dockerfile directly and let its later compose build reuse the cached layers. */
 async function prebuildImages(taskIds: string[]): Promise<void> {
   const [name, version] = DATASET.split('==')
   if (!name || !version) {
@@ -376,18 +371,19 @@ async function prebuildImages(taskIds: string[]): Promise<void> {
   }
   const cacheRoot = join(homedir(), '.cache', 'terminal-bench', name, version)
   for (const taskId of taskIds) {
-    const composePath = join(cacheRoot, taskId, 'docker-compose.yaml')
+    const taskDir = join(cacheRoot, taskId)
+    const dockerfile = join(taskDir, 'Dockerfile')
     try {
-      await stat(composePath)
+      await stat(dockerfile)
     } catch {
-      console.log(`  prebuild: no compose at ${composePath}; tb will materialize ${taskId} on first run`)
+      console.log(`  prebuild: no Dockerfile at ${dockerfile}; tb will materialize ${taskId} on first run`)
       continue
     }
     process.stdout.write(`  prebuild: ${taskId} … `)
     await new Promise<void>((resolve, reject) => {
       execFile(
         'docker',
-        ['compose', '-p', `tbprebuild-${taskId}`, '-f', composePath, 'build'],
+        ['build', '--tag', `tbprebuild-${taskId}`.toLowerCase().replace(/[^a-z0-9_.-]/g, '-'), '--file', dockerfile, taskDir],
         { maxBuffer: 1024 * 1024 * 256, timeout: ROUND_CAP_MS },
         (err) => (err ? reject(new Error(`prebuild ${taskId} failed: ${err.message}`)) : resolve()),
       )

package/tb_agents/opencode_router_agent.py

@@ -202,7 +202,7 @@ class OpenCodeRouterAgent(OpenCodeAgent):
             "$schema": "https://opencode.ai/config.json",
             # Headless benchmark runs cannot answer interactive permission prompts.
             # Keep this identical for raw and supervisor arms.
-            "permission": self._profile.get("permission", {}),
+            "permission": self._profile.get("permissions", {}),
             "provider": {
                 self._provider: {
                     "npm": "@ai-sdk/openai-compatible",