@tangle-network/agent-bench 0.13.13 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.14.0
+
+Require Eval 0.201 through 0.203, Interface 2.15, Knowledge 18, and Sandbox 0.58.4 through the shared catalog.
+Consume Runtime 0.287 through the workspace dependency.
+
 ## 0.13.13
 
 Require Knowledge 17.1.6 and Eval `>=0.191.0 <0.194.0` through the shared dependency catalog.

package/HARNESS.md

@@ -118,9 +118,9 @@ cd bench
 pnpm tsx src/swe-self-improve.mts
 ```
 
-This fixture uses `runStrategyEvolution` with SWE-bench tasks and a frozen holdout.
-It does not exercise `improve`, and it deletes its temporary run directory on exit.
-It therefore cannot provide retained improvement or lineage evidence.
+This fixture uses `runStrategyEvolution` with SWE-bench tasks split into train, selection and sealed test slices.
+It does not exercise `improve`.
+It keeps its run directory, whose search ledger is the checkpoint and the lineage record of one search.
 Use `examples/improve` for the maintained offline API fixture.
 Use the consuming labs for registered learning campaigns with retained execution and comparison evidence.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.13.13",
+  "version": "0.14.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.191.0 <0.194.0",
-    "@tangle-network/agent-interface": "^2.11.0",
-    "@tangle-network/agent-knowledge": "^17.1.6",
-    "@tangle-network/sandbox": ">=0.36.4 <0.48.0",
-    "@tangle-network/agent-runtime": "^0.277.0"
+    "@tangle-network/agent-eval": ">=0.201.0 <0.204.0",
+    "@tangle-network/agent-interface": "^2.15.0",
+    "@tangle-network/agent-knowledge": "^18.0.0",
+    "@tangle-network/sandbox": ">=0.58.4 <0.59.0",
+    "@tangle-network/agent-runtime": "^0.287.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/selector.ts

@@ -1,5 +1,5 @@
 /**
- * Deployable, non-oracle selector (docs/roadmap-rsi.md Phase 1).
+ * Deployable, non-oracle selector.
  *
  * best-of-N only pays if you can pick the good attempt WITHOUT the judge. Today
  * the loop's winner is judge-selected (verdict.score) — an oracle upper bound, not

package/src/swe-bench-env.ts

@@ -277,8 +277,9 @@ async function resolveInstanceImage(ws: Ws, expected?: SweImageIdentity): Promis
 }
 
 /** Build the SWE-bench Environment + a DISJOINT-slice task supplier over the Verified split. The
- *  supplier keys tasks by dataset offset so `runStrategyEvolution`'s train [0,trainN) and holdout
- *  [trainN+off,…) never overlap. Verified is loaded once; instances carry their repo/base_commit. */
+ *  supplier keys tasks by dataset offset, so consecutive slices (train, selection and test for
+ *  `runStrategyEvolution`) never overlap. Verified is loaded once; instances carry their
+ *  repo/base_commit. */
 export async function createSweBenchEnvironment(
   poolN = 80,
   opts: {

package/src/swe-self-improve.mts

@@ -1,20 +1,22 @@
 /**
- * SWE-bench self-improvement — the PROPER, no-cheating run: a frontier worker over the SWE-bench
- * `Environment`, with `runStrategyEvolution` enforcing the train→freeze→holdout split (the substrate
- * draws a disjoint holdout slice and gates once — adaptive reuse is impossible). CONTAMINATION CAVEAT
- * applies (public fixes may be memorized) — reported, never claimed clean.
+ * SWE-bench self-improvement: a frontier worker over the SWE-bench `Environment`, with
+ * `runStrategyEvolution` searching strategies on disjoint train, selection and test slices. The
+ * author reads train results only, strategies are ranked on the private selection slice, and the
+ * claim runs once on the sealed test slice. CONTAMINATION CAVEAT applies (public fixes may be
+ * memorized) — reported, never claimed clean.
  *
  *   CALIBRATE first (cost gate):  TANGLE_API_KEY=… CALIBRATE=1 N=3 tsx bench/src/swe-self-improve.mts
- *   Full run:                     TANGLE_API_KEY=… TRAIN_N=6 HOLDOUT_N=8 GENERATIONS=2 tsx bench/src/swe-self-improve.mts
+ *   Full run:                     TANGLE_API_KEY=… TRAIN_N=4 SELECTION_N=12 TEST_N=12 tsx bench/src/swe-self-improve.mts
+ *
+ * The run directory is kept (`OUT_DIR`, default `.swe-run`): its ledger is the checkpoint, so the
+ * same command continues an interrupted search.
  */
-import { mkdtempSync, rmSync } from 'node:fs'
 import { join } from 'node:path'
-import type { AgentProfile } from '@tangle-network/agent-interface'
+import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface'
 import {
   refine,
   runAgentic,
   runStrategyEvolution,
-  sample,
   strategyAuthorSystemPrompt,
 } from '@tangle-network/agent-runtime/kernel'
 import { createSweBenchEnvironment } from './swe-bench-env'
@@ -72,46 +74,52 @@ async function main(): Promise<void> {
     return
   }
 
-  const report = await (async () => {
-    const outDir = mkdtempSync(join(process.cwd(), '.swe-run-'))
-    try {
-      return await runStrategyEvolution({
-        environment,
-        tasks,
-        trainN: Number(process.env.TRAIN_N ?? 6),
-        holdoutN: Number(process.env.HOLDOUT_N ?? 8),
-        worker: { routerBaseUrl, routerKey, workerProfile },
-        author: {
-          profile: authorProfile(authorModel, 'swe-strategy-author'),
-          executor: { backend: 'router', routerBaseUrl, routerKey },
-          fallbackProfile: authorProfile(
-            process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash',
-            'swe-strategy-author-fallback',
-          ),
-        },
-        baselines: [sample, refine],
-        budget: Number(process.env.BUDGET ?? 2),
-        generations: Number(process.env.GENERATIONS ?? 2),
-        populationSize: Number(process.env.POP ?? 2),
-        outDir,
-      })
-    } finally {
-      rmSync(outDir, { recursive: true, force: true })
-    }
-  })()
+  const trainN = Number(process.env.TRAIN_N ?? 4)
+  const selectionN = Number(process.env.SELECTION_N ?? 12)
+  const testN = Number(process.env.TEST_N ?? 12)
+  const all = await tasks(0, trainN + selectionN + testN)
+  const report = await runStrategyEvolution({
+    environment,
+    train: all.slice(0, trainN),
+    selection: all.slice(trainN, trainN + selectionN),
+    test: all.slice(trainN + selectionN),
+    claim: {
+      use: 'comparison',
+      population: { id: 'swe-bench-verified', description: 'SWE-bench Verified instances' },
+      samplingFrame: 'consecutive SWE-bench Verified instances from the loaded pool',
+      independentUnit: 'id',
+      generalization: 'new-units',
+      minimumEffect: Number(process.env.MIN_EFFECT ?? 0.15),
+    },
+    executionRef: canonicalCandidateDigest({
+      environment: 'bench/src/swe-bench-env.ts',
+      harness: 'swebench-docker',
+      innerTurns,
+    }),
+    worker: { routerBaseUrl, routerKey, workerProfile },
+    author: {
+      profile: authorProfile(authorModel, 'swe-strategy-author'),
+      executor: { backend: 'router', routerBaseUrl, routerKey },
+      fallbackProfile: authorProfile(
+        process.env.AUTHOR_FALLBACK ?? 'deepseek-v4-flash',
+        'swe-strategy-author-fallback',
+      ),
+    },
+    root: refine,
+    budget: Number(process.env.BUDGET ?? 2),
+    maxExpansions: Number(process.env.EXPANSIONS ?? 4),
+    outDir: join(process.cwd(), process.env.OUT_DIR ?? '.swe-run'),
+  })
 
-  const v = report.verdict
-  console.log('\n═══ SWE-bench SELF-IMPROVEMENT — certified on a FROZEN holdout (CONTAMINATION-flagged) ═══')
-  console.log(`worker=${workerModel}  author=${authorModel}`)
-  console.log(`gen0 champion:   ${report.gen0Champion.name}`)
-  console.log(`final champion:  ${report.finalChampion.name}`)
-  console.log(`PROMOTED:        ${v.promoted}  (${v.reason})`)
-  console.log(`held-out lift:   mean ${v.lift.mean.toFixed(3)}  95% CI [${v.lift.low.toFixed(3)}, ${v.lift.high.toFixed(3)}]  n=${v.n}`)
-  console.log(
-    v.promoted
-      ? '\n>>> The search taught the agent a strategy that resolves MORE real bugs it never trained on, beyond luck. (Report the contamination caveat: public fixes may be memorized.)'
-      : '\n>>> No promotion: the evolved strategy did not beat gen0 on the fresh holdout beyond noise (honest null).',
-  )
+  const shipped = report.claim.finalists.find((f) => f.nodeId === report.claim.selected)?.test
+  console.log('\n═══ SWE-bench SELF-IMPROVEMENT — claimed on a SEALED test slice (CONTAMINATION-flagged) ═══')
+  console.log(`worker=${workerModel}  author=${authorModel}  ledger=${report.ledger}`)
+  console.log(`strategies:  ${report.strategies.map((s) => `${s.name} (${s.status})`).join(', ')}`)
+  console.log(`selected:    ${report.selected.name}`)
+  console.log(`DECISION:    ${report.decision}  (${report.reason})`)
+  if (shipped) {
+    console.log(`test lift:   ${shipped.delta.toFixed(3)}  [${shipped.interval[0].toFixed(3)}, ${shipped.interval[1].toFixed(3)}]  n=${shipped.pairs}`)
+  }
 }
 
 main().catch((e) => {