@tangle-network/agent-bench 0.4.1 → 0.4.4

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.4.1",
+  "version": "0.4.4",
   "type": "module",
-  "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
+  "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/tangle-network/agent-runtime.git",
@@ -25,20 +25,22 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "0.126.6",
-    "@tangle-network/agent-interface": "0.32.0",
-    "@tangle-network/agent-knowledge": "5.0.1",
-    "@tangle-network/sandbox": "^0.12.0",
-    "@tangle-network/agent-runtime": "0.105.0"
+    "@tangle-network/agent-eval": "0.133.0",
+    "@tangle-network/agent-interface": "0.36.0",
+    "@tangle-network/agent-knowledge": "6.1.2",
+    "@tangle-network/sandbox": "0.15.0",
+    "@tangle-network/agent-runtime": "0.107.2"
   },
   "devDependencies": {
-    "@types/node": "^25.9.3",
-    "tsx": "^4.22.4",
-    "tsup": "^8.5.1",
-    "typescript": "^5.7.0"
+    "@arethetypeswrong/cli": "0.18.5",
+    "@types/node": "26.1.1",
+    "publint": "0.3.22",
+    "tsdown": "0.22.14",
+    "tsx": "4.23.1",
+    "typescript": "6.0.3"
   },
   "engines": {
-    "node": ">=20.19.0"
+    "node": ">=22.13.0"
   },
   "files": [
     "dist",
@@ -61,7 +63,7 @@
   },
   "scripts": {
     "prebuild": "pnpm --dir .. build",
-    "build": "tsup",
+    "build": "tsdown",
     "gate": "tsx src/corpus-replay.mts corpus/finsearch.jsonl --selector",
     "gate-cli": "tsx src/gate-cli.mts",
     "run-benchmarks": "tsx src/run-benchmarks-cli.mts",
@@ -69,8 +71,9 @@
     "terminal-compare": "tsx src/terminal-compare.ts",
     "test": "node scripts/run-package-tests.mjs",
     "typecheck:public": "tsc -p tsconfig.public.json",
-    "verify:package": "node scripts/verify-packed-consumer.mjs",
-    "verify:package:local-runtime": "node scripts/verify-packed-consumer.mjs --local-runtime",
+    "verify:package": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs",
+    "verify:package:local-runtime": "pnpm run verify:package:static && node scripts/verify-packed-consumer.mjs --local-runtime",
+    "verify:package:static": "pnpm run build && publint && attw --pack --profile esm-only .",
     "verify:pier": "tsx scripts/verify-pier-pair.mts"
   }
 }

package/scripts/verify-packed-consumer.mjs

@@ -52,11 +52,13 @@ async function resolveRuntimePackage(packDir) {
 
 try {
   const packDir = path.join(scratch, 'pack')
+  const unpackDir = path.join(scratch, 'unpack')
   const runtimePackDir = path.join(scratch, 'runtime-pack')
   const consumerDir = path.join(scratch, 'consumer')
   const terminalBenchVenv = path.join(scratch, 'terminal-bench-venv')
   const terminalBenchBinDir = path.join(terminalBenchVenv, 'bin')
   await mkdir(packDir)
+  await mkdir(unpackDir)
   await mkdir(runtimePackDir)
   await mkdir(consumerDir)
   await mkdir(terminalBenchBinDir, { recursive: true })
@@ -115,21 +117,14 @@ writeFileSync(
     throw new Error(`expected one packed agent-bench tarball, found ${packedFiles.length}`)
   }
   const tarball = path.join(packDir, packedFiles[0])
+  await run('tar', ['-xzf', tarball, '-C', unpackDir], benchDir)
   const runtimePackage = await resolveRuntimePackage(runtimePackDir)
-  const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8'))
-  const devDependencies = manifest.devDependencies
-  if (
-    typeof devDependencies?.['@types/node'] !== 'string' ||
-    typeof devDependencies.typescript !== 'string' ||
-    typeof devDependencies.tsx !== 'string' ||
-    !devDependencies['@types/node'] ||
-    !devDependencies.typescript ||
-    !devDependencies.tsx
-  ) {
-    throw new Error(
-      'package verification requires @types/node, typescript, and tsx devDependencies',
-    )
-  }
+  const packedManifest = JSON.parse(
+    await readFile(path.join(unpackDir, 'package', 'package.json'), 'utf8'),
+  )
+  const nodeTypes = requiredPackedDevelopmentDependency(packedManifest, '@types/node')
+  requiredPackedDevelopmentDependency(packedManifest, 'typescript')
+  const tsx = requiredPackedDevelopmentDependency(packedManifest, 'tsx')
   const publicTsconfig = JSON.parse(
     await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8'),
   )
@@ -150,9 +145,9 @@ writeFileSync(
             : {}),
         },
         devDependencies: {
-          '@types/node': devDependencies['@types/node'],
+          '@types/node': nodeTypes,
           typescript: TYPESCRIPT_5,
-          tsx: devDependencies.tsx,
+          tsx,
         },
       },
       null,
@@ -292,8 +287,16 @@ for name in sorted(expected):
     })
   }
   console.log(
-    `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}, absolute Terminal-Bench venv; prepared ${prepareProof.executionPlanDigest}`,
+    `packed consumer verified: ${packedManifest.name}@${packedManifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}, TypeScript ${TYPESCRIPT_5} and ${TYPESCRIPT_6}, absolute Terminal-Bench venv; prepared ${prepareProof.executionPlanDigest}`,
   )
 } finally {
   await rm(scratch, { recursive: true, force: true })
 }
+
+function requiredPackedDevelopmentDependency(packageJson, name) {
+  const version = packageJson.devDependencies?.[name]
+  if (typeof version !== 'string' || version.length === 0 || version.startsWith('catalog:')) {
+    throw new Error(`packed consumer requires a resolved ${name} development dependency`)
+  }
+  return version
+}

package/src/corpus.test.mts

@@ -109,6 +109,8 @@ const baseRec = (attempts: AttemptRecord[], over: Partial<RunRecord> = {}): RunR
   assert.equal(r0?.commitSha, 'abc123')
   assert.equal(r0?.tokenUsage.input, 100, 'real tokens carried (not zeroed)')
   assert.equal(r0?.costUsd, 0.01, 'real cost carried')
+  assert.deepEqual(r0?.costProvenance, { kind: 'observed', usd: 0.01 }, 'reported cost remains observed')
+  assert.equal(r0?.terminalOutcome, 'succeeded', 'failed task quality is not an execution failure')
   assert.equal(r0?.prompt, 'q0', 'verbatim prompt survives the validator')
   assert.equal(r0?.completion, 'alpha', 'verbatim completion survives the validator')
   assert.equal(r0?.outcome.searchScore, 0, 'search split → searchScore from attempt score')
@@ -117,6 +119,17 @@ const baseRec = (attempts: AttemptRecord[], over: Partial<RunRecord> = {}): RunR
   assert.notEqual(r0?.runId, r1?.runId, 'per-attempt runIds are distinct')
 }
 
+// --- attempt error is execution state, not a fabricated zero-quality label ---
+{
+  const { records, unmappable } = await benchRecordToCorpusRecords(
+    baseRec([{ ...measuredAttempt(0, 'partial output', false), error: 'provider disconnected' }]),
+    { commitSha: 'abc123', model: 'gpt-5-2025-08-07' },
+  )
+  assert.equal(unmappable.length, 0)
+  assert.equal(records[0]?.terminalOutcome, 'failed')
+  assert.equal(records[0]?.outcome.searchScore, 0, 'task quality remains independently recorded')
+}
+
 // --- unmeasured economics → unmappable, never forged with phantom zeros ---
 {
   const bare: AttemptRecord = { round: 0, prompt: 'q', output: 'x', valid: true, eventCount: 0, eventTypes: {} }

package/src/corpus.ts

@@ -288,7 +288,11 @@ export async function benchRecordToCorpusRecords(
       commitSha: opts.commitSha,
       wallMs: a.wallMs as number,
       costUsd: a.costUsd as number,
+      // Attempt costs are present only when the worker reported them. Missing
+      // costs exited above rather than becoming a made-up zero.
+      costProvenance: { kind: 'observed', usd: a.costUsd as number },
       tokenUsage: { input: a.tokensIn as number, output: a.tokensOut as number },
+      terminalOutcome: a.error === undefined ? 'succeeded' : 'failed',
       outcome: {
         ...(splitTag === 'holdout' ? { holdoutScore: score } : { searchScore: score }),
         raw: { valid: a.valid === true ? 1 : 0, score },

package/src/pier-trial-controller.test.mts

@@ -85,18 +85,20 @@ test('the Pier result wait does not add time beyond an expired deadline', async
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream } from 'node:fs'
-import { writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const directory = process.argv[2]
 process.on('SIGTERM', () => {
-  writeFileSync(path.join(directory, 'terminal.json'), JSON.stringify({
+  const target = path.join(directory, 'terminal.json')
+  const temporary = path.join(directory, '.terminal.json.tmp')
+  writeFileSync(temporary, JSON.stringify({
     schemaVersion: 1,
     kind: 'pier-trial-terminal',
     status: 'stopped',
     processExited: true,
     containersRemoved: true,
   }))
+  renameSync(temporary, target)
   process.exit(0)
 })
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
@@ -156,7 +158,7 @@ test('the supervisor receives only launch data and no inherited evaluator enviro
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 const chunks = []
 for await (const chunk of createReadStream('', { fd: 3 })) chunks.push(Buffer.from(chunk))
@@ -165,7 +167,9 @@ const isolated = process.env.PIER_SUPERVISOR_MODE === 'explicit' &&
   process.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_PARENT_SECRET === undefined &&
   launch.env.PIER_CHILD_SECRET === 'child-only'
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: isolated ? 'completed' : 'failed',
@@ -173,6 +177,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   ...(isolated ? {} : { error: 'supervisor inherited evaluator environment' }),
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({
@@ -224,10 +229,12 @@ test('terminal acknowledgements reject unknown fields', async () => {
     await chmod(fakeDocker, 0o700)
     await writeFile(
       supervisorPath,
-      `import { createReadStream, writeFileSync } from 'node:fs'
+      `import { createReadStream, renameSync, writeFileSync } from 'node:fs'
 import path from 'node:path'
 for await (const _chunk of createReadStream('', { fd: 3 })) {}
-writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
+const target = path.join(process.argv[2], 'terminal.json')
+const temporary = path.join(process.argv[2], '.terminal.json.tmp')
+writeFileSync(temporary, JSON.stringify({
   schemaVersion: 1,
   kind: 'pier-trial-terminal',
   status: 'completed',
@@ -235,6 +242,7 @@ writeFileSync(path.join(process.argv[2], 'terminal.json'), JSON.stringify({
   containersRemoved: true,
   containerRemoved: true,
 }))
+renameSync(temporary, target)
 `,
     )
     const controller = new FilePierCandidateTrialController({

package/src/profile-coordinates.ts

@@ -14,8 +14,8 @@
  */
 import { readFileSync, readdirSync } from 'node:fs'
 import { join } from 'node:path'
-import type { AgentProfile } from '@tangle-network/sandbox'
-import { defineInlineResource } from '@tangle-network/sandbox'
+import type { AgentProfile } from '@tangle-network/agent-interface'
+import { defineInlineResource } from '@tangle-network/agent-interface'
 
 export interface ProfileCoordinate {
   /** Coordinate id (the COORDINATE= knob value). */

package/src/rollout-ledger/backfill-swe-arena.mts

@@ -141,10 +141,11 @@ interface Ctx {
   stats: BackfillStats
 }
 
-function baseLine(ctx: Ctx): Pick<RolloutLine, 'schema' | 'run_id' | 'provenance'> {
+function baseLine(ctx: Ctx): Pick<RolloutLine, 'schema' | 'run_id' | 'experiment_id' | 'provenance'> {
   return {
     schema: ROLLOUT_SCHEMA,
     run_id: ctx.runId,
+    experiment_id: null,
     provenance: { captured_at: ctx.capturedAt, capture: 'backfill' },
   }
 }
@@ -216,6 +217,7 @@ async function emitCellLines(
       reward,
       reward_source: reward === null ? null : OFFICIAL_JUDGE,
       verdict: judgeVerdict,
+      realness_gated: false,
       metrics: {
         resolved: artifact?.resolved ?? null,
         verify_pass: artifact?.verifyPass ?? null,
@@ -335,6 +337,7 @@ function workerLine(
       reward,
       reward_source: reward === null ? null : `${OFFICIAL_JUDGE}/inherited`,
       verdict: null,
+      realness_gated: false,
       metrics: {
         worker_cwd: cwd,
         session_agent: session?.agent ?? null,
@@ -504,6 +507,7 @@ async function emitProposerLines(ctx: Ctx, lines: RolloutLine[], entries: Rollou
         reward,
         reward_source: reward === null ? null : `${OFFICIAL_JUDGE}/candidate-resolved-fraction`,
         verdict: null,
+        realness_gated: false,
         metrics: {
           resolved_count: entry?.outcome.resolvedCount ?? null,
           instance_count: ctx.instanceCount,

package/src/rollout-ledger/settle-capture.mts

@@ -194,9 +194,12 @@ export function createSettleCapture(opts: SettleCaptureOptions): SettleCapture {
   const log = opts.log ?? (() => {})
   const dbPath = opts.opencodeDb ?? DEFAULT_OPENCODE_DB
 
-  const base = (capturedAt: string): Pick<RolloutLine, 'schema' | 'run_id'> & { provenance: RolloutLine['provenance'] } => ({
+  const base = (
+    capturedAt: string,
+  ): Pick<RolloutLine, 'schema' | 'run_id' | 'experiment_id'> & { provenance: RolloutLine['provenance'] } => ({
     schema: ROLLOUT_SCHEMA,
     run_id: opts.runId,
+    experiment_id: null,
     provenance: { captured_at: capturedAt, capture: 'settle-time' },
   })
 
@@ -232,6 +235,7 @@ export function createSettleCapture(opts: SettleCaptureOptions): SettleCapture {
           reward,
           reward_source: reward === null ? null : OFFICIAL_JUDGE,
           verdict: args.judgeVerdict,
+          realness_gated: false,
           metrics: {
             ...args.metrics,
             ...(args.splitVisibility !== null ? { split_visibility: args.splitVisibility } : {}),
@@ -311,6 +315,7 @@ export function createSettleCapture(opts: SettleCaptureOptions): SettleCapture {
                 reward: v2.reward,
                 reward_source: v2.reward === null && v2.deliveredMatch !== 'unknown' ? null : WORKER_REWARD_SOURCE_V2,
                 verdict: null,
+                realness_gated: false,
                 metrics: {
                   worker_label: worker.label,
                   worker_cwd: worker.cwd,
@@ -400,6 +405,7 @@ export function createSettleCapture(opts: SettleCaptureOptions): SettleCapture {
           reward,
           reward_source: PROPOSER_REWARD_SOURCE_V2,
           verdict: null,
+          realness_gated: false,
           metrics: {
             resolved_count: args.candResolved,
             baseline_resolved_count: args.baselineResolved,

package/src/search-bench/profiles.ts

@@ -17,7 +17,7 @@
  * shape — a `transport:'http'` server at the router's `/v1/search/mcp` endpoint,
  * provider pinned via the `?provider=` query param.
  */
-import type { AgentProfile } from '@tangle-network/sandbox'
+import type { AgentProfile } from '@tangle-network/agent-interface'
 
 export type SearchArm = 'native' | 'off' | { provider: string }
 

package/src/skill-sandbox-smoke.mts

@@ -14,7 +14,8 @@
  *
  * Run: dotenvx run -f ~/company/devops/secrets/.env.keys -- pnpm exec tsx bench/src/skill-sandbox-smoke.mts
  */
-import { Sandbox, defineInlineResource } from '@tangle-network/sandbox'
+import { defineInlineResource } from '@tangle-network/agent-interface'
+import { Sandbox } from '@tangle-network/sandbox'
 
 const must = (k: string): string => {
   const v = process.env[k]

package/src/swe-arena/gepa-seat.mts

@@ -233,7 +233,7 @@ export function innerSmokeJudge(): JudgeConfig<SmokeVerdict, GepaSeatScenario> {
 // ---------------------------------------------------------------------------
 
 export const GEPA_PYTHON_INSTALL_HINT =
-  'install `agent-eval-rpc==0.126.6`, then install ' +
+  'install `agent-eval-rpc==0.131.0`, then install ' +
   '`gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f`'
 
 export type GepaMethodFactory = (

package/src/swe-arena/ledger-orphans.test.mts

@@ -7,7 +7,8 @@
  * path improve() rides.
  */
 
-import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, readFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { describe, expect, it } from 'vitest'
@@ -22,40 +23,41 @@ import {
 
 const ATTRIBUTION = { channel: 'agent', phase: 'search.baseline', model: 'm' }
 
-/** A durable event line in the exact shape cost-ledger.jsonl persists. */
-const line = (record: Record<string, unknown>): string =>
-  `${JSON.stringify({ version: 1, record })}\n`
-
-/** Write a synthetic ledger: one settled pair + one crash-orphaned pending. */
+/**
+ * Write one settled receipt and then terminate a process with a real pending
+ * call. The child uses the public ledger API so this fixture cannot shadow its
+ * private JSONL event format.
+ */
 function writeCrashedLedger(dir: string): void {
-  const settledPending = {
-    status: 'pending',
-    callId: 'settled-1',
-    ...ATTRIBUTION,
-    actor: 'worker:astropy#r0',
-    timestamp: 1_000,
-  }
-  const settledReceipt = {
-    ...settledPending,
-    status: 'settled',
-    inputTokens: 10,
-    outputTokens: 5,
-    costUsd: 0.01,
-    costUnknown: false,
-    actualCostUsd: 0.01,
-  }
-  const orphanPending = {
-    status: 'pending',
-    callId: 'orphan-1',
-    ...ATTRIBUTION,
-    actor: 'worker:xarray#r0',
-    timestamp: 2_000,
-    tags: { cellId: 'pydata__xarray-4687:0' },
-  }
-  writeFileSync(
-    join(dir, 'cost-ledger.jsonl'),
-    line(settledPending) + line(settledReceipt) + line(orphanPending),
-  )
+  const child = `
+    import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign'
+    const runDir = process.env.COST_LEDGER_RUN_DIR
+    if (typeof runDir !== 'string' || runDir.length === 0) throw new Error('missing COST_LEDGER_RUN_DIR')
+    const attribution = { channel: 'agent', phase: 'search.baseline', model: 'm' }
+    const ledger = createRunCostLedger({ storage: fsCampaignStorage(), runDir })
+    await ledger.runPaidCall({
+      ...attribution,
+      callId: 'settled-1',
+      actor: 'worker:astropy#r0',
+      execute: async () => 'settled',
+      receipt: () => ({ model: 'm', inputTokens: 10, outputTokens: 5, actualCostUsd: 0.01 }),
+    })
+    let started
+    const pending = new Promise((resolve) => { started = resolve })
+    void ledger.runPaidCall({
+      ...attribution,
+      callId: 'orphan-1',
+      actor: 'worker:xarray#r0',
+      tags: { cellId: 'pydata__xarray-4687:0' },
+      execute: async () => { started(); return await new Promise(() => {}) },
+      receipt: () => ({ model: 'm', inputTokens: 0, outputTokens: 0, actualCostUsd: 0 }),
+    })
+    await pending
+  `
+  execFileSync(process.execPath, ['--input-type=module', '--eval', child], {
+    cwd: process.cwd(),
+    env: { ...process.env, COST_LEDGER_RUN_DIR: dir },
+  })
 }
 
 const openLedger = (dir: string) =>

package/dist/benchmarks/cadbench.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/cadgenbench.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/benchmarks/types.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}