@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/swe-arena/score-split.test.mts

@@ -0,0 +1,123 @@
+/**
+ * Gen-5 public/private score split: determinism (seeded by runId), persisted
+ * resume stability, and the never-surfaced invariant's checkable pieces.
+ */
+
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import {
+  leaksPrivateInstance,
+  loadOrCreateScoreSplit,
+  publicOnly,
+  SCORE_SPLIT_FILENAME,
+  splitInstances,
+  subScores,
+} from './score-split.mts'
+
+const SIX = [
+  'astropy__astropy-13033',
+  'django__django-11532',
+  'matplotlib__matplotlib-20826',
+  'pydata__xarray-4687',
+  'pytest-dev__pytest-6197',
+  'sphinx-doc__sphinx-9658',
+]
+
+describe('splitInstances', () => {
+  it('is deterministic: same (runId, instances, publicCount) → identical split', () => {
+    const a = splitInstances('r4-abc123', SIX, 4)
+    const b = splitInstances('r4-abc123', SIX, 4)
+    expect(a).toEqual(b)
+  })
+
+  it('is instance-ORDER-insensitive (ranking is content-derived)', () => {
+    const a = splitInstances('r4-abc123', SIX, 4)
+    const b = splitInstances('r4-abc123', [...SIX].reverse(), 4)
+    expect(a).toEqual(b)
+  })
+
+  it('respects publicCount and partitions exactly (4 public / 2 private of 6)', () => {
+    const split = splitInstances('r4-abc123', SIX, 4)
+    expect(split.publicInstances).toHaveLength(4)
+    expect(split.privateInstances).toHaveLength(2)
+    expect([...split.publicInstances, ...split.privateInstances].sort()).toEqual([...SIX].sort())
+    expect(split.publicInstances.filter((i) => split.privateInstances.includes(i))).toEqual([])
+  })
+
+  it('is SEEDED by runId: different runIds produce different splits', () => {
+    const splits = new Set(
+      Array.from({ length: 10 }, (_, i) => splitInstances(`r4-run${i}`, SIX, 4).privateInstances.join('|')),
+    )
+    expect(splits.size).toBeGreaterThan(1)
+  })
+
+  it('rejects publicCount outside (0, n) and duplicate instance ids', () => {
+    expect(() => splitInstances('r', SIX, 0)).toThrow(/publicCount/)
+    expect(() => splitInstances('r', SIX, 6)).toThrow(/publicCount/)
+    expect(() => splitInstances('r', SIX, 2.5)).toThrow(/publicCount/)
+    expect(() => splitInstances('r', ['a', 'a', 'b'], 1)).toThrow(/duplicate/)
+  })
+})
+
+describe('loadOrCreateScoreSplit (resume stability)', () => {
+  let outDir: string
+  beforeEach(async () => {
+    outDir = await mkdtemp(join(tmpdir(), 'split-'))
+  })
+  afterEach(async () => {
+    await rm(outDir, { recursive: true, force: true })
+  })
+
+  it('persists on first computation and RELOADS verbatim under a NEW runId (a resume cannot rotate private into view)', async () => {
+    const first = await loadOrCreateScoreSplit({ outDir, runId: 'r4-first', instances: SIX, publicCount: 4 })
+    expect(JSON.parse(await readFile(join(outDir, SCORE_SPLIT_FILENAME), 'utf8'))).toEqual(first)
+    const resumed = await loadOrCreateScoreSplit({ outDir, runId: 'r4-DIFFERENT', instances: SIX, publicCount: 4 })
+    expect(resumed).toEqual(first)
+    expect(resumed.seededBy).toBe('r4-first')
+  })
+
+  it('fails loud when the persisted split covers different instances or a different publicCount', async () => {
+    await loadOrCreateScoreSplit({ outDir, runId: 'r4-first', instances: SIX, publicCount: 4 })
+    await expect(
+      loadOrCreateScoreSplit({ outDir, runId: 'r4-x', instances: [...SIX.slice(0, 5), 'other__iid-1'], publicCount: 4 }),
+    ).rejects.toThrow(/refusing to silently re-split/)
+    await expect(
+      loadOrCreateScoreSplit({ outDir, runId: 'r4-x', instances: SIX, publicCount: 3 }),
+    ).rejects.toThrow(/refusing to silently re-split/)
+  })
+
+  it('fails loud on an unknown schema in the persisted file', async () => {
+    await writeFile(join(outDir, SCORE_SPLIT_FILENAME), JSON.stringify({ schema: 'nope' }))
+    await expect(
+      loadOrCreateScoreSplit({ outDir, runId: 'r', instances: SIX, publicCount: 4 }),
+    ).rejects.toThrow(/unknown schema/)
+  })
+})
+
+describe('never-surfaced invariant helpers', () => {
+  const split = splitInstances('r4-abc123', SIX, 4)
+
+  it('leaksPrivateInstance names every private id present in a text', () => {
+    expect(leaksPrivateInstance('nothing to see', split)).toEqual([])
+    const leakText = `evidence for ${split.privateInstances[0]} here`
+    expect(leaksPrivateInstance(leakText, split)).toEqual([split.privateInstances[0]])
+  })
+
+  it('publicOnly drops private-instance records and is identity without a split', () => {
+    const records = SIX.map((iid) => ({ iid, x: 1 }))
+    const filtered = publicOnly(records, (r) => r.iid, split)
+    expect(filtered.map((r) => r.iid).sort()).toEqual(split.publicInstances)
+    expect(publicOnly(records, (r) => r.iid, null)).toEqual(records)
+  })
+
+  it('subScores counts fail-closed AND-verdicts per half (missing verdict = 0)', () => {
+    const verdicts: Record<string, boolean> = {}
+    verdicts[split.publicInstances[0]!] = true
+    verdicts[split.publicInstances[1]!] = false
+    verdicts[split.privateInstances[0]!] = true
+    // privateInstances[1] has NO verdict — counts 0.
+    expect(subScores(verdicts, split)).toEqual({ publicResolvedCount: 1, privateResolvedCount: 1 })
+  })
+})

package/src/swe-arena/scratch-worktree.test.mts

@@ -0,0 +1,55 @@
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { runOk } from './proc.ts'
+import {
+  createDetachedWorktree,
+  pruneDetachedWorktrees,
+  removeDetachedWorktree,
+} from './scratch-worktree.ts'
+
+describe('scratch worktrees', () => {
+  const roots: string[] = []
+
+  afterEach(async () => {
+    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+  })
+
+  it('creates and removes a parallel batch without pruning live metadata', async () => {
+    const repository = await mkdtemp(join(tmpdir(), 'scratch-worktree-repo-'))
+    const output = await mkdtemp(join(tmpdir(), 'scratch-worktree-out-'))
+    roots.push(repository, output)
+    await runOk('git', ['init', '-q', '-b', 'main', repository])
+    await runOk('git', ['-C', repository, 'config', 'user.email', 'test@example.com'])
+    await runOk('git', ['-C', repository, 'config', 'user.name', 'Test'])
+    await writeFile(join(repository, 'seed.txt'), 'seed\n')
+    await runOk('git', ['-C', repository, 'add', 'seed.txt'])
+    await runOk('git', ['-C', repository, 'commit', '-q', '-m', 'seed'])
+    const commit = (
+      await runOk('git', ['-C', repository, 'rev-parse', 'HEAD'])
+    ).stdout.trim()
+    for (let batch = 0; batch < 5; batch += 1) {
+      const worktrees = Array.from(
+        { length: 8 },
+        (_, index) => join(output, `batch-${batch}-candidate-${index}`),
+      )
+      await pruneDetachedWorktrees(repository)
+      await Promise.all(
+        worktrees.map((worktree) => createDetachedWorktree(repository, commit, worktree)),
+      )
+      expect(
+        await Promise.all(
+          worktrees.map((worktree) => readFile(join(worktree, 'seed.txt'), 'utf8')),
+        ),
+      ).toEqual(Array.from({ length: 8 }, () => 'seed\n'))
+      await Promise.all(
+        worktrees.map((worktree) => removeDetachedWorktree(repository, worktree)),
+      )
+    }
+    const listed = (
+      await runOk('git', ['-C', repository, 'worktree', 'list', '--porcelain'])
+    ).stdout
+    expect(listed.match(/^worktree /gmu)).toHaveLength(1)
+  })
+})

package/src/swe-arena/scratch-worktree.ts

@@ -0,0 +1,34 @@
+import { rm } from 'node:fs/promises'
+import { run, runOk } from './proc'
+
+export async function createDetachedWorktree(
+  repository: string,
+  commit: string,
+  destination: string,
+): Promise<void> {
+  await run('git', ['-C', repository, 'worktree', 'remove', '--force', '--', destination])
+  await rm(destination, { recursive: true, force: true })
+  await runOk('git', ['-C', repository, 'worktree', 'add', '--detach', destination, commit])
+}
+
+export async function pruneDetachedWorktrees(repository: string): Promise<void> {
+  await runOk('git', ['-C', repository, 'worktree', 'prune'])
+}
+
+export async function removeDetachedWorktree(
+  repository: string,
+  destination: string,
+): Promise<void> {
+  const result = await run('git', [
+    '-C',
+    repository,
+    'worktree',
+    'remove',
+    '--force',
+    '--',
+    destination,
+  ])
+  if (result.code !== 0) {
+    await rm(destination, { recursive: true, force: true })
+  }
+}

package/src/swe-arena/types.ts

@@ -30,6 +30,58 @@ export interface SweInstance {
   environment_setup_commit: string | null
 }
 
+/**
+ * One factory-bench instance — a merged feature PR from our own repo history
+ * turned into a gradable end-to-end feature-building task (see
+ * supervisor-lab/factory-bench/docs/design.md). The worker sees only the tree
+ * at `base_commit` (archive export, synthetic git history) plus the rewritten
+ * spec; the PR's own added test files are the hidden judge, overlaid from
+ * `judge_ref` at judge time. Field names mirror the instance dirs'
+ * `manifest.json` byte-for-byte, same policy as SweInstance vs task-meta.json.
+ */
+export interface FactoryInstance {
+  /** `factory.<repo>.<pr>` */
+  id: string
+  /** `owner/name` */
+  repo: string
+  /** Judge-side local mirror; NEVER exposed to the worker workspace. */
+  repo_local_mirror: string
+  /** The worker's world — the PR's base commit. */
+  base_commit: string
+  /** Merge commit the judge tests are read from (`git show <judge_ref>:<path>`). */
+  judge_ref: string
+  /** Instance-dir-relative spec file (PM-ticket grade rewrite of the PR body). */
+  spec_md: string
+  /** Hidden judge test files, overlaid at judge time only. */
+  judge_tests: string[]
+  /** Flaky/env-dependent tests excluded at calibration, reasons in calibration.md. */
+  excluded_tests: string[]
+  /** Immutable Node container image used for setup and judge commands. */
+  command_image: string
+  setup_cmds: string[]
+  judge_cmds: string[]
+  /** e.g. "all 30 judge tests pass; partial score = passed/30" — the /NN is parsed. */
+  resolved_criterion: string
+  timeout_s: number
+  worker_visible_paths_note?: string
+  runtime?: string
+  calibration?: { gold: string; base: string; receipts: string }
+}
+
+/**
+ * Discriminated instance union for the seams that used to assume SweInstance.
+ * A tagged wrapper (not a structural union) because both shapes mirror their
+ * on-disk artifacts byte-for-byte and neither may grow a discriminant field.
+ */
+export type ArenaInstance =
+  | { kind: 'swe'; instance: SweInstance }
+  | { kind: 'factory'; instance: FactoryInstance }
+
+/** The ledger/judge identity: SWE `instance_id` or factory `id`. */
+export function arenaInstanceId(a: ArenaInstance): string {
+  return a.kind === 'swe' ? a.instance.instance_id : a.instance.id
+}
+
 /** Per-run opencode usage breakdown captured on the SOLO arm. */
 export interface SoloUsage {
   steps: number

package/src/swe-code-improve.mts

@@ -33,7 +33,6 @@ import { spawn, spawnSync } from 'node:child_process'
 import { existsSync, mkdirSync, symlinkSync } from 'node:fs'
 import { join } from 'node:path'
 import { improve, agenticGenerator } from '@tangle-network/agent-runtime'
-import type { AgentProfile } from '@tangle-network/agent-interface'
 import type { DispatchContext, JudgeConfig, Scenario } from '@tangle-network/agent-eval/contract'
 import { createSweBenchAdapter } from './benchmarks/swe-bench'
 import type { BenchTask } from './benchmarks/types'
@@ -108,7 +107,6 @@ async function main(): Promise<void> {
   if (!routerKey) throw new Error('TANGLE_API_KEY required (worker calls the router)')
   const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1'
   const workerModel = process.env.WORKER_MODEL ?? 'glm-4.6'
-  const reflectModel = process.env.REFLECT_MODEL ?? 'glm-4.6'
   const trainIds = (process.env.TRAIN_IDS ?? 'psf__requests-2931,pallets__flask-5014').split(',').map((s) => s.trim()).filter(Boolean)
   const holdoutIds = (process.env.HOLDOUT_IDS ?? 'psf__requests-1142,psf__requests-1921').split(',').map((s) => s.trim()).filter(Boolean)
   const generations = Number(process.env.GENERATIONS ?? 1)
@@ -127,7 +125,7 @@ async function main(): Promise<void> {
   const allIds = [...new Set([...trainIds, ...holdoutIds])]
 
   console.log('=== META-HARNESS on the SWE scaffold — improve(surface:code) ===')
-  console.log(`worker=${workerModel} reflect=${reflectModel} router=${routerBaseUrl} runTool=${enableRun}`)
+  console.log(`worker=${workerModel} router=${routerBaseUrl} runTool=${enableRun}`)
   console.log(`train=[${trainIds.join(', ')}] holdout=[${holdoutIds.join(', ')}]`)
   console.log(`generations=${generations} population=${population} innerTurns=${innerTurns} maxTokens=${maxTokens}`)
   console.log(`repoRoot=${REPO_ROOT} baseRef=${baseRef}`)
@@ -156,15 +154,25 @@ async function main(): Promise<void> {
     const worktreeRef = isCode ? String((surface as { worktreeRef?: string }).worktreeRef ?? '') : ''
     const rootDir = isCode && worktreeRef && existsSync(worktreeRef) ? worktreeRef : SWE_MAIN_ROOT
     const t0 = Date.now()
-    const r = await runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs)
+    const paid = await ctx.cost.runPaidCall({
+      channel: 'agent',
+      actor: 'swe-scaffold-worker',
+      model: workerModel,
+      execute: () => runEmit(rootDir, scenario.id, workerEnv, emitTimeoutMs),
+      receipt: (result) => {
+        const usageUnknown = result.tokIn === 0 && result.tokOut === 0
+        return {
+          model: workerModel,
+          inputTokens: result.tokIn,
+          outputTokens: result.tokOut,
+          ...(result.usd > 0 ? { actualCostUsd: result.usd } : {}),
+          ...(usageUnknown ? { usageUnknown: true, costUnknown: result.usd <= 0 } : {}),
+        }
+      },
+    })
+    if (!paid.succeeded) throw paid.error
+    const r = paid.value
     const hasPatch = r.patch.trim().length > 0
-    // Report real usage; floor a patch-bearing zero-usage cell so the stub-guard cannot abort on a
-    // router telemetry gap (lift is judge-derived, so this only affects cost accounting).
-    const zeroUsage = r.tokIn === 0 && r.tokOut === 0
-    ctx.cost.observe(zeroUsage && hasPatch ? Math.max(r.usd, 0.0001) : r.usd, workerModel)
-    ctx.cost.observeTokens(
-      zeroUsage && hasPatch ? { input: Math.max(r.tokIn, 1), output: Math.max(r.tokOut, 1) } : { input: r.tokIn, output: r.tokOut },
-    )
     const files = hasPatch ? [...r.patch.matchAll(/^diff --git a\/(\S+)/gm)].map((m) => m[1]) : []
     console.log(
       `  [measure] ${isCode ? 'cand' : 'base'} ${scenario.id} patch=${r.patch.length}b files=[${files.join(', ') || 'none'}] ` +
@@ -173,7 +181,7 @@ async function main(): Promise<void> {
     return hasPatch ? r.patch : null
   }
 
-  const judge: JudgeConfig<string, Scenario> = {
+  const judge: JudgeConfig<string | null, Scenario> = {
     name: 'swebench-docker',
     dimensions: [{ key: 'resolved', description: 'FAIL_TO_PASS + PASS_TO_PASS resolved by the official swebench Docker harness' }],
     async score({ artifact, scenario }) {
@@ -286,11 +294,10 @@ async function main(): Promise<void> {
     runHarness: runHarness as any,
   })
 
-  const profile: AgentProfile = { name: 'swe-scaffold', prompt: { systemPrompt: '' } }
   const scenarios: Scenario[] = allIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
   const holdoutScenarios: Scenario[] = holdoutIds.map((id) => ({ id, kind: 'swe-bench-verified' }))
 
-  const out = await improve(profile, [], {
+  const out = await improve({
     surface: 'code',
     gate: 'holdout',
     code: { repoRoot: REPO_ROOT, baseRef, worktreeDir, generator },
@@ -301,25 +308,17 @@ async function main(): Promise<void> {
     agent,
     expectUsage: 'warn',
     budget: { generations, populationSize: population, holdoutScenarios, maxConcurrency: 1, reps: 1 },
-    llm: { baseUrl: routerBaseUrl, apiKey: routerKey, model: reflectModel },
   })
 
   console.log('\n=== RESULT ===')
-  console.log(`gateDecision=${out.gateDecision} shipped=${out.shipped} lift=${out.lift}`)
+  console.log(`decision=${out.decision} lift=${out.lift}`)
   console.log(`baseline holdout composite = ${out.raw.baseline.compositeMean}`)
   console.log(`winner   holdout composite = ${out.raw.winner.compositeMean}`)
   console.log(`baseline per-scenario: ${JSON.stringify(out.raw.baseline.perScenario)}`)
   console.log(`winner   per-scenario: ${JSON.stringify(out.raw.winner.perScenario)}`)
   if (out.raw.winner.label) console.log(`winner label: ${out.raw.winner.label}`)
-  if (out.raw.winner.summary) console.log(`winner summary: ${out.raw.winner.summary}`)
-  for (const gen of out.raw.generations ?? []) {
-    console.log(`\n-- generation ${gen.record.generationIndex} candidates --`)
-    for (const c of gen.record.candidates) {
-      const perScenario = (c as { scenarios?: Array<{ scenarioId: string; composite: number }> }).scenarios ?? []
-      const detail = perScenario.map((s) => `${s.scenarioId}=${s.composite}`).join(' ')
-      console.log(`  candidate ${c.surfaceHash.slice(0, 8)} composite=${c.composite}${c.label ? ` "${c.label}"` : ''} [${detail}]`)
-    }
-  }
+  if (out.raw.winner.rationale) console.log(`winner rationale: ${out.raw.winner.rationale}`)
+  console.log(`generations explored: ${out.generationsExplored ?? 0}`)
 }
 
 main().catch((e) => {