@tangle-network/agent-bench 0.11.3 → 0.13.2

package/src/swe-arena/holdout-certify.mts

@@ -1,408 +0,0 @@
-/**
- * 2-rep holdout certification — the operator-approved run that decides whether
- * a would-be-KEEP winner ships. Encodes the gen-2 postmortem: the winner
- * failed certification 3/6 vs 4/6 on a ONE-REP holdout with exactly one
- * discordant cell — a known single-rep noise class — so the protocol now
- * pins:
- *
- *   - `holdoutRepsPerInstance` (default 2) replicate cells per instance,
- *   - resolved = ALL reps resolved (fail-closed, identical to the improvement
- *     set's AND-rule in cell-evidence.mts),
- *   - the certification bar compares against a SAME-PROTOCOL parent
- *     measurement: `config.holdoutBaseline` is either an explicit
- *     `{iid -> bool}` verdict map measured under this exact protocol, or
- *     'measure' — the incumbent runs the same 2-rep holdout FIRST in this run.
- *
- * Certification rule (pre-registered, gen-2 bar unchanged in kind):
- * non-regression — the candidate's AND-resolved count must be >= the parent's,
- * with complete conclusive coverage on the candidate side. The improvement-set
- * gate already demanded a strict gain; the holdout certifies that the gain did
- * not cost held-out performance.
- *
- * The operator-release valve is unchanged: the outer loop always runs with
- * `holdout: 'deferred'` and only reports a would-be-KEEP brief; THIS module is
- * the separate, operator-fired run:
- *
- *   tsx src/swe-arena/holdout-certify.mts <config.json> --candidate <loops-commit> [--out <dir>]
- */
-
-import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'
-import { existsSync } from 'node:fs'
-import { join } from 'node:path'
-import { pathToFileURL } from 'node:url'
-import process from 'node:process'
-import { createSweBenchAdapter } from '../benchmarks/swe-bench.ts'
-import { loadExcludes, runSupervisorArm, type SecretsEnv, type SupervisorArmSpec } from './arms.ts'
-import { gatesForArmKind, waitForCapacity } from './capacity.ts'
-import {
-  replicateCoverageComplete,
-  resolvedInstanceCount,
-  type ReplicateRun,
-} from './cell-evidence.mts'
-import {
-  acquireInstanceLock,
-  addEvalWorktree,
-  assertFrozenArm,
-  assertLaunchEnv,
-  removeEvalWorktree,
-  runWithPostGateClock,
-  type OuterLoopConfig,
-} from './outer-loop.mts'
-import { installProcessSignalAbort, runOk } from './proc.ts'
-import { loadInstanceImages } from './run-experiment.mts'
-import { createSerializedJudge, type SerializedJudge } from './serialized-judge.ts'
-
-// ---------------------------------------------------------------------------
-// Pure protocol pieces — unit-tested against synthetic replicate runs.
-// ---------------------------------------------------------------------------
-
-export const DEFAULT_HOLDOUT_REPS = 2
-
-export function holdoutReps(config: Pick<OuterLoopConfig, 'holdoutRepsPerInstance'>): number {
-  const reps = config.holdoutRepsPerInstance ?? DEFAULT_HOLDOUT_REPS
-  if (!Number.isInteger(reps) || reps < 1) {
-    throw new Error(`holdout-certify: holdoutRepsPerInstance must be a positive integer, got ${JSON.stringify(reps)}`)
-  }
-  return reps
-}
-
-/** Validate an explicit same-protocol parent verdict map: every holdout
- *  instance must carry a boolean (a partial map cannot anchor the bar). */
-export function assertHoldoutBaselineMap(
-  baseline: Record<string, boolean>,
-  holdoutInstances: string[],
-): void {
-  const missing = holdoutInstances.filter((iid) => typeof baseline[iid] !== 'boolean')
-  if (missing.length > 0) {
-    throw new Error(
-      `holdout-certify: holdoutBaseline map is missing same-protocol verdicts for: ${missing.join(', ')} — ` +
-        "provide all instances or use holdoutBaseline: 'measure'",
-    )
-  }
-  const extras = Object.keys(baseline).filter((iid) => !holdoutInstances.includes(iid))
-  if (extras.length > 0) {
-    throw new Error(`holdout-certify: holdoutBaseline names non-holdout instances: ${extras.join(', ')}`)
-  }
-}
-
-export interface HoldoutCertification {
-  candResolved: number
-  parentResolved: number
-  coverageComplete: boolean
-  certified: boolean
-  reasons: string[]
-  perInstance: Array<{ iid: string; candidate: boolean | null; parent: boolean | null; discordant: boolean }>
-}
-
-/** The certification decision over fail-closed AND-verdicts. Pure. */
-export function decideHoldoutCertification(input: {
-  candidateRuns: ReplicateRun[]
-  /** Parent AND-verdict per instance, measured under the SAME protocol
-   *  (reps, fail-closed rule). */
-  parentVerdicts: Record<string, boolean>
-  iids: string[]
-  reps: number
-}): HoldoutCertification {
-  const { candidateRuns, parentVerdicts, iids, reps } = input
-  assertHoldoutBaselineMap(parentVerdicts, iids)
-  const candResolved = resolvedInstanceCount(candidateRuns, iids, reps)
-  const parentResolved = iids.filter((iid) => parentVerdicts[iid] === true).length
-  const coverageComplete = replicateCoverageComplete(candidateRuns, iids, reps)
-  const perInstance = iids.map((iid) => {
-    const mine = candidateRuns.filter((r) => r.iid === iid)
-    const candidate =
-      mine.length === reps && mine.every((r) => r.resolved !== null)
-        ? mine.every((r) => r.resolved === true)
-        : null
-    const parent = parentVerdicts[iid] ?? null
-    return { iid, candidate, parent, discordant: candidate !== null && parent !== null && candidate !== parent }
-  })
-  const reasons: string[] = []
-  if (!coverageComplete) reasons.push(`candidate coverage incomplete: not every instance has ${reps} conclusive reps`)
-  if (candResolved < parentResolved) {
-    reasons.push(`regression: candidate ${candResolved} < parent ${parentResolved} (same-protocol AND-verdicts)`)
-  }
-  const certified = coverageComplete && candResolved >= parentResolved
-  if (certified) reasons.push(`non-regression met: candidate ${candResolved} >= parent ${parentResolved} at ${reps} reps, fail-closed`)
-  return { candResolved, parentResolved, coverageComplete, certified, reasons, perInstance }
-}
-
-// ---------------------------------------------------------------------------
-// The runner — real arms + the official judge, resumable per (commit,iid,rep).
-// ---------------------------------------------------------------------------
-
-export interface HoldoutCellRow {
-  schema: 'swe-arena.holdout-cell.v1'
-  at: string
-  commit: string
-  side: 'candidate' | 'parent'
-  iid: string
-  rep: number
-  resolved: boolean | null
-  verifyPass: boolean | null
-  wallS: number | null
-  runDir: string
-  error?: string
-}
-
-/** Read prior cells (resume): conclusive rows are reused, inconclusive re-run. */
-export async function loadHoldoutCells(cellsPath: string): Promise<HoldoutCellRow[]> {
-  if (!existsSync(cellsPath)) return []
-  const raw = await readFile(cellsPath, 'utf8')
-  return raw
-    .split('\n')
-    .filter((l) => l.trim().length > 0)
-    .map((l) => JSON.parse(l) as HoldoutCellRow)
-    .filter((r) => r.schema === 'swe-arena.holdout-cell.v1')
-}
-
-const log = (msg: string): void => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`)
-
-export async function runHoldoutCertification(
-  config: OuterLoopConfig,
-  opts: { candidateCommit: string; outDir?: string },
-  signal?: AbortSignal,
-): Promise<HoldoutCertification> {
-  signal?.throwIfAborted()
-  assertFrozenArm(config.arm)
-  const reps = holdoutReps(config)
-  const iids = config.holdoutInstances
-  if (iids.length === 0) throw new Error('holdout-certify: empty holdout set')
-  const outDir = opts.outDir ?? join(config.outDir, 'holdout-certification')
-  await mkdir(outDir, { recursive: true })
-  const cellsPath = join(outDir, 'cells.jsonl')
-
-  const secrets: SecretsEnv = { secretsDir: config.secretsDir, envFiles: config.envFiles }
-  const excludes = await loadExcludes()
-  const images = await loadInstanceImages(config.instanceImagesPath)
-  const adapter = createSweBenchAdapter()
-  const tasks = await adapter.loadTasks({ ids: iids, split: 'test' })
-  const problemById = new Map<string, string>()
-  for (const iid of iids) {
-    const task = tasks.find((t) => t.id === iid)
-    if (!task) throw new Error(`holdout-certify: ${iid} not found in SWE-bench_Verified`)
-    const problem = String(task.metadata?.problem_statement ?? '')
-    if (!problem) throw new Error(`holdout-certify: ${iid} has an empty problem_statement`)
-    if (!images[iid]) throw new Error(`holdout-certify: ${iid} has no image mapping`)
-    if (!existsSync(join(config.verifyDir, `${iid}.sh`))) {
-      throw new Error(`holdout-certify: missing verify script ${join(config.verifyDir, `${iid}.sh`)}`)
-    }
-    problemById.set(iid, problem)
-  }
-
-  const judge: SerializedJudge = createSerializedJudge(
-    config.judgeTimeoutMs !== undefined ? { timeoutMs: config.judgeTimeoutMs } : {},
-  )
-  const prior = await loadHoldoutCells(cellsPath)
-
-  const awaitGates = async (gateSignal: AbortSignal | undefined = signal): Promise<void> => {
-    for (const gate of gatesForArmKind('supervisor', secrets, {
-      ...(config.gateWaitCeilingMs !== undefined ? { waitCeilingMs: config.gateWaitCeilingMs } : {}),
-      ...(config.capacityModel !== undefined ? { model: config.capacityModel } : {}),
-      onStatus: log,
-    })) {
-      if (!(await waitForCapacity(gate, gateSignal))) throw new Error(`no capacity on ${gate.name} within ceiling`)
-    }
-  }
-
-  const runSide = async (side: 'candidate' | 'parent', commit: string): Promise<ReplicateRun[]> => {
-    signal?.throwIfAborted()
-    const runs: ReplicateRun[] = []
-    for (const iid of iids) {
-      for (let rep = 0; rep < reps; rep++) {
-        signal?.throwIfAborted()
-        const cached = prior.find(
-          (r) => r.side === side && r.commit === commit && r.iid === iid && r.rep === rep && r.resolved !== null,
-        )
-        if (cached) {
-          log(`holdout ${side} ${iid} rep=${rep}: cached resolved=${cached.resolved}`)
-          runs.push({ iid, resolved: cached.resolved })
-          continue
-        }
-        const tag = `${side}-${commit.slice(0, 10)}`
-        const evalWt = join(outDir, 'eval-wt', `${tag}-${iid}-r${rep}`)
-        const armOutDir = join(outDir, 'arm-runs', tag, iid, `rep-${rep}`)
-        let row: HoldoutCellRow
-        try {
-          try {
-            await addEvalWorktree(config.loopsRepo, commit, evalWt, signal)
-            const spec: SupervisorArmSpec = {
-              kind: 'supervisor',
-              name: config.armName,
-              workerModel: config.arm.workerModel,
-              driverModel: config.arm.driverModel,
-              budget: config.arm.budget,
-              maxSandboxes: config.arm.maxSandboxes,
-              maxUsd: config.arm.maxUsd,
-              maxDepth: config.arm.maxDepth,
-              ...(config.arm.envKnobs ? { envKnobs: config.arm.envKnobs } : {}),
-              loopsRepo: evalWt,
-              extensionPath: join(evalWt, 'extensions', 'pi', 'loops.ts'),
-              timeoutMs: config.arm.timeoutMs,
-            }
-            log(`>>> holdout ${side} ${iid} rep=${rep} @ ${commit.slice(0, 10)}`)
-            const entry = images[iid]!
-            const { armRes, verdict } = await runWithPostGateClock({
-              awaitGates,
-              work: async (signal) => {
-                const armRes = await runSupervisorArm(spec, {
-                  instanceId: iid,
-                  image: entry.image,
-                  baseCommit: entry.base_commit,
-                  problemStatement: problemById.get(iid)!,
-                  verifyCmd: `bash ${join(config.verifyDir, `${iid}.sh`)}`,
-                  outDir: armOutDir,
-                  secrets,
-                  excludes,
-                  signal,
-                })
-                signal.throwIfAborted()
-                const verdict = await judge.judge(iid, armRes.patchPath, `holdout-${tag}`, signal)
-                return { armRes, verdict }
-              },
-              timeoutMs: config.dispatchTimeoutMs,
-              label: `holdout ${side} ${iid} r${rep}`,
-              signal,
-            })
-            row = {
-              schema: 'swe-arena.holdout-cell.v1',
-              at: new Date().toISOString(),
-              commit,
-              side,
-              iid,
-              rep,
-              resolved: verdict.resolved,
-              verifyPass: armRes.verify_pass,
-              wallS: armRes.wall_s,
-              runDir: join(armOutDir, 'runs', iid, config.armName),
-              ...(verdict.resolved === null ? { error: verdict.error ?? 'inconclusive judge verdict' } : {}),
-            }
-          } finally {
-            await removeEvalWorktree(config.loopsRepo, evalWt)
-          }
-        } catch (cause) {
-          if (signal?.aborted) throw cause
-          // An errored cell is an INCONCLUSIVE replicate (resolved: null) —
-          // fail-closed downstream, never a fabricated boolean.
-          row = {
-            schema: 'swe-arena.holdout-cell.v1',
-            at: new Date().toISOString(),
-            commit,
-            side,
-            iid,
-            rep,
-            resolved: null,
-            verifyPass: null,
-            wallS: null,
-            runDir: armOutDir,
-            error: (cause as Error).message,
-          }
-        }
-        await appendFile(cellsPath, JSON.stringify(row) + '\n')
-        log(`holdout ${side} ${iid} rep=${rep}: resolved=${row.resolved}${row.error ? ` (${row.error})` : ''}`)
-        runs.push({ iid, resolved: row.resolved })
-      }
-    }
-    return runs
-  }
-
-  // SAME-PROTOCOL parent verdicts: an explicit operator-provided map, or the
-  // incumbent measured FIRST under the identical 2-rep fail-closed protocol.
-  let parentVerdicts: Record<string, boolean>
-  let parentSource: 'provided' | 'measured'
-  let parentCommit: string | null = null
-  const baseline = config.holdoutBaseline ?? 'measure'
-  if (baseline !== 'measure') {
-    assertHoldoutBaselineMap(baseline, iids)
-    parentVerdicts = baseline
-    parentSource = 'provided'
-  } else {
-    parentCommit = (
-      await runOk('git', ['-C', config.loopsRepo, 'rev-parse', config.loopsBaseRef], { signal })
-    ).stdout.trim()
-    log(`holdout parent: measuring incumbent ${parentCommit.slice(0, 10)} under the same ${reps}-rep protocol`)
-    const parentRuns = await runSide('parent', parentCommit)
-    parentVerdicts = {}
-    for (const iid of iids) {
-      const mine = parentRuns.filter((r) => r.iid === iid)
-      // Fail-closed: an inconclusive/missing parent replicate reads as NOT
-      // resolved for the parent — it can only make the bar EASIER, never
-      // stricter, so a broken parent measurement cannot fail a good candidate.
-      parentVerdicts[iid] = mine.length === reps && mine.every((r) => r.resolved === true)
-    }
-    parentSource = 'measured'
-  }
-
-  const candidateRuns = await runSide('candidate', opts.candidateCommit)
-  signal?.throwIfAborted()
-  const decision = decideHoldoutCertification({ candidateRuns, parentVerdicts, iids, reps })
-
-  const record = {
-    schema: 'swe-arena.holdout-certification.v1',
-    at: new Date().toISOString(),
-    round: config.round,
-    reps,
-    instances: iids,
-    candidateCommit: opts.candidateCommit,
-    parent: { source: parentSource, commit: parentCommit, verdicts: parentVerdicts },
-    ...decision,
-  }
-  const recordPath = join(outDir, `certification-${opts.candidateCommit.slice(0, 10)}.json`)
-  await writeFile(recordPath, JSON.stringify(record, null, 2))
-  log(
-    `holdout certification: candidate ${decision.candResolved}/${iids.length} vs parent ` +
-      `${decision.parentResolved}/${iids.length} at ${reps} reps — ${decision.certified ? 'CERTIFIED' : 'NOT CERTIFIED'}`,
-  )
-  for (const r of decision.reasons) log(`  - ${r}`)
-  log(`certification record → ${recordPath}`)
-  signal?.throwIfAborted()
-  return decision
-}
-
-// ---------------------------------------------------------------------------
-// CLI.
-// ---------------------------------------------------------------------------
-
-const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
-
-if (isMain) {
-  const interrupt = installProcessSignalAbort('holdout-certify')
-  try {
-    const argv = process.argv.slice(2)
-    const configPath = argv[0]
-    const flag = (name: string): string | undefined => {
-      const i = argv.indexOf(name)
-      return i !== -1 ? argv[i + 1] : undefined
-    }
-    const candidate = flag('--candidate')
-    if (!configPath || configPath.startsWith('--') || !candidate) {
-      console.error(
-        'usage: tsx src/swe-arena/holdout-certify.mts <config.json> --candidate <loops-commit> [--out <dir>]  # SPENDS: arms + judges',
-      )
-      process.exit(2)
-    }
-    const config = JSON.parse(await readFile(configPath, 'utf8')) as OuterLoopConfig
-    assertLaunchEnv()
-    interrupt.signal.throwIfAborted()
-    const out = flag('--out')
-    const lockDir = out ?? join(config.outDir, 'holdout-certification')
-    await mkdir(lockDir, { recursive: true })
-    const lock = await acquireInstanceLock(lockDir)
-    try {
-      const decision = await runHoldoutCertification(config, {
-        candidateCommit: candidate,
-        ...(out ? { outDir: out } : {}),
-      }, interrupt.signal)
-      interrupt.signal.throwIfAborted()
-      process.exitCode = decision.certified ? 0 : 1
-    } finally {
-      await lock.release()
-    }
-  } catch (cause) {
-    if (!interrupt.signal.aborted) throw cause
-    const detail = cause instanceof Error ? cause.message : String(cause)
-    console.error(`holdout-certify stopped after cleanup: ${detail}`)
-  } finally {
-    interrupt.dispose()
-  }
-}

package/src/swe-arena/holdout-certify.test.mts

@@ -1,160 +0,0 @@
-import { mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import type { ReplicateRun } from './cell-evidence.mts'
-import {
-  assertHoldoutBaselineMap,
-  decideHoldoutCertification,
-  DEFAULT_HOLDOUT_REPS,
-  holdoutReps,
-  loadHoldoutCells,
-  type HoldoutCellRow,
-} from './holdout-certify.mts'
-
-const IIDS = ['a', 'b', 'c']
-
-const runs = (spec: Record<string, Array<boolean | null>>): ReplicateRun[] =>
-  Object.entries(spec).flatMap(([iid, reps]) => reps.map((resolved) => ({ iid, resolved })))
-
-describe('holdoutReps', () => {
-  it('defaults to 2 — the gen-2 postmortem protocol', () => {
-    expect(DEFAULT_HOLDOUT_REPS).toBe(2)
-    expect(holdoutReps({})).toBe(2)
-    expect(holdoutReps({ holdoutRepsPerInstance: 3 })).toBe(3)
-  })
-
-  it('rejects non-positive/non-integer reps', () => {
-    expect(() => holdoutReps({ holdoutRepsPerInstance: 0 })).toThrow(/positive integer/)
-    expect(() => holdoutReps({ holdoutRepsPerInstance: 1.5 })).toThrow(/positive integer/)
-  })
-})
-
-describe('assertHoldoutBaselineMap', () => {
-  it('requires a same-protocol verdict for EVERY holdout instance', () => {
-    expect(() => assertHoldoutBaselineMap({ a: true }, IIDS)).toThrow(/missing same-protocol verdicts.*b, c/)
-  })
-
-  it('rejects verdicts for non-holdout instances', () => {
-    expect(() => assertHoldoutBaselineMap({ a: true, b: false, c: true, z: true }, IIDS)).toThrow(/non-holdout/)
-  })
-
-  it('accepts a complete map', () => {
-    expect(() => assertHoldoutBaselineMap({ a: true, b: false, c: true }, IIDS)).not.toThrow()
-  })
-})
-
-describe('decideHoldoutCertification — 2-rep fail-closed aggregation', () => {
-  const parent = { a: true, b: false, c: true } // parent 2/3 same-protocol
-
-  it('an instance resolves only when BOTH reps resolve (T/F counts as unresolved)', () => {
-    const d = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true, true], b: [true, false], c: [true, true] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 2,
-    })
-    expect(d.candResolved).toBe(2) // a + c; b's discordant pair is fail-closed
-    expect(d.parentResolved).toBe(2)
-    expect(d.coverageComplete).toBe(true)
-    expect(d.certified).toBe(true) // non-regression: 2 >= 2
-    expect(d.perInstance.find((p) => p.iid === 'b')).toMatchObject({ candidate: false, parent: false, discordant: false })
-  })
-
-  it('regression fails certification even with complete coverage', () => {
-    const d = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true, true], b: [false, false], c: [false, true] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 2,
-    })
-    expect(d.candResolved).toBe(1)
-    expect(d.certified).toBe(false)
-    expect(d.reasons.join(' ')).toMatch(/regression: candidate 1 < parent 2/)
-    expect(d.perInstance.find((p) => p.iid === 'c')).toMatchObject({ candidate: false, parent: true, discordant: true })
-  })
-
-  it('an inconclusive replicate (errored cell) blocks coverage and certification — fail-closed', () => {
-    const d = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true, true], b: [false, false], c: [true, null] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 2,
-    })
-    expect(d.coverageComplete).toBe(false)
-    expect(d.certified).toBe(false)
-    expect(d.reasons.join(' ')).toMatch(/coverage incomplete/)
-    expect(d.perInstance.find((p) => p.iid === 'c')).toMatchObject({ candidate: null, discordant: false })
-  })
-
-  it('a missing replicate (only 1 of 2 reps ran) blocks coverage', () => {
-    const d = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true, true], b: [true, true], c: [true] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 2,
-    })
-    expect(d.coverageComplete).toBe(false)
-    expect(d.certified).toBe(false)
-  })
-
-  it('gen-2 shape: one discordant instance under 1-rep would have failed; the same data at 2 reps needs both reps', () => {
-    // At 1 rep a single flipped cell decided the gen-2 verdict. The 2-rep
-    // protocol requires the flip to REPLICATE before it counts either way.
-    const oneRep = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true], b: [false], c: [false] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 1,
-    })
-    expect(oneRep.certified).toBe(false) // 1 < 2 — the gen-2 outcome
-    const twoRep = decideHoldoutCertification({
-      candidateRuns: runs({ a: [true, true], b: [false, false], c: [true, true] }),
-      parentVerdicts: parent,
-      iids: IIDS,
-      reps: 2,
-    })
-    expect(twoRep.certified).toBe(true) // c held up under replication
-  })
-
-  it('validates the parent map before deciding', () => {
-    expect(() =>
-      decideHoldoutCertification({
-        candidateRuns: runs({ a: [true, true] }),
-        parentVerdicts: { a: true },
-        iids: IIDS,
-        reps: 2,
-      }),
-    ).toThrow(/missing same-protocol verdicts/)
-  })
-})
-
-describe('loadHoldoutCells', () => {
-  let dir: string
-  beforeEach(async () => {
-    dir = await mkdtemp(join(tmpdir(), 'holdout-cells-'))
-  })
-  afterEach(async () => {
-    await rm(dir, { recursive: true, force: true })
-  })
-
-  it('returns [] for a missing file and round-trips schema-valid rows', async () => {
-    const path = join(dir, 'cells.jsonl')
-    expect(await loadHoldoutCells(path)).toEqual([])
-    const row: HoldoutCellRow = {
-      schema: 'swe-arena.holdout-cell.v1',
-      at: new Date(0).toISOString(),
-      commit: 'abc123',
-      side: 'candidate',
-      iid: 'a',
-      rep: 0,
-      resolved: true,
-      verifyPass: true,
-      wallS: 100,
-      runDir: '/tmp/x',
-    }
-    await writeFile(path, `${JSON.stringify(row)}\n${JSON.stringify({ schema: 'other' })}\n`)
-    const rows = await loadHoldoutCells(path)
-    expect(rows).toEqual([row]) // foreign schemas are dropped
-  })
-})

package/src/swe-arena/implementation-ref.test.mts

@@ -1,64 +0,0 @@
-import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, describe, expect, it } from 'vitest'
-import {
-  fileTreeImplementationRef,
-  pythonDistributionImplementationRef,
-} from './implementation-ref.ts'
-
-describe('implementation refs', () => {
-  const roots: string[] = []
-
-  afterEach(async () => {
-    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
-  })
-
-  it('is stable across creation order and changes with file content or symlink targets', async () => {
-    const first = await mkdtemp(join(tmpdir(), 'implementation-ref-a-'))
-    const second = await mkdtemp(join(tmpdir(), 'implementation-ref-b-'))
-    roots.push(first, second)
-    await mkdir(join(first, 'nested'))
-    await writeFile(join(first, 'nested', 'b.ts'), 'export const b = 2\n')
-    await writeFile(join(first, 'a.ts'), 'export const a = 1\n')
-    await symlink('a.ts', join(first, 'entry.ts'))
-    await writeFile(join(second, 'a.ts'), 'export const a = 1\n')
-    await symlink('a.ts', join(second, 'entry.ts'))
-    await mkdir(join(second, 'nested'))
-    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n')
-
-    const initial = await fileTreeImplementationRef(first)
-    expect(await fileTreeImplementationRef(second)).toBe(initial)
-
-    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 3\n')
-    expect(await fileTreeImplementationRef(second)).not.toBe(initial)
-    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n')
-    await rm(join(second, 'entry.ts'))
-    await symlink('nested/b.ts', join(second, 'entry.ts'))
-    expect(await fileTreeImplementationRef(second)).not.toBe(initial)
-  })
-
-  it('binds a Python distribution to its complete manifest', async () => {
-    const runPython = async (_script: string, args: string[]): Promise<string> =>
-      `import warning\n${JSON.stringify({
-        distribution: args[0],
-        version: '1.0.0',
-        python: '3.12.4',
-        files: [{ path: '__init__.py', sha256: 'a'.repeat(64) }],
-      })}`
-    const original = await pythonDistributionImplementationRef('swebench', runPython)
-    const changed = await pythonDistributionImplementationRef(
-      'swebench',
-      async (_script, args) =>
-        JSON.stringify({
-          distribution: args[0],
-          version: '1.0.1',
-          python: '3.12.4',
-          files: [{ path: '__init__.py', sha256: 'b'.repeat(64) }],
-        }),
-    )
-
-    expect(original).toMatch(/^sha256:[a-f0-9]{64}$/)
-    expect(changed).not.toBe(original)
-  })
-})