@tangle-network/agent-bench 0.3.7 → 0.4.0

package/src/swe-arena/implementation-ref.test.mts

@@ -0,0 +1,64 @@
+import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import {
+  fileTreeImplementationRef,
+  pythonDistributionImplementationRef,
+} from './implementation-ref.ts'
+
+describe('implementation refs', () => {
+  const roots: string[] = []
+
+  afterEach(async () => {
+    await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+  })
+
+  it('is stable across creation order and changes with file content or symlink targets', async () => {
+    const first = await mkdtemp(join(tmpdir(), 'implementation-ref-a-'))
+    const second = await mkdtemp(join(tmpdir(), 'implementation-ref-b-'))
+    roots.push(first, second)
+    await mkdir(join(first, 'nested'))
+    await writeFile(join(first, 'nested', 'b.ts'), 'export const b = 2\n')
+    await writeFile(join(first, 'a.ts'), 'export const a = 1\n')
+    await symlink('a.ts', join(first, 'entry.ts'))
+    await writeFile(join(second, 'a.ts'), 'export const a = 1\n')
+    await symlink('a.ts', join(second, 'entry.ts'))
+    await mkdir(join(second, 'nested'))
+    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n')
+
+    const initial = await fileTreeImplementationRef(first)
+    expect(await fileTreeImplementationRef(second)).toBe(initial)
+
+    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 3\n')
+    expect(await fileTreeImplementationRef(second)).not.toBe(initial)
+    await writeFile(join(second, 'nested', 'b.ts'), 'export const b = 2\n')
+    await rm(join(second, 'entry.ts'))
+    await symlink('nested/b.ts', join(second, 'entry.ts'))
+    expect(await fileTreeImplementationRef(second)).not.toBe(initial)
+  })
+
+  it('binds a Python distribution to its complete manifest', async () => {
+    const runPython = async (_script: string, args: string[]): Promise<string> =>
+      `import warning\n${JSON.stringify({
+        distribution: args[0],
+        version: '1.0.0',
+        python: '3.12.4',
+        files: [{ path: '__init__.py', sha256: 'a'.repeat(64) }],
+      })}`
+    const original = await pythonDistributionImplementationRef('swebench', runPython)
+    const changed = await pythonDistributionImplementationRef(
+      'swebench',
+      async (_script, args) =>
+        JSON.stringify({
+          distribution: args[0],
+          version: '1.0.1',
+          python: '3.12.4',
+          files: [{ path: '__init__.py', sha256: 'b'.repeat(64) }],
+        }),
+    )
+
+    expect(original).toMatch(/^sha256:[a-f0-9]{64}$/)
+    expect(changed).not.toBe(original)
+  })
+})

package/src/swe-arena/implementation-ref.ts

@@ -0,0 +1,62 @@
+import { createHash } from 'node:crypto'
+import { lstat, readFile, readdir, readlink, realpath } from 'node:fs/promises'
+import { join, relative } from 'node:path'
+import { canonicalCandidateDigest } from '@tangle-network/agent-interface'
+
+const sha256Content = (value: Uint8Array): string =>
+  `sha256:${createHash('sha256').update(value).digest('hex')}`
+
+export async function fileTreeImplementationRef(root: string): Promise<string> {
+  const canonicalRoot = await realpath(root)
+  const files: Array<{ path: string; sha256?: string; symlink?: string }> = []
+
+  const walk = async (directory: string): Promise<void> => {
+    const entries = await readdir(directory, { withFileTypes: true })
+    entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+    for (const entry of entries) {
+      const path = join(directory, entry.name)
+      const relativePath = relative(canonicalRoot, path)
+      if (entry.isDirectory()) {
+        await walk(path)
+      } else if (entry.isSymbolicLink()) {
+        files.push({ path: relativePath, symlink: await readlink(path) })
+      } else if (entry.isFile() || (await lstat(path)).isFile()) {
+        files.push({ path: relativePath, sha256: sha256Content(await readFile(path)) })
+      }
+    }
+  }
+
+  await walk(canonicalRoot)
+  return canonicalCandidateDigest(files)
+}
+
+export async function pythonDistributionImplementationRef(
+  distribution: string,
+  runPython: (script: string, args: string[]) => Promise<string>,
+): Promise<string> {
+  const output = await runPython(
+    [
+      'import hashlib, importlib, importlib.metadata, json, pathlib, sys',
+      'name = sys.argv[1]',
+      'module = importlib.import_module(name)',
+      'root = pathlib.Path(module.__file__).resolve().parent',
+      'files = []',
+      'for path in sorted(p for p in root.rglob("*") if p.is_file()):',
+      '    if "__pycache__" in path.parts or path.suffix == ".pyc":',
+      '        continue',
+      '    files.append({"path": str(path.relative_to(root)), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()})',
+      'try:',
+      '    version = importlib.metadata.version(name)',
+      'except importlib.metadata.PackageNotFoundError:',
+      '    version = None',
+      'print(json.dumps({"distribution": name, "version": version, "python": sys.version, "files": files}, sort_keys=True))',
+    ].join('\n'),
+    [distribution],
+  )
+  const line = output.trim().split('\n').at(-1)
+  if (line === undefined) {
+    throw new Error(`python distribution ${distribution} returned no implementation manifest`)
+  }
+  const manifest = JSON.parse(line) as unknown
+  return canonicalCandidateDigest(manifest)
+}

package/src/swe-arena/ledger-orphans.mts

@@ -0,0 +1,77 @@
+/**
+ * Crash-orphan reconciliation for the improve-run CostLedger.
+ *
+ * A killed outer-loop process leaves its in-flight paid call as a durable
+ * 'pending' record with no receipt. On resume the ledger's fail-closed
+ * admission guard ("N unresolved call(s) must be reconciled before new paid
+ * work") then refuses EVERY new paid call, so a single crash permanently
+ * blocks the run. This module settles those orphans at startup through the
+ * ledger's dedicated crash-recovery verb, `CostLedger.reconcile()`.
+ *
+ * Safety contract — never settle a live run's call:
+ *  - The caller must hold the outDir single-instance pid lock
+ *    (`acquireInstanceLock` in outer-loop.mts) before invoking this, so no
+ *    concurrent outer-loop shares the ledger file. Every pending record
+ *    restored from disk is therefore from a dead process.
+ *  - Belt and braces, only `state: 'interrupted'` pendings are touched:
+ *    restored-from-persistence calls this process's ledger instance never
+ *    started. An 'active'/'late' pending belongs to an in-flight
+ *    `runPaidCall` and settles through its own path.
+ *  - The ledger's persistence append is compare-and-swap on the file
+ *    revision, so even an unexpected concurrent writer makes reconciliation
+ *    fail loud (`CostCallConflictError`) instead of corrupting the log.
+ *
+ * The orphan settles as a FAILED receipt with zero usage and a fully-known
+ * $0 actual cost (`actualCostUsd: 0`), reason 'process-crash-orphan'. A
+ * `costUnknown`/`usageUnknown` receipt would be marginally more honest about
+ * the crashed call's true spend, but an unknown-cost receipt marks the
+ * ledger's accounting incomplete forever, which blocks all paid work in
+ * capped runs — the same permanent lockout this module exists to fix.
+ */
+
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import type { CostReceipt, PendingCostCallView } from '@tangle-network/agent-eval'
+import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign'
+
+export const CRASH_ORPHAN_REASON = 'process-crash-orphan'
+
+/** The two ledger verbs reconciliation needs, structural for testability. */
+export interface OrphanReconcilableLedger {
+  listPending(): PendingCostCallView[]
+  reconcile(
+    callId: string,
+    observed: {
+      model: string
+      inputTokens: number
+      outputTokens: number
+      actualCostUsd?: number
+    },
+    options?: { error?: string },
+  ): CostReceipt
+}
+
+/** Settle every crash-orphaned ('interrupted') pending call as a $0 failure. */
+export function reconcileCrashOrphanCalls(ledger: OrphanReconcilableLedger): CostReceipt[] {
+  return ledger
+    .listPending()
+    .filter((pending) => pending.state === 'interrupted')
+    .map((pending) =>
+      ledger.reconcile(
+        pending.callId,
+        { model: pending.model, inputTokens: 0, outputTokens: 0, actualCostUsd: 0 },
+        { error: CRASH_ORPHAN_REASON },
+      ),
+    )
+}
+
+/**
+ * Open the durable ledger under `improveRunDir` and settle crash orphans.
+ * No-op (empty result) when no ledger file exists yet — a fresh run.
+ * PRECONDITION: the caller holds the outDir instance lock (see module doc).
+ */
+export function reconcileCrashOrphansOnDisk(improveRunDir: string): CostReceipt[] {
+  if (!existsSync(join(improveRunDir, 'cost-ledger.jsonl'))) return []
+  const ledger = createRunCostLedger({ storage: fsCampaignStorage(), runDir: improveRunDir })
+  return reconcileCrashOrphanCalls(ledger)
+}

package/src/swe-arena/ledger-orphans.test.mts

@@ -0,0 +1,147 @@
+/**
+ * Crash-orphan ledger reconciliation: a synthetic durable ledger with one
+ * orphaned pending call (crashed process — call opened, no receipt) and one
+ * resolved call. Reconciliation must settle ONLY the orphan, as a $0
+ * failure receipt tagged 'process-crash-orphan', after which the ledger's
+ * fail-closed admission guard passes new paid work again — the exact resume
+ * path improve() rides.
+ */
+
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import type { CostReceipt, PendingCostCallView } from '@tangle-network/agent-eval'
+import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign'
+import {
+  CRASH_ORPHAN_REASON,
+  reconcileCrashOrphanCalls,
+  reconcileCrashOrphansOnDisk,
+  type OrphanReconcilableLedger,
+} from './ledger-orphans.mts'
+
+const ATTRIBUTION = { channel: 'agent', phase: 'search.baseline', model: 'm' }
+
+/** A durable event line in the exact shape cost-ledger.jsonl persists. */
+const line = (record: Record<string, unknown>): string =>
+  `${JSON.stringify({ version: 1, record })}\n`
+
+/** Write a synthetic ledger: one settled pair + one crash-orphaned pending. */
+function writeCrashedLedger(dir: string): void {
+  const settledPending = {
+    status: 'pending',
+    callId: 'settled-1',
+    ...ATTRIBUTION,
+    actor: 'worker:astropy#r0',
+    timestamp: 1_000,
+  }
+  const settledReceipt = {
+    ...settledPending,
+    status: 'settled',
+    inputTokens: 10,
+    outputTokens: 5,
+    costUsd: 0.01,
+    costUnknown: false,
+    actualCostUsd: 0.01,
+  }
+  const orphanPending = {
+    status: 'pending',
+    callId: 'orphan-1',
+    ...ATTRIBUTION,
+    actor: 'worker:xarray#r0',
+    timestamp: 2_000,
+    tags: { cellId: 'pydata__xarray-4687:0' },
+  }
+  writeFileSync(
+    join(dir, 'cost-ledger.jsonl'),
+    line(settledPending) + line(settledReceipt) + line(orphanPending),
+  )
+}
+
+const openLedger = (dir: string) =>
+  createRunCostLedger({ storage: fsCampaignStorage(), runDir: dir })
+
+describe('reconcileCrashOrphansOnDisk', () => {
+  it('settles only the orphan, and the admission guard passes afterwards', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    writeCrashedLedger(dir)
+
+    // The bug being fixed: with the orphan unresolved, ALL new paid work is refused.
+    const blocked = await openLedger(dir).runPaidCall({
+      ...ATTRIBUTION,
+      actor: 'worker:next',
+      execute: async () => 'ok',
+      receipt: () => ({ model: 'm', inputTokens: 1, outputTokens: 1, actualCostUsd: 0 }),
+    })
+    expect(blocked.succeeded).toBe(false)
+    if (!blocked.succeeded) {
+      expect(blocked.error.message).toContain('unresolved call(s) must be reconciled')
+    }
+
+    const receipts = reconcileCrashOrphansOnDisk(dir)
+    expect(receipts.map((r) => r.callId)).toEqual(['orphan-1'])
+    expect(receipts[0]).toMatchObject({
+      status: 'settled',
+      error: CRASH_ORPHAN_REASON,
+      inputTokens: 0,
+      outputTokens: 0,
+      costUsd: 0,
+      costUnknown: false,
+    })
+
+    // Durable: a fresh ledger instance sees zero unresolved calls, the
+    // settled call untouched, and admits new paid work again.
+    const reopened = openLedger(dir)
+    const summary = reopened.summary()
+    expect(summary.unresolvedCalls).toBe(0)
+    expect(summary.pendingCalls).toBe(0)
+    expect(summary.accountingComplete).toBe(true)
+    const settled = reopened.list().find((r) => r.callId === 'settled-1')
+    expect(settled?.costUsd).toBe(0.01)
+    expect(settled?.error).toBeUndefined()
+    const admitted = await reopened.runPaidCall({
+      ...ATTRIBUTION,
+      actor: 'worker:next',
+      execute: async () => 'ok',
+      receipt: () => ({ model: 'm', inputTokens: 1, outputTokens: 1, actualCostUsd: 0 }),
+    })
+    expect(admitted.succeeded).toBe(true)
+
+    // The durable log carries the orphan's failure receipt verbatim.
+    const persisted = readFileSync(join(dir, 'cost-ledger.jsonl'), 'utf8')
+    expect(persisted).toContain(CRASH_ORPHAN_REASON)
+  })
+
+  it('is idempotent — a second startup pass finds nothing to settle', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    writeCrashedLedger(dir)
+    expect(reconcileCrashOrphansOnDisk(dir)).toHaveLength(1)
+    expect(reconcileCrashOrphansOnDisk(dir)).toHaveLength(0)
+  })
+
+  it('is a no-op on a fresh run dir with no ledger file', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    expect(reconcileCrashOrphansOnDisk(dir)).toEqual([])
+  })
+})
+
+describe('reconcileCrashOrphanCalls', () => {
+  it("never settles a live in-process call — only 'interrupted' pendings", () => {
+    const base = { ...ATTRIBUTION, status: 'pending' as const, timestamp: 1 }
+    const pendings: PendingCostCallView[] = [
+      { ...base, callId: 'live-1', actor: 'a', state: 'active' },
+      { ...base, callId: 'late-1', actor: 'b', state: 'late' },
+      { ...base, callId: 'orphan-1', actor: 'c', state: 'interrupted' },
+    ]
+    const settled: string[] = []
+    const ledger: OrphanReconcilableLedger = {
+      listPending: () => pendings,
+      reconcile: (callId) => {
+        settled.push(callId)
+        return { callId } as CostReceipt
+      },
+    }
+    reconcileCrashOrphanCalls(ledger)
+    expect(settled).toEqual(['orphan-1'])
+  })
+})