@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/ledger-orphans.test.mts

@@ -0,0 +1,147 @@
+/**
+ * Crash-orphan ledger reconciliation: a synthetic durable ledger with one
+ * orphaned pending call (crashed process — call opened, no receipt) and one
+ * resolved call. Reconciliation must settle ONLY the orphan, as a $0
+ * failure receipt tagged 'process-crash-orphan', after which the ledger's
+ * fail-closed admission guard passes new paid work again — the exact resume
+ * path improve() rides.
+ */
+
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import type { CostReceipt, PendingCostCallView } from '@tangle-network/agent-eval'
+import { createRunCostLedger, fsCampaignStorage } from '@tangle-network/agent-eval/campaign'
+import {
+  CRASH_ORPHAN_REASON,
+  reconcileCrashOrphanCalls,
+  reconcileCrashOrphansOnDisk,
+  type OrphanReconcilableLedger,
+} from './ledger-orphans.mts'
+
+const ATTRIBUTION = { channel: 'agent', phase: 'search.baseline', model: 'm' }
+
+/** A durable event line in the exact shape cost-ledger.jsonl persists. */
+const line = (record: Record<string, unknown>): string =>
+  `${JSON.stringify({ version: 1, record })}\n`
+
+/** Write a synthetic ledger: one settled pair + one crash-orphaned pending. */
+function writeCrashedLedger(dir: string): void {
+  const settledPending = {
+    status: 'pending',
+    callId: 'settled-1',
+    ...ATTRIBUTION,
+    actor: 'worker:astropy#r0',
+    timestamp: 1_000,
+  }
+  const settledReceipt = {
+    ...settledPending,
+    status: 'settled',
+    inputTokens: 10,
+    outputTokens: 5,
+    costUsd: 0.01,
+    costUnknown: false,
+    actualCostUsd: 0.01,
+  }
+  const orphanPending = {
+    status: 'pending',
+    callId: 'orphan-1',
+    ...ATTRIBUTION,
+    actor: 'worker:xarray#r0',
+    timestamp: 2_000,
+    tags: { cellId: 'pydata__xarray-4687:0' },
+  }
+  writeFileSync(
+    join(dir, 'cost-ledger.jsonl'),
+    line(settledPending) + line(settledReceipt) + line(orphanPending),
+  )
+}
+
+const openLedger = (dir: string) =>
+  createRunCostLedger({ storage: fsCampaignStorage(), runDir: dir })
+
+describe('reconcileCrashOrphansOnDisk', () => {
+  it('settles only the orphan, and the admission guard passes afterwards', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    writeCrashedLedger(dir)
+
+    // The bug being fixed: with the orphan unresolved, ALL new paid work is refused.
+    const blocked = await openLedger(dir).runPaidCall({
+      ...ATTRIBUTION,
+      actor: 'worker:next',
+      execute: async () => 'ok',
+      receipt: () => ({ model: 'm', inputTokens: 1, outputTokens: 1, actualCostUsd: 0 }),
+    })
+    expect(blocked.succeeded).toBe(false)
+    if (!blocked.succeeded) {
+      expect(blocked.error.message).toContain('unresolved call(s) must be reconciled')
+    }
+
+    const receipts = reconcileCrashOrphansOnDisk(dir)
+    expect(receipts.map((r) => r.callId)).toEqual(['orphan-1'])
+    expect(receipts[0]).toMatchObject({
+      status: 'settled',
+      error: CRASH_ORPHAN_REASON,
+      inputTokens: 0,
+      outputTokens: 0,
+      costUsd: 0,
+      costUnknown: false,
+    })
+
+    // Durable: a fresh ledger instance sees zero unresolved calls, the
+    // settled call untouched, and admits new paid work again.
+    const reopened = openLedger(dir)
+    const summary = reopened.summary()
+    expect(summary.unresolvedCalls).toBe(0)
+    expect(summary.pendingCalls).toBe(0)
+    expect(summary.accountingComplete).toBe(true)
+    const settled = reopened.list().find((r) => r.callId === 'settled-1')
+    expect(settled?.costUsd).toBe(0.01)
+    expect(settled?.error).toBeUndefined()
+    const admitted = await reopened.runPaidCall({
+      ...ATTRIBUTION,
+      actor: 'worker:next',
+      execute: async () => 'ok',
+      receipt: () => ({ model: 'm', inputTokens: 1, outputTokens: 1, actualCostUsd: 0 }),
+    })
+    expect(admitted.succeeded).toBe(true)
+
+    // The durable log carries the orphan's failure receipt verbatim.
+    const persisted = readFileSync(join(dir, 'cost-ledger.jsonl'), 'utf8')
+    expect(persisted).toContain(CRASH_ORPHAN_REASON)
+  })
+
+  it('is idempotent — a second startup pass finds nothing to settle', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    writeCrashedLedger(dir)
+    expect(reconcileCrashOrphansOnDisk(dir)).toHaveLength(1)
+    expect(reconcileCrashOrphansOnDisk(dir)).toHaveLength(0)
+  })
+
+  it('is a no-op on a fresh run dir with no ledger file', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'ledger-orphans-'))
+    expect(reconcileCrashOrphansOnDisk(dir)).toEqual([])
+  })
+})
+
+describe('reconcileCrashOrphanCalls', () => {
+  it("never settles a live in-process call — only 'interrupted' pendings", () => {
+    const base = { ...ATTRIBUTION, status: 'pending' as const, timestamp: 1 }
+    const pendings: PendingCostCallView[] = [
+      { ...base, callId: 'live-1', actor: 'a', state: 'active' },
+      { ...base, callId: 'late-1', actor: 'b', state: 'late' },
+      { ...base, callId: 'orphan-1', actor: 'c', state: 'interrupted' },
+    ]
+    const settled: string[] = []
+    const ledger: OrphanReconcilableLedger = {
+      listPending: () => pendings,
+      reconcile: (callId) => {
+        settled.push(callId)
+        return { callId } as CostReceipt
+      },
+    }
+    reconcileCrashOrphanCalls(ledger)
+    expect(settled).toEqual(['orphan-1'])
+  })
+})

package/src/swe-arena/lineage-record.mts

@@ -0,0 +1,164 @@
+/**
+ * Gen-5 lineage DAG wiring (SOTA adoption #3) — the hand-rolled parent fields
+ * stop being the only ancestry record: every candidate becomes a
+ * `LineageNode` in agent-eval's Lineage DAG (campaign/lineage.ts —
+ * multi-parent merges, tracks, deterministic content-hash ids, append-only
+ * fsLineageStore), every generation appends, and the library's
+ * `heuristicGovernor` reads the graph to decide per-generation continuation
+ * (extend the winner / branch on stall / merge frontier tips / prune). The
+ * governor's decision is RECORDED in the round summary — the operator (or a
+ * future automated outer loop) acts on it; this run never acts on it itself.
+ *
+ * Store location: `<outDir>/.evolve/lineage.jsonl` (an .evolve-compatible
+ * path). The existing staircase rows are UNCHANGED and still written — the
+ * observatory depends on them; the DAG is additive.
+ *
+ * Node semantics:
+ *  - the BASELINE incumbent is the root (track 'baseline', proposer
+ *    'baseline'),
+ *  - configured Pareto parents (prior-run frontier commits) become nodes off
+ *    the root under their own tracks, so a merge can name them as parents,
+ *  - each evaluated candidate is a node under its proposer-named track:
+ *    single-parent (root) for regular authors, MULTI-PARENT (the Pareto
+ *    parent nodes) for the merge seat,
+ *  - score = combined resolved fraction; scoreVector = per-instance
+ *    fail-closed AND-verdicts over the lexicographically sorted instance ids.
+ */
+
+import { join } from 'node:path'
+import {
+  Lineage,
+  fsLineageStore,
+  heuristicGovernor,
+  lineageNodeId,
+  type GovernorOp,
+  type LineageNode,
+} from '@tangle-network/agent-eval/campaign'
+
+export const LINEAGE_STORE_RELPATH = join('.evolve', 'lineage.jsonl')
+
+export interface LineageCandidateInput {
+  /** Proposer name — the node's track and proposer. */
+  label: string
+  /** Loops commit; null (never-committed candidate) is skipped with a note. */
+  commit: string | null
+  resolvedCount: number
+  /** Per-instance AND-verdicts (missing instance = fail-closed false). */
+  verdicts: Record<string, boolean>
+  merge: boolean
+  /** Staircase verdict — recorded as the node's rationale. */
+  verdict: string
+}
+
+export interface LineageRecordArgs {
+  outDir: string
+  runId: string
+  instances: readonly string[]
+  baseline: { commit: string; resolvedCount: number; verdicts: Record<string, boolean> }
+  paretoParents: Array<{ label: string; commit: string; resolvedInstances: string[] }>
+  candidates: LineageCandidateInput[]
+}
+
+export interface LineageRecordResult {
+  path: string
+  /** Node ids appended by THIS call (idempotent re-runs append zero). */
+  appended: string[]
+  /** Nodes skipped because the candidate never became a commit. */
+  skipped: string[]
+  governor: GovernorOp
+  nodesTotal: number
+}
+
+const scoreVectorOf = (instances: readonly string[], verdicts: Record<string, boolean>): number[] =>
+  [...instances].sort().map((iid) => (verdicts[iid] === true ? 1 : 0))
+
+const scoreOf = (resolvedCount: number, instanceCount: number): number =>
+  instanceCount > 0 ? resolvedCount / instanceCount : 0
+
+/** Record one generation into the lineage DAG and ask the governor for the
+ *  continuation decision. Append-only + idempotent: node ids are content
+ *  hashes, so re-recording an identical generation appends nothing. */
+export async function recordLineageGeneration(args: LineageRecordArgs): Promise<LineageRecordResult> {
+  const path = join(args.outDir, LINEAGE_STORE_RELPATH)
+  const store = fsLineageStore(path)
+  const lineage = await store.load()
+  const appended: string[] = []
+  const skipped: string[] = []
+
+  const ensure = async (input: Parameters<Lineage['addNode']>[0]): Promise<LineageNode> => {
+    const id = lineageNodeId({
+      parentIds: input.parentIds,
+      track: input.track,
+      surface: input.surface,
+      proposer: input.proposer,
+    })
+    const existed = lineage.all().some((n) => n.id === id)
+    const node = lineage.addNode(input)
+    if (!existed) {
+      await store.append(node)
+      appended.push(node.id)
+    }
+    return node
+  }
+
+  const n = args.instances.length
+  const root = await ensure({
+    parentIds: [],
+    track: 'baseline',
+    surface: `loops@${args.baseline.commit}`,
+    score: scoreOf(args.baseline.resolvedCount, n),
+    scoreVector: scoreVectorOf(args.instances, args.baseline.verdicts),
+    proposer: 'baseline',
+  })
+
+  // Prior-run frontier commits become referenceable parent nodes.
+  const parentNodeByLabel = new Map<string, LineageNode>()
+  for (const parent of args.paretoParents) {
+    const verdicts: Record<string, boolean> = {}
+    for (const iid of parent.resolvedInstances) verdicts[iid] = true
+    const node = await ensure({
+      parentIds: [root.id],
+      track: parent.label,
+      surface: `loops@${parent.commit}`,
+      score: scoreOf(parent.resolvedInstances.length, n),
+      scoreVector: scoreVectorOf(args.instances, verdicts),
+      proposer: parent.label,
+    })
+    parentNodeByLabel.set(parent.label, node)
+  }
+
+  for (const cand of args.candidates) {
+    if (cand.commit === null) {
+      skipped.push(cand.label)
+      continue
+    }
+    const parentIds = cand.merge
+      ? [...parentNodeByLabel.values()].map((p) => p.id)
+      : [root.id]
+    if (cand.merge && parentIds.length < 2) {
+      throw new Error(
+        `lineage: merge candidate ${cand.label} needs >=2 pareto parent nodes, got ${parentIds.length}`,
+      )
+    }
+    await ensure({
+      parentIds,
+      track: cand.label,
+      surface: `loops@${cand.commit}`,
+      score: scoreOf(cand.resolvedCount, n),
+      scoreVector: scoreVectorOf(args.instances, cand.verdicts),
+      proposer: cand.merge ? 'merge' : cand.label,
+      rationale: `run ${args.runId}: ${cand.verdict}`,
+    })
+  }
+
+  // The continuation decision — recorded, not acted on here. budgetRemaining=1:
+  // the operator approves one next generation at a time.
+  const governor = await heuristicGovernor().decide({
+    lineage,
+    step: lineage.all().length,
+    budgetRemaining: 1,
+    prunedTracks: [],
+  })
+
+  return { path, appended, skipped, governor, nodesTotal: lineage.all().length }
+}

package/src/swe-arena/lineage-record.test.mts

@@ -0,0 +1,115 @@
+/**
+ * Gen-5 lineage DAG wiring: baseline root + pareto-parent nodes + candidate
+ * nodes (multi-parent merge), append-only idempotence at the
+ * .evolve-compatible store, and the recorded governor decision.
+ */
+
+import { readFile, rm } from 'node:fs/promises'
+import { mkdtemp } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { Lineage } from '@tangle-network/agent-eval/campaign'
+import { LINEAGE_STORE_RELPATH, recordLineageGeneration, type LineageRecordArgs } from './lineage-record.mts'
+
+const SIX = ['a-1', 'b-2', 'c-3', 'd-4', 'e-5', 'f-6']
+
+describe('recordLineageGeneration', () => {
+  let outDir: string
+
+  beforeEach(async () => {
+    outDir = await mkdtemp(join(tmpdir(), 'lineage-rec-'))
+  })
+
+  afterEach(async () => {
+    await rm(outDir, { recursive: true, force: true })
+  })
+
+  const args = (): LineageRecordArgs => ({
+    outDir,
+    runId: 'r4-test',
+    instances: SIX,
+    baseline: { commit: 'base'.repeat(10), resolvedCount: 1, verdicts: { 'a-1': true } },
+    paretoParents: [
+      { label: 'default-author', commit: 'p1p1'.repeat(10), resolvedInstances: ['d-4', 'f-6'] },
+      { label: 'prompts-author', commit: 'p2p2'.repeat(10), resolvedInstances: ['b-2', 'd-4'] },
+    ],
+    candidates: [
+      {
+        label: 'claude-author',
+        commit: 'c1c1'.repeat(10),
+        resolvedCount: 2,
+        verdicts: { 'a-1': true, 'd-4': true },
+        merge: false,
+        verdict: 'accepted',
+      },
+      {
+        label: 'merge-author',
+        commit: 'c2c2'.repeat(10),
+        resolvedCount: 3,
+        verdicts: { 'a-1': true, 'b-2': true, 'd-4': true },
+        merge: true,
+        verdict: 'accepted',
+      },
+      {
+        label: 'killed-author',
+        commit: null,
+        resolvedCount: 0,
+        verdicts: {},
+        merge: false,
+        verdict: 'rejected-prefilter',
+      },
+    ],
+  })
+
+  it('records root + parents + candidates at the .evolve-compatible store; merge is MULTI-PARENT', async () => {
+    const result = await recordLineageGeneration(args())
+    expect(result.path).toBe(join(outDir, LINEAGE_STORE_RELPATH))
+    expect(result.path).toContain('.evolve')
+    // root + 2 parents + 2 committed candidates (killed-author skipped).
+    expect(result.nodesTotal).toBe(5)
+    expect(result.appended).toHaveLength(5)
+    expect(result.skipped).toEqual(['killed-author'])
+
+    const raw = await readFile(result.path, 'utf8')
+    const lineage = Lineage.fromJSONL(raw)
+    const root = lineage.roots()
+    expect(root).toHaveLength(1)
+    expect(root[0]!.track).toBe('baseline')
+    expect(root[0]!.score).toBeCloseTo(1 / 6)
+
+    const merge = lineage.all().find((n) => n.track === 'merge-author')!
+    expect(merge.parentIds).toHaveLength(2)
+    const parentTracks = merge.parentIds.map((id) => lineage.all().find((n) => n.id === id)!.track).sort()
+    expect(parentTracks).toEqual(['default-author', 'prompts-author'])
+    expect(merge.proposer).toBe('merge')
+    expect(merge.score).toBeCloseTo(3 / 6)
+
+    const single = lineage.all().find((n) => n.track === 'claude-author')!
+    expect(single.parentIds).toEqual([root[0]!.id])
+    expect(single.rationale).toContain('accepted')
+    // scoreVector: sorted instance order, fail-closed 0 for missing verdicts.
+    expect(single.scoreVector).toEqual([1, 0, 0, 1, 0, 0])
+  })
+
+  it('is idempotent: re-recording the identical generation appends ZERO nodes', async () => {
+    const first = await recordLineageGeneration(args())
+    const second = await recordLineageGeneration(args())
+    expect(first.appended).toHaveLength(5)
+    expect(second.appended).toHaveLength(0)
+    expect(second.nodesTotal).toBe(5)
+    const raw = await readFile(second.path, 'utf8')
+    expect(raw.trim().split('\n')).toHaveLength(5)
+  })
+
+  it('returns a governor continuation decision (recorded, never acted on here)', async () => {
+    const result = await recordLineageGeneration(args())
+    expect(['extend', 'branch', 'merge', 'prune', 'stop']).toContain(result.governor.op)
+  })
+
+  it('fails loud when a merge candidate has fewer than 2 parent nodes', async () => {
+    const bad = args()
+    bad.paretoParents = bad.paretoParents.slice(0, 1)
+    await expect(recordLineageGeneration(bad)).rejects.toThrow(/needs >=2/)
+  })
+})