@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/proposer-provenance.mts

@@ -0,0 +1,177 @@
+/**
+ * GEN-4 proposer-model provenance — pin each author seat's MODEL IDENTITY in
+ * the run record at t=0, before any author shot fires.
+ *
+ * Why: a proposer spec may pin a model explicitly (`spec.model` → the harness
+ * CLI's `-m` flag via the author profile's `model.default`), but the claude
+ * seat deliberately does NOT pass `-m` — the CLI runs on its logged-in
+ * account, whose resolved model comes from its own settings. Verified against
+ * claude CLI 2.1.217: `--model` IS supported headless (`-p`), but the run's
+ * provenance must not depend on a flag we chose not to send. So the capture
+ * records what is VERIFIABLE at launch for every configured harness:
+ *
+ *   - `<harness> --version` output (the exact CLI build that authored),
+ *   - the claude settings default model (`~/.claude/settings.json` `model`),
+ *   - `codex login status` (the codex seat must be authed or the run refuses
+ *     at t=0 — a mid-run auth failure would silently kill one candidate slot),
+ *   - the spec's pinned model id (null when the seat rides the CLI default).
+ *
+ * The capture FAILS LOUD on a missing/broken harness binary: populationSize
+ * must equal proposers.length, so a dead seat cannot be skipped at runtime —
+ * the config decides membership, this guard proves it at launch.
+ */
+
+import { readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+import {
+  DEFAULT_GEPA_PYTHON,
+  isGepaSeat,
+  loadGepaMethodFactory,
+  probeGepaRuntime,
+  type CampaignModuleImport,
+} from './gepa-seat.mts'
+import { run } from './proc.ts'
+import type { ProposerSpec } from './proposer-fanout.mts'
+
+export interface ProposerModelProvenance {
+  name: string
+  /** Absent on a GEN-6 engine seat (see `engine`). */
+  harness: ProposerSpec['harness']
+  /** Explicit model pin from the spec (threaded as `-m`), or null when the
+   *  seat runs the CLI's own resolved default. */
+  pinnedModel: string | null
+  /** `<harness> --version` stdout (trimmed). For a GEN-6 gepa seat this is
+   *  the bridge python's `--version` output — the runtime that authors. */
+  harnessVersion: string
+  /** claude seats only: the settings default model the logged-in CLI resolves
+   *  when no `-m` is passed. Null when unreadable (recorded, never fatal —
+   *  the version capture is the hard gate). */
+  settingsModel: string | null
+  /** codex seats only: `codex login status` stdout (trimmed). */
+  authStatus: string | null
+  merge: boolean
+  /** GEN-6 gepa seat: the engine name from the spec. */
+  engine?: 'gepa' | 'omni'
+  /** GEN-6 gepa seat: the ONE change-space file GEPA optimizes. */
+  surface?: string
+  /** GEN-6 gepa seat: installed gepa version ('source' for a source pin). */
+  gepaVersion?: string
+  /** GEN-6 gepa seat: the Python bridge module the seat runs. */
+  bridge?: string
+}
+
+export interface ProvenanceCaptureRecord {
+  schema: 'swe-arena.proposer-provenance.v1'
+  capturedAt: string
+  proposers: ProposerModelProvenance[]
+}
+
+/** Exec seam — test-injectable. Mirrors `run` from proc.ts. */
+export type VersionExec = (
+  command: string,
+  args: string[],
+) => Promise<{ code: number | null; stdout: string; stderr: string }>
+
+const defaultExec: VersionExec = async (command, args) => {
+  const res = await run(command, args, { timeoutMs: 30_000 })
+  return { code: res.code, stdout: res.stdout, stderr: res.stderr }
+}
+
+/** Read the claude CLI's settings default model. Pure over injected reader. */
+export function claudeSettingsModel(
+  readFile: (path: string) => string = (p) => readFileSync(p, 'utf8'),
+  settingsPath = join(homedir(), '.claude', 'settings.json'),
+): string | null {
+  try {
+    const parsed = JSON.parse(readFile(settingsPath)) as { model?: unknown }
+    return typeof parsed.model === 'string' && parsed.model.length > 0 ? parsed.model : null
+  } catch {
+    return null
+  }
+}
+
+/** Capture per-proposer model provenance. Throws when any configured harness
+ *  binary is missing/broken, when a codex seat is not logged in, or — GEN-6 —
+ *  when a gepa seat's runtime is incomplete: the installed agent-eval must
+ *  export `gepaOptimizationMethod` and the Python bridge + GEPA engine must
+ *  import (probeGepaRuntime carries the exact install instructions). A dead
+ *  seat fails the launch at t=0, never a mid-run candidate slot. */
+export async function captureProposerProvenance(
+  proposers: ProposerSpec[],
+  deps: {
+    exec?: VersionExec
+    readSettingsModel?: () => string | null
+    importCampaign?: CampaignModuleImport
+  } = {},
+): Promise<ProvenanceCaptureRecord> {
+  const exec = deps.exec ?? defaultExec
+  const readSettingsModel = deps.readSettingsModel ?? (() => claudeSettingsModel())
+  const versionByHarness = new Map<string, string>()
+  const authByHarness = new Map<string, string>()
+  const gepaBySeat = new Map<string, { pythonVersion: string; gepaVersion: string }>()
+  const gepaSeats = proposers.filter(isGepaSeat)
+  if (gepaSeats.length > 0) {
+    // Node side first: the adapter export (fails loud with upgrade hint).
+    await loadGepaMethodFactory(...(deps.importCampaign ? [deps.importCampaign] : []))
+    for (const seat of gepaSeats) {
+      gepaBySeat.set(seat.name, await probeGepaRuntime(seat.python ?? DEFAULT_GEPA_PYTHON, exec, seat.name))
+    }
+  }
+  const harnesses = [...new Set(proposers.map((p) => p.harness))].filter(
+    (h): h is NonNullable<ProposerSpec['harness']> => h !== undefined,
+  )
+  for (const harness of harnesses) {
+    const res = await exec(harness, ['--version'])
+    if (res.code !== 0) {
+      throw new Error(
+        `proposer provenance: '${harness} --version' failed (rc=${res.code}) — the ${harness} seat cannot author. ` +
+          `stderr: ${res.stderr.slice(0, 300)}`,
+      )
+    }
+    versionByHarness.set(harness, res.stdout.trim())
+    if (harness === 'codex') {
+      const auth = await exec('codex', ['login', 'status'])
+      const authOut = auth.stdout + auth.stderr
+      const authed = /logged in/i.test(authOut) && !/not logged in/i.test(authOut)
+      if (auth.code !== 0 || !authed) {
+        throw new Error(
+          `proposer provenance: codex seat configured but 'codex login status' says not authed ` +
+            `(rc=${auth.code}, out=${(auth.stdout + auth.stderr).trim().slice(0, 200)})`,
+        )
+      }
+      authByHarness.set('codex', (auth.stdout + auth.stderr).trim())
+    }
+  }
+  return {
+    schema: 'swe-arena.proposer-provenance.v1',
+    capturedAt: new Date().toISOString(),
+    proposers: proposers.map((spec): ProposerModelProvenance => {
+      if (isGepaSeat(spec)) {
+        const probe = gepaBySeat.get(spec.name)!
+        return {
+          name: spec.name,
+          harness: undefined,
+          pinnedModel: null,
+          harnessVersion: probe.pythonVersion,
+          settingsModel: null,
+          authStatus: null,
+          merge: false,
+          engine: spec.engine,
+          surface: spec.surface,
+          gepaVersion: probe.gepaVersion,
+          bridge: 'agent_eval_rpc.gepa_bridge',
+        }
+      }
+      return {
+        name: spec.name,
+        harness: spec.harness,
+        pinnedModel: spec.model ?? null,
+        harnessVersion: versionByHarness.get(spec.harness!)!,
+        settingsModel: spec.harness === 'claude' && !spec.model ? readSettingsModel() : null,
+        authStatus: (spec.harness !== undefined ? authByHarness.get(spec.harness) : undefined) ?? null,
+        merge: spec.merge === true,
+      }
+    }),
+  }
+}

package/src/swe-arena/proposer-provenance.test.mts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from 'vitest'
+import {
+  captureProposerProvenance,
+  claudeSettingsModel,
+  type VersionExec,
+} from './proposer-provenance.mts'
+import type { ProposerSpec } from './proposer-fanout.mts'
+
+const ok = (stdout: string) => ({ code: 0, stdout, stderr: '' })
+
+const gen4ish: ProposerSpec[] = [
+  { name: 'claude-author', profile: 'default-author.profile.json', harness: 'claude' },
+  { name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' },
+  { name: 'codex-author', harness: 'codex' },
+  { name: 'merge-author', profile: 'default-author.profile.json', harness: 'claude', merge: true },
+]
+
+describe('captureProposerProvenance', () => {
+  const exec: VersionExec = async (command, args) => {
+    if (args[0] === '--version') return ok(`${command}-version 9.9.9`)
+    if (command === 'codex' && args[0] === 'login') return ok('Logged in using ChatGPT')
+    throw new Error(`unexpected exec ${command} ${args.join(' ')}`)
+  }
+
+  it('records pinned model, harness version, settings model, and codex auth per seat', async () => {
+    const record = await captureProposerProvenance(gen4ish, {
+      exec,
+      readSettingsModel: () => 'claude-fable-5',
+    })
+    expect(record.schema).toBe('swe-arena.proposer-provenance.v1')
+    const byName = Object.fromEntries(record.proposers.map((p) => [p.name, p]))
+    // Claude seat: no pin — the CLI's resolved settings model is the record.
+    expect(byName['claude-author']).toMatchObject({
+      harness: 'claude',
+      pinnedModel: null,
+      settingsModel: 'claude-fable-5',
+      harnessVersion: 'claude-version 9.9.9',
+      authStatus: null,
+      merge: false,
+    })
+    // Pinned opencode seat: explicit model id, settings model not consulted.
+    expect(byName['glm-author']).toMatchObject({
+      harness: 'opencode',
+      pinnedModel: 'zai-coding-plan/glm-5.2',
+      settingsModel: null,
+      harnessVersion: 'opencode-version 9.9.9',
+    })
+    // Codex seat: version + auth status captured.
+    expect(byName['codex-author']).toMatchObject({
+      harness: 'codex',
+      pinnedModel: null,
+      authStatus: 'Logged in using ChatGPT',
+    })
+    expect(byName['merge-author']!.merge).toBe(true)
+  })
+
+  it('fails loud when a configured harness binary is missing', async () => {
+    const broken: VersionExec = async (command, args) =>
+      command === 'codex' ? { code: 127, stdout: '', stderr: 'not found' } : exec(command, args)
+    await expect(
+      captureProposerProvenance(gen4ish, { exec: broken, readSettingsModel: () => null }),
+    ).rejects.toThrow(/'codex --version' failed/)
+  })
+
+  it('fails loud when the codex seat is not logged in', async () => {
+    const loggedOut: VersionExec = async (command, args) => {
+      if (args[0] === '--version') return ok(`${command} 1.0.0`)
+      return ok('Not logged in')
+    }
+    await expect(
+      captureProposerProvenance([{ name: 'codex-author', harness: 'codex' }], {
+        exec: loggedOut,
+        readSettingsModel: () => null,
+      }),
+    ).rejects.toThrow(/not authed/)
+  })
+
+  it('runs one version probe per harness, not per proposer', async () => {
+    const calls: string[] = []
+    const counting: VersionExec = async (command, args) => {
+      calls.push(`${command} ${args.join(' ')}`)
+      if (args[0] === '--version') return ok(`${command} 1`)
+      return ok('Logged in using ChatGPT')
+    }
+    await captureProposerProvenance(gen4ish, { exec: counting, readSettingsModel: () => null })
+    expect(calls.filter((c) => c === 'claude --version')).toHaveLength(1)
+    expect(calls.filter((c) => c === 'codex --version')).toHaveLength(1)
+    expect(calls.filter((c) => c === 'codex login status')).toHaveLength(1)
+  })
+})
+
+describe('claudeSettingsModel', () => {
+  it('reads the settings model field', () => {
+    expect(claudeSettingsModel(() => JSON.stringify({ model: 'claude-fable-5' }), '/x')).toBe('claude-fable-5')
+  })
+
+  it('returns null on unreadable/missing/blank settings', () => {
+    expect(
+      claudeSettingsModel(() => {
+        throw new Error('ENOENT')
+      }, '/x'),
+    ).toBeNull()
+    expect(claudeSettingsModel(() => JSON.stringify({}), '/x')).toBeNull()
+    expect(claudeSettingsModel(() => JSON.stringify({ model: '' }), '/x')).toBeNull()
+  })
+})

package/src/swe-arena/replay.mts

@@ -0,0 +1,183 @@
+/**
+ * Replay CLI: `tsx src/swe-arena/replay.mts`
+ *
+ * Reproduces the reference `fixtures/analyze.py` output from the committed
+ * fixtures (section 1 matches its printed lines exactly — pinned in
+ * replay.test.mts), then prints what the reference script never did:
+ * the reconciled valid-denominator verdict, the true SUP spend including
+ * worker tokens, the supervisor evolution rounds, and the holdout registry.
+ */
+
+import { pathToFileURL } from 'node:url'
+import {
+  loadHoldout,
+  loadLedger,
+  loadPreregisterLog,
+  loadRejudge,
+  loadRematchRounds,
+  loadSupJournalTrue,
+  loadWorkerTokens,
+} from './fixtures.ts'
+import {
+  costRollup,
+  ledgerOutcomes,
+  pairedSignTest,
+  reconciledOutcomes,
+  roundsProgression,
+  splitPairs,
+  type CostRollup,
+  type DiscordantSplit,
+  type RoundState,
+  type SignTestResult,
+} from './analyze.ts'
+import { reconcile, type PairedTable } from './reconcile.ts'
+import type { HoldoutRegistry, LedgerRow, RematchRow } from './types.ts'
+
+/** Python-style list repr (single quotes) so section 1 matches analyze.py byte-for-byte. */
+const pyList = (xs: string[]): string => `[${xs.map((x) => `'${x}'`).join(', ')}]`
+const signed = (x: number, digits?: number): string =>
+  (x >= 0 ? '+' : '') + (digits === undefined ? String(x) : x.toFixed(digits))
+
+export interface Replay {
+  ledger: LedgerRow[]
+  raw: { split: DiscordantSplit; sign: SignTestResult; soloResolved: number; supResolved: number }
+  cost: CostRollup
+  table: PairedTable
+  valid: { split: DiscordantSplit; sign: SignTestResult; soloResolved: number; supResolved: number }
+  rounds: Map<string, RoundState[]>
+  rematchRounds: RematchRow[][]
+  holdout: HoldoutRegistry
+  preregisterLog: string[]
+}
+
+export function buildReplay(): Replay {
+  const ledger = loadLedger()
+  const rejudge = loadRejudge()
+  const rematchRounds = loadRematchRounds()
+
+  const rawOutcomes = ledgerOutcomes(ledger)
+  const table = reconcile(ledger, rejudge)
+  const validOutcomes = reconciledOutcomes(table.valid)
+
+  return {
+    ledger,
+    raw: {
+      split: splitPairs(rawOutcomes),
+      sign: pairedSignTest(rawOutcomes),
+      soloResolved: rawOutcomes.filter((o) => o.solo).length,
+      supResolved: rawOutcomes.filter((o) => o.sup).length,
+    },
+    cost: costRollup(ledger, loadSupJournalTrue(), loadWorkerTokens()),
+    table,
+    valid: {
+      split: splitPairs(validOutcomes),
+      sign: pairedSignTest(validOutcomes),
+      soloResolved: validOutcomes.filter((o) => o.solo).length,
+      supResolved: validOutcomes.filter((o) => o.sup).length,
+    },
+    rounds: roundsProgression(ledger, rematchRounds),
+    rematchRounds,
+    holdout: loadHoldout(),
+    preregisterLog: loadPreregisterLog(),
+  }
+}
+
+/** Section 1 — byte-faithful reproduction of analyze.py's printed analysis. */
+export function renderReference(r: Replay): string[] {
+  const rows = [...r.ledger].sort((a, b) => (a.iid < b.iid ? -1 : a.iid > b.iid ? 1 : 0))
+  const n = rows.length
+  const { raw, cost } = r
+  const bar = '='.repeat(80)
+  const lines: string[] = []
+  lines.push(bar)
+  lines.push(`PAIRED HEAD-TO-HEAD: glm-5.2 SOLO vs glm-5.2 SUPERVISOR  (N=${n} paired instances)`)
+  lines.push(bar)
+  lines.push(`SOLO resolved: ${raw.soloResolved}/${n} = ${((100 * raw.soloResolved) / n).toFixed(1)}%`)
+  lines.push(`SUP  resolved: ${raw.supResolved}/${n} = ${((100 * raw.supResolved) / n).toFixed(1)}%`)
+  const delta = raw.supResolved - raw.soloResolved
+  lines.push(`delta (SUP-SOLO): ${signed(delta)} instances (${signed((100 * delta) / n, 1)} pts)`)
+  lines.push('')
+  lines.push('DISCORDANT PAIRS (the signal):')
+  lines.push(`  SUP-only wins (SUP✓ SOLO✗): ${raw.split.supOnly.length}  ${pyList(raw.split.supOnly)}`)
+  lines.push(`  SOLO-only wins (SOLO✓ SUP✗): ${raw.split.soloOnly.length}  ${pyList(raw.split.soloOnly)}`)
+  lines.push(`  both resolved: ${raw.split.both.length}  |  neither: ${raw.split.neither.length}  ${pyList(raw.split.neither)}`)
+  lines.push(`  exact two-sided sign test on discordant pairs: p=${raw.sign.pValue.toFixed(4)}`)
+  lines.push('')
+  lines.push(
+    `COST (measured tokens; USD via shared blended rate $${(cost.blendedRatePerTok * 1e6).toFixed(3)}/1M from SUP accounting):`,
+  )
+  lines.push(`  SOLO total tokens: ${cost.soloTokens.toLocaleString('en-US')}  -> derived $${cost.soloUsdDerived.toFixed(4)}`)
+  lines.push(`  SUP  total tokens: ${cost.supBrainTokens.toLocaleString('en-US')}  -> runtime $${cost.supUsd.toFixed(4)}`)
+  lines.push(`  SUP/SOLO token ratio: ${cost.brainTokenRatio.toFixed(2)}x`)
+  lines.push(`  SUP/SOLO cost ratio (token-derived): ${(cost.supUsd / cost.soloUsdDerived).toFixed(2)}x`)
+  lines.push(`  [telemetry] instances where runtime spentTokens != journal-true (no-winner zeroing): ${pyList(cost.telemetryGaps)}`)
+  lines.push(`  WALL: SOLO ${cost.soloWallS}s total vs SUP ${cost.supWallS}s total -> SUP ${cost.wallRatio.toFixed(2)}x wall`)
+  lines.push('')
+  lines.push(bar)
+  lines.push('PER-INSTANCE')
+  lines.push(bar)
+  lines.push(
+    `${'instance'.padEnd(32)} ${'SOLO'.padEnd(5)} ${'SUP'.padEnd(5)} ${'v_s'.padEnd(3)} ${'v_p'.padEnd(3)} ${'wrk'.padEnd(3)} ${'soloTok'.padEnd(8)} ${'supTok'.padEnd(8)} ${'supUSD'.padEnd(7)} ${'soloW'.padEnd(5)} ${'supW'.padEnd(5)}`,
+  )
+  for (const row of rows) {
+    const pyBool = (v: boolean): string => (v ? 'True' : 'False')
+    lines.push(
+      `${row.iid.padEnd(32)} ${pyBool(row.solo_resolved).slice(0, 5).padEnd(5)} ${pyBool(row.sup_resolved).slice(0, 5).padEnd(5)} ` +
+        `${pyBool(row.solo_verify_pass)[0].padEnd(3)} ${pyBool(row.sup_verify_pass)[0].padEnd(3)} ` +
+        `${String(row.sup_workers ?? '?').padEnd(3)} ${String(row.solo_tokens).padEnd(8)} ${String(row.sup_spentTokens ?? 0).padEnd(8)} ` +
+        `${(row.sup_spentUsd ?? 0).toFixed(4)} ${String(row.solo_wall_s).padEnd(5)} ${String(row.sup_wall_s).padEnd(5)}`,
+    )
+  }
+  return lines
+}
+
+/** Sections 2-5 — the analysis that lived in session lore, now typed. */
+export function renderReconciled(r: Replay): string[] {
+  const bar = '='.repeat(80)
+  const lines: string[] = []
+  const { table, valid, cost } = r
+  const n = table.valid.length
+
+  lines.push(bar)
+  lines.push('RECONCILED VERDICT (re-judged, gold-gated denominator)')
+  lines.push(bar)
+  for (const e of table.excluded) lines.push(`  EXCLUDED ${e.iid}: ${e.excludeReason}`)
+  for (const v of table.valid) {
+    const src = [v.solo.source !== 'ledger' ? `solo:${v.solo.source}` : null, v.sup.source !== 'ledger' ? `sup:${v.sup.source}` : null]
+      .filter(Boolean)
+      .join(' ')
+    if (src) lines.push(`  RE-JUDGED ${v.iid}: ${src}`)
+  }
+  lines.push(`SOLO resolved: ${valid.soloResolved}/${n}`)
+  lines.push(`SUP  resolved: ${valid.supResolved}/${n}`)
+  lines.push(`discordant: SUP-only ${pyList(valid.split.supOnly)} | SOLO-only ${pyList(valid.split.soloOnly)}`)
+  lines.push(`exact two-sided sign test: p=${valid.sign.pValue.toFixed(4)}`)
+  lines.push('')
+  lines.push('TRUE SUP SPEND (brain + workers; analyze.py printed brain only):')
+  lines.push(`  brain ${cost.supBrainTokens.toLocaleString('en-US')} + workers ${cost.supWorkerTokens.toLocaleString('en-US')} = ${cost.supTotalTokens.toLocaleString('en-US')} tokens`)
+  lines.push(`  SUP/SOLO true token ratio: ${cost.totalTokenRatio.toFixed(2)}x  (brain-only ratio: ${cost.brainTokenRatio.toFixed(2)}x)`)
+  lines.push('')
+  lines.push('SUP EVOLUTION ROUNDS (SUP = original head-to-head run):')
+  for (const [iid, states] of r.rounds) {
+    const cells = states.map(
+      (s) => `${s.round}:${s.resolved ? 'RESOLVED' : 'unresolved'}(${s.patchLines}L,${s.verdict ?? 'null'})`,
+    )
+    lines.push(`  ${iid.padEnd(32)} ${cells.join(' -> ')}`)
+  }
+  lines.push('')
+  lines.push(`HOLDOUT REGISTRY (pre-registered at loops@${r.holdout.selectedAtCommit.slice(0, 10)}, untouched):`)
+  for (const e of r.holdout.entries) {
+    lines.push(`  ${e.iid.padEnd(36)} gold_official_resolved=${e.gold_official_resolved} verify_calibrated=${e.verify_calibrated}`)
+  }
+  return lines
+}
+
+export function renderReplay(r: Replay = buildReplay()): string {
+  return [...renderReference(r), '', ...renderReconciled(r)].join('\n')
+}
+
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
+
+if (isMain) {
+  console.log(renderReplay())
+}