@tangle-network/agent-bench 0.11.3 → 0.13.0

package/src/swe-arena/proposer-provenance.mts

@@ -1,176 +0,0 @@
-/**
- * GEN-4 proposer-model provenance — pin each author seat's MODEL IDENTITY in
- * the run record at t=0, before any author shot fires.
- *
- * Why: a proposer spec may pin a model explicitly (`spec.model` → the harness
- * CLI's `-m` flag via the author profile's `model.default`), but the claude
- * seat deliberately does NOT pass `-m` — the CLI runs on its logged-in
- * account, whose resolved model comes from its own settings. Verified against
- * claude CLI 2.1.217: `--model` IS supported headless (`-p`), but the run's
- * provenance must not depend on a flag we chose not to send. So the capture
- * records what is VERIFIABLE at launch for every configured harness:
- *
- *   - `<harness> --version` output (the exact CLI build that authored),
- *   - the claude settings default model (`~/.claude/settings.json` `model`),
- *   - `codex login status` (the codex seat must be authed or the run refuses
- *     at t=0 — a mid-run auth failure would silently kill one candidate slot),
- *   - the spec's pinned model id (null when the seat rides the CLI default).
- *
- * The capture FAILS LOUD on a missing/broken harness binary: populationSize
- * must equal proposers.length, so a dead seat cannot be skipped at runtime —
- * the config decides membership, this guard proves it at launch.
- */
-
-import { readFileSync } from 'node:fs'
-import { homedir } from 'node:os'
-import { join } from 'node:path'
-import { localHarnessExecutable } from '@tangle-network/agent-runtime/mcp'
-import {
-  DEFAULT_GEPA_PYTHON,
-  isGepaSeat,
-  probeGepaRuntime,
-} from './gepa-seat.mts'
-import { run } from './proc.ts'
-import type { ProposerSpec } from './proposer-fanout.mts'
-
-export interface ProposerModelProvenance {
-  name: string
-  /** Absent on an engine seat (see `engine`). */
-  harness: ProposerSpec['harness']
-  /** Explicit model pin from the spec (threaded as `-m`), or null when the
-   *  seat runs the CLI's own resolved default. */
-  pinnedModel: string | null
-  /** `<harness> --version` stdout (trimmed). For a GEPA seat this is
-   *  the bridge python's `--version` output — the runtime that authors. */
-  harnessVersion: string
-  /** claude seats only: the settings default model the logged-in CLI resolves
-   *  when no `-m` is passed. Null when unreadable (recorded, never fatal —
-   *  the version capture is the hard gate). */
-  settingsModel: string | null
-  /** codex seats only: `codex login status` stdout (trimmed). */
-  authStatus: string | null
-  merge: boolean
-  /** GEPA seat: the engine name from the spec. */
-  engine?: 'gepa' | 'omni'
-  /** GEPA seat: the one change-space file GEPA optimizes. */
-  surface?: string
-  /** GEPA seat: installed GEPA version ('source' for a source pin). */
-  gepaVersion?: string
-  /** GEPA seat: the Python bridge module the seat runs. */
-  bridge?: string
-}
-
-export interface ProvenanceCaptureRecord {
-  schema: 'swe-arena.proposer-provenance.v1'
-  capturedAt: string
-  proposers: ProposerModelProvenance[]
-}
-
-/** Exec seam — test-injectable. Mirrors `run` from proc.ts. */
-export type VersionExec = (
-  command: string,
-  args: string[],
-) => Promise<{ code: number | null; stdout: string; stderr: string }>
-
-const defaultExec: VersionExec = async (command, args) => {
-  const res = await run(command, args, { timeoutMs: 30_000 })
-  return { code: res.code, stdout: res.stdout, stderr: res.stderr }
-}
-
-/** Read the claude CLI's settings default model. Pure over injected reader. */
-export function claudeSettingsModel(
-  readFile: (path: string) => string = (p) => readFileSync(p, 'utf8'),
-  settingsPath = join(homedir(), '.claude', 'settings.json'),
-): string | null {
-  try {
-    const parsed = JSON.parse(readFile(settingsPath)) as { model?: unknown }
-    return typeof parsed.model === 'string' && parsed.model.length > 0 ? parsed.model : null
-  } catch {
-    return null
-  }
-}
-
-/** Capture per-proposer model provenance. Throws when any configured harness
- *  binary is missing/broken, when a codex seat is not logged in, or when
- *  a GEPA seat's Python bridge or engine cannot import
- *  (`probeGepaRuntime` carries the exact install instructions). A dead seat
- *  fails the launch at t=0, never a mid-run candidate slot. The TypeScript
- *  adapter is a pinned package dependency and therefore checked at install. */
-export async function captureProposerProvenance(
-  proposers: ProposerSpec[],
-  deps: {
-    exec?: VersionExec
-    readSettingsModel?: () => string | null
-  } = {},
-): Promise<ProvenanceCaptureRecord> {
-  const exec = deps.exec ?? defaultExec
-  const readSettingsModel = deps.readSettingsModel ?? (() => claudeSettingsModel())
-  const versionByHarness = new Map<string, string>()
-  const authByHarness = new Map<string, string>()
-  const gepaBySeat = new Map<string, { pythonVersion: string; gepaVersion: string }>()
-  const gepaSeats = proposers.filter(isGepaSeat)
-  if (gepaSeats.length > 0) {
-    for (const seat of gepaSeats) {
-      gepaBySeat.set(seat.name, await probeGepaRuntime(seat.python ?? DEFAULT_GEPA_PYTHON, exec, seat.name))
-    }
-  }
-  const harnesses = [...new Set(proposers.map((p) => p.harness))].filter(
-    (h): h is NonNullable<ProposerSpec['harness']> => h !== undefined,
-  )
-  for (const harness of harnesses) {
-    // The harness id is not the binary name (`claude-code` runs `claude`); read the executable
-    // from the runtime's harness table rather than spawning the id.
-    const executable = harness === 'pi' ? 'pi' : localHarnessExecutable(harness)
-    const res = await exec(executable, ['--version'])
-    if (res.code !== 0) {
-      throw new Error(
-        `proposer provenance: '${executable} --version' failed (rc=${res.code}) — the ${harness} seat cannot author. ` +
-          `stderr: ${res.stderr.slice(0, 300)}`,
-      )
-    }
-    versionByHarness.set(harness, res.stdout.trim())
-    if (harness === 'codex') {
-      const auth = await exec('codex', ['login', 'status'])
-      const authOut = auth.stdout + auth.stderr
-      const authed = /logged in/i.test(authOut) && !/not logged in/i.test(authOut)
-      if (auth.code !== 0 || !authed) {
-        throw new Error(
-          `proposer provenance: codex seat configured but 'codex login status' says not authed ` +
-            `(rc=${auth.code}, out=${(auth.stdout + auth.stderr).trim().slice(0, 200)})`,
-        )
-      }
-      authByHarness.set('codex', (auth.stdout + auth.stderr).trim())
-    }
-  }
-  return {
-    schema: 'swe-arena.proposer-provenance.v1',
-    capturedAt: new Date().toISOString(),
-    proposers: proposers.map((spec): ProposerModelProvenance => {
-      if (isGepaSeat(spec)) {
-        const probe = gepaBySeat.get(spec.name)!
-        return {
-          name: spec.name,
-          harness: undefined,
-          pinnedModel: null,
-          harnessVersion: probe.pythonVersion,
-          settingsModel: null,
-          authStatus: null,
-          merge: false,
-          engine: spec.engine,
-          surface: spec.surface,
-          gepaVersion: probe.gepaVersion,
-          bridge: 'agent_eval_rpc.gepa_bridge',
-        }
-      }
-      return {
-        name: spec.name,
-        harness: spec.harness,
-        pinnedModel: spec.model ?? null,
-        harnessVersion: versionByHarness.get(spec.harness!)!,
-        settingsModel: spec.harness === 'claude-code' && !spec.model ? readSettingsModel() : null,
-        authStatus: (spec.harness !== undefined ? authByHarness.get(spec.harness) : undefined) ?? null,
-        merge: spec.merge === true,
-      }
-    }),
-  }
-}

package/src/swe-arena/proposer-provenance.test.mts

@@ -1,106 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import {
-  captureProposerProvenance,
-  claudeSettingsModel,
-  type VersionExec,
-} from './proposer-provenance.mts'
-import type { ProposerSpec } from './proposer-fanout.mts'
-
-const ok = (stdout: string) => ({ code: 0, stdout, stderr: '' })
-
-const gen4ish: ProposerSpec[] = [
-  { name: 'claude-author', profile: 'default-author.profile.json', harness: 'claude-code' },
-  { name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' },
-  { name: 'codex-author', harness: 'codex' },
-  { name: 'merge-author', profile: 'default-author.profile.json', harness: 'claude-code', merge: true },
-]
-
-describe('captureProposerProvenance', () => {
-  const exec: VersionExec = async (command, args) => {
-    if (args[0] === '--version') return ok(`${command}-version 9.9.9`)
-    if (command === 'codex' && args[0] === 'login') return ok('Logged in using ChatGPT')
-    throw new Error(`unexpected exec ${command} ${args.join(' ')}`)
-  }
-
-  it('records pinned model, harness version, settings model, and codex auth per seat', async () => {
-    const record = await captureProposerProvenance(gen4ish, {
-      exec,
-      readSettingsModel: () => 'claude-fable-5',
-    })
-    expect(record.schema).toBe('swe-arena.proposer-provenance.v1')
-    const byName = Object.fromEntries(record.proposers.map((p) => [p.name, p]))
-    // Claude seat: no pin — the CLI's resolved settings model is the record.
-    expect(byName['claude-author']).toMatchObject({
-      harness: 'claude-code',
-      pinnedModel: null,
-      settingsModel: 'claude-fable-5',
-      harnessVersion: 'claude-version 9.9.9',
-      authStatus: null,
-      merge: false,
-    })
-    // Pinned opencode seat: explicit model id, settings model not consulted.
-    expect(byName['glm-author']).toMatchObject({
-      harness: 'opencode',
-      pinnedModel: 'zai-coding-plan/glm-5.2',
-      settingsModel: null,
-      harnessVersion: 'opencode-version 9.9.9',
-    })
-    // Codex seat: version + auth status captured.
-    expect(byName['codex-author']).toMatchObject({
-      harness: 'codex',
-      pinnedModel: null,
-      authStatus: 'Logged in using ChatGPT',
-    })
-    expect(byName['merge-author']!.merge).toBe(true)
-  })
-
-  it('fails loud when a configured harness binary is missing', async () => {
-    const broken: VersionExec = async (command, args) =>
-      command === 'codex' ? { code: 127, stdout: '', stderr: 'not found' } : exec(command, args)
-    await expect(
-      captureProposerProvenance(gen4ish, { exec: broken, readSettingsModel: () => null }),
-    ).rejects.toThrow(/'codex --version' failed/)
-  })
-
-  it('fails loud when the codex seat is not logged in', async () => {
-    const loggedOut: VersionExec = async (command, args) => {
-      if (args[0] === '--version') return ok(`${command} 1.0.0`)
-      return ok('Not logged in')
-    }
-    await expect(
-      captureProposerProvenance([{ name: 'codex-author', harness: 'codex' }], {
-        exec: loggedOut,
-        readSettingsModel: () => null,
-      }),
-    ).rejects.toThrow(/not authed/)
-  })
-
-  it('runs one version probe per harness, not per proposer', async () => {
-    const calls: string[] = []
-    const counting: VersionExec = async (command, args) => {
-      calls.push(`${command} ${args.join(' ')}`)
-      if (args[0] === '--version') return ok(`${command} 1`)
-      return ok('Logged in using ChatGPT')
-    }
-    await captureProposerProvenance(gen4ish, { exec: counting, readSettingsModel: () => null })
-    expect(calls.filter((c) => c === 'claude --version')).toHaveLength(1)
-    expect(calls.filter((c) => c === 'codex --version')).toHaveLength(1)
-    expect(calls.filter((c) => c === 'codex login status')).toHaveLength(1)
-  })
-})
-
-describe('claudeSettingsModel', () => {
-  it('reads the settings model field', () => {
-    expect(claudeSettingsModel(() => JSON.stringify({ model: 'claude-fable-5' }), '/x')).toBe('claude-fable-5')
-  })
-
-  it('returns null on unreadable/missing/blank settings', () => {
-    expect(
-      claudeSettingsModel(() => {
-        throw new Error('ENOENT')
-      }, '/x'),
-    ).toBeNull()
-    expect(claudeSettingsModel(() => JSON.stringify({}), '/x')).toBeNull()
-    expect(claudeSettingsModel(() => JSON.stringify({ model: '' }), '/x')).toBeNull()
-  })
-})

package/src/swe-arena/replay.mts

@@ -1,183 +0,0 @@
-/**
- * Replay CLI: `tsx src/swe-arena/replay.mts`
- *
- * Reproduces the reference `fixtures/analyze.py` output from the committed
- * fixtures (section 1 matches its printed lines exactly — pinned in
- * replay.test.mts), then prints what the reference script never did:
- * the reconciled valid-denominator verdict, the true SUP spend including
- * worker tokens, the supervisor evolution rounds, and the holdout registry.
- */
-
-import { pathToFileURL } from 'node:url'
-import {
-  loadHoldout,
-  loadLedger,
-  loadPreregisterLog,
-  loadRejudge,
-  loadRematchRounds,
-  loadSupJournalTrue,
-  loadWorkerTokens,
-} from './fixtures.ts'
-import {
-  costRollup,
-  ledgerOutcomes,
-  pairedSignTest,
-  reconciledOutcomes,
-  roundsProgression,
-  splitPairs,
-  type CostRollup,
-  type DiscordantSplit,
-  type RoundState,
-  type SignTestResult,
-} from './analyze.ts'
-import { reconcile, type PairedTable } from './reconcile.ts'
-import type { HoldoutRegistry, LedgerRow, RematchRow } from './types.ts'
-
-/** Python-style list repr (single quotes) so section 1 matches analyze.py byte-for-byte. */
-const pyList = (xs: string[]): string => `[${xs.map((x) => `'${x}'`).join(', ')}]`
-const signed = (x: number, digits?: number): string =>
-  (x >= 0 ? '+' : '') + (digits === undefined ? String(x) : x.toFixed(digits))
-
-export interface Replay {
-  ledger: LedgerRow[]
-  raw: { split: DiscordantSplit; sign: SignTestResult; soloResolved: number; supResolved: number }
-  cost: CostRollup
-  table: PairedTable
-  valid: { split: DiscordantSplit; sign: SignTestResult; soloResolved: number; supResolved: number }
-  rounds: Map<string, RoundState[]>
-  rematchRounds: RematchRow[][]
-  holdout: HoldoutRegistry
-  preregisterLog: string[]
-}
-
-export function buildReplay(): Replay {
-  const ledger = loadLedger()
-  const rejudge = loadRejudge()
-  const rematchRounds = loadRematchRounds()
-
-  const rawOutcomes = ledgerOutcomes(ledger)
-  const table = reconcile(ledger, rejudge)
-  const validOutcomes = reconciledOutcomes(table.valid)
-
-  return {
-    ledger,
-    raw: {
-      split: splitPairs(rawOutcomes),
-      sign: pairedSignTest(rawOutcomes),
-      soloResolved: rawOutcomes.filter((o) => o.solo).length,
-      supResolved: rawOutcomes.filter((o) => o.sup).length,
-    },
-    cost: costRollup(ledger, loadSupJournalTrue(), loadWorkerTokens()),
-    table,
-    valid: {
-      split: splitPairs(validOutcomes),
-      sign: pairedSignTest(validOutcomes),
-      soloResolved: validOutcomes.filter((o) => o.solo).length,
-      supResolved: validOutcomes.filter((o) => o.sup).length,
-    },
-    rounds: roundsProgression(ledger, rematchRounds),
-    rematchRounds,
-    holdout: loadHoldout(),
-    preregisterLog: loadPreregisterLog(),
-  }
-}
-
-/** Section 1 — byte-faithful reproduction of analyze.py's printed analysis. */
-export function renderReference(r: Replay): string[] {
-  const rows = [...r.ledger].sort((a, b) => (a.iid < b.iid ? -1 : a.iid > b.iid ? 1 : 0))
-  const n = rows.length
-  const { raw, cost } = r
-  const bar = '='.repeat(80)
-  const lines: string[] = []
-  lines.push(bar)
-  lines.push(`PAIRED HEAD-TO-HEAD: glm-5.2 SOLO vs glm-5.2 SUPERVISOR  (N=${n} paired instances)`)
-  lines.push(bar)
-  lines.push(`SOLO resolved: ${raw.soloResolved}/${n} = ${((100 * raw.soloResolved) / n).toFixed(1)}%`)
-  lines.push(`SUP  resolved: ${raw.supResolved}/${n} = ${((100 * raw.supResolved) / n).toFixed(1)}%`)
-  const delta = raw.supResolved - raw.soloResolved
-  lines.push(`delta (SUP-SOLO): ${signed(delta)} instances (${signed((100 * delta) / n, 1)} pts)`)
-  lines.push('')
-  lines.push('DISCORDANT PAIRS (the signal):')
-  lines.push(`  SUP-only wins (SUP✓ SOLO✗): ${raw.split.supOnly.length}  ${pyList(raw.split.supOnly)}`)
-  lines.push(`  SOLO-only wins (SOLO✓ SUP✗): ${raw.split.soloOnly.length}  ${pyList(raw.split.soloOnly)}`)
-  lines.push(`  both resolved: ${raw.split.both.length}  |  neither: ${raw.split.neither.length}  ${pyList(raw.split.neither)}`)
-  lines.push(`  exact two-sided sign test on discordant pairs: p=${raw.sign.pValue.toFixed(4)}`)
-  lines.push('')
-  lines.push(
-    `COST (measured tokens; USD via shared blended rate $${(cost.blendedRatePerTok * 1e6).toFixed(3)}/1M from SUP accounting):`,
-  )
-  lines.push(`  SOLO total tokens: ${cost.soloTokens.toLocaleString('en-US')}  -> derived $${cost.soloUsdDerived.toFixed(4)}`)
-  lines.push(`  SUP  total tokens: ${cost.supBrainTokens.toLocaleString('en-US')}  -> runtime $${cost.supUsd.toFixed(4)}`)
-  lines.push(`  SUP/SOLO token ratio: ${cost.brainTokenRatio.toFixed(2)}x`)
-  lines.push(`  SUP/SOLO cost ratio (token-derived): ${(cost.supUsd / cost.soloUsdDerived).toFixed(2)}x`)
-  lines.push(`  [telemetry] instances where runtime spentTokens != journal-true (no-winner zeroing): ${pyList(cost.telemetryGaps)}`)
-  lines.push(`  WALL: SOLO ${cost.soloWallS}s total vs SUP ${cost.supWallS}s total -> SUP ${cost.wallRatio.toFixed(2)}x wall`)
-  lines.push('')
-  lines.push(bar)
-  lines.push('PER-INSTANCE')
-  lines.push(bar)
-  lines.push(
-    `${'instance'.padEnd(32)} ${'SOLO'.padEnd(5)} ${'SUP'.padEnd(5)} ${'v_s'.padEnd(3)} ${'v_p'.padEnd(3)} ${'wrk'.padEnd(3)} ${'soloTok'.padEnd(8)} ${'supTok'.padEnd(8)} ${'supUSD'.padEnd(7)} ${'soloW'.padEnd(5)} ${'supW'.padEnd(5)}`,
-  )
-  for (const row of rows) {
-    const pyBool = (v: boolean): string => (v ? 'True' : 'False')
-    lines.push(
-      `${row.iid.padEnd(32)} ${pyBool(row.solo_resolved).slice(0, 5).padEnd(5)} ${pyBool(row.sup_resolved).slice(0, 5).padEnd(5)} ` +
-        `${pyBool(row.solo_verify_pass)[0].padEnd(3)} ${pyBool(row.sup_verify_pass)[0].padEnd(3)} ` +
-        `${String(row.sup_workers ?? '?').padEnd(3)} ${String(row.solo_tokens).padEnd(8)} ${String(row.sup_spentTokens ?? 0).padEnd(8)} ` +
-        `${(row.sup_spentUsd ?? 0).toFixed(4)} ${String(row.solo_wall_s).padEnd(5)} ${String(row.sup_wall_s).padEnd(5)}`,
-    )
-  }
-  return lines
-}
-
-/** Sections 2-5 — the analysis that lived in session lore, now typed. */
-export function renderReconciled(r: Replay): string[] {
-  const bar = '='.repeat(80)
-  const lines: string[] = []
-  const { table, valid, cost } = r
-  const n = table.valid.length
-
-  lines.push(bar)
-  lines.push('RECONCILED VERDICT (re-judged, gold-gated denominator)')
-  lines.push(bar)
-  for (const e of table.excluded) lines.push(`  EXCLUDED ${e.iid}: ${e.excludeReason}`)
-  for (const v of table.valid) {
-    const src = [v.solo.source !== 'ledger' ? `solo:${v.solo.source}` : null, v.sup.source !== 'ledger' ? `sup:${v.sup.source}` : null]
-      .filter(Boolean)
-      .join(' ')
-    if (src) lines.push(`  RE-JUDGED ${v.iid}: ${src}`)
-  }
-  lines.push(`SOLO resolved: ${valid.soloResolved}/${n}`)
-  lines.push(`SUP  resolved: ${valid.supResolved}/${n}`)
-  lines.push(`discordant: SUP-only ${pyList(valid.split.supOnly)} | SOLO-only ${pyList(valid.split.soloOnly)}`)
-  lines.push(`exact two-sided sign test: p=${valid.sign.pValue.toFixed(4)}`)
-  lines.push('')
-  lines.push('TRUE SUP SPEND (brain + workers; analyze.py printed brain only):')
-  lines.push(`  brain ${cost.supBrainTokens.toLocaleString('en-US')} + workers ${cost.supWorkerTokens.toLocaleString('en-US')} = ${cost.supTotalTokens.toLocaleString('en-US')} tokens`)
-  lines.push(`  SUP/SOLO true token ratio: ${cost.totalTokenRatio.toFixed(2)}x  (brain-only ratio: ${cost.brainTokenRatio.toFixed(2)}x)`)
-  lines.push('')
-  lines.push('SUP EVOLUTION ROUNDS (SUP = original head-to-head run):')
-  for (const [iid, states] of r.rounds) {
-    const cells = states.map(
-      (s) => `${s.round}:${s.resolved ? 'RESOLVED' : 'unresolved'}(${s.patchLines}L,${s.verdict ?? 'null'})`,
-    )
-    lines.push(`  ${iid.padEnd(32)} ${cells.join(' -> ')}`)
-  }
-  lines.push('')
-  lines.push(`HOLDOUT REGISTRY (pre-registered at loops@${r.holdout.selectedAtCommit.slice(0, 10)}, untouched):`)
-  for (const e of r.holdout.entries) {
-    lines.push(`  ${e.iid.padEnd(36)} gold_official_resolved=${e.gold_official_resolved} verify_calibrated=${e.verify_calibrated}`)
-  }
-  return lines
-}
-
-export function renderReplay(r: Replay = buildReplay()): string {
-  return [...renderReference(r), '', ...renderReconciled(r)].join('\n')
-}
-
-const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
-
-if (isMain) {
-  console.log(renderReplay())
-}