@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/swe-arena/activation.test.mts

@@ -1,300 +0,0 @@
-/**
- * Gen-5 activation gate: predicate validation, execution over run artifacts
- * (grep + script, ws/ skip with ws/.loops searched), the prefilter kill for a
- * missing/invalid predicate, and the quarantine verdict path (an improved
- * score with a never-fired mechanism is quarantined, not promoted).
- */
-
-import { existsSync } from 'node:fs'
-import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import type { ProposalFinding } from '@tangle-network/agent-eval'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import {
-  ACTIVATION_PREDICATE_RELPATH,
-  activationPredicateInstruction,
-  parseActivationPredicate,
-  runActivationPredicate,
-} from './activation.mts'
-import { decideVerdict } from './cell-evidence.mts'
-import { defaultRound4Config, parseStaircaseRow, STAIRCASE_SCHEMA, type OuterLoopConfig, type StaircaseRow } from './outer-loop.mts'
-import { fanOutLoopsGenerator, type ProposerSpec } from './proposer-fanout.mts'
-import { runOk } from './proc.ts'
-
-describe('parseActivationPredicate', () => {
-  it('accepts a valid grep predicate (with optional files filters)', () => {
-    const parsed = parseActivationPredicate(
-      JSON.stringify({
-        description: 'patchRiskWarnings emitted in at least one settle',
-        kind: 'grep',
-        pattern: 'patchRiskWarnings|patch-risk',
-        files: ['journal.jsonl', 'workers/'],
-      }),
-    )
-    expect(parsed.ok).toBe(true)
-  })
-
-  it('accepts a valid script predicate', () => {
-    const parsed = parseActivationPredicate(
-      JSON.stringify({ description: 'x', kind: 'script', script: 'grep -rq X .' }),
-    )
-    expect(parsed.ok).toBe(true)
-  })
-
-  it.each([
-    ['not json', 'not valid JSON'],
-    ['[]', 'JSON object'],
-    [JSON.stringify({ description: '', kind: 'grep', pattern: 'a' }), 'description'],
-    [JSON.stringify({ description: 'x', kind: 'grep' }), 'pattern'],
-    [JSON.stringify({ description: 'x', kind: 'grep', pattern: '(' }), 'RegExp'],
-    [JSON.stringify({ description: 'x', kind: 'grep', pattern: 'a', files: [1] }), 'files'],
-    [JSON.stringify({ description: 'x', kind: 'script' }), 'script'],
-    [JSON.stringify({ description: 'x', kind: 'sql' }), 'kind'],
-  ])('rejects %s', (raw, want) => {
-    const parsed = parseActivationPredicate(raw)
-    expect(parsed.ok).toBe(false)
-    if (!parsed.ok) expect(parsed.error).toContain(want)
-  })
-
-  it('the prompt instruction carries the template and the quarantine warning', () => {
-    const text = activationPredicateInstruction()
-    expect(text).toContain(ACTIVATION_PREDICATE_RELPATH)
-    expect(text).toContain('QUARANTINED')
-    expect(text).toContain('"kind": "grep"')
-  })
-})
-
-describe('runActivationPredicate', () => {
-  let runA: string
-  let runB: string
-
-  beforeEach(async () => {
-    runA = await mkdtemp(join(tmpdir(), 'act-a-'))
-    runB = await mkdtemp(join(tmpdir(), 'act-b-'))
-    await writeFile(join(runA, 'driver.log'), 'boot\nno mechanism here\n')
-    await writeFile(join(runB, 'driver.log'), 'boot\npatchRiskWarnings: 2 warnings emitted\n')
-  })
-
-  afterEach(async () => {
-    await rm(runA, { recursive: true, force: true })
-    await rm(runB, { recursive: true, force: true })
-  })
-
-  const grep = (pattern: string, files?: string[]) => ({
-    description: 'd',
-    kind: 'grep' as const,
-    pattern,
-    ...(files ? { files } : {}),
-  })
-
-  it('fires with path:line evidence when the pattern appears in ANY run dir', async () => {
-    const res = await runActivationPredicate(grep('patchRiskWarnings'), [runA, runB])
-    expect(res.fired).toBe(true)
-    expect(res.checkedRunDirs).toBe(2)
-    expect(res.evidence[0]).toContain(join(runB, 'driver.log'))
-    expect(res.evidence[0]).toContain('patchRiskWarnings')
-  })
-
-  it('does not fire when the pattern never appears (fail-closed)', async () => {
-    const res = await runActivationPredicate(grep('neverEverPresent'), [runA, runB])
-    expect(res.fired).toBe(false)
-    expect(res.evidence).toEqual([])
-  })
-
-  it('honors files filters (relative-path substring)', async () => {
-    await writeFile(join(runB, 'other.txt'), 'patchRiskWarnings\n')
-    const onlyOther = await runActivationPredicate(grep('patchRiskWarnings', ['other.txt']), [runB])
-    expect(onlyOther.fired).toBe(true)
-    const onlyMissing = await runActivationPredicate(grep('patchRiskWarnings', ['nope.bin']), [runB])
-    expect(onlyMissing.fired).toBe(false)
-  })
-
-  it('skips the ws/ workspace subtree EXCEPT ws/.loops (supervisor artifacts)', async () => {
-    // Marker only inside ws/ (a checked-out repo) → must NOT count as fired.
-    await mkdir(join(runA, 'ws', 'src'), { recursive: true })
-    await writeFile(join(runA, 'ws', 'src', 'code.py'), 'patchRiskWarnings in the repo source\n')
-    const inWs = await runActivationPredicate(grep('patchRiskWarnings'), [runA])
-    expect(inWs.fired).toBe(false)
-    // Marker in ws/.loops → the supervisor's own artifacts, searched.
-    await mkdir(join(runA, 'ws', '.loops', 'supervisor', 's1'), { recursive: true })
-    await writeFile(join(runA, 'ws', '.loops', 'supervisor', 's1', 'journal.jsonl'), '{"k":"patchRiskWarnings"}\n')
-    const inLoops = await runActivationPredicate(grep('patchRiskWarnings'), [runA])
-    expect(inLoops.fired).toBe(true)
-  })
-
-  it('script kind fires on rc=0 in any run dir, not-fired otherwise', async () => {
-    const script = (s: string) => ({ description: 'd', kind: 'script' as const, script: s })
-    const hit = await runActivationPredicate(script('grep -q patchRiskWarnings driver.log'), [runA, runB])
-    expect(hit.fired).toBe(true)
-    expect(hit.evidence[0]).toContain(runB)
-    const miss = await runActivationPredicate(script('grep -q neverEverPresent driver.log'), [runA, runB])
-    expect(miss.fired).toBe(false)
-  })
-})
-
-describe('quarantine verdict path', () => {
-  const base = {
-    violations: [],
-    coverageComplete: true,
-    resolvedCount: 3,
-    parentResolvedCount: 1,
-    costRatio: 1.0,
-    costGuardRatio: 1.2,
-  }
-
-  it('quarantines a candidate whose mechanism never fired EVEN when its score improved', () => {
-    expect(decideVerdict({ ...base, activationFired: false })).toBe('quarantined-inactive')
-  })
-
-  it('accepts an improved candidate whose mechanism fired; gate-not-applicable behaves as before', () => {
-    expect(decideVerdict({ ...base, activationFired: true })).toBe('accepted')
-    expect(decideVerdict({ ...base, activationFired: null })).toBe('accepted')
-    expect(decideVerdict(base)).toBe('accepted')
-  })
-
-  it('out-of-space still wins over quarantine; quarantine wins over no-gain/cost/coverage', () => {
-    expect(decideVerdict({ ...base, violations: ['judge.py'], activationFired: false })).toBe('rejected-out-of-space')
-    expect(decideVerdict({ ...base, coverageComplete: false, activationFired: false })).toBe('quarantined-inactive')
-    expect(decideVerdict({ ...base, resolvedCount: 1, activationFired: false })).toBe('quarantined-inactive')
-  })
-
-  it('parseStaircaseRow accepts a quarantined row with gen-5 split + activation fields', () => {
-    const row: StaircaseRow = {
-      schema: STAIRCASE_SCHEMA,
-      round: 4,
-      generation: 0,
-      runId: 'r4-x',
-      at: new Date(0).toISOString(),
-      candidate: 'hash',
-      candidateCommit: 'c'.repeat(40),
-      parent: 'baseline',
-      parentResolvedCount: 1,
-      changedFiles: [],
-      changeSpaceViolations: [],
-      perInstance: [],
-      resolvedCount: 3,
-      coverageComplete: true,
-      wallS: 10,
-      baselineWallS: 10,
-      costRatio: 1,
-      costGuardRatio: 1.2,
-      internallyPromoted: true,
-      verdict: 'quarantined-inactive',
-      holdout: 'operator-approval-required',
-      armProvenance: null,
-      diffPath: null,
-      diffSha256: null,
-      split: {
-        publicInstances: ['a', 'b'],
-        privateInstances: ['c'],
-        publicResolvedCount: 2,
-        privateResolvedCount: 1,
-      },
-      activation: { present: true, description: 'd', fired: false, evidence: [], warnings: [] },
-    }
-    expect(parseStaircaseRow(JSON.stringify(row))).toEqual(row)
-  })
-})
-
-// ---------------------------------------------------------------------------
-// Prefilter enforcement — a candidate without a parseable predicate is killed
-// before any evaluation spend.
-// ---------------------------------------------------------------------------
-
-describe('activation-predicate prefilter', () => {
-  let loopsRepo: string
-  let outDir: string
-  let driverWt: string
-
-  beforeEach(async () => {
-    loopsRepo = await mkdtemp(join(tmpdir(), 'act-repo-'))
-    outDir = await mkdtemp(join(tmpdir(), 'act-out-'))
-    await runOk('git', ['init', '-q', '-b', 'main', loopsRepo])
-    await runOk('git', ['-C', loopsRepo, 'config', 'core.hooksPath', '/dev/null'])
-    await runOk('git', ['-C', loopsRepo, 'config', 'user.email', 't@t.dev'])
-    await runOk('git', ['-C', loopsRepo, 'config', 'user.name', 'T'])
-    await writeFile(join(loopsRepo, 'src.ts'), 'base\n')
-    await runOk('git', ['-C', loopsRepo, 'add', '-A'])
-    await runOk('git', ['-C', loopsRepo, 'commit', '-q', '-m', 'init'])
-    driverWt = join(outDir, 'driver-wt')
-    await runOk('git', ['-C', loopsRepo, 'worktree', 'add', '--detach', driverWt, 'HEAD'])
-  })
-
-  afterEach(async () => {
-    await rm(outDir, { recursive: true, force: true })
-    await rm(loopsRepo, { recursive: true, force: true })
-  })
-
-  const config = (proposers: ProposerSpec[]): OuterLoopConfig => ({
-    ...defaultRound4Config(),
-    loopsRepo,
-    outDir,
-    populationSize: proposers.length,
-    proposers,
-    activationGate: true,
-  })
-
-  const generatorArgs = (candidateIndex: number) => ({
-    worktreePath: driverWt,
-    findings: [] as ProposalFinding[],
-    maxShots: 1,
-    signal: new AbortController().signal,
-    generation: 0,
-    candidateIndex,
-  })
-
-  it('kills a candidate without .improve/activation.json (stage activation-predicate) and passes one WITH it', async () => {
-    const proposers: ProposerSpec[] = [
-      { name: 'with-predicate', harness: 'claude-code' },
-      { name: 'without-predicate', harness: 'claude-code' },
-      { name: 'invalid-predicate', harness: 'claude-code' },
-    ]
-    const gen = fanOutLoopsGenerator(config(proposers), {
-      author: async (proposer, args) => {
-        await mkdir(join(args.worktreePath, 'extensions', 'pi'), { recursive: true })
-        await writeFile(join(args.worktreePath, 'extensions', 'pi', `${proposer.name}.ts`), 'x\n')
-        if (proposer.name === 'with-predicate') {
-          await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
-          await writeFile(
-            join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH),
-            JSON.stringify({ description: 'd', kind: 'grep', pattern: 'x' }),
-          )
-        } else if (proposer.name === 'invalid-predicate') {
-          await mkdir(join(args.worktreePath, '.improve'), { recursive: true })
-          await writeFile(join(args.worktreePath, ACTIVATION_PREDICATE_RELPATH), '{}')
-        }
-        return { applied: true, summary: `${proposer.name} edit` }
-      },
-    })
-
-    const survivor = await gen.generate(generatorArgs(0))
-    const missing = await gen.generate(generatorArgs(1))
-    const invalid = await gen.generate(generatorArgs(2))
-
-    expect(survivor.applied).toBe(true)
-    expect(missing.applied).toBe(false)
-    expect(invalid.applied).toBe(false)
-    const kills = gen.drainPrefilterKills()
-    expect(kills).toHaveLength(2)
-    expect(kills.find((k) => k.proposer === 'without-predicate')).toMatchObject({ stage: 'activation-predicate' })
-    expect(kills.find((k) => k.proposer === 'without-predicate')!.reason).toContain('missing')
-    expect(kills.find((k) => k.proposer === 'invalid-predicate')!.reason).toContain('invalid')
-    // The survivor's predicate landed on the driver worktree with the patch.
-    expect(existsSync(join(driverWt, ACTIVATION_PREDICATE_RELPATH))).toBe(true)
-  })
-
-  it('does not require a predicate when the gate is off (gen-4 behavior unchanged)', async () => {
-    const cfg = config([{ name: 'legacy', harness: 'claude-code' }])
-    cfg.activationGate = false
-    const gen = fanOutLoopsGenerator(cfg, {
-      author: async (_p, args) => {
-        await mkdir(join(args.worktreePath, 'extensions', 'pi'), { recursive: true })
-        await writeFile(join(args.worktreePath, 'extensions', 'pi', 'l.ts'), 'x\n')
-        return { applied: true, summary: 'edit' }
-      },
-    })
-    expect((await gen.generate(generatorArgs(0))).applied).toBe(true)
-    expect(gen.drainPrefilterKills()).toEqual([])
-  })
-})

package/src/swe-arena/analyze.ts

@@ -1,211 +0,0 @@
-/**
- * Typed port of the reference `fixtures/analyze.py` statistics: discordant
- * pair extraction, exact two-sided sign test, and cost rollups.
- *
- * The sign test is NOT re-implemented: `mcnemar` from
- * `@tangle-network/agent-eval` computes the identical exact two-sided
- * binomial p on discordant pairs (`min(1, 2·P(X ≤ min(b,c)))`, X ~
- * Binomial(b+c, 0.5)) — the same formula as `analyze.py::sign_test`.
- *
- * Token accounting has TWO deliberately distinct rollups:
- *  - `supBrainTokens` — what analyze.py prints as "SUP total tokens":
- *    supervisor-journal metered spend (the supervisor "brain"), journal-true
- *    where captured, else the runtime's `sup_spentTokens`.
- *  - `supTotalTokens` — brain + worker-session tokens (worker-tokens.json).
- *    This is the true SUP-arm spend (1,722,390 for the 12-run set) and the
- *    number the 2.55x SUP/SOLO ratio refers to. analyze.py never printed it;
- *    its printed 0.83x ratio is brain-only.
- */
-
-import { mcnemar } from '@tangle-network/agent-eval'
-import type { ReconciledInstance } from './reconcile'
-import type { LedgerRow, RematchRow, WorkerTokens } from './types'
-
-export interface PairedOutcome {
-  iid: string
-  solo: boolean
-  sup: boolean
-}
-
-/** Raw ledger outcomes — analyze.py semantics (strict `is True`), sorted by iid. */
-export function ledgerOutcomes(ledger: LedgerRow[]): PairedOutcome[] {
-  return [...ledger]
-    .sort((a, b) => (a.iid < b.iid ? -1 : a.iid > b.iid ? 1 : 0))
-    .map((r) => ({ iid: r.iid, solo: r.solo_resolved === true, sup: r.sup_resolved === true }))
-}
-
-/** Reconciled (re-judged, gold-gated) outcomes, sorted by iid. */
-export function reconciledOutcomes(instances: ReconciledInstance[]): PairedOutcome[] {
-  return [...instances]
-    .sort((a, b) => (a.iid < b.iid ? -1 : a.iid > b.iid ? 1 : 0))
-    .map((r) => ({ iid: r.iid, solo: r.solo.resolved, sup: r.sup.resolved }))
-}
-
-export interface DiscordantSplit {
-  /** SUP resolved, SOLO not. */
-  supOnly: string[]
-  /** SOLO resolved, SUP not. */
-  soloOnly: string[]
-  both: string[]
-  neither: string[]
-}
-
-export function splitPairs(outcomes: PairedOutcome[]): DiscordantSplit {
-  const supOnly: string[] = []
-  const soloOnly: string[] = []
-  const both: string[] = []
-  const neither: string[] = []
-  for (const o of outcomes) {
-    if (o.sup && !o.solo) supOnly.push(o.iid)
-    else if (o.solo && !o.sup) soloOnly.push(o.iid)
-    else if (o.solo && o.sup) both.push(o.iid)
-    else neither.push(o.iid)
-  }
-  return { supOnly, soloOnly, both, neither }
-}
-
-export interface SignTestResult {
-  n: number
-  /** Discordant pairs where SUP (treatment) won. */
-  b: number
-  /** Discordant pairs where SOLO (control) won. */
-  c: number
-  nDiscordant: number
-  /** Exact two-sided binomial sign-test p on discordant pairs. */
-  pValue: number
-}
-
-/** Exact paired sign test, SOLO as control and SUP as treatment. */
-export function pairedSignTest(outcomes: PairedOutcome[]): SignTestResult {
-  const result = mcnemar(
-    outcomes.map((o) => o.solo),
-    outcomes.map((o) => o.sup),
-  )
-  const { n, b, c, nDiscordant, pValue } = result
-  return { n, b, c, nDiscordant, pValue }
-}
-
-export interface CostRollup {
-  soloTokens: number
-  /** analyze.py's "SUP total tokens": journal-true brain spend, else sup_spentTokens. */
-  supBrainTokens: number
-  /** Σ worker-session tokens (worker-tokens.json; instances without workers contribute 0). */
-  supWorkerTokens: number
-  /** Brain + workers — the true SUP-arm token spend. */
-  supTotalTokens: number
-  supUsd: number
-  /** Blended $ per token from SUP runtime accounting (analyze.py's `rate`). */
-  blendedRatePerTok: number
-  soloUsdDerived: number
-  /** supBrainTokens / soloTokens — the ratio analyze.py prints (0.83x). */
-  brainTokenRatio: number
-  /** supTotalTokens / soloTokens — the true-spend ratio (2.55x). */
-  totalTokenRatio: number
-  soloWallS: number
-  supWallS: number
-  wallRatio: number
-  /** iids where the runtime's sup_spentTokens disagrees with journal-true (no-winner zeroing). */
-  telemetryGaps: string[]
-}
-
-/**
- * Port of analyze.py's cost section over the full ledger (all 12 paired runs
- * — cost is a property of the runs, not of the gradeable subset), extended
- * with the worker-token rollup.
- */
-export function costRollup(
-  ledger: LedgerRow[],
-  supJournalTrue: Record<string, number | null>,
-  workerTokens: Record<string, WorkerTokens>,
-): CostRollup {
-  const rows = [...ledger].sort((a, b) => (a.iid < b.iid ? -1 : a.iid > b.iid ? 1 : 0))
-  let soloTokens = 0
-  let supBrainTokens = 0
-  let supUsd = 0
-  let soloWallS = 0
-  let supWallS = 0
-  const telemetryGaps: string[] = []
-  for (const r of rows) {
-    soloTokens += r.solo_tokens
-    const journalTrue = supJournalTrue[r.iid] ?? null
-    supBrainTokens += journalTrue ?? r.sup_spentTokens ?? 0
-    supUsd += r.sup_spentUsd ?? 0
-    soloWallS += r.solo_wall_s
-    supWallS += r.sup_wall_s
-    if (journalTrue !== null && (r.sup_spentTokens ?? 0) !== journalTrue) telemetryGaps.push(r.iid)
-  }
-  const supWorkerTokens = Object.values(workerTokens).reduce((s, w) => s + w.worker_tok, 0)
-  const supTotalTokens = supBrainTokens + supWorkerTokens
-  const blendedRatePerTok = supBrainTokens > 0 ? supUsd / supBrainTokens : 0
-  return {
-    soloTokens,
-    supBrainTokens,
-    supWorkerTokens,
-    supTotalTokens,
-    supUsd,
-    blendedRatePerTok,
-    soloUsdDerived: soloTokens * blendedRatePerTok,
-    brainTokenRatio: soloTokens > 0 ? supBrainTokens / soloTokens : 0,
-    totalTokenRatio: soloTokens > 0 ? supTotalTokens / soloTokens : 0,
-    soloWallS,
-    supWallS,
-    wallRatio: soloWallS > 0 ? supWallS / soloWallS : 0,
-    telemetryGaps,
-  }
-}
-
-/** Round label for the original head-to-head SUP run in the ledger. */
-export type SupRound = 'SUP' | 'SUP2' | 'SUP3' | 'SUP4'
-
-export interface RoundState {
-  round: SupRound
-  resolved: boolean
-  verifyPass: boolean
-  patchLines: number
-  verdict: 'delivered' | 'no-winner' | 'best-effort' | null
-  delivered: boolean | null
-  spentTokens: number | null
-}
-
-/**
- * Supervisor evolution progression: the original SUP run (round 0, from the
- * ledger) followed by each rematch round, for every instance that appears in
- * at least one rematch file.
- */
-export function roundsProgression(
-  ledger: LedgerRow[],
-  rounds: RematchRow[][],
-): Map<string, RoundState[]> {
-  const byIid = new Map<string, RoundState[]>()
-  const rematchIids = new Set(rounds.flat().map((r) => r.iid))
-  for (const r of ledger) {
-    if (!rematchIids.has(r.iid)) continue
-    byIid.set(r.iid, [
-      {
-        round: 'SUP',
-        resolved: r.sup_resolved === true,
-        verifyPass: r.sup_verify_pass === true,
-        patchLines: r.sup_patch_lines,
-        verdict: r.sup_verdict,
-        delivered: r.sup_delivered,
-        spentTokens: r.sup_spentTokens,
-      },
-    ])
-  }
-  for (const round of rounds) {
-    for (const r of round) {
-      const states = byIid.get(r.iid)
-      if (!states) throw new Error(`rematch row for ${r.iid} has no ledger baseline`)
-      states.push({
-        round: r.arm,
-        resolved: r.resolved === true,
-        verifyPass: r.verify_pass === true,
-        patchLines: r.patch_lines,
-        verdict: r.sup_verdict,
-        delivered: r.delivered,
-        spentTokens: r.spentTokens,
-      })
-    }
-  }
-  return byIid
-}