@tangle-network/agent-bench 0.11.3 → 0.13.2

package/src/swe-arena/calibrate.ts

@@ -1,217 +0,0 @@
-/**
- * Dual calibration — an instance may enter an experiment only if BOTH gates
- * hold, mirroring the experiment's `calibrate.sh` (repro gate) plus the
- * gold-family judge rows M1 reconciles on (official-judge gate):
- *
- *  1. REPRO GATE (`verifyCalibrated`): on a pristine image-materialized
- *     workspace the self-repro verify command must FAIL at base_commit and
- *     PASS once the official gold patch is applied (git apply, then
- *     `patch --fuzz=3` fallback — several Verified gold patches only apply
- *     fuzzily to their own base). A verify that can't see the gold fix can't
- *     grade an arm's fix.
- *
- *  2. OFFICIAL-JUDGE GOLD GATE (`goldOfficialResolved`): the official swebench
- *     judge (via serialized-judge → adapter.judge) must resolve the gold patch
- *     itself. psf__requests-2931/-2317 proved a judge can be blind on an
- *     instance whose verify calibrates fine — those became the excluded
- *     "gold-ungradeable" rows in the M1 denominator.
- */
-
-import { rm, writeFile } from 'node:fs/promises'
-import { join } from 'node:path'
-import { pathToFileURL } from 'node:url'
-import { judgeFactoryPatch } from './factory-judge-child.mts'
-import { loadFactoryInstance, loadFactoryInstances, type LoadedFactoryInstance } from './fixtures'
-import { materializeWorkspace } from './materialize'
-import { run, runOk, shq } from './proc'
-import type { SerializedJudge } from './serialized-judge'
-
-export interface CalibrateOptions {
-  instanceId: string
-  image: string
-  baseCommit: string
-  /** The official gold patch text (task-meta `patch`). */
-  goldPatch: string
-  /**
-   * Self-repro verify command, run via `bash -c` with cwd = the workspace
-   * (calibrate.sh ran `bash verify/<iid>.sh` from inside the tree).
-   */
-  verifyCmd: string
-  /** Scratch root; two throwaway workspaces are created and removed under it. */
-  workDir: string
-  /** Judge for the official gold gate. */
-  judge: SerializedJudge
-  /** Ceiling for one verify run (repro scripts self-limit at 180s; this is a backstop). */
-  verifyTimeoutMs?: number
-  /** Keep the calibration workspaces for post-mortem. Default: removed. */
-  keepWorkspaces?: boolean
-}
-
-export interface CalibrationResult {
-  iid: string
-  /** rc of verify on pristine base (must be nonzero). */
-  baseRc: number
-  /** rc of applying the gold patch (0 via git apply or the fuzz fallback). */
-  goldApplyRc: number
-  /** rc of verify with gold applied (must be zero). */
-  goldRc: number
-  /** base FAILS and gold PASSES. */
-  verifyCalibrated: boolean
-  /** Official judge resolves the gold patch. */
-  goldOfficialResolved: boolean
-  /** verifyCalibrated && goldOfficialResolved — the experiment admission bar. */
-  experimentValid: boolean
-}
-
-async function runVerify(verifyCmd: string, ws: string, timeoutMs: number): Promise<number> {
-  const res = await run('bash', ['-c', verifyCmd], { cwd: ws, timeoutMs })
-  return res.code
-}
-
-/**
- * Apply a patch file with calibrate.sh's exact fallback chain:
- * `git apply --whitespace=nowarn`, then `patch -p1 --fuzz=3` on failure.
- * Returns the rc of the LAST attempt (0 = applied).
- */
-export async function applyPatchWithFallback(ws: string, patchFile: string): Promise<number> {
-  const gitApply = await run('git', ['apply', '--whitespace=nowarn', patchFile], { cwd: ws })
-  if (gitApply.code === 0) return 0
-  const fuzz = await run('bash', ['-c', `patch -p1 --fuzz=3 < ${shq(patchFile)}`], { cwd: ws })
-  return fuzz.code
-}
-
-export async function calibrateInstance(opts: CalibrateOptions): Promise<CalibrationResult> {
-  const { instanceId, image, baseCommit, verifyCmd, judge } = opts
-  const verifyTimeoutMs = opts.verifyTimeoutMs ?? 600_000
-  const baseWs = join(opts.workDir, `cal-base-${instanceId}`)
-  const goldWs = join(opts.workDir, `cal-gold-${instanceId}`)
-  const goldPatchFile = join(opts.workDir, `${instanceId}.gold.patch`)
-
-  try {
-    await materializeWorkspace({ instanceId, image, baseCommit, dest: baseWs })
-    const baseRc = await runVerify(verifyCmd, baseWs, verifyTimeoutMs)
-
-    await materializeWorkspace({ instanceId, image, baseCommit, dest: goldWs })
-    await writeFile(goldPatchFile, opts.goldPatch)
-    const goldApplyRc = await applyPatchWithFallback(goldWs, goldPatchFile)
-    const goldRc = await runVerify(verifyCmd, goldWs, verifyTimeoutMs)
-
-    const verifyCalibrated = baseRc !== 0 && goldRc === 0
-
-    const goldVerdict = await judge.judge(instanceId, goldPatchFile, 'gold')
-    const goldOfficialResolved = goldVerdict.resolved === true
-
-    return {
-      iid: instanceId,
-      baseRc,
-      goldApplyRc,
-      goldRc,
-      verifyCalibrated,
-      goldOfficialResolved,
-      experimentValid: verifyCalibrated && goldOfficialResolved,
-    }
-  } finally {
-    if (!opts.keepWorkspaces) {
-      await rm(baseWs, { recursive: true, force: true })
-      await rm(goldWs, { recursive: true, force: true })
-    }
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Factory-bench admission gate — the same "calibrate through the OFFICIAL
-// judge" lesson, generalized: gold (the real PR's impl-only diff) must judge
-// resolved, and the bare base (empty patch) must judge unresolved. Both runs
-// go through the SAME judge code the arena uses (judgeFactoryPatch — the
-// factory-judge-child body), deliberately bypassing serialized-judge's
-// empty-patch short-circuit so the base direction really executes the judge
-// tests on the bare tree instead of trivially returning false.
-// ---------------------------------------------------------------------------
-
-export interface FactoryCalibrationResult {
-  iid: string
-  /** Gold = impl-only PR diff. Must be resolved with full score. */
-  goldResolved: boolean
-  goldPassed: number
-  /** Base = empty patch. Must be unresolved. */
-  baseResolved: boolean
-  basePassed: number
-  total: number
-  /** goldResolved && !baseResolved — the pool admission bar. */
-  admitted: boolean
-}
-
-/**
- * The real PR's impl-only patch: full first-parent diff base→judge_ref minus
- * the judge test files (they are the hidden judge, not the deliverable).
- */
-export async function goldImplPatch(inst: LoadedFactoryInstance): Promise<string> {
-  const res = await runOk('git', [
-    '-C', inst.repo_local_mirror,
-    'diff', inst.base_commit, inst.judge_ref,
-    '--', '.',
-    ...inst.judge_tests.map((t) => `:(exclude)${t}`),
-  ])
-  if (res.stdout.trim().length === 0) {
-    throw new Error(`calibrate ${inst.id}: impl-only gold diff is empty — judge_tests exclude everything?`)
-  }
-  return res.stdout
-}
-
-/** Run both admission directions for one instance. Throws only on infra failure. */
-export async function calibrateFactoryInstance(inst: LoadedFactoryInstance): Promise<FactoryCalibrationResult> {
-  const gold = await judgeFactoryPatch(inst, await goldImplPatch(inst))
-  const base = await judgeFactoryPatch(inst, '')
-  return {
-    iid: inst.id,
-    goldResolved: gold.result.resolved,
-    goldPassed: gold.result.passed,
-    baseResolved: base.result.resolved,
-    basePassed: base.result.passed,
-    total: inst.judgeTestTotal,
-    admitted: gold.result.resolved && !base.result.resolved,
-  }
-}
-
-// ---------------------------------------------------------------------------
-// CLI:  tsx src/swe-arena/calibrate.ts --factory <instancesDirOrInstanceDir> [id ...]
-// Rejection is LOUD: any instance failing either direction exits nonzero.
-// ---------------------------------------------------------------------------
-
-const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
-
-if (isMain) {
-  const [mode, root, ...ids] = process.argv.slice(2)
-  if (mode !== '--factory' || !root) {
-    console.error('usage: tsx src/swe-arena/calibrate.ts --factory <instancesDir|instanceDir> [id ...]')
-    process.exit(2)
-  }
-  let instances: LoadedFactoryInstance[]
-  try {
-    instances = loadFactoryInstances(root)
-  } catch {
-    instances = [loadFactoryInstance(root)]
-  }
-  if (ids.length > 0) {
-    const byId = new Map(instances.map((i) => [i.id, i]))
-    instances = ids.map((id) => {
-      const inst = byId.get(id)
-      if (!inst) throw new Error(`unknown instance id ${id} (have: ${[...byId.keys()].join(', ')})`)
-      return inst
-    })
-  }
-  let rejected = 0
-  for (const inst of instances) {
-    const r = await calibrateFactoryInstance(inst)
-    const verdict = r.admitted ? 'ADMITTED' : 'REJECTED'
-    console.log(
-      `CALIBRATE ${r.iid}: gold ${r.goldPassed}/${r.total} resolved=${r.goldResolved}; ` +
-        `base ${r.basePassed}/${r.total} resolved=${r.baseResolved} → ${verdict}`,
-    )
-    if (!r.admitted) rejected += 1
-  }
-  if (rejected > 0) {
-    console.error(`calibration gate: ${rejected} instance(s) REJECTED (gold must pass AND base must fail)`)
-    process.exit(1)
-  }
-}

package/src/swe-arena/capabilities.mts

@@ -1,76 +0,0 @@
-/**
- * Substrate passthrough guard for the improve-loop options this harness
- * depends on: `selfImprove` forwarding `premeasuredBaseline` and
- * `budget.maxImprovementShots` into the loop (merged to agent-eval main).
- *
- * The remaining risk is a STALE INSTALL: the bench consumes agent-eval via a
- * pnpm `file:` dependency, which snapshots the checkout at install time — a
- * rebuilt-but-never-reinstalled substrate silently reverts to a bundle whose
- * `selfImprove` DROPS both options. Silent drop is the worst failure mode
- * here (a "premeasured" baseline would quietly re-run and re-spend; the depth
- * dial would quietly pin to the lib default), so the guard FAILS LOUD instead
- * of falling back.
- *
- * The probe reads the RESOLVED `@tangle-network/agent-eval/contract` module
- * text (the bundle that contains the compiled `selfImprove`) and requires
- * both option names. Verified against the pre-merge build: that bundle
- * contained ZERO occurrences of either literal (selfImprove never named them;
- * `runOptimization`'s own seam compiles into a different chunk), so the probe
- * cannot false-positive on a stale substrate.
- */
-
-import { readFileSync } from 'node:fs'
-import { createRequire } from 'node:module'
-import { fileURLToPath } from 'node:url'
-
-export interface ImproveLoopPassthroughCaps {
-  /** `selfImprove` forwards `premeasuredBaseline` into the loop. */
-  premeasuredBaseline: boolean
-  /** `selfImprove` forwards `budget.maxImprovementShots` into the loop. */
-  maxImprovementShots: boolean
-}
-
-/** Pure probe over the contract module text — unit-testable. */
-export function detectPassthroughCaps(contractModuleText: string): ImproveLoopPassthroughCaps {
-  return {
-    premeasuredBaseline: contractModuleText.includes('premeasuredBaseline'),
-    maxImprovementShots: contractModuleText.includes('maxImprovementShots'),
-  }
-}
-
-/** Resolve the installed agent-eval contract bundle's file path. */
-export function resolveContractModulePath(): string {
-  const require = createRequire(import.meta.url)
-  const url = import.meta.resolve?.('@tangle-network/agent-eval/contract')
-  if (typeof url === 'string' && url.startsWith('file:')) return fileURLToPath(url)
-  return require.resolve('@tangle-network/agent-eval/contract')
-}
-
-/** Fail-loud stale-install guard: throws unless the resolved substrate names
- *  BOTH passthrough options in its contract bundle. An unreadable bundle also
- *  throws — nothing here ever downgrades to a silent fallback. */
-export function assertSubstratePassthroughs(log: (msg: string) => void = () => {}): void {
-  let path: string
-  let caps: ImproveLoopPassthroughCaps
-  try {
-    path = resolveContractModulePath()
-    caps = detectPassthroughCaps(readFileSync(path, 'utf8'))
-  } catch (cause) {
-    throw new Error(
-      'substrate passthrough probe failed — cannot prove the installed agent-eval forwards ' +
-        `premeasuredBaseline/maxImprovementShots: ${(cause as Error).message}`,
-      { cause },
-    )
-  }
-  log(
-    `substrate caps (${path}): premeasuredBaseline=${caps.premeasuredBaseline} maxImprovementShots=${caps.maxImprovementShots}`,
-  )
-  const missing = (Object.keys(caps) as Array<keyof ImproveLoopPassthroughCaps>).filter((k) => !caps[k])
-  if (missing.length > 0) {
-    throw new Error(
-      `stale substrate install: the resolved agent-eval contract bundle (${path}) never names ` +
-        `${missing.join(' + ')}, so selfImprove would silently drop the option(s). ` +
-        'Rebuild the checkout (cd ~/code/agent-eval && git pull && pnpm build), then `pnpm install --force` in the bench.',
-    )
-  }
-}

package/src/swe-arena/capabilities.test.mts

@@ -1,57 +0,0 @@
-/**
- * Substrate passthrough guard: the pure probe over module text plus the live
- * contract check against the installed agent-eval bundle. The guard must pass
- * cleanly on a substrate that threads premeasuredBaseline +
- * maxImprovementShots and THROW LOUD (never fall back) on one that drops
- * them — so a run against a stale install dies at t≈0, before it can
- * silently re-spend its baseline.
- */
-
-import { readFileSync } from 'node:fs'
-import { describe, expect, it } from 'vitest'
-import {
-  assertSubstratePassthroughs,
-  detectPassthroughCaps,
-  resolveContractModulePath,
-} from './capabilities.mts'
-
-describe('detectPassthroughCaps (pure)', () => {
-  it('an option name absent from the bundle reads as no capability', () => {
-    expect(detectPassthroughCaps('')).toEqual({ premeasuredBaseline: false, maxImprovementShots: false })
-    expect(detectPassthroughCaps('function selfImprove(opts) { return runSelfImprove(opts) }')).toEqual({
-      premeasuredBaseline: false,
-      maxImprovementShots: false,
-    })
-  })
-
-  it('the forwarding literals flip their capability independently', () => {
-    expect(detectPassthroughCaps('premeasuredBaseline: opts.premeasuredBaseline,')).toEqual({
-      premeasuredBaseline: true,
-      maxImprovementShots: false,
-    })
-    expect(detectPassthroughCaps('maxImprovementShots: budget.maxImprovementShots,')).toEqual({
-      premeasuredBaseline: false,
-      maxImprovementShots: true,
-    })
-  })
-})
-
-describe('live substrate guard', () => {
-  it('passes on a substrate that names both passthroughs, throws loud otherwise', () => {
-    const path = resolveContractModulePath()
-    expect(path).toMatch(/agent-eval/)
-    const caps = detectPassthroughCaps(readFileSync(path, 'utf8'))
-    // The guard's contract holds against whatever substrate is installed:
-    // both literals present ⇒ clean pass with the caps logged; anything less
-    // ⇒ a loud actionable error naming the stale bundle (never a silent
-    // fallback). A run against a stale install dies HERE, at t≈0.
-    if (caps.premeasuredBaseline && caps.maxImprovementShots) {
-      const logged: string[] = []
-      expect(() => assertSubstratePassthroughs((msg) => logged.push(msg))).not.toThrow()
-      expect(logged.join('\n')).toContain('premeasuredBaseline=true')
-      expect(logged.join('\n')).toContain('maxImprovementShots=true')
-    } else {
-      expect(() => assertSubstratePassthroughs()).toThrow(/stale substrate install/)
-    }
-  })
-})

package/src/swe-arena/capacity.ts

@@ -1,198 +0,0 @@
-/**
- * Endpoint capacity gate — the typed port of `probe-capacity.sh`, generalized
- * after a proven blind spot: the bash probe watched ONLY the z.ai coding
- * endpoint while the supervisor BRAIN rides router.tangle.tools — three
- * evolution rounds went infra-null because the gate said "capacity" while the
- * router 503-stormed. Rule encoded here: gate every arm on the endpoint that
- * arm actually calls; supervisor arms MUST include the router-path probe.
- *
- * The containing experiment is launched through dotenvx, so probes read the already-scoped key
- * from this process and enter Runtime through one exact AgentProfile.
- */
-
-import { runBenchRouterTurn } from '../router-turn'
-import type { SecretsEnv } from './arms'
-
-export type CapacityProbe = (signal?: AbortSignal) => Promise<boolean>
-
-export interface EndpointCapacityGate {
-  /** Human label for status lines (e.g. 'z.ai-coding', 'router'). */
-  name: string
-  probe: CapacityProbe
-  /** Window passes when >= k of n probes succeed (bash: 3 of 4). */
-  kOfN: { k: number; n: number }
-  /** Consecutive passing windows required before opening. Default 1. */
-  steadyM?: number
-  /** Total wait budget; exceeded → gate reports no-capacity (orchestrate: 300 min). */
-  waitCeilingMs: number
-  /** Pause between probes inside a window (bash: 1s). */
-  probeIntervalMs?: number
-  /** Pause between windows while waiting (orchestrate: 30s). */
-  retryDelayMs?: number
-  onStatus?: (msg: string) => void
-}
-
-export function sleepWithSignal(ms: number, signal?: AbortSignal): Promise<void> {
-  signal?.throwIfAborted()
-  return new Promise((resolve, reject) => {
-    let timer: NodeJS.Timeout | undefined
-    const onAbort = () => {
-      if (timer) clearTimeout(timer)
-      signal?.removeEventListener('abort', onAbort)
-      reject(signal?.reason ?? new Error('capacity wait aborted'))
-    }
-    timer = setTimeout(() => {
-      signal?.removeEventListener('abort', onAbort)
-      resolve()
-    }, ms)
-    signal?.addEventListener('abort', onAbort, { once: true })
-  })
-}
-
-/** One k-of-n probe window. Exported for direct reuse (bash probe-capacity.sh body). */
-export async function probeWindow(
-  gate: EndpointCapacityGate,
-  signal?: AbortSignal,
-): Promise<{ ok: number; passed: boolean }> {
-  const { k, n } = gate.kOfN
-  let ok = 0
-  for (let i = 0; i < n; i++) {
-    signal?.throwIfAborted()
-    try {
-      if (await gate.probe(signal)) ok += 1
-    } catch {
-      // Endpoint failures count as a failed probe; caller cancellation does not.
-      signal?.throwIfAborted()
-    }
-    signal?.throwIfAborted()
-    if (i < n - 1) await sleepWithSignal(gate.probeIntervalMs ?? 1000, signal)
-  }
-  return { ok, passed: ok >= k }
-}
-
-/**
- * Block until the endpoint shows steady capacity (steadyM consecutive passing
- * k-of-n windows) or the ceiling elapses. Returns whether capacity was found —
- * callers decide whether a closed gate skips the instance or aborts the run.
- */
-export async function waitForCapacity(gate: EndpointCapacityGate, signal?: AbortSignal): Promise<boolean> {
-  signal?.throwIfAborted()
-  const steadyM = gate.steadyM ?? 1
-  const deadline = Date.now() + gate.waitCeilingMs
-  let consecutive = 0
-  for (;;) {
-    signal?.throwIfAborted()
-    const { ok, passed } = await probeWindow(gate, signal)
-    signal?.throwIfAborted()
-    gate.onStatus?.(`[${gate.name}] capacity: ${ok}/${gate.kOfN.n}${passed ? '' : ' (below k)'} steady=${passed ? consecutive + 1 : 0}/${steadyM}`)
-    if (passed) {
-      consecutive += 1
-      if (consecutive >= steadyM) return true
-    } else {
-      consecutive = 0
-    }
-    if (Date.now() >= deadline) return false
-    await sleepWithSignal(gate.retryDelayMs ?? 30_000, signal)
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Probe functions.
-// ---------------------------------------------------------------------------
-
-export interface HttpProbeSpec {
-  url: string
-  provider: string
-  /** NAME of the env var holding the bearer key (resolved in the dotenvx child). */
-  apiKeyEnv: string
-  model: string
-  /** curl --max-time, seconds. Default 40 (probe-capacity.sh). */
-  maxTimeS?: number
-  /**
-   * max_tokens in the probe body. Default 8000 — glm-5.2 returns empty content
-   * below that (measured), and an empty-content 200 would be a lying probe.
-   */
-  maxTokens?: number
-}
-
-export const ZAI_CODING_ENDPOINT = 'https://api.z.ai/api/coding/paas/v4'
-export const ROUTER_ENDPOINT = 'https://router.tangle.tools/v1'
-
-/**
- * Generic chat-completions probe through Runtime: true only when the selected model emits `OK`
- * within the time budget.
- */
-export function httpCapacityProbe(spec: HttpProbeSpec): CapacityProbe {
-  if (!/^[A-Z_][A-Z0-9_]*$/.test(spec.apiKeyEnv)) {
-    throw new Error(`invalid apiKeyEnv name: ${spec.apiKeyEnv}`)
-  }
-  const maxTime = spec.maxTimeS ?? 40
-  return async (signal?: AbortSignal) => {
-    signal?.throwIfAborted()
-    const routerKey = process.env[spec.apiKeyEnv]
-    if (!routerKey) throw new Error(`${spec.apiKeyEnv} is required; launch through dotenvx`)
-    const turn = await runBenchRouterTurn(
-      {
-        routerBaseUrl: spec.url.replace(/\/chat\/completions\/?$/u, ''),
-        routerKey,
-        profile: {
-          name: `capacity-${spec.model}`,
-          harness: 'cli-base',
-          model: {
-            provider: spec.provider,
-            default: spec.model,
-            metadata: { temperature: 0 },
-            maxVisibleOutputTokens: spec.maxTokens ?? 8000,
-          },
-          prompt: { systemPrompt: 'Reply with the single word OK.' },
-        },
-        timeoutMs: maxTime * 1000,
-        signal,
-      },
-      'Capacity probe.',
-    )
-    return turn.finalText.trim() === 'OK'
-  }
-}
-
-/** probe-capacity.sh's z.ai coding-plan probe (the WORKER path). */
-export function zaiCodingProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
-  void secrets
-  return httpCapacityProbe({
-    url: ZAI_CODING_ENDPOINT,
-    apiKeyEnv: 'ZAI_API_KEY',
-    provider: 'zai',
-    model,
-  })
-}
-
-/**
- * Router-path probe (the BRAIN path — router.tangle.tools with TANGLE_API_KEY).
- * Supervisor arms must gate on this; probing only z.ai is the proven blind spot.
- */
-export function routerProbe(secrets: SecretsEnv, model = 'glm-5.2'): CapacityProbe {
-  void secrets
-  return httpCapacityProbe({
-    url: ROUTER_ENDPOINT,
-    apiKeyEnv: 'TANGLE_API_KEY',
-    provider: 'tangle-router',
-    model,
-  })
-}
-
-/** The gates an arm must pass, by kind: solo → worker path; supervisor → BOTH paths. */
-export function gatesForArmKind(
-  kind: 'solo' | 'supervisor',
-  secrets: SecretsEnv,
-  opts: { waitCeilingMs?: number; model?: string; onStatus?: (msg: string) => void } = {},
-): EndpointCapacityGate[] {
-  const base = {
-    kOfN: { k: 3, n: 4 },
-    waitCeilingMs: opts.waitCeilingMs ?? 300 * 60_000,
-    ...(opts.onStatus ? { onStatus: opts.onStatus } : {}),
-  }
-  const worker: EndpointCapacityGate = { name: 'z.ai-coding', probe: zaiCodingProbe(secrets, opts.model), ...base }
-  if (kind === 'solo') return [worker]
-  const brain: EndpointCapacityGate = { name: 'router', probe: routerProbe(secrets, opts.model), ...base }
-  return [worker, brain]
-}