@tangle-network/agent-bench 0.11.2 → 0.13.0

package/src/rollout-ledger/settle-capture.test.mts

@@ -1,270 +0,0 @@
-/**
- * Settle-time capture with LABEL v2: the contribution rule (delivered worker
- * vs sibling bystander vs identity gap), baseline-relative proposer rewards,
- * campaign-path attribution, and end-to-end line validity on a synthetic
- * supervisor run dir (opencode store absent → labeled gap lines).
- */
-
-import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { readRolloutLedger } from '@tangle-network/agent-eval/rollout'
-import {
-  campaignCoordsFromCellPath,
-  createSettleCapture,
-  deliveredWorkerLabels,
-  PROPOSER_REWARD_SOURCE_V2,
-  proposerRewardV2,
-  readWorkerEvidence,
-  WORKER_REWARD_SOURCE_V2,
-  workerRewardV2,
-  type CellCaptureArgs,
-} from './settle-capture.mts'
-
-describe('workerRewardV2 (contribution rule)', () => {
-  it('rewards ONLY the delivering worker in a resolved cell', () => {
-    expect(workerRewardV2({ resolved: true, isDelivered: true, identityKnown: true })).toEqual({
-      reward: 1,
-      bystander: false,
-      deliveredMatch: 'delivered',
-    })
-  })
-
-  it('marks resolved-cell siblings as bystanders with reward 0', () => {
-    expect(workerRewardV2({ resolved: true, isDelivered: false, identityKnown: true })).toEqual({
-      reward: 0,
-      bystander: true,
-      deliveredMatch: 'bystander',
-    })
-  })
-
-  it('gives every worker 0 in an unresolved cell (no bystander flag)', () => {
-    expect(workerRewardV2({ resolved: false, isDelivered: false, identityKnown: true })).toEqual({
-      reward: 0,
-      bystander: false,
-      deliveredMatch: 'unresolved',
-    })
-  })
-
-  it('labels an identity gap as reward null (never fabricated credit) and an inconclusive cell as null', () => {
-    expect(workerRewardV2({ resolved: true, isDelivered: false, identityKnown: false })).toEqual({
-      reward: null,
-      bystander: false,
-      deliveredMatch: 'unknown',
-    })
-    expect(workerRewardV2({ resolved: null, isDelivered: false, identityKnown: false }).reward).toBeNull()
-  })
-})
-
-describe('deliveredWorkerLabels', () => {
-  const workers = [
-    { label: 'w1', cwd: '/tmp/w1', patch: 'diff --git a/x b/x\n+fix\n' },
-    { label: 'w2', cwd: '/tmp/w2', patch: 'diff --git a/y b/y\n+other\n' },
-    { label: 'w3', cwd: null, patch: null },
-  ]
-
-  it('matches by trimmed patch equality', () => {
-    expect(deliveredWorkerLabels('diff --git a/x b/x\n+fix\n\n', workers)).toEqual(['w1'])
-  })
-
-  it('returns [] for an empty delivery or no match', () => {
-    expect(deliveredWorkerLabels('', workers)).toEqual([])
-    expect(deliveredWorkerLabels('diff --git a/z b/z\n+mystery\n', workers)).toEqual([])
-  })
-})
-
-describe('proposerRewardV2 (baseline-relative)', () => {
-  it('is positive for an improvement, negative for a regression, zero for a tie', () => {
-    expect(proposerRewardV2(3, 1, 6)).toBeCloseTo(2 / 6)
-    expect(proposerRewardV2(0, 1, 6)).toBeCloseTo(-1 / 6)
-    expect(proposerRewardV2(1, 1, 6)).toBe(0)
-  })
-
-  it('rejects a zero instance count', () => {
-    expect(() => proposerRewardV2(1, 0, 0)).toThrow(/instanceCount/)
-  })
-})
-
-describe('campaignCoordsFromCellPath (directory attribution, never dispatch order)', () => {
-  it('parses candidate and baseline cells; unknown shapes return null', () => {
-    expect(campaignCoordsFromCellPath('/x/improve-run/gen-0/candidate-2/cell-1/arm-summary.json')).toEqual({
-      generation: 0,
-      candidateIndex: 2,
-    })
-    expect(campaignCoordsFromCellPath('/x/improve-run/baseline/cell-1/arm-summary.json')).toEqual({
-      generation: -1,
-      candidateIndex: -1,
-    })
-    expect(campaignCoordsFromCellPath('/x/somewhere/else.json')).toBeNull()
-  })
-})
-
-describe('readWorkerEvidence', () => {
-  let supRunDir: string
-
-  beforeEach(async () => {
-    supRunDir = await mkdtemp(join(tmpdir(), 'sup-'))
-    const workers = join(supRunDir, 'workers')
-    await mkdir(workers, { recursive: true })
-    await writeFile(
-      join(workers, 'w1.ndjson'),
-      `${JSON.stringify({ kind: 'started', cwd: '/tmp/clone-w1' })}\n${JSON.stringify({ kind: 'settled' })}\n`,
-    )
-    await writeFile(join(workers, 'w1.patch'), 'diff --git a/x b/x\n+fix\n')
-    await writeFile(join(workers, 'w2.ndjson'), `${JSON.stringify({ kind: 'started', cwd: '/tmp/clone-w2' })}\n`)
-    // Inbox files must not create phantom workers.
-    await writeFile(join(workers, 'w1.inbox.ndjson'), '{}\n')
-  })
-
-  afterEach(async () => {
-    await rm(supRunDir, { recursive: true, force: true })
-  })
-
-  it('joins per-worker cwd + patch; a worker without a patch reads patch:null', async () => {
-    const records = await readWorkerEvidence(supRunDir)
-    expect(records).toEqual([
-      { label: 'w1', cwd: '/tmp/clone-w1', patch: 'diff --git a/x b/x\n+fix\n' },
-      { label: 'w2', cwd: '/tmp/clone-w2', patch: null },
-    ])
-  })
-
-  it('returns [] for a run dir without worker evidence', async () => {
-    expect(await readWorkerEvidence(join(supRunDir, 'nope'))).toEqual([])
-  })
-})
-
-describe('createSettleCapture end-to-end (opencode store absent)', () => {
-  let root: string
-  let supRunDir: string
-  let ledgerPath: string
-
-  const cellArgs = (over: Partial<CellCaptureArgs> = {}): CellCaptureArgs => ({
-    generation: 0,
-    candidateIndex: 1,
-    iid: 'pydata__xarray-4687',
-    rep: 0,
-    seed: 42,
-    splitVisibility: 'public',
-    commit: 'c'.repeat(40),
-    resolved: true,
-    judgeVerdict: { resolved: true, attempts: 1 },
-    runDir: join(root, 'runs', 'pydata__xarray-4687', 'R4'),
-    patchPath: join(root, 'delivered.patch'),
-    supRunDir,
-    deliveredPatch: 'diff --git a/x b/x\n+fix\n',
-    workerModel: 'zai-coding-plan/glm-5.2',
-    metrics: { resolved: true, verify_pass: true },
-    cost: { usd: 1.25, wallS: 900, spentTokens: 1000 },
-    ...over,
-  })
-
-  beforeEach(async () => {
-    root = await mkdtemp(join(tmpdir(), 'settle-'))
-    ledgerPath = join(root, 'rollout-ledger.jsonl')
-    supRunDir = join(root, 'ws', '.loops', 'supervisor', 's1')
-    const workers = join(supRunDir, 'workers')
-    await mkdir(workers, { recursive: true })
-    await writeFile(join(workers, 'w1.ndjson'), `${JSON.stringify({ kind: 'started', cwd: '/tmp/clone-w1' })}\n`)
-    await writeFile(join(workers, 'w1.patch'), 'diff --git a/x b/x\n+fix\n')
-    await writeFile(join(workers, 'w2.ndjson'), `${JSON.stringify({ kind: 'started', cwd: '/tmp/clone-w2' })}\n`)
-    await writeFile(join(workers, 'w2.patch'), 'diff --git a/y b/y\n+other\n')
-  })
-
-  afterEach(async () => {
-    await rm(root, { recursive: true, force: true })
-  })
-
-  const capture = () =>
-    createSettleCapture({
-      ledgerPath,
-      runId: 'r4-test',
-      instanceCount: 6,
-      opencodeDb: join(root, 'no-such.db'),
-      now: () => new Date('2026-07-23T00:00:00Z'),
-    })
-
-  it('emits schema-valid supervisor + worker lines with v2 labels: delivered worker 1, bystander 0', async () => {
-    const { lines } = await capture().captureCell(cellArgs())
-    expect(lines).toBe(3)
-    const ledger = await readRolloutLedger(ledgerPath) // validates every line
-    expect(ledger).toHaveLength(3)
-
-    const sup = ledger.find((l) => l.role === 'supervisor')!
-    expect(sup.outcome.reward).toBe(1)
-    expect(sup.outcome.reward_source).toBe('swe-arena-official-judge')
-    expect(sup.generation).toBe(0)
-    expect(sup.candidate_index).toBe(1)
-    expect(sup.candidate_id).toBe('gen0-cand1')
-    // The canonical trainable split, not the legacy `train` alias.
-    expect(sup.task.split).toBe('search')
-    expect(sup.outcome.metrics.split_visibility).toBe('public')
-    expect(sup.provenance.capture).toBe('settle-time')
-
-    const workers = ledger.filter((l) => l.role === 'worker')
-    expect(workers).toHaveLength(2)
-    for (const w of workers) {
-      expect(w.parent_rollout_id).toBe(sup.rollout_id)
-      expect(w.candidate_id).toBe('gen0-cand1')
-      expect(w.task.split).toBe('search')
-      expect(w.outcome.reward_source).toBe(WORKER_REWARD_SOURCE_V2)
-      // Store absent → labeled gap, never a silent drop.
-      expect(w.messages).toEqual([])
-      expect(w.provenance.gap).toBeTruthy()
-      // No session at all, so no completed invocation to claim.
-      expect(w.outcome.metrics.has_session).toBe(false)
-      expect(w.outcome.is_completed).toBe(false)
-    }
-    const delivered = workers.find((w) => w.outcome.metrics.worker_label === 'w1')!
-    const bystander = workers.find((w) => w.outcome.metrics.worker_label === 'w2')!
-    expect(delivered.outcome.reward).toBe(1)
-    expect(delivered.outcome.metrics.bystander).toBe(false)
-    expect(delivered.outcome.metrics.delivered_match).toBe('delivered')
-    expect(bystander.outcome.reward).toBe(0)
-    expect(bystander.outcome.metrics.bystander).toBe(true)
-    expect(bystander.outcome.metrics.delivered_match).toBe('bystander')
-  })
-
-  it('gives all workers 0 in an unresolved cell', async () => {
-    await capture().captureCell(cellArgs({ resolved: false }))
-    const workers = (await readRolloutLedger(ledgerPath)).filter((l) => l.role === 'worker')
-    expect(workers.map((w) => w.outcome.reward)).toEqual([0, 0])
-    expect(workers.every((w) => w.outcome.metrics.bystander === false)).toBe(true)
-  })
-
-  it('labels an unmatched delivery as an identity gap: worker rewards null', async () => {
-    await capture().captureCell(cellArgs({ deliveredPatch: 'diff --git a/z b/z\n+mystery\n' }))
-    const workers = (await readRolloutLedger(ledgerPath)).filter((l) => l.role === 'worker')
-    expect(workers.map((w) => w.outcome.reward)).toEqual([null, null])
-    expect(workers.every((w) => w.outcome.metrics.delivered_match === 'unknown')).toBe(true)
-  })
-
-  it('emits a baseline-relative proposer line (improvement positive, v2 source)', async () => {
-    await capture().captureProposer({
-      generation: 0,
-      candidateIndex: 1,
-      proposer: 'glm-author',
-      harness: 'opencode',
-      commit: 'c'.repeat(40),
-      candResolved: 3,
-      baselineResolved: 1,
-      shotReceiptPaths: [],
-      diffPath: null,
-    })
-    const [line] = await readRolloutLedger(ledgerPath)
-    expect(line!.role).toBe('proposer')
-    expect(line!.outcome.reward).toBeCloseTo(2 / 6)
-    expect(line!.outcome.reward_source).toBe(PROPOSER_REWARD_SOURCE_V2)
-    expect(line!.outcome.metrics.baseline_resolved_count).toBe(1)
-    expect(line!.task.instance_id).toBe('gen0-cand1-glm-author')
-  })
-
-  it('appends across cells (one ledger, many flushes) and keeps every line valid', async () => {
-    const c = capture()
-    await c.captureCell(cellArgs())
-    await c.captureCell(cellArgs({ rep: 1, resolved: false, splitVisibility: 'private' }))
-    const ledger = await readRolloutLedger(ledgerPath)
-    expect(ledger).toHaveLength(6)
-    expect(ledger.filter((l) => l.outcome.metrics.split_visibility === 'private')).toHaveLength(3)
-  })
-})

package/src/swe-arena/activation.mts

@@ -1,225 +0,0 @@
-/**
- * Gen-5 activation gate (SOTA adoption #2, GSME-style "verify the mechanism
- * fired before the score counts").
- *
- * Every proposer's deliverable must include a MACHINE-CHECKABLE ACTIVATION
- * PREDICATE at `.improve/activation.json` (inside the change-space's metadata
- * prefix, so the improvement driver's finalize commits it with the candidate):
- * a grep pattern or script over the candidate's OWN campaign run artifacts
- * that proves its mechanism actually fired (e.g. "the new prompt section
- * rendered in worker prompts", "patchRiskWarnings emitted in >=1 settle").
- *
- * Enforcement is two-stage, both fail-closed:
- *   1. PREFILTER — a candidate without a parseable predicate is killed before
- *      any evaluation spend (proposer-fanout.mts, stage 'activation-predicate').
- *   2. POST-EVAL — the evaluator runs the predicate over the candidate's own
- *      cell run dirs; a candidate whose mechanism NEVER fired is QUARANTINED
- *      (staircase verdict 'quarantined-inactive': recorded, never promoted)
- *      even when its score improved — a score with an inactive mechanism is
- *      indistinguishable from luck or from gaming the visible instances.
- */
-
-import { readFile, readdir, stat } from 'node:fs/promises'
-import { join, relative } from 'node:path'
-import { run } from './proc.ts'
-
-export const ACTIVATION_PREDICATE_RELPATH = '.improve/activation.json'
-
-export interface ActivationPredicate {
-  /** One sentence: which mechanism this proves fired. */
-  description: string
-  kind: 'grep' | 'script'
-  /** kind 'grep': JS RegExp source tested line-by-line over run artifacts. */
-  pattern?: string
-  /** kind 'grep': optional relative-path substring filters (a file is searched
-   *  when its run-dir-relative path contains ANY entry). Empty/absent = all. */
-  files?: string[]
-  /** kind 'script': bash script; run once per run dir with cwd=<runDir> and
-   *  $RUN_DIR set; exit 0 in ANY run dir = mechanism fired. */
-  script?: string
-}
-
-export type ParsedPredicate = { ok: true; predicate: ActivationPredicate } | { ok: false; error: string }
-
-export function parseActivationPredicate(raw: string): ParsedPredicate {
-  let value: unknown
-  try {
-    value = JSON.parse(raw)
-  } catch (cause) {
-    return { ok: false, error: `not valid JSON: ${(cause as Error).message}` }
-  }
-  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
-    return { ok: false, error: 'must be a JSON object' }
-  }
-  const p = value as Record<string, unknown>
-  if (typeof p.description !== 'string' || p.description.trim().length === 0) {
-    return { ok: false, error: 'description must be a non-empty string' }
-  }
-  if (p.kind === 'grep') {
-    if (typeof p.pattern !== 'string' || p.pattern.length === 0) {
-      return { ok: false, error: 'kind "grep" requires a non-empty pattern' }
-    }
-    try {
-      new RegExp(p.pattern)
-    } catch (cause) {
-      return { ok: false, error: `pattern is not a valid RegExp: ${(cause as Error).message}` }
-    }
-    if (p.files !== undefined && (!Array.isArray(p.files) || p.files.some((f) => typeof f !== 'string'))) {
-      return { ok: false, error: 'files must be an array of strings when present' }
-    }
-  } else if (p.kind === 'script') {
-    if (typeof p.script !== 'string' || p.script.trim().length === 0) {
-      return { ok: false, error: 'kind "script" requires a non-empty script' }
-    }
-  } else {
-    return { ok: false, error: `kind must be "grep" or "script", got ${JSON.stringify(p.kind)}` }
-  }
-  return { ok: true, predicate: p as unknown as ActivationPredicate }
-}
-
-/** The prompt-visible contract: template + worked example. */
-export function activationPredicateInstruction(): string {
-  return [
-    'ACTIVATION PREDICATE (required deliverable — a candidate without one is rejected before evaluation):',
-    `Write ${ACTIVATION_PREDICATE_RELPATH} in this worktree: a machine-checkable proof that YOUR mechanism`,
-    "actually fired during evaluation. The evaluator runs it over your candidate's own run artifacts",
-    '(each arm run dir: driver.log, brain.jsonl, result.json, ws/.loops/** journal + worker evidence);',
-    'if it never fires, your candidate is QUARANTINED even when its score improved.',
-    '',
-    'Template (kind "grep" — a RegExp tested over the run artifacts):',
-    '  {',
-    '    "description": "patchRiskWarnings emitted in at least one settle",',
-    '    "kind": "grep",',
-    '    "pattern": "patchRiskWarnings|patch-risk",',
-    '    "files": ["journal.jsonl", "workers/"]',
-    '  }',
-    'Or kind "script": {"description":"...","kind":"script","script":"grep -rq NEW_SECTION ws/.loops"}',
-    '(exit 0 in any run dir = fired).',
-    'Pick a pattern that can ONLY appear when your mechanism ran — not one that matches the diff itself.',
-  ].join('\n')
-}
-
-// ---------------------------------------------------------------------------
-// Predicate execution over run dirs.
-// ---------------------------------------------------------------------------
-
-export interface ActivationResult {
-  fired: boolean
-  /** Bounded evidence: matching `path:line` refs (grep) or script stdout tails. */
-  evidence: string[]
-  checkedRunDirs: number
-  checkedFiles: number
-  /** Bounded notes on skipped inputs (oversized files, walk caps). */
-  warnings: string[]
-}
-
-const MAX_FILES_PER_RUN_DIR = 4000
-const MAX_FILE_BYTES = 32 * 1024 * 1024
-const MAX_EVIDENCE = 10
-const SCRIPT_TIMEOUT_MS = 120_000
-
-/** Walk one run dir. The `ws/` workspace subtree (a whole checked-out repo) is
- *  skipped EXCEPT `ws/.loops/**` — the supervisor's own artifacts live there
- *  and are exactly where mechanism traces land. */
-async function walkRunDir(runDir: string, warnings: string[]): Promise<string[]> {
-  const files: string[] = []
-  const queue: string[] = [runDir]
-  while (queue.length > 0) {
-    const dir = queue.shift()!
-    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
-    for (const entry of entries) {
-      const abs = join(dir, entry.name)
-      const rel = relative(runDir, abs)
-      if (entry.isDirectory()) {
-        if (rel === 'ws') {
-          queue.push(join(abs, '.loops'))
-          continue
-        }
-        queue.push(abs)
-      } else if (entry.isFile()) {
-        files.push(abs)
-        if (files.length >= MAX_FILES_PER_RUN_DIR) {
-          warnings.push(`${runDir}: file walk capped at ${MAX_FILES_PER_RUN_DIR} files`)
-          return files
-        }
-      }
-    }
-  }
-  return files
-}
-
-/** Run the predicate over the candidate's run dirs. Fail-closed: an unreadable
- *  artifact contributes nothing (with a warning) — it can never count as
- *  "fired". */
-export async function runActivationPredicate(
-  predicate: ActivationPredicate,
-  runDirs: readonly string[],
-): Promise<ActivationResult> {
-  const result: ActivationResult = { fired: false, evidence: [], checkedRunDirs: 0, checkedFiles: 0, warnings: [] }
-  for (const runDir of runDirs) {
-    result.checkedRunDirs += 1
-    if (predicate.kind === 'script') {
-      const res = await run('bash', ['-c', predicate.script!], {
-        cwd: runDir,
-        timeoutMs: SCRIPT_TIMEOUT_MS,
-        env: { ...process.env, RUN_DIR: runDir },
-      })
-      if (res.code === 0) {
-        result.fired = true
-        if (result.evidence.length < MAX_EVIDENCE) {
-          result.evidence.push(`${runDir}: script rc=0${res.stdout.trim() ? ` — ${res.stdout.trim().slice(0, 200)}` : ''}`)
-        }
-      } else if (res.timedOut) {
-        result.warnings.push(`${runDir}: script timed out after ${SCRIPT_TIMEOUT_MS}ms (counted as not-fired)`)
-      }
-      continue
-    }
-    const regex = new RegExp(predicate.pattern!)
-    const filters = (predicate.files ?? []).filter((f) => f.length > 0)
-    for (const file of await walkRunDir(runDir, result.warnings)) {
-      const rel = relative(runDir, file)
-      if (filters.length > 0 && !filters.some((f) => rel.includes(f))) continue
-      const info = await stat(file).catch(() => null)
-      if (info === null) continue
-      if (info.size > MAX_FILE_BYTES) {
-        result.warnings.push(`${rel}: skipped (${info.size} bytes > ${MAX_FILE_BYTES})`)
-        continue
-      }
-      result.checkedFiles += 1
-      const content = await readFile(file, 'utf8').catch(() => null)
-      if (content === null) continue
-      const lines = content.split('\n')
-      for (let i = 0; i < lines.length; i++) {
-        if (regex.test(lines[i]!)) {
-          result.fired = true
-          if (result.evidence.length < MAX_EVIDENCE) {
-            result.evidence.push(`${file}:${i + 1}: ${lines[i]!.trim().slice(0, 200)}`)
-          }
-          break // one match per file is enough evidence
-        }
-      }
-    }
-  }
-  return result
-}
-
-/** Read + parse the predicate committed at a candidate's loops commit. */
-export async function readCommittedPredicate(
-  loopsRepo: string,
-  commit: string,
-): Promise<{ raw: string; parsed: ParsedPredicate } | null> {
-  const show = await run('git', ['-C', loopsRepo, 'show', `${commit}:${ACTIVATION_PREDICATE_RELPATH}`])
-  if (show.code !== 0) return null
-  return { raw: show.stdout, parsed: parseActivationPredicate(show.stdout) }
-}
-
-/** The staircase row's activation record. */
-export interface ActivationRecord {
-  /** Whether a parseable predicate was present on the candidate commit. */
-  present: boolean
-  description: string | null
-  /** null = not evaluated (no predicate / gate disabled / baseline). */
-  fired: boolean | null
-  evidence: string[]
-  warnings: string[]
-}