@tangle-network/agent-bench 0.3.7 → 0.3.8

package/src/swe-arena/fixtures/run-report/gen3-rollup.md

@@ -0,0 +1,64 @@
+# Round rollup — gen3
+
+- Cells: 51
+- **Steers across all cells: 0** (cells with ≥1 steer: 0; cells where the steer count is unavailable: 9)
+- Waves per cell (mean): 3.137
+- Max concurrency observed: 2
+- Worker utilization (mean): 0.474
+- Idle share (mean): 52.618%
+- Workers spawned: 161 · accepted: 35
+- Spend: $3.258497 · judged resolved: 17/51
+
+| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |
+|---|---|---:|---:|---:|---:|---|---:|
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.803 | 19.7 | false | 0.049903 |
+| django__django-11532 | R4 | 0 | 2 | 0.702 | 29.8 | false | 0.036822 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 10 | 0.801 | 19.9 | false | 0.224134 |
+| pydata__xarray-4687 | R4 | 0 | 5 | 0.831 | 16.9 | false | 0.096077 |
+| pytest-dev__pytest-6197 | R4 | 0 | 14 | 0.308 | 69.9 | false | 0.189169 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 4 | 0.574 | 42.6 | true | 0.085226 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.833 | 16.7 | false | 0.033692 |
+| django__django-11532 | R4 | 0 | 2 | 0.546 | 45.4 | false | 0.079202 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 4 | 0.749 | 25.1 | false | 0.107538 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.722 | 27.8 | true | 0.040083 |
+| pytest-dev__pytest-6197 | R4 | 0 | 4 | 0.458 | 54.2 | false | 0.072682 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 5 | 0.611 | 38.9 | true | 0.063746 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.817 | 18.3 | true | 0.064968 |
+| django__django-11532 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-4b0be0 | 0 | 0 | 100 | false | 0 |
+| matplotlib__matplotlib-20826 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-2c15b1 | 0 | 0 | 100 | false | 0 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.473 | 52.7 | true | 0.043101 |
+| pytest-dev__pytest-6197 | R4 | 0 | 13 | 0.352 | 64.8 | false | 0.222542 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-cd137b | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | 0 | 1 | 0.321 | 67.9 | false | 0.014523 |
+| django__django-11532 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-c84e92 | 0 | 0 | 100 | false | 0 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 2 | 0.305 | 69.5 | true | 0.028989 |
+| pydata__xarray-4687 | R4 | 0 | 4 | 0.648 | 35.2 | true | 0.081441 |
+| pytest-dev__pytest-6197 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/pytest-dev__pytest-6197/R4/ws/.loops/supervisor/sup-1-d7b670 | 0 | 0 | 100 | false | 0 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-37e35d | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/astropy__astropy-13033/R4/ws/.loops/supervisor/sup-1-4b8aff | 0 | 0 | 100 | false | 0 |
+| django__django-11532 | R4 | 0 | 2 | 0.447 | 55.3 | true | 0.042501 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 2 | 0.233 | 76.7 | false | 0.028501 |
+| pydata__xarray-4687 | R4 | 0 | 3 | 0.847 | 15.3 | true | 0.070831 |
+| pytest-dev__pytest-6197 | R4 | 0 | 2 | 0.14 | 86 | false | 0.029033 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-d97029 | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.482 | 51.8 | false | 0.037866 |
+| django__django-11532 | R4 | 0 | 3 | 0.388 | 61.2 | true | 0.059868 |
+| matplotlib__matplotlib-20826 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-1/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-eab778 | 0 | 0 | 100 | false | 0 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.714 | 28.6 | true | 0.044491 |
+| pytest-dev__pytest-6197 | R4 | 0 | 5 | 0.566 | 43.4 | false | 0.108311 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 4 | 0.359 | 64.1 | true | 0.060977 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.834 | 16.6 | false | 0.04263 |
+| django__django-11532 | R4 | 0 | 3 | 0.394 | 60.6 | false | 0.068403 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 5 | 0.588 | 41.2 | false | 0.110963 |
+| pydata__xarray-4687 | R4 | 0 | 3 | 0.82 | 18 | true | 0.084503 |
+| pytest-dev__pytest-6197 | R4 | 0 | 3 | 0.281 | 71.9 | false | 0.079547 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 5 | 0.649 | 35.1 | true | 0.068044 |
+| astropy__astropy-13033 | R4 | 0 | 4 | 0.85 | 15 | false | 0.096642 |
+| django__django-11532 | R4 | 0 | 2 | 0.244 | 75.6 | false | 0.047655 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 7 | 0.76 | 24 | false | 0.161885 |
+| pydata__xarray-4687 | R4 | 0 | 5 | 0.528 | 47.2 | true | 0.084818 |
+| pytest-dev__pytest-6197 | R4 | 0 | 5 | 0.263 | 73.7 | true | 0.122129 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 2 | 0.514 | 48.6 | true | 0.0709 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.789 | 21.1 | unavailable — judge.json absent | 0.079491 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.846 | 15.4 | unavailable — judge.json absent | 0.040525 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.782 | 21.8 | unavailable — judge.json absent | 0.084145 |

package/src/swe-arena/fixtures.ts

@@ -1,13 +1,15 @@
 /**
- * Hermetic fixture loaders. All artifacts live under `./fixtures/` (copied
+ * Pinned fixture loaders. All artifacts live under `./fixtures/` (copied
  * from the live experiment's scratchpad on 2026-07-15 so the replay survives
  * scratchpad deletion). Loaders fail loud on shape drift — a silent default
  * here would let the pinned reproduction pass against corrupted fixtures.
  */
 
-import { readFileSync } from 'node:fs'
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type {
+  FactoryInstance,
   HoldoutEntry,
   HoldoutRegistry,
   LedgerRow,
@@ -79,6 +81,137 @@ export function loadPreregisterLog(): string[] {
     .filter((l) => l.trim().length > 0)
 }
 
+// ---------------------------------------------------------------------------
+// Factory-bench instance source (manifest dirs, one per harvested PR).
+// The 3 calibrated pilots ship under ./fixtures/factory/; any directory with
+// the same layout (manifest.json + spec.md) loads identically.
+// ---------------------------------------------------------------------------
+
+/** Default shipped instance root: `./fixtures/factory`. */
+export const FACTORY_INSTANCES_DIR = fixturePath('factory')
+
+/**
+ * A FactoryInstance plus what the runner needs beyond the raw manifest: its
+ * directory (the judge child re-loads it by path), the spec text, and the
+ * calibrated judge-test denominator parsed from `resolved_criterion` — kept
+ * OUT of vitest's reported total so a judge file that fails to collect still
+ * scores against the full calibrated count instead of shrinking the divisor.
+ */
+export interface LoadedFactoryInstance extends FactoryInstance {
+  dir: string
+  spec: string
+  judgeTestTotal: number
+}
+
+const SHA_RE = /^[0-9a-f]{40}$/
+const IMAGE_DIGEST_RE = /^.+@sha256:[0-9a-f]{64}$/
+
+function requireStringArray(dir: string, field: string, v: unknown, minLen: number): string[] {
+  if (!Array.isArray(v) || v.length < minLen || v.some((x) => typeof x !== 'string' || x.length === 0)) {
+    throw new Error(`${dir}/manifest.json: ${field} must be a string[] with ≥${minLen} non-empty entries`)
+  }
+  return v as string[]
+}
+
+/** Load + validate one instance dir (fail-loud on any shape drift). */
+export function loadFactoryInstance(dir: string): LoadedFactoryInstance {
+  const manifestPath = join(dir, 'manifest.json')
+  let raw: unknown
+  try {
+    raw = JSON.parse(readFileSync(manifestPath, 'utf8'))
+  } catch (err) {
+    throw new Error(`${manifestPath}: unreadable or invalid JSON — ${(err as Error).message}`)
+  }
+  const m = raw as Record<string, unknown>
+  for (const field of [
+    'id',
+    'repo',
+    'repo_local_mirror',
+    'base_commit',
+    'judge_ref',
+    'spec_md',
+    'command_image',
+    'resolved_criterion',
+  ] as const) {
+    if (typeof m[field] !== 'string' || (m[field] as string).length === 0) {
+      throw new Error(`${manifestPath}: missing or empty required string field '${field}'`)
+    }
+  }
+  if (!(m.id as string).startsWith('factory.')) {
+    throw new Error(`${manifestPath}: id '${m.id as string}' must start with 'factory.'`)
+  }
+  for (const field of ['base_commit', 'judge_ref'] as const) {
+    if (!SHA_RE.test(m[field] as string)) {
+      throw new Error(`${manifestPath}: ${field} '${m[field] as string}' is not a full 40-hex sha`)
+    }
+  }
+  if (!IMAGE_DIGEST_RE.test(m.command_image as string)) {
+    throw new Error(
+      `${manifestPath}: command_image must be pinned as '<image>@sha256:<64 hex>'`,
+    )
+  }
+  const judge_tests = requireStringArray(dir, 'judge_tests', m.judge_tests, 1)
+  const excluded_tests = requireStringArray(dir, 'excluded_tests', m.excluded_tests ?? [], 0)
+  const setup_cmds = requireStringArray(dir, 'setup_cmds', m.setup_cmds ?? [], 0)
+  const judge_cmds = requireStringArray(dir, 'judge_cmds', m.judge_cmds, 1)
+  if (typeof m.timeout_s !== 'number' || !Number.isFinite(m.timeout_s) || m.timeout_s <= 0) {
+    throw new Error(`${manifestPath}: timeout_s must be a positive number`)
+  }
+  const totalMatch = /passed\s*\/\s*(\d+)/.exec(m.resolved_criterion as string)
+  const judgeTestTotal = totalMatch ? Number(totalMatch[1]) : 0
+  if (judgeTestTotal <= 0) {
+    throw new Error(
+      `${manifestPath}: resolved_criterion '${m.resolved_criterion as string}' must state the calibrated ` +
+        `denominator as 'passed/<N>' — partial credit needs a stable divisor`,
+    )
+  }
+  const specPath = join(dir, m.spec_md as string)
+  let spec: string
+  try {
+    spec = readFileSync(specPath, 'utf8')
+  } catch (err) {
+    throw new Error(`${specPath}: unreadable spec — ${(err as Error).message}`)
+  }
+  if (spec.trim().length === 0) throw new Error(`${specPath}: spec is empty`)
+  return {
+    id: m.id as string,
+    repo: m.repo as string,
+    repo_local_mirror: m.repo_local_mirror as string,
+    base_commit: m.base_commit as string,
+    judge_ref: m.judge_ref as string,
+    spec_md: m.spec_md as string,
+    judge_tests,
+    excluded_tests,
+    command_image: m.command_image as string,
+    setup_cmds,
+    judge_cmds,
+    resolved_criterion: m.resolved_criterion as string,
+    timeout_s: m.timeout_s,
+    ...(typeof m.worker_visible_paths_note === 'string' ? { worker_visible_paths_note: m.worker_visible_paths_note } : {}),
+    ...(typeof m.runtime === 'string' ? { runtime: m.runtime } : {}),
+    ...(m.calibration !== undefined ? { calibration: m.calibration as FactoryInstance['calibration'] } : {}),
+    dir,
+    spec,
+    judgeTestTotal,
+  }
+}
+
+/** Load every `manifest.json` instance dir under `rootDir`. Fail-loud, ids must be unique. */
+export function loadFactoryInstances(rootDir: string = FACTORY_INSTANCES_DIR): LoadedFactoryInstance[] {
+  const entries = readdirSync(rootDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => join(rootDir, e.name))
+    .sort()
+  const instances = entries.map((dir) => loadFactoryInstance(dir))
+  if (instances.length === 0) throw new Error(`${rootDir}: no factory instance directories found`)
+  const ids = new Set<string>()
+  for (const inst of instances) {
+    if (ids.has(inst.id)) throw new Error(`${rootDir}: duplicate instance id ${inst.id}`)
+    ids.add(inst.id)
+  }
+  return instances
+}
+
 /** Per-instance worker-session token spend (instances with 0 workers are absent). */
 export function loadWorkerTokens(): Record<string, WorkerTokens> {
   return JSON.parse(readFixture('worker-tokens.json')) as Record<string, WorkerTokens>