@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/fixtures/verify/django__django-11532.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Self-repro verify for django__django-11532 — AUTHORED FROM THE PROBLEM
+# STATEMENT ONLY (honesty firewall: gold patch never read by the author; it is
+# applied mechanically by calibration). Exit 0 = fixed, nonzero = bug present.
+#
+# Issue: email messages crash with UnicodeEncodeError when the hostname
+# (DNS_NAME) is non-ASCII and the email encoding is non-unicode
+# (iso-8859-1); the domain should be converted to punycode. The issue's own
+# acceptance test: with DNS_NAME patched to "漢字", the Message-ID header must
+# contain the punycode form 'xn--p8s937b'.
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.django_1776_django-11532:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/repro.py" <<'PY'
+import sys
+from unittest.mock import patch
+
+from django.conf import settings
+
+settings.configure()
+
+from django.core.mail import EmailMessage
+
+# Simulate the issue's exact trigger — "Set hostname to non iso-8859-1 value
+# (i.e. hostname 正宗)" — at its true source: DNS_NAME lazily derives the
+# domain from socket.getfqdn() on first use, so patching getfqdn BEFORE the
+# first message build runs the full production path through whatever
+# conversion the fix adds.
+try:
+    with patch('socket.getfqdn', return_value='漢字'):
+        email = EmailMessage('subject', '', 'from@example.com', ['to@example.com'])
+        email.encoding = 'iso-8859-1'
+        message = email.message()
+        msgid = str(message['Message-ID'])
+        print("Message-ID:", msgid)
+        if 'xn--p8s937b' in msgid:
+            print("VERIFY PASS: non-ASCII hostname converted to punycode in Message-ID")
+            sys.exit(0)
+        print("VERIFY FAIL: Message-ID does not contain the punycode domain")
+        sys.exit(1)
+except UnicodeEncodeError as e:
+    print("VERIFY FAIL: UnicodeEncodeError (bug present):", e)
+    sys.exit(1)
+PY
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && timeout -s KILL 180 env PYTHONPATH=/testbed python /repro/repro.py'

package/src/swe-arena/fixtures/verify/matplotlib__matplotlib-20826.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Self-repro verify for matplotlib__matplotlib-20826 — AUTHORED FROM THE
+# PROBLEM STATEMENT ONLY (honesty firewall: gold patch never read by the
+# author; it is applied mechanically by calibration). Exit 0 = fixed,
+# nonzero = bug present.
+#
+# Issue: on shared axes (plt.subplots(2, 2, sharex=True, sharey=True)),
+# ax.clear() un-hides tick labels that sharing should hide and adds extra
+# top/right ticks. The problem statement's own control: with the clear() call
+# removed, the plot is identical to the correct output. So the check is a
+# paired comparison — per-subplot tick/label visibility state must be
+# IDENTICAL between a cleared and an uncleared figure.
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-20826:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/repro.py" <<'PY'
+import sys
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import numpy as np
+
+x = np.arange(0.0, 2 * np.pi, 0.01)
+y = np.sin(x)
+
+
+def make(clear):
+    fig, axes = plt.subplots(2, 2, sharex=True, sharey=True)
+    for ax in axes.flatten():
+        if clear:
+            ax.clear()
+        ax.plot(x, y)
+    fig.canvas.draw()
+    return fig, axes
+
+
+def tick_state(ax):
+    def axis_state(axis):
+        return [
+            (
+                bool(t.tick1line.get_visible()),
+                bool(t.tick2line.get_visible()),
+                bool(t.label1.get_visible()),
+                bool(t.label2.get_visible()),
+            )
+            for t in axis.get_major_ticks()
+        ]
+    return {'x': axis_state(ax.xaxis), 'y': axis_state(ax.yaxis)}
+
+
+fig_ref, axes_ref = make(clear=False)
+fig_test, axes_test = make(clear=True)
+
+mismatches = []
+for (i, j) in [(0, 0), (0, 1), (1, 0), (1, 1)]:
+    ref = tick_state(axes_ref[i][j])
+    test = tick_state(axes_test[i][j])
+    if ref != test:
+        mismatches.append((i, j, ref, test))
+
+if mismatches:
+    for i, j, ref, test in mismatches:
+        print(f"VERIFY FAIL: axes[{i}][{j}] tick/label visibility differs after clear()")
+        print("  without clear:", ref)
+        print("  with clear:   ", test)
+    sys.exit(1)
+print("VERIFY PASS: clear() preserves shared-axis tick/label visibility")
+sys.exit(0)
+PY
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && timeout -s KILL 180 env PYTHONPATH=/testbed python /repro/repro.py'

package/src/swe-arena/fixtures/verify/pydata__xarray-4687.sh

@@ -0,0 +1,44 @@
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.pydata_1776_xarray-4687:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/repro.py" <<'PY'
+import sys
+
+import xarray as xr
+
+# Issue: xr.where drops the attributes of its DataArray argument. Expected
+# behavior per the report: "Attributes should be preserved or at least there
+# should be a choice (e.g. ... keep_attrs=True)". Accept either design: attrs
+# preserved on a plain call (from whichever DataArray argument carries them),
+# or preserved when explicitly requested via keep_attrs=True. At the buggy
+# base ALL variants drop attrs (and keep_attrs is not accepted), so this
+# fails; a fix satisfying the issue makes at least one variant preserve them.
+da = xr.DataArray(1)
+da.attrs['foo'] = 'bar'
+
+def attempts():
+    yield 'default, attrs on y', lambda: xr.where(da == 0, -1, da)
+    yield 'default, attrs on x', lambda: xr.where(da == 1, da, -1)
+    yield 'keep_attrs=True, attrs on y', lambda: xr.where(da == 0, -1, da, keep_attrs=True)
+    yield 'keep_attrs=True, attrs on x', lambda: xr.where(da == 1, da, -1, keep_attrs=True)
+
+for label, call in attempts():
+    try:
+        res = call()
+    except TypeError as err:
+        print(f"variant skipped ({label}): {err}")
+        continue
+    if res.attrs.get('foo') == 'bar':
+        print(f"VERIFY PASS: xr.where preserves DataArray attributes ({label})")
+        sys.exit(0)
+    print(f"variant dropped attrs ({label}): got {dict(res.attrs)!r}")
+
+print("VERIFY FAIL: no xr.where invocation preserves DataArray attributes")
+sys.exit(1)
+PY
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && timeout -s KILL 180 env PYTHONPATH=/testbed python /repro/repro.py'

package/src/swe-arena/fixtures/verify/pytest-dev__pytest-6197.sh

@@ -0,0 +1,32 @@
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/repro.sh" <<'SH'
+set -u
+# Issue repro: a non-test package __init__.py under rootdir must NOT be
+# imported during collection. At the regressed base, `pytest` errors out
+# collecting foobar/__init__.py (assert False) and exits nonzero.
+PROJ="$(mktemp -d)"
+cd "$PROJ"
+mkdir foobar
+echo 'assert False' > foobar/__init__.py
+cat > test_foo.py <<'EOF'
+def test_foo():
+    assert True
+EOF
+# pytest 5.2.x is a src layout; put the workspace checkout first on the path.
+PYTHONPATH=/testbed/src python -m pytest -p no:cacheprovider
+rc=$?
+if [ "$rc" -ne 0 ]; then
+    echo "VERIFY FAIL: pytest collected/imported foobar/__init__.py (rc=$rc)"
+    exit 1
+fi
+echo "VERIFY PASS: pytest ran the test without touching the stray __init__.py"
+exit 0
+SH
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && timeout -s KILL 180 bash /repro/repro.sh'

package/src/swe-arena/fixtures/verify/sphinx-doc__sphinx-9658.sh

@@ -0,0 +1,51 @@
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+mkdir -p "$TMP/proj"
+cat > "$TMP/proj/conf.py" <<'PY'
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('.'))
+extensions = ['sphinx.ext.autodoc']
+# The issue: a class inheriting from a MOCKED base is documented with a
+# truncated "Bases" entry ("mocklib.nn." instead of "mocklib.nn.Module").
+autodoc_mock_imports = ['mocklib']
+PY
+cat > "$TMP/proj/mymod.py" <<'PY'
+import mocklib
+
+
+class DeepKernel(mocklib.nn.Module):
+    """A kernel inheriting a mocked torch-style base class."""
+PY
+cat > "$TMP/proj/index.rst" <<'RST'
+Test
+====
+
+.. autoclass:: mymod.DeepKernel
+   :show-inheritance:
+RST
+cat > "$TMP/repro.sh" <<'SH'
+set -u
+PROJ="$(mktemp -d)"
+cp /repro/proj/* "$PROJ"/
+OUT="$(mktemp -d)"
+PYTHONPATH=/testbed python -m sphinx -b text -q "$PROJ" "$OUT" || {
+    echo "VERIFY FAIL: sphinx build errored"
+    exit 1
+}
+cat "$OUT/index.txt"
+if grep -q 'mocklib\.nn\.Module' "$OUT/index.txt"; then
+    echo "VERIFY PASS: mocked base class documented as mocklib.nn.Module"
+    exit 0
+fi
+echo "VERIFY FAIL: Bases entry does not name mocklib.nn.Module (mocked base truncated)"
+exit 1
+SH
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && timeout -s KILL 180 bash /repro/repro.sh'

package/src/swe-arena/fixtures/worker-tokens.json

@@ -0,0 +1,42 @@
+{
+ "astropy__astropy-13033": {
+  "worker_sessions": 3,
+  "worker_tok": 180000
+ },
+ "django__django-11532": {
+  "worker_sessions": 4,
+  "worker_tok": 287038
+ },
+ "matplotlib__matplotlib-20826": {
+  "worker_sessions": 3,
+  "worker_tok": 242121
+ },
+ "pallets__flask-5014": {
+  "worker_sessions": 1,
+  "worker_tok": 46737
+ },
+ "psf__requests-1766": {
+  "worker_sessions": 2,
+  "worker_tok": 67541
+ },
+ "psf__requests-2317": {
+  "worker_sessions": 1,
+  "worker_tok": 57555
+ },
+ "psf__requests-2931": {
+  "worker_sessions": 2,
+  "worker_tok": 124545
+ },
+ "pydata__xarray-4687": {
+  "worker_sessions": 1,
+  "worker_tok": 64243
+ },
+ "sympy__sympy-14976": {
+  "worker_sessions": 1,
+  "worker_tok": 55937
+ },
+ "sympy__sympy-23534": {
+  "worker_sessions": 1,
+  "worker_tok": 36119
+ }
+}

package/src/swe-arena/fixtures.ts

@@ -0,0 +1,237 @@
+/**
+ * Pinned fixture loaders. All artifacts live under `./fixtures/` (copied
+ * from the live experiment's scratchpad on 2026-07-15 so the replay survives
+ * scratchpad deletion). Loaders fail loud on shape drift — a silent default
+ * here would let the pinned reproduction pass against corrupted fixtures.
+ */
+
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type {
+  FactoryInstance,
+  HoldoutEntry,
+  HoldoutRegistry,
+  LedgerRow,
+  RejudgeRow,
+  RematchRow,
+  WorkerTokens,
+} from './types'
+
+const fixturePath = (name: string): string =>
+  fileURLToPath(new URL(`./fixtures/${name}`, import.meta.url))
+
+export function readFixture(name: string): string {
+  return readFileSync(fixturePath(name), 'utf8')
+}
+
+function parseJsonl<T>(name: string): T[] {
+  return readFixture(name)
+    .split('\n')
+    .filter((l) => l.trim().length > 0)
+    .map((l, i) => {
+      try {
+        return JSON.parse(l) as T
+      } catch (err) {
+        throw new Error(`${name}:${i + 1}: invalid JSON — ${(err as Error).message}`)
+      }
+    })
+}
+
+export function loadLedger(): LedgerRow[] {
+  const rows = parseJsonl<LedgerRow>('ledger.jsonl')
+  for (const r of rows) {
+    if (typeof r.iid !== 'string' || typeof r.solo_resolved !== 'boolean') {
+      throw new Error(`ledger.jsonl: malformed row ${JSON.stringify(r).slice(0, 80)}`)
+    }
+  }
+  return rows
+}
+
+export function loadRejudge(): RejudgeRow[] {
+  const rows = parseJsonl<RejudgeRow>('rejudge.jsonl')
+  for (const r of rows) {
+    if (typeof r.iid !== 'string' || typeof r.tag !== 'string') {
+      throw new Error(`rejudge.jsonl: malformed row ${JSON.stringify(r).slice(0, 80)}`)
+    }
+  }
+  return rows
+}
+
+/** Evolution rounds in order: SUP2, SUP3, SUP4. */
+export function loadRematchRounds(): RematchRow[][] {
+  return ['rematch.jsonl', 'rematch2.jsonl', 'rematch3.jsonl'].map((f) => parseJsonl<RematchRow>(f))
+}
+
+export function loadHoldout(): HoldoutRegistry {
+  const entries = JSON.parse(readFixture('holdout.json')) as HoldoutEntry[]
+  if (!Array.isArray(entries) || entries.length === 0) {
+    throw new Error('holdout.json: expected a non-empty array')
+  }
+  const commits = new Set(entries.map((e) => e.selected_at_commit))
+  if (commits.size !== 1) {
+    throw new Error(`holdout.json: expected one selection commit, got ${[...commits].join(', ')}`)
+  }
+  return { entries, selectedAtCommit: entries[0].selected_at_commit }
+}
+
+export function loadPreregisterLog(): string[] {
+  return readFixture('holdout-preregister.log')
+    .split('\n')
+    .filter((l) => l.trim().length > 0)
+}
+
+// ---------------------------------------------------------------------------
+// Factory-bench instance source (manifest dirs, one per harvested PR).
+// The 3 calibrated pilots ship under ./fixtures/factory/; any directory with
+// the same layout (manifest.json + spec.md) loads identically.
+// ---------------------------------------------------------------------------
+
+/** Default shipped instance root: `./fixtures/factory`. */
+export const FACTORY_INSTANCES_DIR = fixturePath('factory')
+
+/**
+ * A FactoryInstance plus what the runner needs beyond the raw manifest: its
+ * directory (the judge child re-loads it by path), the spec text, and the
+ * calibrated judge-test denominator parsed from `resolved_criterion` — kept
+ * OUT of vitest's reported total so a judge file that fails to collect still
+ * scores against the full calibrated count instead of shrinking the divisor.
+ */
+export interface LoadedFactoryInstance extends FactoryInstance {
+  dir: string
+  spec: string
+  judgeTestTotal: number
+}
+
+const SHA_RE = /^[0-9a-f]{40}$/
+const IMAGE_DIGEST_RE = /^.+@sha256:[0-9a-f]{64}$/
+
+function requireStringArray(dir: string, field: string, v: unknown, minLen: number): string[] {
+  if (!Array.isArray(v) || v.length < minLen || v.some((x) => typeof x !== 'string' || x.length === 0)) {
+    throw new Error(`${dir}/manifest.json: ${field} must be a string[] with ≥${minLen} non-empty entries`)
+  }
+  return v as string[]
+}
+
+/** Load + validate one instance dir (fail-loud on any shape drift). */
+export function loadFactoryInstance(dir: string): LoadedFactoryInstance {
+  const manifestPath = join(dir, 'manifest.json')
+  let raw: unknown
+  try {
+    raw = JSON.parse(readFileSync(manifestPath, 'utf8'))
+  } catch (err) {
+    throw new Error(`${manifestPath}: unreadable or invalid JSON — ${(err as Error).message}`)
+  }
+  const m = raw as Record<string, unknown>
+  for (const field of [
+    'id',
+    'repo',
+    'repo_local_mirror',
+    'base_commit',
+    'judge_ref',
+    'spec_md',
+    'command_image',
+    'resolved_criterion',
+  ] as const) {
+    if (typeof m[field] !== 'string' || (m[field] as string).length === 0) {
+      throw new Error(`${manifestPath}: missing or empty required string field '${field}'`)
+    }
+  }
+  if (!(m.id as string).startsWith('factory.')) {
+    throw new Error(`${manifestPath}: id '${m.id as string}' must start with 'factory.'`)
+  }
+  for (const field of ['base_commit', 'judge_ref'] as const) {
+    if (!SHA_RE.test(m[field] as string)) {
+      throw new Error(`${manifestPath}: ${field} '${m[field] as string}' is not a full 40-hex sha`)
+    }
+  }
+  if (!IMAGE_DIGEST_RE.test(m.command_image as string)) {
+    throw new Error(
+      `${manifestPath}: command_image must be pinned as '<image>@sha256:<64 hex>'`,
+    )
+  }
+  const judge_tests = requireStringArray(dir, 'judge_tests', m.judge_tests, 1)
+  const excluded_tests = requireStringArray(dir, 'excluded_tests', m.excluded_tests ?? [], 0)
+  const setup_cmds = requireStringArray(dir, 'setup_cmds', m.setup_cmds ?? [], 0)
+  const judge_cmds = requireStringArray(dir, 'judge_cmds', m.judge_cmds, 1)
+  if (typeof m.timeout_s !== 'number' || !Number.isFinite(m.timeout_s) || m.timeout_s <= 0) {
+    throw new Error(`${manifestPath}: timeout_s must be a positive number`)
+  }
+  const totalMatch = /passed\s*\/\s*(\d+)/.exec(m.resolved_criterion as string)
+  const judgeTestTotal = totalMatch ? Number(totalMatch[1]) : 0
+  if (judgeTestTotal <= 0) {
+    throw new Error(
+      `${manifestPath}: resolved_criterion '${m.resolved_criterion as string}' must state the calibrated ` +
+        `denominator as 'passed/<N>' — partial credit needs a stable divisor`,
+    )
+  }
+  const specPath = join(dir, m.spec_md as string)
+  let spec: string
+  try {
+    spec = readFileSync(specPath, 'utf8')
+  } catch (err) {
+    throw new Error(`${specPath}: unreadable spec — ${(err as Error).message}`)
+  }
+  if (spec.trim().length === 0) throw new Error(`${specPath}: spec is empty`)
+  return {
+    id: m.id as string,
+    repo: m.repo as string,
+    repo_local_mirror: m.repo_local_mirror as string,
+    base_commit: m.base_commit as string,
+    judge_ref: m.judge_ref as string,
+    spec_md: m.spec_md as string,
+    judge_tests,
+    excluded_tests,
+    command_image: m.command_image as string,
+    setup_cmds,
+    judge_cmds,
+    resolved_criterion: m.resolved_criterion as string,
+    timeout_s: m.timeout_s,
+    ...(typeof m.worker_visible_paths_note === 'string' ? { worker_visible_paths_note: m.worker_visible_paths_note } : {}),
+    ...(typeof m.runtime === 'string' ? { runtime: m.runtime } : {}),
+    ...(m.calibration !== undefined ? { calibration: m.calibration as FactoryInstance['calibration'] } : {}),
+    dir,
+    spec,
+    judgeTestTotal,
+  }
+}
+
+/** Load every `manifest.json` instance dir under `rootDir`. Fail-loud, ids must be unique. */
+export function loadFactoryInstances(rootDir: string = FACTORY_INSTANCES_DIR): LoadedFactoryInstance[] {
+  const entries = readdirSync(rootDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => join(rootDir, e.name))
+    .sort()
+  const instances = entries.map((dir) => loadFactoryInstance(dir))
+  if (instances.length === 0) throw new Error(`${rootDir}: no factory instance directories found`)
+  const ids = new Set<string>()
+  for (const inst of instances) {
+    if (ids.has(inst.id)) throw new Error(`${rootDir}: duplicate instance id ${inst.id}`)
+    ids.add(inst.id)
+  }
+  return instances
+}
+
+/** Per-instance worker-session token spend (instances with 0 workers are absent). */
+export function loadWorkerTokens(): Record<string, WorkerTokens> {
+  return JSON.parse(readFixture('worker-tokens.json')) as Record<string, WorkerTokens>
+}
+
+/**
+ * Journal-true supervisor brain tokens per instance — extracted from the
+ * supervisor journals' `metered` events with the same logic as
+ * `analyze.py::_true_sup_tok`, captured into a fixture because the raw
+ * journals (runs/) are too large to commit. `null` = no journal found.
+ */
+export function loadSupJournalTrue(): Record<string, number | null> {
+  const raw = JSON.parse(readFixture('sup-journal-true.json')) as Record<string, unknown>
+  const out: Record<string, number | null> = {}
+  for (const [k, v] of Object.entries(raw)) {
+    if (k === '_provenance') continue
+    if (v !== null && typeof v !== 'number') {
+      throw new Error(`sup-journal-true.json: ${k} must be number|null`)
+    }
+    out[k] = v as number | null
+  }
+  return out
+}