@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/fixtures/run-report/gen3-rollup.json

@@ -0,0 +1,551 @@
+{
+ "schema": "swe-arena/run-report-rollup@1",
+ "cells": 51,
+ "steersTotal": 0,
+ "cellsWithSteers": 0,
+ "cellsWithUnavailableSteers": 9,
+ "wavesMean": 3.137,
+ "maxConcurrencyMax": 2,
+ "utilizationMean": 0.474,
+ "idlePctMean": 52.618,
+ "workersSpawnedTotal": 161,
+ "acceptedTotal": 35,
+ "usdTotal": 3.258497,
+ "resolvedCount": 17,
+ "perCell": [
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.803,
+   "idlePct": 19.7,
+   "resolved": false,
+   "usd": 0.049903
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.702,
+   "idlePct": 29.8,
+   "resolved": false,
+   "usd": 0.036822
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 10,
+   "utilization": 0.801,
+   "idlePct": 19.9,
+   "resolved": false,
+   "usd": 0.224134
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.831,
+   "idlePct": 16.9,
+   "resolved": false,
+   "usd": 0.096077
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 14,
+   "utilization": 0.308,
+   "idlePct": 69.9,
+   "resolved": false,
+   "usd": 0.189169
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.574,
+   "idlePct": 42.6,
+   "resolved": true,
+   "usd": 0.085226
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.833,
+   "idlePct": 16.7,
+   "resolved": false,
+   "usd": 0.033692
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.546,
+   "idlePct": 45.4,
+   "resolved": false,
+   "usd": 0.079202
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.749,
+   "idlePct": 25.1,
+   "resolved": false,
+   "usd": 0.107538
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.722,
+   "idlePct": 27.8,
+   "resolved": true,
+   "usd": 0.040083
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.458,
+   "idlePct": 54.2,
+   "resolved": false,
+   "usd": 0.072682
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.611,
+   "idlePct": 38.9,
+   "resolved": true,
+   "usd": 0.063746
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.817,
+   "idlePct": 18.3,
+   "resolved": true,
+   "usd": 0.064968
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-4b0be0"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-2c15b1"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.473,
+   "idlePct": 52.7,
+   "resolved": true,
+   "usd": 0.043101
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 13,
+   "utilization": 0.352,
+   "idlePct": 64.8,
+   "resolved": false,
+   "usd": 0.222542
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-cd137b"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 1,
+   "utilization": 0.321,
+   "idlePct": 67.9,
+   "resolved": false,
+   "usd": 0.014523
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-c84e92"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.305,
+   "idlePct": 69.5,
+   "resolved": true,
+   "usd": 0.028989
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.648,
+   "idlePct": 35.2,
+   "resolved": true,
+   "usd": 0.081441
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/pytest-dev__pytest-6197/R4/ws/.loops/supervisor/sup-1-d7b670"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-37e35d"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/astropy__astropy-13033/R4/ws/.loops/supervisor/sup-1-4b8aff"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.447,
+   "idlePct": 55.3,
+   "resolved": true,
+   "usd": 0.042501
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.233,
+   "idlePct": 76.7,
+   "resolved": false,
+   "usd": 0.028501
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.847,
+   "idlePct": 15.3,
+   "resolved": true,
+   "usd": 0.070831
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.14,
+   "idlePct": 86,
+   "resolved": false,
+   "usd": 0.029033
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-d97029"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.482,
+   "idlePct": 51.8,
+   "resolved": false,
+   "usd": 0.037866
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.388,
+   "idlePct": 61.2,
+   "resolved": true,
+   "usd": 0.059868
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": {
+    "unavailable": "workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-1/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-eab778"
+   },
+   "waves": 0,
+   "utilization": 0,
+   "idlePct": 100,
+   "resolved": false,
+   "usd": 0
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.714,
+   "idlePct": 28.6,
+   "resolved": true,
+   "usd": 0.044491
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.566,
+   "idlePct": 43.4,
+   "resolved": false,
+   "usd": 0.108311
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.359,
+   "idlePct": 64.1,
+   "resolved": true,
+   "usd": 0.060977
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.834,
+   "idlePct": 16.6,
+   "resolved": false,
+   "usd": 0.04263
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.394,
+   "idlePct": 60.6,
+   "resolved": false,
+   "usd": 0.068403
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.588,
+   "idlePct": 41.2,
+   "resolved": false,
+   "usd": 0.110963
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.82,
+   "idlePct": 18,
+   "resolved": true,
+   "usd": 0.084503
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.281,
+   "idlePct": 71.9,
+   "resolved": false,
+   "usd": 0.079547
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.649,
+   "idlePct": 35.1,
+   "resolved": true,
+   "usd": 0.068044
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 4,
+   "utilization": 0.85,
+   "idlePct": 15,
+   "resolved": false,
+   "usd": 0.096642
+  },
+  {
+   "instanceId": "django__django-11532",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.244,
+   "idlePct": 75.6,
+   "resolved": false,
+   "usd": 0.047655
+  },
+  {
+   "instanceId": "matplotlib__matplotlib-20826",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 7,
+   "utilization": 0.76,
+   "idlePct": 24,
+   "resolved": false,
+   "usd": 0.161885
+  },
+  {
+   "instanceId": "pydata__xarray-4687",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.528,
+   "idlePct": 47.2,
+   "resolved": true,
+   "usd": 0.084818
+  },
+  {
+   "instanceId": "pytest-dev__pytest-6197",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 5,
+   "utilization": 0.263,
+   "idlePct": 73.7,
+   "resolved": true,
+   "usd": 0.122129
+  },
+  {
+   "instanceId": "sphinx-doc__sphinx-9658",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.514,
+   "idlePct": 48.6,
+   "resolved": true,
+   "usd": 0.0709
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.789,
+   "idlePct": 21.1,
+   "resolved": {
+    "unavailable": "judge.json absent"
+   },
+   "usd": 0.079491
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 2,
+   "utilization": 0.846,
+   "idlePct": 15.4,
+   "resolved": {
+    "unavailable": "judge.json absent"
+   },
+   "usd": 0.040525
+  },
+  {
+   "instanceId": "astropy__astropy-13033",
+   "arm": "R4",
+   "steers": 0,
+   "waves": 3,
+   "utilization": 0.782,
+   "idlePct": 21.8,
+   "resolved": {
+    "unavailable": "judge.json absent"
+   },
+   "usd": 0.084145
+  }
+ ]
+}

package/src/swe-arena/fixtures/run-report/gen3-rollup.md

@@ -0,0 +1,64 @@
+# Round rollup — gen3
+
+- Cells: 51
+- **Steers across all cells: 0** (cells with ≥1 steer: 0; cells where the steer count is unavailable: 9)
+- Waves per cell (mean): 3.137
+- Max concurrency observed: 2
+- Worker utilization (mean): 0.474
+- Idle share (mean): 52.618%
+- Workers spawned: 161 · accepted: 35
+- Spend: $3.258497 · judged resolved: 17/51
+
+| Instance | Arm | Steers | Waves | Utilization | Idle % | Resolved | USD |
+|---|---|---:|---:|---:|---:|---|---:|
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.803 | 19.7 | false | 0.049903 |
+| django__django-11532 | R4 | 0 | 2 | 0.702 | 29.8 | false | 0.036822 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 10 | 0.801 | 19.9 | false | 0.224134 |
+| pydata__xarray-4687 | R4 | 0 | 5 | 0.831 | 16.9 | false | 0.096077 |
+| pytest-dev__pytest-6197 | R4 | 0 | 14 | 0.308 | 69.9 | false | 0.189169 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 4 | 0.574 | 42.6 | true | 0.085226 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.833 | 16.7 | false | 0.033692 |
+| django__django-11532 | R4 | 0 | 2 | 0.546 | 45.4 | false | 0.079202 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 4 | 0.749 | 25.1 | false | 0.107538 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.722 | 27.8 | true | 0.040083 |
+| pytest-dev__pytest-6197 | R4 | 0 | 4 | 0.458 | 54.2 | false | 0.072682 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 5 | 0.611 | 38.9 | true | 0.063746 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.817 | 18.3 | true | 0.064968 |
+| django__django-11532 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-4b0be0 | 0 | 0 | 100 | false | 0 |
+| matplotlib__matplotlib-20826 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-2c15b1 | 0 | 0 | 100 | false | 0 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.473 | 52.7 | true | 0.043101 |
+| pytest-dev__pytest-6197 | R4 | 0 | 13 | 0.352 | 64.8 | false | 0.222542 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-cd137b | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | 0 | 1 | 0.321 | 67.9 | false | 0.014523 |
+| django__django-11532 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/django__django-11532/R4/ws/.loops/supervisor/sup-1-c84e92 | 0 | 0 | 100 | false | 0 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 2 | 0.305 | 69.5 | true | 0.028989 |
+| pydata__xarray-4687 | R4 | 0 | 4 | 0.648 | 35.2 | true | 0.081441 |
+| pytest-dev__pytest-6197 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/pytest-dev__pytest-6197/R4/ws/.loops/supervisor/sup-1-d7b670 | 0 | 0 | 100 | false | 0 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/31fc08bd3a/rep-1/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-37e35d | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/astropy__astropy-13033/R4/ws/.loops/supervisor/sup-1-4b8aff | 0 | 0 | 100 | false | 0 |
+| django__django-11532 | R4 | 0 | 2 | 0.447 | 55.3 | true | 0.042501 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 2 | 0.233 | 76.7 | false | 0.028501 |
+| pydata__xarray-4687 | R4 | 0 | 3 | 0.847 | 15.3 | true | 0.070831 |
+| pytest-dev__pytest-6197 | R4 | 0 | 2 | 0.14 | 86 | false | 0.029033 |
+| sphinx-doc__sphinx-9658 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-0/runs/sphinx-doc__sphinx-9658/R4/ws/.loops/supervisor/sup-1-d97029 | 0 | 0 | 100 | false | 0 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.482 | 51.8 | false | 0.037866 |
+| django__django-11532 | R4 | 0 | 3 | 0.388 | 61.2 | true | 0.059868 |
+| matplotlib__matplotlib-20826 | R4 | unavailable — workers/ directory absent under /tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gen3/arm-runs/a7a2a982e5/rep-1/runs/matplotlib__matplotlib-20826/R4/ws/.loops/supervisor/sup-1-eab778 | 0 | 0 | 100 | false | 0 |
+| pydata__xarray-4687 | R4 | 0 | 2 | 0.714 | 28.6 | true | 0.044491 |
+| pytest-dev__pytest-6197 | R4 | 0 | 5 | 0.566 | 43.4 | false | 0.108311 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 4 | 0.359 | 64.1 | true | 0.060977 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.834 | 16.6 | false | 0.04263 |
+| django__django-11532 | R4 | 0 | 3 | 0.394 | 60.6 | false | 0.068403 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 5 | 0.588 | 41.2 | false | 0.110963 |
+| pydata__xarray-4687 | R4 | 0 | 3 | 0.82 | 18 | true | 0.084503 |
+| pytest-dev__pytest-6197 | R4 | 0 | 3 | 0.281 | 71.9 | false | 0.079547 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 5 | 0.649 | 35.1 | true | 0.068044 |
+| astropy__astropy-13033 | R4 | 0 | 4 | 0.85 | 15 | false | 0.096642 |
+| django__django-11532 | R4 | 0 | 2 | 0.244 | 75.6 | false | 0.047655 |
+| matplotlib__matplotlib-20826 | R4 | 0 | 7 | 0.76 | 24 | false | 0.161885 |
+| pydata__xarray-4687 | R4 | 0 | 5 | 0.528 | 47.2 | true | 0.084818 |
+| pytest-dev__pytest-6197 | R4 | 0 | 5 | 0.263 | 73.7 | true | 0.122129 |
+| sphinx-doc__sphinx-9658 | R4 | 0 | 2 | 0.514 | 48.6 | true | 0.0709 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.789 | 21.1 | unavailable — judge.json absent | 0.079491 |
+| astropy__astropy-13033 | R4 | 0 | 2 | 0.846 | 15.4 | unavailable — judge.json absent | 0.040525 |
+| astropy__astropy-13033 | R4 | 0 | 3 | 0.782 | 21.8 | unavailable — judge.json absent | 0.084145 |

package/src/swe-arena/fixtures/sup-journal-true.json

@@ -0,0 +1,19 @@
+{
+ "_provenance": {
+  "source": "runs/<iid>/SUP/ws/.loops/supervisor/*/journal.jsonl metered events, input+output summed",
+  "extraction": "identical to analyze.py _true_sup_tok; captured 2026-07-15 before scratchpad deletion",
+  "note": "null = no supervisor journal found; sphinx-doc__sphinx-9658 had a journal with zero metered events (true 0)"
+ },
+ "astropy__astropy-13033": 122399,
+ "django__django-11532": 125857,
+ "matplotlib__matplotlib-20826": 62872,
+ "pallets__flask-5014": 15731,
+ "psf__requests-1766": 38224,
+ "psf__requests-2317": 26076,
+ "psf__requests-2931": 54894,
+ "pydata__xarray-4687": 27341,
+ "pytest-dev__pytest-6197": 32192,
+ "sphinx-doc__sphinx-9658": 0,
+ "sympy__sympy-14976": 28694,
+ "sympy__sympy-23534": 26274
+}

package/src/swe-arena/fixtures/verify/astropy__astropy-13033.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Self-repro verify for astropy__astropy-13033 — AUTHORED FROM THE PROBLEM
+# STATEMENT ONLY (honesty firewall: gold patch never read by the author; it is
+# applied mechanically by calibration). Exit 0 = fixed, nonzero = bug present.
+#
+# Issue: removing a required column from a TimeSeries raises a MISLEADING
+# ValueError ("expected 'time' as the first columns but found 'time'") instead
+# of telling the user which required columns are missing.
+#
+# Runs inside the instance's swebench image (conda testbed env) with the
+# CURRENT worktree ($PWD) mounted read-only over /testbed — the same jail
+# pattern as the repro calibrator (PYTHONPATH=/testbed is load-bearing).
+set -euo pipefail
+IMG="swebench/sweb.eval.x86_64.astropy_1776_astropy-13033:latest"
+WS="${1:-$PWD}"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/repro.py" <<'PY'
+import sys
+import numpy as np
+from astropy.time import Time
+from astropy.timeseries import TimeSeries
+
+time = Time(np.arange(100000, 100003), format='jd')
+ts = TimeSeries(time=time, data={"flux": [99.9, 99.8, 99.7]})
+ts._required_columns = ["time", "flux"]
+try:
+    ts.remove_column("flux")
+except ValueError as e:
+    msg = str(e)
+    print("ValueError:", msg)
+    # Fixed behavior: the exception tells the user required column(s) are
+    # missing — it must name the missing column ('flux') or say "missing".
+    if "flux" in msg or "missing" in msg.lower():
+        print("VERIFY PASS: exception names the missing required column")
+        sys.exit(0)
+    print("VERIFY FAIL: misleading exception (does not name the missing required column)")
+    sys.exit(1)
+except Exception as e:  # any other exception type is not the informative error either
+    print("VERIFY FAIL: unexpected exception:", type(e).__name__, e)
+    sys.exit(1)
+print("VERIFY FAIL: no exception raised on removing a required column")
+sys.exit(1)
+PY
+exec timeout -k 10 240 docker run --rm --network none \
+  -v "$WS":/testbed:ro -v "$TMP":/repro:ro -w /testbed \
+  -e PYTHONDONTWRITEBYTECODE=1 \
+  "$IMG" bash -lc 'source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && timeout -s KILL 180 env PYTHONPATH=/testbed python /repro/repro.py'