@tangle-network/agent-bench 0.3.6 → 0.3.8

package/src/swe-arena/fixtures/instances.json

@@ -0,0 +1,146 @@
+{
+ "astropy__astropy-13033": {
+  "base_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752",
+  "environment_setup_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752",
+  "image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-13033:latest",
+  "repo": "astropy/astropy"
+ },
+ "astropy__astropy-14182": {
+  "base_commit": "a5917978be39d13cd90b517e1de4e7a539ffaa48",
+  "environment_setup_commit": "5f74eacbcc7fff707a44d8eb58adaa514cb7dcb5",
+  "image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-14182:latest",
+  "repo": "astropy/astropy"
+ },
+ "astropy__astropy-14369": {
+  "base_commit": "fa4e8d1cd279acf9b24560813c8652494ccd5922",
+  "environment_setup_commit": "5f74eacbcc7fff707a44d8eb58adaa514cb7dcb5",
+  "image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-14369:latest",
+  "repo": "astropy/astropy"
+ },
+ "astropy__astropy-14508": {
+  "base_commit": "a3f4ae6cd24d5ecdf49f213d77b3513dd509a06c",
+  "environment_setup_commit": "5f74eacbcc7fff707a44d8eb58adaa514cb7dcb5",
+  "image": "swebench/sweb.eval.x86_64.astropy_1776_astropy-14508:latest",
+  "repo": "astropy/astropy"
+ },
+ "django__django-11532": {
+  "base_commit": "a5308514fb4bc5086c9a16a8a24a945eeebb073c",
+  "environment_setup_commit": "419a78300f7cd27611196e1e464d50fd0385ff27",
+  "image": "swebench/sweb.eval.x86_64.django_1776_django-11532:latest",
+  "repo": "django/django"
+ },
+ "django__django-12774": {
+  "base_commit": "67f9d076cfc1858b94f9ed6d1a5ce2327dcc8d0d",
+  "environment_setup_commit": "0668164b4ac93a5be79f5b87fae83c657124d9ab",
+  "image": "swebench/sweb.eval.x86_64.django_1776_django-12774:latest",
+  "repo": "django/django"
+ },
+ "django__django-14140": {
+  "base_commit": "45814af6197cfd8f4dc72ee43b90ecde305a1d5a",
+  "environment_setup_commit": "475cffd1d64c690cdad16ede4d5e81985738ceb4",
+  "image": "swebench/sweb.eval.x86_64.django_1776_django-14140:latest",
+  "repo": "django/django"
+ },
+ "matplotlib__matplotlib-20826": {
+  "base_commit": "a0d2e399729d36499a1924e5ca5bc067c8396810",
+  "environment_setup_commit": "f93c0a3dcb82feed0262d758626c90d4002685f3",
+  "image": "swebench/sweb.eval.x86_64.matplotlib_1776_matplotlib-20826:latest",
+  "repo": "matplotlib/matplotlib"
+ },
+ "mwaskom__seaborn-3187": {
+  "base_commit": "22cdfb0c93f8ec78492d87edb810f10cb7f57a31",
+  "environment_setup_commit": "d25872b0fc99dbf7e666a91f59bd4ed125186aa1",
+  "image": "swebench/sweb.eval.x86_64.mwaskom_1776_seaborn-3187:latest",
+  "repo": "mwaskom/seaborn"
+ },
+ "pallets__flask-5014": {
+  "base_commit": "7ee9ceb71e868944a46e1ff00b506772a53a4f1d",
+  "environment_setup_commit": "182ce3dd15dfa3537391c3efaf9c3ff407d134d4",
+  "image": "swebench/sweb.eval.x86_64.pallets_1776_flask-5014:latest",
+  "repo": "pallets/flask"
+ },
+ "psf__requests-1766": {
+  "base_commit": "847735553aeda6e6633f2b32e14ba14ba86887a4",
+  "environment_setup_commit": "4bceb312f1b99d36a25f2985b5606e98b6f0d8cd",
+  "image": "swebench/sweb.eval.x86_64.psf_1776_requests-1766:latest",
+  "repo": "psf/requests"
+ },
+ "psf__requests-2317": {
+  "base_commit": "091991be0da19de9108dbe5e3752917fea3d7fdc",
+  "environment_setup_commit": "091991be0da19de9108dbe5e3752917fea3d7fdc",
+  "image": "swebench/sweb.eval.x86_64.psf_1776_requests-2317:latest",
+  "repo": "psf/requests"
+ },
+ "psf__requests-2931": {
+  "base_commit": "5f7a3a74aab1625c2bb65f643197ee885e3da576",
+  "environment_setup_commit": "bbeb0001cdc657ac8c7fef98e154229bc392db0e",
+  "image": "swebench/sweb.eval.x86_64.psf_1776_requests-2931:latest",
+  "repo": "psf/requests"
+ },
+ "pydata__xarray-4687": {
+  "base_commit": "d3b6aa6d8b997df115a53c001d00222a0f92f63a",
+  "environment_setup_commit": "1c198a191127c601d091213c4b3292a8bb3054e1",
+  "image": "swebench/sweb.eval.x86_64.pydata_1776_xarray-4687:latest",
+  "repo": "pydata/xarray"
+ },
+ "pylint-dev__pylint-8898": {
+  "base_commit": "1f8c4d9eb185c16a2c1d881c054f015e1c2eb334",
+  "environment_setup_commit": "a0ce6e424e3a208f3aed1cbf6e16c40853bec3c0",
+  "image": "swebench/sweb.eval.x86_64.pylint-dev_1776_pylint-8898:latest",
+  "repo": "pylint-dev/pylint"
+ },
+ "pytest-dev__pytest-6197": {
+  "base_commit": "e856638ba086fcf5bebf1bebea32d5cf78de87b4",
+  "environment_setup_commit": "f36ea240fe3579f945bf5d6cc41b5e45a572249d",
+  "image": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-6197:latest",
+  "repo": "pytest-dev/pytest"
+ },
+ "pytest-dev__pytest-7236": {
+  "base_commit": "c98bc4cd3d687fe9b392d8eecd905627191d4f06",
+  "environment_setup_commit": "678c1a0745f1cf175c442c719906a1f13e496910",
+  "image": "swebench/sweb.eval.x86_64.pytest-dev_1776_pytest-7236:latest",
+  "repo": "pytest-dev/pytest"
+ },
+ "scikit-learn__scikit-learn-14087": {
+  "base_commit": "a5743ed36fbd3fbc8e351bdab16561fbfca7dfa1",
+  "environment_setup_commit": "7e85a6d1f038bbb932b36f18d75df6be937ed00d",
+  "image": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-14087:latest",
+  "repo": "scikit-learn/scikit-learn"
+ },
+ "scikit-learn__scikit-learn-14894": {
+  "base_commit": "fdbaa58acbead5a254f2e6d597dc1ab3b947f4c6",
+  "environment_setup_commit": "7e85a6d1f038bbb932b36f18d75df6be937ed00d",
+  "image": "swebench/sweb.eval.x86_64.scikit-learn_1776_scikit-learn-14894:latest",
+  "repo": "scikit-learn/scikit-learn"
+ },
+ "sphinx-doc__sphinx-9658": {
+  "base_commit": "232dbe41c5250eb7d559d40438c4743483e95f15",
+  "environment_setup_commit": "6c6cc8a6f50b18331cb818160d168d7bb9c03e55",
+  "image": "swebench/sweb.eval.x86_64.sphinx-doc_1776_sphinx-9658:latest",
+  "repo": "sphinx-doc/sphinx"
+ },
+ "sympy__sympy-14976": {
+  "base_commit": "9cbea134220b0b951587e11b63e2c832c7246cbc",
+  "environment_setup_commit": "e53e809176de9aa0fb62e85689f8cdb669d4cacb",
+  "image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-14976:latest",
+  "repo": "sympy/sympy"
+ },
+ "sympy__sympy-17318": {
+  "base_commit": "d4e0231b08147337745dcf601e62de7eefe2fb2d",
+  "environment_setup_commit": "70381f282f2d9d039da860e391fe51649df2779d",
+  "image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-17318:latest",
+  "repo": "sympy/sympy"
+ },
+ "sympy__sympy-20438": {
+  "base_commit": "33b47e4bd60e2302e42616141e76285038b724d6",
+  "environment_setup_commit": "3ac1464b8840d5f8b618a654f9fbf09c452fe969",
+  "image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-20438:latest",
+  "repo": "sympy/sympy"
+ },
+ "sympy__sympy-23534": {
+  "base_commit": "832c24fec1046eaa544a4cab4c69e3af3e651759",
+  "environment_setup_commit": "9a6104eab0ea7ac191a09c24f3e2d79dcd66bda5",
+  "image": "swebench/sweb.eval.x86_64.sympy_1776_sympy-23534:latest",
+  "repo": "sympy/sympy"
+ }
+}

package/src/swe-arena/fixtures/ledger.jsonl

@@ -0,0 +1,12 @@
+{"iid": "pallets__flask-5014", "solo_resolved": true, "sup_resolved": true, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 36, "sup_patch_lines": 14, "solo_wall_s": 116, "sup_wall_s": 261, "solo_tokens": 30738, "solo_usage": {"steps": 20, "in": 26753, "out": 3035, "reasoning": 950, "cache_w": 0, "cache_r": 532096, "max_ctx": 33285, "oc_cost": 0.0, "total_io": 30738}, "sup_spentTokens": 15731, "sup_spentUsd": 0.012379399999999999, "sup_spawned": 2, "sup_settled": 1, "sup_subtasks": [], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pallets__flask-5014.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pallets__flask-5014.sup.patch", "_note": "sup patch re-extracted vs BASE commit (supervisor commits delivery); re-judged resolved=true"}
+{"iid": "psf__requests-2931", "solo_resolved": false, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 15, "sup_patch_lines": 96, "solo_wall_s": 166, "sup_wall_s": 1058, "solo_tokens": 23145, "solo_usage": {"steps": 25, "in": 16970, "out": 3037, "reasoning": 3138, "cache_w": 0, "cache_r": 695104, "max_ctx": 35652, "oc_cost": 0.0, "total_io": 23145}, "sup_spentTokens": 54894, "sup_spentUsd": 0.0426564, "sup_spawned": 3, "sup_workers": 2, "sup_settled": 2, "sup_subtasks": ["fix-binary-payload-to_native_string", "refine-actual-to_native_string-fix"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2931.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2931.sup.patch"}
+{"iid": "psf__requests-2317", "solo_resolved": false, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 13, "sup_patch_lines": 68, "solo_wall_s": 131, "sup_wall_s": 682, "solo_tokens": 38719, "solo_usage": {"steps": 22, "in": 34502, "out": 2802, "reasoning": 1415, "cache_w": 0, "cache_r": 738816, "max_ctx": 41281, "oc_cost": 0.0, "total_io": 38719}, "sup_spentTokens": 26076, "sup_spentUsd": 0.0216536, "sup_spawned": 2, "sup_workers": 1, "sup_settled": 1, "sup_subtasks": ["fix-bytes-method-handling"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.sup.patch"}
+{"iid": "psf__requests-1766", "solo_resolved": true, "sup_resolved": true, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 13, "sup_patch_lines": 13, "solo_wall_s": 34, "sup_wall_s": 352, "solo_tokens": 19500, "solo_usage": {"steps": 8, "in": 18828, "out": 596, "reasoning": 76, "cache_w": 0, "cache_r": 176832, "max_ctx": 25468, "oc_cost": 0.0, "total_io": 19500}, "sup_spentTokens": 38224, "sup_spentUsd": 0.02752, "sup_spawned": 3, "sup_workers": 2, "sup_settled": 2, "sup_subtasks": ["survey-digest-auth", "fix-qop-quoting"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-1766.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-1766.sup.patch", "_note": "sup patch byte-identical to solo patch; original sup judge false was a flake; re-judged 2x stable resolved=true"}
+{"iid": "sympy__sympy-23534", "solo_resolved": true, "sup_resolved": true, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 13, "sup_patch_lines": 13, "solo_wall_s": 150, "sup_wall_s": 212, "solo_tokens": 39965, "solo_usage": {"steps": 25, "in": 34557, "out": 4085, "reasoning": 1323, "cache_w": 0, "cache_r": 761984, "max_ctx": 40877, "oc_cost": 0.0, "total_io": 39965}, "sup_spentTokens": 26274, "sup_spentUsd": 0.0238716, "sup_spawned": 2, "sup_workers": 1, "sup_settled": 1, "sup_subtasks": ["fix-symbols-cls-propagation"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sympy__sympy-23534.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sympy__sympy-23534.sup.patch"}
+{"iid": "sympy__sympy-14976", "solo_resolved": true, "sup_resolved": true, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 17, "sup_patch_lines": 16, "solo_wall_s": 212, "sup_wall_s": 337, "solo_tokens": 60193, "solo_usage": {"steps": 42, "in": 52108, "out": 5940, "reasoning": 2145, "cache_w": 0, "cache_r": 1951168, "max_ctx": 57875, "oc_cost": 0.0, "total_io": 60193}, "sup_spentTokens": 28694, "sup_spentUsd": 0.0270036, "sup_spawned": 2, "sup_workers": 1, "sup_settled": 1, "sup_subtasks": ["fix-mpmath-rational-wrapping"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sympy__sympy-14976.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sympy__sympy-14976.sup.patch"}
+{"iid": "astropy__astropy-13033", "solo_resolved": true, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": false, "solo_patch_lines": 34, "sup_patch_lines": 0, "solo_wall_s": 350, "sup_wall_s": 1550, "solo_tokens": 59496, "solo_usage": {"steps": 35, "in": 48041, "out": 6296, "reasoning": 5159, "cache_w": 0, "cache_r": 1401408, "max_ctx": 53915, "oc_cost": 0.0, "total_io": 59496}, "sup_spentTokens": 0, "sup_spentUsd": 0, "sup_spawned": 4, "sup_workers": 3, "sup_settled": 3, "sup_subtasks": ["fix-timeseries-misleading-exception", "apply-and-verify-timeseries-fix", "apply-and-verify-timeseries-fix-v2"], "sup_delivered": false, "sup_status": "completed", "sup_verdict": "no-winner", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/astropy__astropy-13033.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/astropy__astropy-13033.sup.patch"}
+{"iid": "pydata__xarray-4687", "solo_resolved": false, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": true, "solo_patch_lines": 32, "sup_patch_lines": 33, "solo_wall_s": 322, "sup_wall_s": 504, "solo_tokens": 56098, "solo_usage": {"steps": 45, "in": 47437, "out": 5678, "reasoning": 2983, "cache_w": 0, "cache_r": 1823296, "max_ctx": 53147, "oc_cost": 0.0, "total_io": 56098}, "sup_spentTokens": 27341, "sup_spentUsd": 0.020335799999999998, "sup_spawned": 2, "sup_workers": 1, "sup_settled": 1, "sup_subtasks": ["where-keep-attrs-fix"], "sup_delivered": true, "sup_status": "completed", "sup_verdict": "delivered", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pydata__xarray-4687.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pydata__xarray-4687.sup.patch"}
+{"iid": "django__django-11532", "solo_resolved": true, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": false, "solo_patch_lines": 40, "sup_patch_lines": 0, "solo_wall_s": 293, "sup_wall_s": 1704, "solo_tokens": 65706, "solo_usage": {"steps": 23, "in": 53078, "out": 4289, "reasoning": 8339, "cache_w": 0, "cache_r": 1005248, "max_ctx": 59360, "oc_cost": 0.0, "total_io": 65706}, "sup_spentTokens": null, "sup_spentUsd": null, "sup_spawned": 5, "sup_workers": 4, "sup_settled": 3, "sup_subtasks": ["fix-unicode-dns-punycode", "apply-and-verify-punycode", "minimal-fix-verify", "edit-files-and-run-tests"], "sup_delivered": null, "sup_status": "running", "sup_verdict": null, "solo_oc_rc": 0, "sup_driver_rc": 3, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/django__django-11532.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/django__django-11532.sup.patch"}
+{"iid": "pytest-dev__pytest-6197", "solo_resolved": false, "sup_resolved": false, "solo_verify_pass": false, "sup_verify_pass": false, "solo_patch_lines": 0, "sup_patch_lines": 0, "solo_wall_s": 755, "sup_wall_s": 248, "solo_tokens": 133032, "solo_usage": {"steps": 37, "in": 90328, "out": 3993, "reasoning": 38711, "cache_w": 0, "cache_r": 1805248, "max_ctx": 96107, "oc_cost": 0.0, "total_io": 133032}, "sup_spentTokens": 0, "sup_spentUsd": 0, "sup_spawned": 1, "sup_workers": 0, "sup_settled": 0, "sup_subtasks": [], "sup_delivered": false, "sup_status": "completed", "sup_verdict": "no-winner", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pytest-dev__pytest-6197.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pytest-dev__pytest-6197.sup.patch"}
+{"iid": "matplotlib__matplotlib-20826", "solo_resolved": true, "sup_resolved": false, "solo_verify_pass": true, "sup_verify_pass": false, "solo_patch_lines": 23, "sup_patch_lines": 0, "solo_wall_s": 1000, "sup_wall_s": 1702, "solo_tokens": 109119, "solo_usage": {"steps": 79, "in": 81255, "out": 14156, "reasoning": 13708, "cache_w": 0, "cache_r": 4371328, "max_ctx": 85881, "oc_cost": 0.0, "total_io": 109119}, "sup_spentTokens": null, "sup_spentUsd": null, "sup_spawned": 4, "sup_workers": 3, "sup_settled": 2, "sup_subtasks": ["survey-bug-location", "fix-axclear-ticks", "apply-and-verify"], "sup_delivered": null, "sup_status": "running", "sup_verdict": null, "solo_oc_rc": 124, "sup_driver_rc": 3, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/matplotlib__matplotlib-20826.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/matplotlib__matplotlib-20826.sup.patch"}
+{"iid": "sphinx-doc__sphinx-9658", "solo_resolved": false, "sup_resolved": false, "solo_verify_pass": false, "sup_verify_pass": false, "solo_patch_lines": 0, "sup_patch_lines": 0, "solo_wall_s": 240, "sup_wall_s": 514, "solo_tokens": 39701, "solo_usage": {"steps": 19, "in": 33537, "out": 1934, "reasoning": 4230, "cache_w": 0, "cache_r": 577152, "max_ctx": 40548, "oc_cost": 0.0, "total_io": 39701}, "sup_spentTokens": 0, "sup_spentUsd": 0, "sup_spawned": 1, "sup_workers": 0, "sup_settled": 0, "sup_subtasks": [], "sup_delivered": false, "sup_status": "completed", "sup_verdict": "no-winner", "solo_oc_rc": 0, "sup_driver_rc": 0, "solo_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sphinx-doc__sphinx-9658.solo.patch", "sup_patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/sphinx-doc__sphinx-9658.sup.patch"}

package/src/swe-arena/fixtures/patches/pallets__flask-5014.solo.patch

@@ -0,0 +1,36 @@
+diff --git a/CHANGES.rst b/CHANGES.rst
+index 498d5e51..c37dd2a2 100644
+--- a/CHANGES.rst
++++ b/CHANGES.rst
+@@ -35,6 +35,7 @@ Unreleased
+ -   Ensure subdomains are applied with nested blueprints. :issue:`4834`
+ -   ``config.from_file`` can use ``text=False`` to indicate that the parser wants a
+     binary file instead. :issue:`4989`
++-   A blueprint named with an empty string raises a ``ValueError``. :issue:`4944`
+ 
+ 
+ Version 2.2.4
+diff --git a/src/flask/blueprints.py b/src/flask/blueprints.py
+index eb50585d..a36722c9 100644
+--- a/src/flask/blueprints.py
++++ b/src/flask/blueprints.py
+@@ -159,6 +159,9 @@ class Blueprint(Scaffold):
+         automatic detection can fail, so the path can be specified
+         manually instead.
+ 
++    .. versionchanged:: 2.3.0
++        Avoids the ``name`` being an empty string, which can cause errors.
++
+     .. versionchanged:: 1.1.0
+         Blueprints have a ``cli`` group to register nested CLI commands.
+         The ``cli_group`` parameter controls the name of the group under
+@@ -182,6 +185,9 @@ class Blueprint(Scaffold):
+         root_path: t.Optional[str] = None,
+         cli_group: t.Optional[str] = _sentinel,  # type: ignore
+     ):
++        if not name:
++            raise ValueError("'name' may not be empty.")
++
+         super().__init__(
+             import_name=import_name,
+             static_folder=static_folder,

package/src/swe-arena/fixtures/patches/pydata__xarray-4687.sup.patch

@@ -0,0 +1,33 @@
+diff --git a/xarray/core/computation.py b/xarray/core/computation.py
+index 9fe93c88..e2dd1c0b 100644
+--- a/xarray/core/computation.py
++++ b/xarray/core/computation.py
+@@ -1727,7 +1727,7 @@ def dot(*arrays, dims=None, **kwargs):
+     return result.transpose(*all_dims, missing_dims="ignore")
+ 
+ 
+-def where(cond, x, y):
++def where(cond, x, y, keep_attrs=None):
+     """Return elements from `x` or `y` depending on `cond`.
+ 
+     Performs xarray-like broadcasting across input arguments.
+@@ -1743,6 +1743,11 @@ def where(cond, x, y):
+         values to choose from where `cond` is True
+     y : scalar, array, Variable, DataArray or Dataset
+         values to choose from where `cond` is False
++    keep_attrs : bool or None, optional
++        If True, the attributes (``attrs``) will be copied
++        from the first argument to the result. If None, the
++        global default (``OPTIONS[keep_attrs]``) is used,
++        which defaults to False.
+ 
+     Returns
+     -------
+@@ -1817,6 +1822,7 @@ def where(cond, x, y):
+         join="exact",
+         dataset_join="exact",
+         dask="allowed",
++        keep_attrs=keep_attrs,
+     )
+ 
+ 

package/src/swe-arena/fixtures/rejudge.jsonl

@@ -0,0 +1,15 @@
+{"iid": "pallets__flask-5014", "tag": "gold-control", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gates/pallets__flask-5014.gold.patch", "resolved": true, "score": 1, "secs": 22, "patch_bytes": 390}
+{"iid": "psf__requests-2931", "tag": "gold", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gates/psf__requests-2931.gold.patch", "resolved": false, "score": 0, "secs": 26, "patch_bytes": 723}
+{"iid": "psf__requests-2317", "tag": "gold", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gates/psf__requests-2317.gold.patch", "resolved": null, "error": "parse", "raw": ""}
+{"iid": "pydata__xarray-4687", "tag": "gold", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gates/pydata__xarray-4687.gold.patch", "resolved": true, "score": 1, "secs": 71, "patch_bytes": 1539}
+{"iid": "astropy__astropy-13033", "tag": "solo-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/astropy__astropy-13033.solo.patch", "resolved": true, "score": 1, "secs": 58, "patch_bytes": 1596}
+{"iid": "django__django-11532", "tag": "solo-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/django__django-11532.solo.patch", "resolved": true, "score": 1, "secs": 29, "patch_bytes": 1349}
+{"iid": "matplotlib__matplotlib-20826", "tag": "solo-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/matplotlib__matplotlib-20826.solo.patch", "resolved": true, "score": 1, "secs": 70, "patch_bytes": 1057}
+{"iid": "psf__requests-2931", "tag": "sup-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2931.sup.patch", "resolved": false, "score": 0, "secs": 21, "patch_bytes": 3385}
+{"iid": "psf__requests-2317", "tag": "solo-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.solo.patch", "resolved": null, "error": "parse", "raw": ""}
+{"iid": "psf__requests-2317", "tag": "sup-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.sup.patch", "resolved": null, "error": "parse", "raw": ""}
+{"iid": "pydata__xarray-4687", "tag": "solo-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pydata__xarray-4687.solo.patch", "resolved": false, "score": 0, "secs": 71, "patch_bytes": 1133}
+{"iid": "pydata__xarray-4687", "tag": "sup-final", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/pydata__xarray-4687.sup.patch", "resolved": false, "score": 0, "secs": 68, "patch_bytes": 1142}
+{"iid": "psf__requests-2317", "tag": "gold2", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/gates/psf__requests-2317.gold.patch", "resolved": false, "score": 0, "secs": 1142, "patch_bytes": 763}
+{"iid": "psf__requests-2317", "tag": "solo-final2", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.solo.patch", "resolved": false, "score": 0, "secs": 1134, "patch_bytes": 396}
+{"iid": "psf__requests-2317", "tag": "sup-final2", "patch": "/tmp/claude-1000/-home-drew-code-supervisor-lab/f06fd156-042a-4ef9-bd88-f2ec7f52b90c/scratchpad/hh/patches/psf__requests-2317.sup.patch", "resolved": false, "score": 0, "secs": 1129, "patch_bytes": 2362}

package/src/swe-arena/fixtures/rematch.jsonl

@@ -0,0 +1,3 @@
+{"iid": "astropy__astropy-13033", "arm": "SUP2", "resolved": false, "verify_pass": true, "patch_lines": 48, "wall_s": 1262, "spawned": 8, "workers": 7, "sup_status": "completed", "sup_verdict": "best-effort", "delivered": true, "spentTokens": 146618}
+{"iid": "django__django-11532", "arm": "SUP2", "resolved": false, "verify_pass": true, "patch_lines": 64, "wall_s": 1070, "spawned": 3, "workers": 2, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 54623}
+{"iid": "matplotlib__matplotlib-20826", "arm": "SUP2", "resolved": false, "verify_pass": false, "patch_lines": 0, "wall_s": 1530, "spawned": 7, "workers": 6, "sup_status": "completed", "sup_verdict": "no-winner", "delivered": false, "spentTokens": 181765}

package/src/swe-arena/fixtures/rematch2.jsonl

@@ -0,0 +1,3 @@
+{"iid": "astropy__astropy-13033", "arm": "SUP3", "resolved": false, "verify_pass": true, "patch_lines": 47, "wall_s": 741, "spawned": 2, "workers": 1, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 28077}
+{"iid": "django__django-11532", "arm": "SUP3", "resolved": false, "verify_pass": true, "patch_lines": 69, "wall_s": 1422, "spawned": 3, "workers": 2, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 56265}
+{"iid": "matplotlib__matplotlib-20826", "arm": "SUP3", "resolved": false, "verify_pass": false, "patch_lines": 0, "wall_s": 1704, "spawned": 3, "workers": 2, "sup_status": "running", "sup_verdict": null, "delivered": null, "spentTokens": null}

package/src/swe-arena/fixtures/rematch3.jsonl

@@ -0,0 +1,3 @@
+{"iid": "astropy__astropy-13033", "arm": "SUP4", "resolved": false, "verify_pass": true, "patch_lines": 38, "wall_s": 555, "spawned": 3, "workers": 2, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 64387}
+{"iid": "django__django-11532", "arm": "SUP4", "resolved": false, "verify_pass": true, "patch_lines": 50, "wall_s": 1031, "spawned": 4, "workers": 3, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 86094}
+{"iid": "matplotlib__matplotlib-20826", "arm": "SUP4", "resolved": true, "verify_pass": true, "patch_lines": 23, "wall_s": 901, "spawned": 3, "workers": 2, "sup_status": "completed", "sup_verdict": "delivered", "delivered": true, "spentTokens": 62611}

package/src/swe-arena/fixtures/run-report/README.md

@@ -0,0 +1,43 @@
+# run-report evidence
+
+Reports over completed runs, committed as the measured record behind the supervisor-behavior
+claims. Both source run directories were read READ-ONLY (`--report-dir` wrote the reports
+here, never into the run).
+
+The reader is `@tangle-network/agent-eval/supervisor-run` — a supervision tree is a rollout
+trace with one more dimension, so it lives in the trace-analysis layer next to single-rollout
+analysis. `src/swe-arena/run-report.mts` is only the CLI over it.
+
+These files were captured before that move, so they are the pre-move record, not current
+library output. Every metric key and value below was reproduced from the same run bytes with
+zero diffs when the reader moved (tangle-network/agent-eval#417); what a re-run today would
+change is the envelope and the metric set, never these numbers:
+
+- `schema` is `swe-arena/run-report@1` / `swe-arena/run-report-rollup@1` here; the library
+  now emits `tangle.supervisor-run@1` / `tangle.supervisor-run-rollup@1`.
+- the per-run envelope names the source `cellDir`; the library names it `runRef`, because the
+  input contract is a reader over bytes rather than a directory.
+- `economics.brainTruncations` did not exist at capture time and is absent here.
+
+Re-capturing them means re-reading the source run directories, which are not part of this
+repo — so they stay as committed evidence rather than being regenerated in place.
+
+| File | Source run | Command |
+|---|---|---|
+| `gen3-rollup.{json,md}` | gen-3 arena, 51 cells (`…/scratchpad/hh/gen3`) | `tsx src/swe-arena/run-report.mts --round <gen3> --report-dir <out>` |
+| `factory-agent-eval-309-FSUP0.{json,md}` | factory gen-0 live, cell `factory.agent-eval.309` rep 0 (`/tmp/factory-gen0-live`) | `tsx src/swe-arena/run-report.mts <cellDir> --patch <p> --ledger <ledger.jsonl> --report-dir <out>` |
+
+Headline facts these files carry:
+
+- **gen-3: 0 steers across all 51 cells.** 42 cells measured a real `0`; 9 cells report
+  `unavailable` (no `workers/` directory — the supervisor never started a worker there).
+  161 workers spawned, 35 accepted, mean 3.1 waves per cell, mean worker utilization 0.47,
+  mean idle share 52.6%, $3.26 brain spend, 17/51 judged resolved.
+- **factory.agent-eval.309 (FSUP0, rep 0): 0 steers, 4 waves `[2,1,1,1]`, 5 workers,
+  utilization 0.663, idle 15.6min of a 32.2min run (48.3%).** 3 accepted / 1 rejected /
+  1 empty-pass; every respawn was preceded by settled evidence (3/3); delegation depth 1;
+  judge 13/30 (score 0.4333, unresolved).
+
+Read together: the supervisor is a dispatcher with a retry loop — spawn → wait → read
+settled evidence → respawn — not a lead steering contributors mid-task. Half the wall clock
+has no worker running at all.

package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.json

@@ -0,0 +1,173 @@
+{
+ "schema": "swe-arena/run-report@1",
+ "cellDir": "/tmp/factory-gen0-live/rep-0/runs/factory.agent-eval.309/FSUP0",
+ "instanceId": "factory.agent-eval.309",
+ "arm": "FSUP0",
+ "supervisorId": "sup-1-8f75ee",
+ "generatedAt": "2026-07-24T00:32:17.691Z",
+ "orchestration": {
+  "workersSpawned": 5,
+  "workersSettled": 5,
+  "workersCancelled": 0,
+  "steers": 0,
+  "steersDelivered": 0,
+  "steersByWorker": [
+   {
+    "worker": "w-0",
+    "queued": 0,
+    "delivered": 0
+   },
+   {
+    "worker": "w-1",
+    "queued": 0,
+    "delivered": 0
+   },
+   {
+    "worker": "w-2",
+    "queued": 0,
+    "delivered": 0
+   },
+   {
+    "worker": "w-3",
+    "queued": 0,
+    "delivered": 0
+   },
+   {
+    "worker": "w-4",
+    "queued": 0,
+    "delivered": 0
+   }
+  ],
+  "driverSteerCalls": 0,
+  "waves": 4,
+  "waveSizes": [
+   2,
+   1,
+   1,
+   1
+  ],
+  "maxConcurrency": 2,
+  "respawns": 3,
+  "repeatedLabels": [],
+  "delegationDepth": 1,
+  "timeToFirstSpawnMs": 831016,
+  "supervisorWallMs": 1934608,
+  "idleMs": 934563,
+  "idlePct": 48.3,
+  "workerUtilization": 0.663
+ },
+ "decision": {
+  "settledByStatus": {
+   "done": 5
+  },
+  "settledVerdicts": {},
+  "accepted": 3,
+  "rejected": 1,
+  "emptyPass": 1,
+  "observeThenRespawn": 3,
+  "respawnWithoutEvidence": 0,
+  "reviewActions": 0,
+  "workerEvidenceBytes": 12153
+ },
+ "economics": {
+  "brain": {
+   "tokensIn": 228169,
+   "tokensOut": 19398,
+   "usd": 0.179577,
+   "source": "journal metered events (n=17)"
+  },
+  "workers": {
+   "tokensIn": 286763,
+   "tokensOut": 69794,
+   "usd": 0,
+   "source": "journal settled spend + opencode sessions (n=5)"
+  },
+  "totalUsd": 0.179577,
+  "totalUsdSource": "state.json result.spentUsd — brain-priced only; worker CLI inference is unpriced (see worker token counts)",
+  "costPerAcceptedPatchUsd": 0.059859,
+  "workerWallMsDistribution": {
+   "n": 5,
+   "min": 70445,
+   "p50": 235872,
+   "p90": 556457,
+   "max": 556457,
+   "sum": 1274117
+  },
+  "perWorker": [
+   {
+    "worker": "w-0",
+    "wallMs": 556457,
+    "tokensIn": 0,
+    "tokensOut": 0,
+    "usd": 0,
+    "patchBytes": 21183,
+    "passed": false
+   },
+   {
+    "worker": "w-1",
+    "wallMs": 279898,
+    "tokensIn": 0,
+    "tokensOut": 0,
+    "usd": 0,
+    "patchBytes": 14573,
+    "passed": true
+   },
+   {
+    "worker": "w-2",
+    "wallMs": 70445,
+    "tokensIn": 0,
+    "tokensOut": 0,
+    "usd": 0,
+    "patchBytes": 0,
+    "passed": true
+   },
+   {
+    "worker": "w-3",
+    "wallMs": 235872,
+    "tokensIn": 0,
+    "tokensOut": 0,
+    "usd": 0,
+    "patchBytes": 21325,
+    "passed": true
+   },
+   {
+    "worker": "w-4",
+    "wallMs": 131445,
+    "tokensIn": 0,
+    "tokensOut": 0,
+    "usd": 0,
+    "patchBytes": 1016,
+    "passed": true
+   }
+  ]
+ },
+ "outcome": {
+  "supStatus": "completed",
+  "supVerdict": "delivered",
+  "delivered": true,
+  "judgeResolved": false,
+  "judgeScore": 0.4333,
+  "judgePassed": 13,
+  "judgeTotal": 30,
+  "verifyPass": {
+   "unavailable": "result.json absent or has no verify_pass"
+  },
+  "verifyRc": {
+   "unavailable": "result.json absent or has no verify_rc"
+  },
+  "patch": {
+   "files": 5,
+   "linesAdded": 939,
+   "linesRemoved": 0,
+   "testFilesTouched": [
+    "src/capability-headroom.test.ts",
+    "src/paired-arms.test.ts"
+   ]
+  },
+  "judgeSource": "/tmp/factory-gen0-live/ledger.jsonl (ledger row)"
+ },
+ "gaps": [
+  "verifyPass: result.json absent or has no verify_pass"
+ ],
+ "traceCommand": "npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>"
+}

package/src/swe-arena/fixtures/run-report/factory-agent-eval-309-FSUP0.md

@@ -0,0 +1,100 @@
+# Run report — factory.agent-eval.309 [FSUP0]
+
+```
+RUN-REPORT factory.agent-eval.309 [FSUP0]
+  steers=0 (spawn→wait→respawn only; no mid-task steering)
+  waves=4 sizes=[2,1,1,1] workers=5 settled=5 cancelled=0
+  concurrency max=2 utilization=0.663 idle=15.6min (48.3%) wall=32.2min
+  respawns=3 evidence→respawn=3 blind-respawn=0 depth=1
+  accepted=3 rejected=1 empty-pass=1
+  brain=$0.179577 total=$0.179577 judge.resolved=false score=0.4333 verify=unavailable — result.json absent or has no verify_pass
+  gaps(1): verifyPass: result.json absent or has no verify_pass
+```
+
+- Cell dir: `/tmp/factory-gen0-live/rep-0/runs/factory.agent-eval.309/FSUP0`
+- Supervisor: `sup-1-8f75ee`
+- Generated: 2026-07-24T00:32:17.691Z
+
+## Orchestration
+
+| Metric | Value |
+|---|---|
+| Workers spawned | 5 |
+| Workers settled | 5 |
+| Workers cancelled | 0 |
+| **Steers (mid-task messages to live workers)** | **0** |
+| Steers delivered | 0 |
+| Outer-driver `supervisor_steer` calls | 0 |
+| Spawn waves | 4 |
+| Wave sizes | [2, 1, 1, 1] |
+| Max concurrency | 2 |
+| Respawns (spawns after first settle) | 3 |
+| Repeated labels | none |
+| Delegation depth | 1 |
+| Time to first spawn | 13.9min |
+| Supervisor wall | 32.2min |
+| Idle (zero live workers) | 15.6min (48.3%) |
+| Worker utilization (Σ worker wall ÷ supervisor wall) | 0.663 |
+
+### Steers per worker
+
+| Worker | Queued | Delivered |
+|---|---:|---:|
+| `w-0` | 0 | 0 |
+| `w-1` | 0 | 0 |
+| `w-2` | 0 | 0 |
+| `w-3` | 0 | 0 |
+| `w-4` | 0 | 0 |
+
+## Decision quality
+
+| Metric | Value |
+|---|---|
+| Settled by status | done=5 |
+| Settled verdicts | none |
+| Accepted (verify green + patch bytes) | 3 |
+| Rejected (verify red) | 1 |
+| Empty pass (green, no patch) | 1 |
+| Evidence → respawn sequences | 3 |
+| Respawn with no settled evidence in front | 0 |
+| Review actions (steers + worker questions) | 0 |
+| Worker evidence returned | 12153 bytes |
+
+## Economics
+
+| Role | Tokens in | Tokens out | USD | Source |
+|---|---:|---:|---:|---|
+| brain | 228169 | 19398 | 0.179577 | journal metered events (n=17) |
+| workers | 286763 | 69794 | 0 | journal settled spend + opencode sessions (n=5) |
+
+- Total USD: 0.179577 (source: state.json result.spentUsd — brain-priced only; worker CLI inference is unpriced (see worker token counts))
+- Cost per accepted patch: 0.059859
+- Worker wall (n=5): min 70.4s / p50 3.9min / p90 9.3min / max 9.3min / Σ 21.2min
+
+| Worker | Wall | Patch bytes | Verify passed |
+|---|---:|---:|---|
+| `w-0` | 9.3min | 21183 | false |
+| `w-1` | 4.7min | 14573 | true |
+| `w-2` | 70.4s | 0 | true |
+| `w-3` | 3.9min | 21325 | true |
+| `w-4` | 2.2min | 1016 | true |
+
+## Outcome
+
+| Metric | Value |
+|---|---|
+| Supervisor status | completed |
+| Supervisor verdict | delivered |
+| Delivered | true |
+| Judge resolved | false |
+| Judge score | 0.4333 |
+| Judge passed / total | 13 / 30 |
+| Judge source | /tmp/factory-gen0-live/ledger.jsonl (ledger row) |
+| Verify gate | pass=unavailable — result.json absent or has no verify_pass rc=unavailable — result.json absent or has no verify_rc |
+| Patch | 5 file(s), +939/-0, test files touched: src/capability-headroom.test.ts, src/paired-arms.test.ts |
+
+## Gaps
+
+- verifyPass: result.json absent or has no verify_pass
+
+> Harness-session view of the same cell (model calls, stuck loops, tool errors): `npx --yes @tangle-network/traces@latest analyze --harness opencode --cwd <worker-clone-cwd>`