@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +47 -62
  3. package/ROADMAP.md +15 -6
  4. package/SECURITY.md +1 -1
  5. package/benchmarks/README.md +48 -0
  6. package/benchmarks/baselines/context-navigation-2026-09-09.json +94 -0
  7. package/benchmarks/baselines/efficiency-2026-09-08.json +233 -0
  8. package/benchmarks/baselines/external-final-2026-09-08.json +588 -0
  9. package/benchmarks/baselines/external-intermediate-2026-09-08.json +116 -0
  10. package/benchmarks/baselines/external-matched-openai-2026-09-08.json +1922 -0
  11. package/benchmarks/baselines/external-matched-qwen-2026-09-08.json +1706 -0
  12. package/benchmarks/baselines/external-pilot-2026-09-08.json +439 -0
  13. package/benchmarks/baselines/external-qwen-flash-2026-09-08.json +1570 -0
  14. package/benchmarks/baselines/external-qwen-remediation-2026-09-09.json +1509 -0
  15. package/benchmarks/baselines/external-qwen-sdk-114-progress-2026-09-09.json +2509 -0
  16. package/benchmarks/baselines/external-sdk-upgrade-2026-09-08.json +564 -0
  17. package/benchmarks/baselines/external-verifier-recovery-2026-09-09.json +794 -0
  18. package/benchmarks/baselines/full-harness-review-2026-09-10.json +134 -0
  19. package/benchmarks/baselines/harness-audit-2026-09-09.json +136 -0
  20. package/benchmarks/baselines/harness-remediation-2026-09-09.json +160 -0
  21. package/benchmarks/baselines/oci-remediation-2026-09-10.json +535 -0
  22. package/benchmarks/baselines/rc14-holdout-2026-09-10.json +854 -0
  23. package/benchmarks/baselines/repair-architecture-audit-2026-09-09.json +75 -0
  24. package/benchmarks/baselines/time-to-safe-fix-live-expanded-2026-08-21.json +186 -0
  25. package/benchmarks/baselines/time-to-safe-fix-live-smoke-2026-08-21.json +186 -0
  26. package/contracts/public-api.json +25 -0
  27. package/contracts/security-controls.json +3 -2
  28. package/contracts/stable-api-signatures.json +58 -54
  29. package/dist/bounded-reads.d.ts +4 -0
  30. package/dist/change-envelope.d.ts +4 -4
  31. package/dist/cli-options.d.ts +2 -1
  32. package/dist/cli-profiles.d.ts +39 -0
  33. package/dist/cli-stream.d.ts +40 -40
  34. package/dist/cli.d.ts +38 -35
  35. package/dist/cli.js +6556 -4030
  36. package/dist/compaction.d.ts +12 -0
  37. package/dist/config.d.ts +6 -1
  38. package/dist/console-context.d.ts +26 -0
  39. package/dist/console-input.d.ts +28 -0
  40. package/dist/context-engineering.d.ts +5 -5
  41. package/dist/context-metrics.d.ts +11 -0
  42. package/dist/edit-contracts.d.ts +2 -2
  43. package/dist/execution-environment.d.ts +9 -5
  44. package/dist/harness.d.ts +52 -18
  45. package/dist/hostile-repository-demo.js +2942 -1088
  46. package/dist/index.d.ts +3 -3
  47. package/dist/index.js +4316 -2435
  48. package/dist/json-contracts.d.ts +292 -236
  49. package/dist/model-budget.d.ts +61 -0
  50. package/dist/operations.d.ts +82 -20
  51. package/dist/orchestration.d.ts +1 -0
  52. package/dist/providers.d.ts +5 -2
  53. package/dist/repair-controller.d.ts +72 -0
  54. package/dist/repair-progress.d.ts +40 -0
  55. package/dist/repair-verifier.d.ts +6 -0
  56. package/dist/replacement-edits.d.ts +9 -0
  57. package/dist/routing.d.ts +1 -1
  58. package/dist/run-interruption.d.ts +4 -0
  59. package/dist/runtime-checkpoints.d.ts +7 -0
  60. package/dist/runtime-diagnostics.d.ts +63 -0
  61. package/dist/runtime-policy.d.ts +36 -0
  62. package/dist/sessions.d.ts +1 -1
  63. package/dist/state-backup.d.ts +2 -2
  64. package/dist/task-memory.d.ts +36 -0
  65. package/dist/terminal-markdown.d.ts +12 -0
  66. package/dist/time-to-safe-fix.d.ts +19 -19
  67. package/dist/tool-registry.d.ts +4 -0
  68. package/dist/verify-historical-migrations.js +6987 -5140
  69. package/dist/workspace-mutation-lock.d.ts +5 -0
  70. package/dist/workspace.d.ts +39 -4
  71. package/dist/zhx.js +6556 -4030
  72. package/docs/CLI.md +89 -3
  73. package/docs/CONTEXT_ENGINEERING.md +160 -0
  74. package/docs/EXECUTION_ENVIRONMENTS.md +21 -0
  75. package/docs/EXTENSIBILITY.md +2 -2
  76. package/docs/GA_READINESS.md +13 -7
  77. package/docs/LIVE_CERTIFICATION.md +12 -2
  78. package/docs/RC13_SECURITY_REVIEW.md +28 -0
  79. package/docs/RC14_DELEGATED_SECURITY_DECISION.md +80 -0
  80. package/docs/RC14_SECURITY_REVIEW.md +40 -0
  81. package/docs/README.md +33 -0
  82. package/docs/RELEASE.md +4 -2
  83. package/docs/REPOSITORY_EDITING.md +47 -0
  84. package/docs/SECURITY_REVIEW_EVIDENCE.md +5 -3
  85. package/docs/THREAT_MODEL.md +3 -3
  86. package/docs/TIME_TO_SAFE_FIX.md +4 -4
  87. package/docs/ga-readiness.json +147 -16
  88. package/docs/reports/EFFICIENCY_REVIEW.md +156 -0
  89. package/docs/reports/EXTERNAL_COMPARISON.md +434 -0
  90. package/docs/reports/FULL_HARNESS_REVIEW_2026-09-10.md +200 -0
  91. package/docs/reports/GA_BUDGET_REMEDIATION_2026-09-19.md +38 -0
  92. package/docs/reports/GA_REPAIR_RECOVERY_2026-09-19.md +700 -0
  93. package/docs/reports/GA_VALIDATION_2026-09-19.md +111 -0
  94. package/docs/reports/HARNESS_ARCHITECTURE_REVIEW_2026-09-09.md +202 -0
  95. package/docs/reports/HARNESS_AUDIT_2026-09-09.md +306 -0
  96. package/docs/reports/HARNESS_REMEDIATION_2026-09-09.md +74 -0
  97. package/docs/reports/HARNESS_REMEDIATION_2026-09-10.md +90 -0
  98. package/docs/reports/MODEL_COMPARISON_2026-09-09.md +184 -0
  99. package/docs/reports/QWEN_REMEDIATION_COMPARISON_2026-09-09.md +142 -0
  100. package/docs/reports/RC14_VALIDATION_2026-09-10.md +45 -0
  101. package/docs/reports/README.md +26 -0
  102. package/docs/reports/evidence/ga-budget-remediation-2026-09-19.json +133 -0
  103. package/docs/reports/evidence/ga-catalog-ablation-2026-09-19.json +699 -0
  104. package/docs/reports/evidence/ga-catalog-ablation-plan-2026-09-19.json +40 -0
  105. package/docs/reports/evidence/ga-catalog-observed-2026-09-19.json +985 -0
  106. package/docs/reports/evidence/ga-catalog-observed-plan-2026-09-19.json +28 -0
  107. package/docs/reports/evidence/ga-catalog-observed-status-2026-09-19.json +15 -0
  108. package/docs/reports/evidence/ga-complete-local-check-2026-09-19.json +47 -0
  109. package/docs/reports/evidence/ga-completion-reminder-2026-09-19.json +22 -0
  110. package/docs/reports/evidence/ga-fresh-cohort-plan-2026-09-19.json +54 -0
  111. package/docs/reports/evidence/ga-fresh-cohort-preparation-2026-09-19.json +242 -0
  112. package/docs/reports/evidence/ga-fresh-cohort-result-2026-09-19.json +4318 -0
  113. package/docs/reports/evidence/ga-local-validation-followup-2026-09-19.json +66 -0
  114. package/docs/reports/evidence/ga-node-consumers-2026-09-19.json +31 -0
  115. package/docs/reports/evidence/ga-package-content-validation-2026-09-19.json +22 -0
  116. package/docs/reports/evidence/ga-plan-completion-2026-09-19.json +39 -0
  117. package/docs/reports/evidence/ga-plan-reserve-2026-09-19.json +573 -0
  118. package/docs/reports/evidence/ga-plan-reserve-plan-2026-09-19.json +24 -0
  119. package/docs/reports/evidence/ga-plan-transition-2026-09-19.json +19 -0
  120. package/docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json +24 -0
  121. package/docs/reports/evidence/ga-private-verifier-root-cause-2026-09-19.json +41 -0
  122. package/docs/reports/evidence/ga-published-sdk-2026-09-19.json +35 -0
  123. package/docs/reports/evidence/ga-python-checkout-imports-2026-09-19.json +95 -0
  124. package/docs/reports/evidence/ga-python-origin-delivery-2026-09-19.json +211 -0
  125. package/docs/reports/evidence/ga-python-origin-delivery-plan-2026-09-19.json +27 -0
  126. package/docs/reports/evidence/ga-qwen-patched-package-2026-09-19.json +1092 -0
  127. package/docs/reports/evidence/ga-qwen-patched-package-plan-2026-09-19.json +18 -0
  128. package/docs/reports/evidence/ga-qwen-published-2026-09-19.json +571 -0
  129. package/docs/reports/evidence/ga-qwen-published-plan-2026-09-19.json +18 -0
  130. package/docs/reports/evidence/ga-qwen-tool-choice-2026-09-19.json +85 -0
  131. package/docs/reports/evidence/ga-qwen-tool-choice-patched-2026-09-19.json +85 -0
  132. package/docs/reports/evidence/ga-recovery-development-2026-09-19.json +1066 -0
  133. package/docs/reports/evidence/ga-recovery-graded-development-2026-09-19.json +1313 -0
  134. package/docs/reports/evidence/ga-recovery-graded-v2-2026-09-19.json +1151 -0
  135. package/docs/reports/evidence/ga-recovery-v3-2026-09-19.json +3943 -0
  136. package/docs/reports/evidence/ga-recovery-v4-2026-09-19.json +1055 -0
  137. package/docs/reports/evidence/ga-recovery-v5-2026-09-19.json +1245 -0
  138. package/docs/reports/evidence/ga-recovery-v6-2026-09-19.json +699 -0
  139. package/docs/reports/evidence/ga-recovery-v7-2026-09-19.json +854 -0
  140. package/docs/reports/evidence/ga-recovery-v8-2026-09-19.json +596 -0
  141. package/docs/reports/evidence/ga-required-delivery-2026-09-19.json +25 -0
  142. package/docs/reports/evidence/ga-required-delivery-live-2026-09-19.json +1337 -0
  143. package/docs/reports/evidence/ga-required-delivery-live-plan-2026-09-19.json +25 -0
  144. package/docs/reports/evidence/ga-retained-candidate-grading-2026-09-19.json +30 -0
  145. package/docs/reports/evidence/ga-tool-selection-live-2026-09-19.json +559 -0
  146. package/docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json +28 -0
  147. package/docs/reports/evidence/ga-tool-selection-recovery-2026-09-19.json +27 -0
  148. package/docs/reports/evidence/ga-validation-2026-09-19.json +236 -0
  149. package/docs/reports/evidence/ga-verifier-discovery-live-2026-09-19.json +1369 -0
  150. package/docs/reports/evidence/ga-verifier-discovery-plan-2026-09-19.json +29 -0
  151. package/docs/reports/evidence/ga-verifier-failure-identification-2026-09-19.json +30 -0
  152. package/docs/reports/evidence/ga-verifier-feedback-2026-09-19.json +65 -0
  153. package/docs/reports/evidence/ga-verifier-hints-live-2026-09-19.json +1297 -0
  154. package/docs/reports/evidence/ga-verifier-hints-plan-2026-09-19.json +27 -0
  155. package/docs/reports/evidence/ga-work-boundary-context-analysis-2026-09-19.json +113 -0
  156. package/docs/reports/evidence/ga-work-boundary-live-2026-09-19.json +301 -0
  157. package/docs/reports/evidence/ga-work-boundary-live-plan-2026-09-19.json +84 -0
  158. package/docs/reports/evidence/ga-work-budget-planning-2026-09-19.json +32 -0
  159. package/docs/reports/evidence/ga-working-context-2026-09-19.json +22 -0
  160. package/docs/reports/evidence/rc14-delegated-security-review-2026-09-20.json +81 -0
  161. package/docs/reports/evidence/rc14-ga-gate-2026-09-20.json +18 -0
  162. package/docs/reports/evidence/rc14-representative-2026-09-20.json +509 -0
  163. package/docs/reports/evidence/rc14-security-regressions-2026-09-20.json +64 -0
  164. package/docs/reports/evidence/security-current-2026-09-19.json +91 -0
  165. package/docs/reports/evidence/security-remediation-2026-09-19.json +97 -0
  166. package/evaluations/repair-controller-experiment.json +24 -0
  167. package/evaluations/representative-assembly-matrix.json +63 -0
  168. package/evaluations/representative-matrix.json +2219 -2
  169. package/fixtures/contracts/v1/init.json +19 -0
  170. package/package.json +25 -16
@@ -0,0 +1,1369 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "kind": "verifier-discovery-development-validation",
4
+ "classification": "Known development case; not holdout or GA acceptance",
5
+ "manifest": {
6
+ "schemaVersion": 1,
7
+ "dataset": "princeton-nlp/SWE-bench_Verified",
8
+ "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a",
9
+ "selectedTasksSha256": "913d7eb61726ca064eeaba236909f071ab1070aa5b603b17523799229afb259c",
10
+ "excludedTaskIds": [
11
+ "astropy__astropy-14995",
12
+ "django__django-11265",
13
+ "django__django-12262",
14
+ "django__django-13410",
15
+ "django__django-14915",
16
+ "django__django-15554",
17
+ "django__django-15569",
18
+ "pydata__xarray-4695",
19
+ "pydata__xarray-6721",
20
+ "pytest-dev__pytest-8399",
21
+ "scikit-learn__scikit-learn-10908",
22
+ "scikit-learn__scikit-learn-14141",
23
+ "sympy__sympy-15809",
24
+ "sympy__sympy-20438",
25
+ "sympy__sympy-21930"
26
+ ],
27
+ "excludedManifestSha256": [
28
+ "44a893b60eb4b495490f687648234223f6eb5b3b92a0ee348ceb7f981e9b74c7"
29
+ ],
30
+ "seed": 20260910,
31
+ "repetitions": 1,
32
+ "model": "qwen3.8-flash",
33
+ "provider": "qwen",
34
+ "reasoning": "none",
35
+ "transport": "qwen-chat-completions",
36
+ "limits": {
37
+ "steps": 24,
38
+ "outputPerTurn": 2048,
39
+ "inputTokens": 100000,
40
+ "outputTokens": 16000,
41
+ "timeoutSeconds": 300,
42
+ "memoryMb": 2048,
43
+ "cpus": 2
44
+ },
45
+ "versions": {
46
+ "mini-swe-agent": "2.4.6",
47
+ "swebench": "4.1.0"
48
+ },
49
+ "tasks": [
50
+ {
51
+ "instance_id": "django__django-15851",
52
+ "repo": "django/django",
53
+ "base_commit": "b4817d20b9e55df30be0b1b2ca8c8bb6d61aab07",
54
+ "problem_statement": "dbshell additional parameters should be passed before dbname on PostgreSQL.\nDescription\n\t\npsql expects all options to proceed the database name, if provided. So, if doing something like `./manage.py dbshell -- -c \"select * from some_table;\" one will get this:\n$ ./manage.py dbshell -- -c \"select * from some_table;\"\npsql: warning: extra command-line argument \"-c\" ignored\npsql: warning: extra command-line argument \"select * from some_table;\" ignored\npsql (10.21)\nType \"help\" for help.\nsome_database=>\nIt appears the args list just need to be constructed in the proper order, leaving the database name for the end of the args list.\n",
55
+ "evalImage": "swebench/sweb.eval.x86_64.django_1776_django-15851:zhx-aa80884e053cf1a2",
56
+ "evalImageId": "sha256:aa80884e053cf1a238df7c4ceec43e4e07929547f658278bd3c3154f0ef8bc90",
57
+ "evalTag": "zhx-aa80884e053cf1a2",
58
+ "image": "sha256:d53dbd1313ddea4f813b0a8328cd740e978ce6ebb05d4d3c88c4e8f66f79bca1",
59
+ "checkoutHead": "ec9331c1d76135610cbfd45a0d08e3388993253a",
60
+ "baseTree": "80258ef9c833e0b2aa85dd884757717adb3b5416",
61
+ "upstreamTree": "e9a3c8b5c5eaf505427370dc2a61c5568be221c9",
62
+ "imageModeDifferences": 6636
63
+ }
64
+ ],
65
+ "environmentDifferences": [
66
+ "Zhivex native OCI: read-only root and writable /workspace snapshot, operator auto-approval; mini: writable /testbed. Both deny container network and use the same derived image and CPU/memory limits.",
67
+ "Zhivex tool-call limit is steps times eight; mini counts model steps and executes each response's actions. Tool counts are reported, not claimed equivalent.",
68
+ "Input/output token ceilings are checked between model requests and can overshoot by one response; outputPerTurn is sent to both APIs."
69
+ ],
70
+ "prepared": true,
71
+ "nodeImage": "sha256:5736e7ef1f3f2109be7ef8aea0cbdf931804aee9a18c6760507b8ded078b25a9",
72
+ "implementation": {
73
+ "gitCommit": "c2b2bb917d0471e61afc5f731a61c00b44fa3731",
74
+ "sources": {
75
+ "bun.lock": "dbabdf161b3dba306f475c81f37e5ab1620fb98052127bb3ed2a0a0cb611ede4",
76
+ "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
77
+ "package.json": "78989c243298fa78066ce1670439277c689a5a538d973b91f7ea7061281dd82a",
78
+ "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
79
+ "scripts/swebench/candidate.py": "f5e3410b91b950acfbe50fc7511891816a8fb1c223082397e2c6300cf1e866cf",
80
+ "scripts/swebench/candidate.ts": "5767ae241c4a58bd731456abe2c3d92f7a57bba764c80e0f0d00748a0f47d02b",
81
+ "scripts/swebench/common.py": "ad790227dac3bec1f592c458c993efff4f9d3de8f0df39b33e3238b58b3f0b3b",
82
+ "scripts/swebench/context-metrics.ts": "7b84da6789413aa634d997c5963ee6796584e06c3eafe5f68f424fb7e27815e9",
83
+ "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
84
+ "scripts/swebench/model-budget.ts": "e46dd406ee5d58a051a15c7d7133513666292303d8c6534f861a6db0e711f47c",
85
+ "scripts/swebench/model-catalog.ts": "eb79e7333f4f5804f96da87d1952382f620e7165a95b60489fcc2588debe64d8",
86
+ "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
87
+ "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
88
+ "scripts/swebench/report.py": "d0b8d45f32c48e0ba2cb977f50838065a497a0ecaf3b4243ed94139393ce4d6b",
89
+ "scripts/swebench/run.py": "94dccd55dfe6f70d8f8f2da70331700ec85eddd958547e0c3afe3dc416c9802c",
90
+ "scripts/swebench/telemetry.ts": "2c0b4828db3a108184df6d47276d7d3dbb6cf291126b40fc38d33f4b29101bc2",
91
+ "scripts/swebench/test_candidate.py": "e77d16379b748baa2daf184a45ea62587d938b6a7d48dcf9f6061f8af21fafd9",
92
+ "scripts/swebench/test_contracts.py": "902e966d0fbf8501dbb15e011849f160331c033cd2a5480f9920d0f365a08d03",
93
+ "scripts/swebench/verifier-observer.ts": "a90544437473218a83477a1cffb59ee9f645c12abdf8960b787f3e21606035bf",
94
+ "scripts/swebench/zhivex-driver.ts": "52268790d1f20c73b340a68c6d13ea8d3250a04eca362f0ace983ced5c7e6035",
95
+ "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
96
+ "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
97
+ "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
98
+ "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
99
+ "src/cli-options.ts": "f48d9be132fed38f2ddbcfecbd8222c664a0df2f20edc28ce8d1b9b1a1900863",
100
+ "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
101
+ "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
102
+ "src/cli.ts": "5a4acb4695a968a58fbeac05c4bb89848a9653d6189590f0ff90e430ffa98d14",
103
+ "src/compaction.ts": "9e06a0abeeef155377f40c160b21c56256bca95ec18c5ee897681a5fc73db169",
104
+ "src/config.ts": "67ed44b55505ff8dd94f60ba89c33c217759cd7c79de45ccda369f248309cdff",
105
+ "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
106
+ "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
107
+ "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
108
+ "src/context-metrics.ts": "3eb30f9db808f918ed8d73d4f8ac19a13acb09ed5c1ee70fdae05201714b9198",
109
+ "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
110
+ "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
111
+ "src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9",
112
+ "src/file-security.ts": "b8c9f4988c4339c73a60ab4fa8d394b3eecc19117a0c61a8d19ab6483ca253df",
113
+ "src/harness.ts": "1d305078e3c56c24bb659c34aac37c96bf11a4e60af6607bb4b9d65964e84791",
114
+ "src/index.ts": "4bbfcd11376d48c4f1c1ec17d6b5c9f51baa9331fa999432be178bae94530a5e",
115
+ "src/json-contracts.ts": "0780145636b47829a1aa6087c574d5654eb31b7534ea7d9c38c188d648c90272",
116
+ "src/mcp.ts": "7722de031921c620ec03b9e42a6596ef8415aa8e4bb5f93ea84d2427c466e554",
117
+ "src/model-budget.ts": "97c00019d7e737fc7acdd04cdac0811dca8d85c3e858de519ebba91888cf1ee7",
118
+ "src/operations.ts": "bcccaf4d1d0ec117c2ed35df92185564d33399e5fdc5c12225ac61f14fd0b4b1",
119
+ "src/orchestration.ts": "b56c8c7d526a99b745cc17769e0a1b7b500b25920e77f34aa0dd01c9758de319",
120
+ "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
121
+ "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
122
+ "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
123
+ "src/repair-controller.ts": "41f1f09032131306f3284edbf105930577504af0bf764ba98a105c4eee70a1e7",
124
+ "src/repair-progress.ts": "cf30117ab69564f2b36ff8f0e86dcb876e6c031647340a7b1bef648d1e0366b9",
125
+ "src/repair-verifier.ts": "b70fb11feee8cdc7933947ccac0b78f8bb5c0eaccc3b29ad527197d77f40ffc9",
126
+ "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
127
+ "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
128
+ "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
129
+ "src/runtime-checkpoints.ts": "e3b75a0b86fb5d21992b1a126b35446110ba5fcaf4317ed5005b1a44555118dc",
130
+ "src/runtime-diagnostics.ts": "e1e5a3a558fa2aaeb477335dc336551f022c1234fe5e742eefc6599c8803e364",
131
+ "src/runtime-policy.ts": "8356acf0102f0e73839b81ed628a5c0d9a89d9c21b0ca416efedde8f1b3414e2",
132
+ "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
133
+ "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
134
+ "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
135
+ "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
136
+ "src/task-memory.ts": "5b62c863be5a23c33608633cd7aac0464f5536d01cba930aca58b11870170731",
137
+ "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
138
+ "src/terminal-ui.ts": "4da1878a76ba608a2517bf5d5c992a9a88702aba3fd800a9980129ba93e157bc",
139
+ "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
140
+ "src/tool-registry.ts": "7e90fff11940e525883c9ec0d2adbea2e6616f8bd9bc4ec77e1b7277e4217f2c",
141
+ "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
142
+ "src/workspace-mutation-lock.ts": "ca201242db16173bd3a4d86c5d8bb0c2b6f3c43bfed3e4fa4ff5f06a75b19e80",
143
+ "src/workspace.ts": "809f8bbfcf72fed10d860c48585089ca65bb3ef7c672ee6df3c918907a968302"
144
+ }
145
+ },
146
+ "host": {
147
+ "system": "Darwin",
148
+ "architecture": "arm64",
149
+ "python": "3.12.13",
150
+ "docker": "29.8.0"
151
+ },
152
+ "pricingUsdPerMillion": null,
153
+ "approvalPolicy": "Automated operator approves supported isolated Zhivex tools; successful approved verification and patch import is a terminal receipt. Human wait is zero. Not a human-UX comparison."
154
+ },
155
+ "samples": [
156
+ {
157
+ "candidate": "zhivex",
158
+ "instanceId": "django__django-15851",
159
+ "repetition": 0,
160
+ "resolved": false,
161
+ "gradingStatus": "completed",
162
+ "usageComplete": true,
163
+ "costUsd": null,
164
+ "candidateCaptureFailure": null,
165
+ "modelTimings": [
166
+ {
167
+ "durationMs": 3315.1819170000003,
168
+ "firstTokenMs": 3314.4135840000004,
169
+ "completed": true
170
+ },
171
+ {
172
+ "durationMs": 1455.2151669999994,
173
+ "firstTokenMs": 1455.0647500000005,
174
+ "completed": true
175
+ },
176
+ {
177
+ "durationMs": 2136.093666999999,
178
+ "firstTokenMs": 2135.8274999999994,
179
+ "completed": true
180
+ },
181
+ {
182
+ "durationMs": 1896.959832999999,
183
+ "firstTokenMs": 1896.7155829999992,
184
+ "completed": true
185
+ },
186
+ {
187
+ "durationMs": 1926.1423750000013,
188
+ "firstTokenMs": 1926.0177089999997,
189
+ "completed": true
190
+ },
191
+ {
192
+ "durationMs": 1986.674916,
193
+ "firstTokenMs": 1986.4686249999977,
194
+ "completed": true
195
+ },
196
+ {
197
+ "durationMs": 5250.132916999999,
198
+ "firstTokenMs": 5249.963416999999,
199
+ "completed": true
200
+ },
201
+ {
202
+ "durationMs": 3030.0514999999978,
203
+ "firstTokenMs": 3029.913166999999,
204
+ "completed": true
205
+ },
206
+ {
207
+ "durationMs": 1911.8876669999954,
208
+ "firstTokenMs": 1911.6447499999995,
209
+ "completed": true
210
+ },
211
+ {
212
+ "durationMs": 5536.965166999995,
213
+ "firstTokenMs": 5536.646124999999,
214
+ "completed": true
215
+ },
216
+ {
217
+ "durationMs": 1598.489791,
218
+ "firstTokenMs": 1598.2655000000013,
219
+ "completed": true
220
+ },
221
+ {
222
+ "durationMs": 1635.8450830000074,
223
+ "firstTokenMs": 1635.7173750000075,
224
+ "completed": true
225
+ },
226
+ {
227
+ "durationMs": 1785.924291999996,
228
+ "firstTokenMs": 1785.8526249999995,
229
+ "completed": true
230
+ }
231
+ ],
232
+ "modelCatalog": {
233
+ "records": [
234
+ {
235
+ "call": 1,
236
+ "offered": [
237
+ "apply_reviewed_edits",
238
+ "apply_reviewed_replacement",
239
+ "inspect_environment_patch",
240
+ "list_files",
241
+ "mutation_audit",
242
+ "read_files",
243
+ "read_task",
244
+ "repair_plan",
245
+ "run_environment_command",
246
+ "run_environment_shell",
247
+ "search_files",
248
+ "search_many",
249
+ "verify_and_apply_environment_patch"
250
+ ],
251
+ "unknownOffered": 0,
252
+ "systemMessages": 1,
253
+ "choice": "required",
254
+ "returned": [
255
+ {
256
+ "name": "search_files",
257
+ "offered": true,
258
+ "registered": true
259
+ }
260
+ ],
261
+ "omittedReturned": 0,
262
+ "completed": true
263
+ },
264
+ {
265
+ "call": 2,
266
+ "offered": [
267
+ "apply_reviewed_edits",
268
+ "apply_reviewed_replacement",
269
+ "inspect_environment_patch",
270
+ "list_files",
271
+ "mutation_audit",
272
+ "read_files",
273
+ "read_task",
274
+ "repair_plan",
275
+ "run_environment_command",
276
+ "run_environment_shell",
277
+ "search_files",
278
+ "search_many",
279
+ "verify_and_apply_environment_patch"
280
+ ],
281
+ "unknownOffered": 0,
282
+ "systemMessages": 1,
283
+ "choice": "required",
284
+ "returned": [
285
+ {
286
+ "name": "read_files",
287
+ "offered": true,
288
+ "registered": true
289
+ }
290
+ ],
291
+ "omittedReturned": 0,
292
+ "completed": true
293
+ },
294
+ {
295
+ "call": 3,
296
+ "offered": [
297
+ "apply_reviewed_edits",
298
+ "apply_reviewed_replacement",
299
+ "inspect_environment_patch",
300
+ "list_files",
301
+ "mutation_audit",
302
+ "read_files",
303
+ "read_task",
304
+ "repair_plan",
305
+ "run_environment_command",
306
+ "run_environment_shell",
307
+ "search_files",
308
+ "search_many",
309
+ "verify_and_apply_environment_patch"
310
+ ],
311
+ "unknownOffered": 0,
312
+ "systemMessages": 1,
313
+ "choice": "required",
314
+ "returned": [
315
+ {
316
+ "name": "search_files",
317
+ "offered": true,
318
+ "registered": true
319
+ }
320
+ ],
321
+ "omittedReturned": 0,
322
+ "completed": true
323
+ },
324
+ {
325
+ "call": 4,
326
+ "offered": [
327
+ "apply_reviewed_edits",
328
+ "apply_reviewed_replacement",
329
+ "inspect_environment_patch",
330
+ "list_files",
331
+ "mutation_audit",
332
+ "read_files",
333
+ "read_task",
334
+ "repair_plan",
335
+ "run_environment_command",
336
+ "run_environment_shell",
337
+ "search_files",
338
+ "search_many",
339
+ "verify_and_apply_environment_patch"
340
+ ],
341
+ "unknownOffered": 0,
342
+ "systemMessages": 1,
343
+ "choice": "required",
344
+ "returned": [
345
+ {
346
+ "name": "search_files",
347
+ "offered": true,
348
+ "registered": true
349
+ }
350
+ ],
351
+ "omittedReturned": 0,
352
+ "completed": true
353
+ },
354
+ {
355
+ "call": 5,
356
+ "offered": [
357
+ "apply_reviewed_edits",
358
+ "apply_reviewed_replacement",
359
+ "inspect_environment_patch",
360
+ "list_files",
361
+ "mutation_audit",
362
+ "read_files",
363
+ "read_task",
364
+ "repair_plan",
365
+ "run_environment_command",
366
+ "run_environment_shell",
367
+ "search_files",
368
+ "search_many",
369
+ "verify_and_apply_environment_patch"
370
+ ],
371
+ "unknownOffered": 0,
372
+ "systemMessages": 1,
373
+ "choice": "required",
374
+ "returned": [
375
+ {
376
+ "name": "read_files",
377
+ "offered": true,
378
+ "registered": true
379
+ }
380
+ ],
381
+ "omittedReturned": 0,
382
+ "completed": true
383
+ },
384
+ {
385
+ "call": 6,
386
+ "offered": [
387
+ "apply_reviewed_edits",
388
+ "apply_reviewed_replacement",
389
+ "inspect_environment_patch",
390
+ "list_files",
391
+ "mutation_audit",
392
+ "read_files",
393
+ "read_task",
394
+ "repair_plan",
395
+ "run_environment_command",
396
+ "run_environment_shell",
397
+ "search_files",
398
+ "search_many",
399
+ "verify_and_apply_environment_patch"
400
+ ],
401
+ "unknownOffered": 0,
402
+ "systemMessages": 1,
403
+ "choice": "required",
404
+ "returned": [
405
+ {
406
+ "name": "read_files",
407
+ "offered": true,
408
+ "registered": true
409
+ }
410
+ ],
411
+ "omittedReturned": 0,
412
+ "completed": true
413
+ },
414
+ {
415
+ "call": 7,
416
+ "offered": [
417
+ "apply_reviewed_edits",
418
+ "apply_reviewed_replacement",
419
+ "inspect_environment_patch",
420
+ "list_files",
421
+ "mutation_audit",
422
+ "read_files",
423
+ "read_task",
424
+ "repair_plan",
425
+ "run_environment_command",
426
+ "run_environment_shell",
427
+ "search_files",
428
+ "search_many",
429
+ "verify_and_apply_environment_patch"
430
+ ],
431
+ "unknownOffered": 0,
432
+ "systemMessages": 1,
433
+ "choice": "required",
434
+ "returned": [
435
+ {
436
+ "name": "repair_plan",
437
+ "offered": true,
438
+ "registered": true
439
+ }
440
+ ],
441
+ "omittedReturned": 0,
442
+ "completed": true
443
+ },
444
+ {
445
+ "call": 8,
446
+ "offered": [
447
+ "apply_reviewed_edits",
448
+ "apply_reviewed_replacement",
449
+ "inspect_environment_patch",
450
+ "list_files",
451
+ "mutation_audit",
452
+ "read_files",
453
+ "read_task",
454
+ "repair_plan",
455
+ "run_environment_command",
456
+ "run_environment_shell",
457
+ "search_files",
458
+ "search_many",
459
+ "verify_and_apply_environment_patch"
460
+ ],
461
+ "unknownOffered": 0,
462
+ "systemMessages": 1,
463
+ "choice": "required",
464
+ "returned": [
465
+ {
466
+ "name": "apply_reviewed_replacement",
467
+ "offered": true,
468
+ "registered": true
469
+ }
470
+ ],
471
+ "omittedReturned": 0,
472
+ "completed": true
473
+ },
474
+ {
475
+ "call": 9,
476
+ "offered": [
477
+ "apply_reviewed_edits",
478
+ "apply_reviewed_replacement",
479
+ "inspect_environment_patch",
480
+ "list_files",
481
+ "mutation_audit",
482
+ "read_files",
483
+ "read_task",
484
+ "repair_plan",
485
+ "run_environment_command",
486
+ "run_environment_shell",
487
+ "search_files",
488
+ "search_many",
489
+ "verify_and_apply_environment_patch"
490
+ ],
491
+ "unknownOffered": 0,
492
+ "systemMessages": 1,
493
+ "choice": "required",
494
+ "returned": [
495
+ {
496
+ "name": "run_environment_shell",
497
+ "offered": true,
498
+ "registered": true
499
+ }
500
+ ],
501
+ "omittedReturned": 0,
502
+ "completed": true
503
+ },
504
+ {
505
+ "call": 10,
506
+ "offered": [
507
+ "read_task",
508
+ "repair_plan"
509
+ ],
510
+ "unknownOffered": 0,
511
+ "systemMessages": 1,
512
+ "choice": "named",
513
+ "returned": [
514
+ {
515
+ "name": "repair_plan",
516
+ "offered": true,
517
+ "registered": true
518
+ }
519
+ ],
520
+ "omittedReturned": 0,
521
+ "completed": true
522
+ },
523
+ {
524
+ "call": 11,
525
+ "offered": [
526
+ "apply_reviewed_edits",
527
+ "apply_reviewed_replacement",
528
+ "inspect_environment_patch",
529
+ "list_files",
530
+ "mutation_audit",
531
+ "read_files",
532
+ "read_task",
533
+ "repair_plan",
534
+ "run_environment_command",
535
+ "run_environment_shell",
536
+ "search_files",
537
+ "search_many",
538
+ "verify_and_apply_environment_patch"
539
+ ],
540
+ "unknownOffered": 0,
541
+ "systemMessages": 1,
542
+ "choice": "required",
543
+ "returned": [
544
+ {
545
+ "name": "run_environment_shell",
546
+ "offered": true,
547
+ "registered": true
548
+ }
549
+ ],
550
+ "omittedReturned": 0,
551
+ "completed": true
552
+ },
553
+ {
554
+ "call": 12,
555
+ "offered": [
556
+ "apply_reviewed_edits",
557
+ "apply_reviewed_replacement",
558
+ "inspect_environment_patch",
559
+ "list_files",
560
+ "mutation_audit",
561
+ "read_files",
562
+ "read_task",
563
+ "repair_plan",
564
+ "run_environment_command",
565
+ "run_environment_shell",
566
+ "search_files",
567
+ "search_many",
568
+ "verify_and_apply_environment_patch"
569
+ ],
570
+ "unknownOffered": 0,
571
+ "systemMessages": 1,
572
+ "choice": "required",
573
+ "returned": [
574
+ {
575
+ "name": "list_files",
576
+ "offered": true,
577
+ "registered": true
578
+ }
579
+ ],
580
+ "omittedReturned": 0,
581
+ "completed": true
582
+ },
583
+ {
584
+ "call": 13,
585
+ "offered": [
586
+ "apply_reviewed_edits",
587
+ "apply_reviewed_replacement",
588
+ "inspect_environment_patch",
589
+ "list_files",
590
+ "mutation_audit",
591
+ "read_files",
592
+ "read_task",
593
+ "repair_plan",
594
+ "run_environment_command",
595
+ "run_environment_shell",
596
+ "search_files",
597
+ "search_many",
598
+ "verify_and_apply_environment_patch"
599
+ ],
600
+ "unknownOffered": 0,
601
+ "systemMessages": 1,
602
+ "choice": "required",
603
+ "returned": [
604
+ {
605
+ "name": "run_environment_shell",
606
+ "offered": true,
607
+ "registered": true
608
+ }
609
+ ],
610
+ "omittedReturned": 0,
611
+ "completed": true
612
+ }
613
+ ],
614
+ "omittedCalls": 0
615
+ },
616
+ "stageTimings": [
617
+ {
618
+ "event": "edit-requested",
619
+ "elapsedMs": 25012.462042
620
+ },
621
+ {
622
+ "event": "verification-requested",
623
+ "elapsedMs": 26891.442375
624
+ },
625
+ {
626
+ "event": "governed-edit-completed",
627
+ "elapsedMs": 26898.649209
628
+ },
629
+ {
630
+ "event": "command-requested",
631
+ "elapsedMs": 34440.264917
632
+ }
633
+ ],
634
+ "approvalTimings": [
635
+ {
636
+ "durationMs": 0.21370800000295276,
637
+ "resolved": true
638
+ },
639
+ {
640
+ "durationMs": 0.018040999999357155,
641
+ "resolved": true
642
+ },
643
+ {
644
+ "durationMs": 0.033290999999735504,
645
+ "resolved": true
646
+ },
647
+ {
648
+ "durationMs": 0.03329200000007404,
649
+ "resolved": true
650
+ },
651
+ {
652
+ "durationMs": 0.023915999998280313,
653
+ "resolved": true
654
+ },
655
+ {
656
+ "durationMs": 0.03429100000357721,
657
+ "resolved": true
658
+ }
659
+ ],
660
+ "toolTimings": [
661
+ {
662
+ "name": "search_files",
663
+ "calls": 3,
664
+ "errors": 0,
665
+ "totalMs": 770.7560420000018,
666
+ "maxMs": 550.4601670000002
667
+ },
668
+ {
669
+ "name": "read_files",
670
+ "calls": 3,
671
+ "errors": 0,
672
+ "totalMs": 3.022541000000274,
673
+ "maxMs": 1.6277909999989788
674
+ },
675
+ {
676
+ "name": "repair_plan",
677
+ "calls": 2,
678
+ "errors": 0,
679
+ "totalMs": 0.15924999999333522,
680
+ "maxMs": 0.10574999999880674
681
+ },
682
+ {
683
+ "name": "apply_reviewed_replacement",
684
+ "calls": 1,
685
+ "errors": 0,
686
+ "totalMs": 8.72650000000067,
687
+ "maxMs": 8.72650000000067
688
+ },
689
+ {
690
+ "name": "verify_and_apply_environment_patch",
691
+ "calls": 2,
692
+ "errors": 2,
693
+ "totalMs": 8934.487166000006,
694
+ "maxMs": 4495.965541000005
695
+ },
696
+ {
697
+ "name": "run_environment_shell",
698
+ "calls": 3,
699
+ "errors": 0,
700
+ "totalMs": 30546.374374000014,
701
+ "maxMs": 15256.078500000003
702
+ }
703
+ ],
704
+ "schemaVersion": 1,
705
+ "status": "failed",
706
+ "failure": "INPUT_TOKEN_BUDGET",
707
+ "phase": "agent",
708
+ "durationMs": 88393.40016700001,
709
+ "approvals": 6,
710
+ "turns": [
711
+ {
712
+ "index": 1,
713
+ "status": "completed",
714
+ "finishReason": "tool-calls",
715
+ "requestedTools": [
716
+ "search_files"
717
+ ],
718
+ "inputTokens": 4652,
719
+ "outputTokens": 65,
720
+ "cachedInputTokens": 0,
721
+ "requestCharacters": 7658,
722
+ "tools": [
723
+ {
724
+ "name": "search_files",
725
+ "isError": false,
726
+ "characters": 346
727
+ }
728
+ ]
729
+ },
730
+ {
731
+ "index": 2,
732
+ "status": "completed",
733
+ "finishReason": "tool-calls",
734
+ "requestedTools": [
735
+ "read_files"
736
+ ],
737
+ "inputTokens": 4883,
738
+ "outputTokens": 36,
739
+ "cachedInputTokens": 4096,
740
+ "requestCharacters": 8394,
741
+ "tools": [
742
+ {
743
+ "name": "read_files",
744
+ "isError": false,
745
+ "characters": 2673
746
+ }
747
+ ]
748
+ },
749
+ {
750
+ "index": 3,
751
+ "status": "completed",
752
+ "finishReason": "tool-calls",
753
+ "requestedTools": [
754
+ "search_files"
755
+ ],
756
+ "inputTokens": 5804,
757
+ "outputTokens": 60,
758
+ "cachedInputTokens": 4096,
759
+ "requestCharacters": 11419,
760
+ "tools": [
761
+ {
762
+ "name": "search_files",
763
+ "isError": false,
764
+ "characters": 164
765
+ }
766
+ ]
767
+ },
768
+ {
769
+ "index": 4,
770
+ "status": "completed",
771
+ "finishReason": "tool-calls",
772
+ "requestedTools": [
773
+ "search_files"
774
+ ],
775
+ "inputTokens": 5940,
776
+ "outputTokens": 64,
777
+ "cachedInputTokens": 5120,
778
+ "requestCharacters": 11957,
779
+ "tools": [
780
+ {
781
+ "name": "search_files",
782
+ "isError": false,
783
+ "characters": 4521
784
+ }
785
+ ]
786
+ },
787
+ {
788
+ "index": 5,
789
+ "status": "completed",
790
+ "finishReason": "tool-calls",
791
+ "requestedTools": [
792
+ "read_files"
793
+ ],
794
+ "inputTokens": 8166,
795
+ "outputTokens": 35,
796
+ "cachedInputTokens": 5120,
797
+ "requestCharacters": 16864,
798
+ "tools": [
799
+ {
800
+ "name": "read_files",
801
+ "isError": false,
802
+ "characters": 4790
803
+ }
804
+ ]
805
+ },
806
+ {
807
+ "index": 6,
808
+ "status": "completed",
809
+ "finishReason": "tool-calls",
810
+ "requestedTools": [
811
+ "read_files"
812
+ ],
813
+ "inputTokens": 9653,
814
+ "outputTokens": 52,
815
+ "cachedInputTokens": 7168,
816
+ "requestCharacters": 21999,
817
+ "tools": [
818
+ {
819
+ "name": "read_files",
820
+ "isError": false,
821
+ "characters": 3399
822
+ }
823
+ ]
824
+ },
825
+ {
826
+ "index": 7,
827
+ "status": "completed",
828
+ "finishReason": "tool-calls",
829
+ "requestedTools": [
830
+ "repair_plan"
831
+ ],
832
+ "inputTokens": 10811,
833
+ "outputTokens": 263,
834
+ "cachedInputTokens": 9216,
835
+ "requestCharacters": 25773,
836
+ "tools": [
837
+ {
838
+ "name": "repair_plan",
839
+ "isError": false,
840
+ "characters": 1002
841
+ }
842
+ ]
843
+ },
844
+ {
845
+ "index": 8,
846
+ "status": "waiting_approval",
847
+ "finishReason": "tool-calls",
848
+ "requestedTools": [
849
+ "apply_reviewed_replacement"
850
+ ],
851
+ "inputTokens": 11383,
852
+ "outputTokens": 164,
853
+ "cachedInputTokens": 10240,
854
+ "requestCharacters": 28386,
855
+ "tools": []
856
+ },
857
+ {
858
+ "index": 9,
859
+ "status": "completed",
860
+ "finishReason": "tool-calls",
861
+ "requestedTools": [
862
+ "verify_and_apply_environment_patch"
863
+ ],
864
+ "inputTokens": 0,
865
+ "outputTokens": 0,
866
+ "cachedInputTokens": 0,
867
+ "requestCharacters": 12176,
868
+ "tools": [
869
+ {
870
+ "name": "apply_reviewed_replacement",
871
+ "isError": false,
872
+ "characters": 483
873
+ },
874
+ {
875
+ "name": "verify_and_apply_environment_patch",
876
+ "isError": true,
877
+ "characters": 436,
878
+ "exitCode": 1,
879
+ "timedOut": false,
880
+ "verificationOutcome": "failed",
881
+ "diagnostic": {
882
+ "code": "EXECUTION_FAILED",
883
+ "category": "execution",
884
+ "failureType": "execution",
885
+ "retryable": false,
886
+ "chain": [
887
+ {
888
+ "validationCode": null,
889
+ "verifierExitCode": 1,
890
+ "providerReason": null,
891
+ "validationIssues": [],
892
+ "status": null,
893
+ "errorClass": "Object",
894
+ "providerHints": [],
895
+ "fingerprint": "d22955ecb1810cd7e960e732636eec570e04ab3b19eab276ebf23e2096a3c941",
896
+ "reason": null
897
+ }
898
+ ]
899
+ }
900
+ }
901
+ ]
902
+ },
903
+ {
904
+ "index": 10,
905
+ "status": "waiting_approval",
906
+ "finishReason": "tool-calls",
907
+ "requestedTools": [
908
+ "run_environment_shell"
909
+ ],
910
+ "inputTokens": 6509,
911
+ "outputTokens": 55,
912
+ "cachedInputTokens": 3072,
913
+ "requestCharacters": 13661,
914
+ "tools": []
915
+ },
916
+ {
917
+ "index": 11,
918
+ "status": "completed",
919
+ "finishReason": "tool-calls",
920
+ "requestedTools": [
921
+ "repair_plan"
922
+ ],
923
+ "inputTokens": 4595,
924
+ "outputTokens": 261,
925
+ "cachedInputTokens": 0,
926
+ "requestCharacters": 14944,
927
+ "tools": [
928
+ {
929
+ "name": "run_environment_shell",
930
+ "isError": false,
931
+ "characters": 475,
932
+ "phaseLatencies": {
933
+ "commandAndAttestationMs": 2755.772083,
934
+ "workspaceExportMs": 7273.672125,
935
+ "totalMs": 12429.716625,
936
+ "sessionCreationMs": 2400.124042
937
+ },
938
+ "exitCode": 0,
939
+ "timedOut": false
940
+ }
941
+ ]
942
+ },
943
+ {
944
+ "index": 12,
945
+ "status": "completed",
946
+ "finishReason": "tool-calls",
947
+ "requestedTools": [
948
+ "verify_and_apply_environment_patch"
949
+ ],
950
+ "inputTokens": 0,
951
+ "outputTokens": 0,
952
+ "cachedInputTokens": 0,
953
+ "requestCharacters": 17236,
954
+ "tools": [
955
+ {
956
+ "name": "repair_plan",
957
+ "isError": false,
958
+ "characters": 988
959
+ },
960
+ {
961
+ "name": "verify_and_apply_environment_patch",
962
+ "isError": true,
963
+ "characters": 465,
964
+ "exitCode": 2,
965
+ "timedOut": false,
966
+ "verificationOutcome": "failed",
967
+ "diagnostic": {
968
+ "code": "EXECUTION_FAILED",
969
+ "category": "execution",
970
+ "failureType": "execution",
971
+ "retryable": false,
972
+ "chain": [
973
+ {
974
+ "validationCode": null,
975
+ "verifierExitCode": 2,
976
+ "providerReason": null,
977
+ "validationIssues": [],
978
+ "status": null,
979
+ "errorClass": "Object",
980
+ "providerHints": [],
981
+ "fingerprint": "4ff86268eac7f800eb2c27715a6c688c0ec016b7f588f60e55561e25acc71aba",
982
+ "reason": null
983
+ }
984
+ ]
985
+ }
986
+ }
987
+ ]
988
+ },
989
+ {
990
+ "index": 13,
991
+ "status": "waiting_approval",
992
+ "finishReason": "tool-calls",
993
+ "requestedTools": [
994
+ "run_environment_shell"
995
+ ],
996
+ "inputTokens": 7626,
997
+ "outputTokens": 56,
998
+ "cachedInputTokens": 5120,
999
+ "requestCharacters": 18386,
1000
+ "tools": []
1001
+ },
1002
+ {
1003
+ "index": 14,
1004
+ "status": "completed",
1005
+ "finishReason": "tool-calls",
1006
+ "requestedTools": [
1007
+ "list_files"
1008
+ ],
1009
+ "inputTokens": 7872,
1010
+ "outputTokens": 48,
1011
+ "cachedInputTokens": 6144,
1012
+ "requestCharacters": 19353,
1013
+ "tools": [
1014
+ {
1015
+ "name": "run_environment_shell",
1016
+ "isError": false,
1017
+ "characters": 574,
1018
+ "phaseLatencies": {
1019
+ "commandAndAttestationMs": 539.834334,
1020
+ "totalMs": 2860.220541,
1021
+ "sessionCreationMs": 2210.532292
1022
+ },
1023
+ "exitCode": 2,
1024
+ "timedOut": false
1025
+ }
1026
+ ]
1027
+ },
1028
+ {
1029
+ "index": 15,
1030
+ "status": "waiting_approval",
1031
+ "finishReason": "tool-calls",
1032
+ "requestedTools": [
1033
+ "run_environment_shell"
1034
+ ],
1035
+ "inputTokens": 7994,
1036
+ "outputTokens": 82,
1037
+ "cachedInputTokens": 6656,
1038
+ "requestCharacters": 19942,
1039
+ "tools": [
1040
+ {
1041
+ "name": "list_files",
1042
+ "isError": true,
1043
+ "characters": 4,
1044
+ "diagnostic": {
1045
+ "code": "EXECUTION_FAILED",
1046
+ "category": "execution",
1047
+ "failureType": "budget",
1048
+ "retryable": false,
1049
+ "chain": [
1050
+ {
1051
+ "validationCode": null,
1052
+ "verifierExitCode": null,
1053
+ "providerReason": null,
1054
+ "validationIssues": [],
1055
+ "status": null,
1056
+ "errorClass": "Object",
1057
+ "providerHints": [],
1058
+ "fingerprint": "f0b8f211b295b6091f99106b1308508d1d6377969ce64e95bb2da008f681224e",
1059
+ "reason": "budget"
1060
+ }
1061
+ ]
1062
+ }
1063
+ },
1064
+ {
1065
+ "name": "run_environment_shell",
1066
+ "isError": false,
1067
+ "characters": 845,
1068
+ "phaseLatencies": {
1069
+ "commandAndAttestationMs": 3144.983958,
1070
+ "workspaceExportMs": 8103.587583,
1071
+ "totalMs": 15255.812625,
1072
+ "sessionCreationMs": 4006.959792
1073
+ },
1074
+ "exitCode": 0,
1075
+ "timedOut": false
1076
+ }
1077
+ ]
1078
+ }
1079
+ ],
1080
+ "inputTokens": 95888,
1081
+ "outputTokens": 1241,
1082
+ "cachedInputTokens": 66048,
1083
+ "modelCalls": 13,
1084
+ "compactions": [
1085
+ {
1086
+ "beforeStep": 9,
1087
+ "estimatedTokensBefore": 6888,
1088
+ "estimatedTokensAfter": 2533
1089
+ },
1090
+ {
1091
+ "beforeStep": 16,
1092
+ "estimatedTokensBefore": 4597,
1093
+ "estimatedTokensAfter": 2682
1094
+ }
1095
+ ],
1096
+ "terminalDiagnostic": {
1097
+ "code": "EXECUTION_FAILED",
1098
+ "category": "execution",
1099
+ "failureType": "budget",
1100
+ "retryable": false,
1101
+ "chain": [
1102
+ {
1103
+ "validationCode": null,
1104
+ "verifierExitCode": null,
1105
+ "providerReason": null,
1106
+ "validationIssues": [],
1107
+ "status": null,
1108
+ "errorClass": "Object",
1109
+ "providerHints": [],
1110
+ "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
1111
+ "reason": "budget"
1112
+ }
1113
+ ]
1114
+ },
1115
+ "thrownDiagnostic": {
1116
+ "code": "EXECUTION_FAILED",
1117
+ "category": "execution",
1118
+ "failureType": "budget",
1119
+ "retryable": false,
1120
+ "chain": [
1121
+ {
1122
+ "validationCode": null,
1123
+ "verifierExitCode": null,
1124
+ "providerReason": null,
1125
+ "validationIssues": [],
1126
+ "status": null,
1127
+ "errorClass": "HarnessExecutionError",
1128
+ "providerHints": [],
1129
+ "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
1130
+ "reason": "budget"
1131
+ },
1132
+ {
1133
+ "validationCode": null,
1134
+ "verifierExitCode": null,
1135
+ "providerReason": null,
1136
+ "validationIssues": [],
1137
+ "status": null,
1138
+ "errorClass": "Error",
1139
+ "providerHints": [],
1140
+ "fingerprint": "064591d1d6249f6eadfb2800eddbdfe8576a334f23f5e1215117dc19107f21f4",
1141
+ "reason": "budget"
1142
+ }
1143
+ ]
1144
+ },
1145
+ "guardrails": [],
1146
+ "repairProgress": {
1147
+ "repeatedResults": 0,
1148
+ "suppressedResults": 0,
1149
+ "blockedBroadCalls": 1,
1150
+ "enteredClosure": true,
1151
+ "phase": "verify",
1152
+ "blockedPhaseCalls": 0,
1153
+ "planRecorded": true
1154
+ },
1155
+ "contextMetrics": [
1156
+ {
1157
+ "systemCharacters": 5073,
1158
+ "userCharacters": 2581,
1159
+ "assistantCharacters": 0,
1160
+ "toolResultCharacters": 0,
1161
+ "otherMessageCharacters": 0,
1162
+ "toolDefinitionCharacters": 9375,
1163
+ "unmeasuredToolDefinitions": 0
1164
+ },
1165
+ {
1166
+ "systemCharacters": 5073,
1167
+ "userCharacters": 2581,
1168
+ "assistantCharacters": 228,
1169
+ "toolResultCharacters": 506,
1170
+ "otherMessageCharacters": 0,
1171
+ "toolDefinitionCharacters": 9375,
1172
+ "unmeasuredToolDefinitions": 0
1173
+ },
1174
+ {
1175
+ "systemCharacters": 5073,
1176
+ "userCharacters": 2581,
1177
+ "assistantCharacters": 420,
1178
+ "toolResultCharacters": 3337,
1179
+ "otherMessageCharacters": 0,
1180
+ "toolDefinitionCharacters": 9375,
1181
+ "unmeasuredToolDefinitions": 0
1182
+ },
1183
+ {
1184
+ "systemCharacters": 5073,
1185
+ "userCharacters": 2581,
1186
+ "assistantCharacters": 632,
1187
+ "toolResultCharacters": 3661,
1188
+ "otherMessageCharacters": 0,
1189
+ "toolDefinitionCharacters": 9375,
1190
+ "unmeasuredToolDefinitions": 0
1191
+ },
1192
+ {
1193
+ "systemCharacters": 5073,
1194
+ "userCharacters": 2581,
1195
+ "assistantCharacters": 856,
1196
+ "toolResultCharacters": 8342,
1197
+ "otherMessageCharacters": 0,
1198
+ "toolDefinitionCharacters": 9375,
1199
+ "unmeasuredToolDefinitions": 0
1200
+ },
1201
+ {
1202
+ "systemCharacters": 5073,
1203
+ "userCharacters": 2581,
1204
+ "assistantCharacters": 1041,
1205
+ "toolResultCharacters": 13290,
1206
+ "otherMessageCharacters": 0,
1207
+ "toolDefinitionCharacters": 9375,
1208
+ "unmeasuredToolDefinitions": 0
1209
+ },
1210
+ {
1211
+ "systemCharacters": 5073,
1212
+ "userCharacters": 2581,
1213
+ "assistantCharacters": 1256,
1214
+ "toolResultCharacters": 16847,
1215
+ "otherMessageCharacters": 0,
1216
+ "toolDefinitionCharacters": 9375,
1217
+ "unmeasuredToolDefinitions": 0
1218
+ },
1219
+ {
1220
+ "systemCharacters": 5073,
1221
+ "userCharacters": 2936,
1222
+ "assistantCharacters": 2351,
1223
+ "toolResultCharacters": 18008,
1224
+ "otherMessageCharacters": 0,
1225
+ "toolDefinitionCharacters": 9375,
1226
+ "unmeasuredToolDefinitions": 0
1227
+ },
1228
+ {
1229
+ "systemCharacters": 5073,
1230
+ "userCharacters": 2460,
1231
+ "assistantCharacters": 4723,
1232
+ "toolResultCharacters": 1397,
1233
+ "otherMessageCharacters": 0,
1234
+ "toolDefinitionCharacters": 9375,
1235
+ "unmeasuredToolDefinitions": 0
1236
+ },
1237
+ {
1238
+ "systemCharacters": 5073,
1239
+ "userCharacters": 2856,
1240
+ "assistantCharacters": 4964,
1241
+ "toolResultCharacters": 2041,
1242
+ "otherMessageCharacters": 0,
1243
+ "toolDefinitionCharacters": 1635,
1244
+ "unmeasuredToolDefinitions": 0
1245
+ },
1246
+ {
1247
+ "systemCharacters": 5073,
1248
+ "userCharacters": 2957,
1249
+ "assistantCharacters": 6385,
1250
+ "toolResultCharacters": 3957,
1251
+ "otherMessageCharacters": 0,
1252
+ "toolDefinitionCharacters": 9375,
1253
+ "unmeasuredToolDefinitions": 0
1254
+ },
1255
+ {
1256
+ "systemCharacters": 5073,
1257
+ "userCharacters": 2900,
1258
+ "assistantCharacters": 6664,
1259
+ "toolResultCharacters": 4700,
1260
+ "otherMessageCharacters": 0,
1261
+ "toolDefinitionCharacters": 9375,
1262
+ "unmeasuredToolDefinitions": 0
1263
+ },
1264
+ {
1265
+ "systemCharacters": 5073,
1266
+ "userCharacters": 2900,
1267
+ "assistantCharacters": 6844,
1268
+ "toolResultCharacters": 5107,
1269
+ "otherMessageCharacters": 0,
1270
+ "toolDefinitionCharacters": 9375,
1271
+ "unmeasuredToolDefinitions": 0
1272
+ }
1273
+ ],
1274
+ "omittedContextMeasurements": 0,
1275
+ "toolCalls": 14,
1276
+ "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
1277
+ "protectedFilesChanged": [],
1278
+ "gradingMethod": "empty-patch-no-solution",
1279
+ "officialResolved": false,
1280
+ "candidateCaptured": true,
1281
+ "candidatePatchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
1282
+ "candidateGradingStatus": "completed",
1283
+ "candidateResolved": true,
1284
+ "repairStages": {
1285
+ "candidatePresent": true,
1286
+ "importedPatchPresent": false,
1287
+ "verificationAttempted": true
1288
+ },
1289
+ "gradingDurationMs": 25452.03741703881,
1290
+ "totalDurationMs": 128156.90558298957
1291
+ },
1292
+ {
1293
+ "candidate": "mini-swe-agent",
1294
+ "instanceId": "django__django-15851",
1295
+ "repetition": 0,
1296
+ "resolved": true,
1297
+ "gradingStatus": "completed",
1298
+ "usageComplete": true,
1299
+ "costUsd": null,
1300
+ "schemaVersion": 1,
1301
+ "status": "Submitted",
1302
+ "failure": null,
1303
+ "phase": "agent",
1304
+ "durationMs": 29804.76358399028,
1305
+ "approvals": 0,
1306
+ "toolCalls": 11,
1307
+ "inputTokens": 32788,
1308
+ "outputTokens": 886,
1309
+ "cachedInputTokens": 24064,
1310
+ "modelCalls": 11,
1311
+ "patchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
1312
+ "protectedFilesChanged": [],
1313
+ "officialResolved": true,
1314
+ "candidateCaptured": true,
1315
+ "candidatePatchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
1316
+ "candidateGradingStatus": "completed",
1317
+ "candidateResolved": true,
1318
+ "repairStages": {
1319
+ "candidatePresent": true,
1320
+ "importedPatchPresent": null,
1321
+ "verificationAttempted": null
1322
+ },
1323
+ "gradingDurationMs": 22976.258167007472,
1324
+ "totalDurationMs": 54962.95591699891
1325
+ }
1326
+ ],
1327
+ "report": {
1328
+ "schemaVersion": 1,
1329
+ "kind": "swebench-comparison",
1330
+ "manifestSha256": "bc1cfdc98246a8fc09b52d43fbf648e15f377fd2839878ead8c81278ce689df5",
1331
+ "complete": true,
1332
+ "candidates": {
1333
+ "zhivex": {
1334
+ "planned": 1,
1335
+ "recorded": 1,
1336
+ "resolved": 0,
1337
+ "candidateCaptured": 1,
1338
+ "candidateResolved": 1,
1339
+ "candidateGraded": 1,
1340
+ "resolutionRate": 0.0,
1341
+ "missing": 0,
1342
+ "gradingFailures": 0,
1343
+ "usageComplete": true,
1344
+ "totalCostUsd": null,
1345
+ "costPerResolvedUsd": null,
1346
+ "resolvedP50Ms": null
1347
+ },
1348
+ "mini-swe-agent": {
1349
+ "planned": 1,
1350
+ "recorded": 1,
1351
+ "resolved": 1,
1352
+ "candidateCaptured": 1,
1353
+ "candidateResolved": 1,
1354
+ "candidateGraded": 1,
1355
+ "resolutionRate": 1.0,
1356
+ "missing": 0,
1357
+ "gradingFailures": 0,
1358
+ "usageComplete": true,
1359
+ "totalCostUsd": null,
1360
+ "costPerResolvedUsd": null,
1361
+ "resolvedP50Ms": 54962.95591699891
1362
+ }
1363
+ },
1364
+ "pairedTaskResolutionDifference": -1.0,
1365
+ "pairedTaskBootstrap95": null,
1366
+ "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
1367
+ },
1368
+ "conclusion": "Candidate passes official grading, but verified import fails after verifier exits 1 and 2 and input budget exhaustion. Candidate capture now reports the actual patch. Prompt change alone did not resolve delivery."
1369
+ }