@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +47 -62
  3. package/ROADMAP.md +15 -6
  4. package/SECURITY.md +1 -1
  5. package/benchmarks/README.md +48 -0
  6. package/benchmarks/baselines/context-navigation-2026-09-09.json +94 -0
  7. package/benchmarks/baselines/efficiency-2026-09-08.json +233 -0
  8. package/benchmarks/baselines/external-final-2026-09-08.json +588 -0
  9. package/benchmarks/baselines/external-intermediate-2026-09-08.json +116 -0
  10. package/benchmarks/baselines/external-matched-openai-2026-09-08.json +1922 -0
  11. package/benchmarks/baselines/external-matched-qwen-2026-09-08.json +1706 -0
  12. package/benchmarks/baselines/external-pilot-2026-09-08.json +439 -0
  13. package/benchmarks/baselines/external-qwen-flash-2026-09-08.json +1570 -0
  14. package/benchmarks/baselines/external-qwen-remediation-2026-09-09.json +1509 -0
  15. package/benchmarks/baselines/external-qwen-sdk-114-progress-2026-09-09.json +2509 -0
  16. package/benchmarks/baselines/external-sdk-upgrade-2026-09-08.json +564 -0
  17. package/benchmarks/baselines/external-verifier-recovery-2026-09-09.json +794 -0
  18. package/benchmarks/baselines/full-harness-review-2026-09-10.json +134 -0
  19. package/benchmarks/baselines/harness-audit-2026-09-09.json +136 -0
  20. package/benchmarks/baselines/harness-remediation-2026-09-09.json +160 -0
  21. package/benchmarks/baselines/oci-remediation-2026-09-10.json +535 -0
  22. package/benchmarks/baselines/rc14-holdout-2026-09-10.json +854 -0
  23. package/benchmarks/baselines/repair-architecture-audit-2026-09-09.json +75 -0
  24. package/benchmarks/baselines/time-to-safe-fix-live-expanded-2026-08-21.json +186 -0
  25. package/benchmarks/baselines/time-to-safe-fix-live-smoke-2026-08-21.json +186 -0
  26. package/contracts/public-api.json +25 -0
  27. package/contracts/security-controls.json +3 -2
  28. package/contracts/stable-api-signatures.json +58 -54
  29. package/dist/bounded-reads.d.ts +4 -0
  30. package/dist/change-envelope.d.ts +4 -4
  31. package/dist/cli-options.d.ts +2 -1
  32. package/dist/cli-profiles.d.ts +39 -0
  33. package/dist/cli-stream.d.ts +40 -40
  34. package/dist/cli.d.ts +38 -35
  35. package/dist/cli.js +6556 -4030
  36. package/dist/compaction.d.ts +12 -0
  37. package/dist/config.d.ts +6 -1
  38. package/dist/console-context.d.ts +26 -0
  39. package/dist/console-input.d.ts +28 -0
  40. package/dist/context-engineering.d.ts +5 -5
  41. package/dist/context-metrics.d.ts +11 -0
  42. package/dist/edit-contracts.d.ts +2 -2
  43. package/dist/execution-environment.d.ts +9 -5
  44. package/dist/harness.d.ts +52 -18
  45. package/dist/hostile-repository-demo.js +2942 -1088
  46. package/dist/index.d.ts +3 -3
  47. package/dist/index.js +4316 -2435
  48. package/dist/json-contracts.d.ts +292 -236
  49. package/dist/model-budget.d.ts +61 -0
  50. package/dist/operations.d.ts +82 -20
  51. package/dist/orchestration.d.ts +1 -0
  52. package/dist/providers.d.ts +5 -2
  53. package/dist/repair-controller.d.ts +72 -0
  54. package/dist/repair-progress.d.ts +40 -0
  55. package/dist/repair-verifier.d.ts +6 -0
  56. package/dist/replacement-edits.d.ts +9 -0
  57. package/dist/routing.d.ts +1 -1
  58. package/dist/run-interruption.d.ts +4 -0
  59. package/dist/runtime-checkpoints.d.ts +7 -0
  60. package/dist/runtime-diagnostics.d.ts +63 -0
  61. package/dist/runtime-policy.d.ts +36 -0
  62. package/dist/sessions.d.ts +1 -1
  63. package/dist/state-backup.d.ts +2 -2
  64. package/dist/task-memory.d.ts +36 -0
  65. package/dist/terminal-markdown.d.ts +12 -0
  66. package/dist/time-to-safe-fix.d.ts +19 -19
  67. package/dist/tool-registry.d.ts +4 -0
  68. package/dist/verify-historical-migrations.js +6987 -5140
  69. package/dist/workspace-mutation-lock.d.ts +5 -0
  70. package/dist/workspace.d.ts +39 -4
  71. package/dist/zhx.js +6556 -4030
  72. package/docs/CLI.md +89 -3
  73. package/docs/CONTEXT_ENGINEERING.md +160 -0
  74. package/docs/EXECUTION_ENVIRONMENTS.md +21 -0
  75. package/docs/EXTENSIBILITY.md +2 -2
  76. package/docs/GA_READINESS.md +13 -7
  77. package/docs/LIVE_CERTIFICATION.md +12 -2
  78. package/docs/RC13_SECURITY_REVIEW.md +28 -0
  79. package/docs/RC14_DELEGATED_SECURITY_DECISION.md +80 -0
  80. package/docs/RC14_SECURITY_REVIEW.md +40 -0
  81. package/docs/README.md +33 -0
  82. package/docs/RELEASE.md +4 -2
  83. package/docs/REPOSITORY_EDITING.md +47 -0
  84. package/docs/SECURITY_REVIEW_EVIDENCE.md +5 -3
  85. package/docs/THREAT_MODEL.md +3 -3
  86. package/docs/TIME_TO_SAFE_FIX.md +4 -4
  87. package/docs/ga-readiness.json +147 -16
  88. package/docs/reports/EFFICIENCY_REVIEW.md +156 -0
  89. package/docs/reports/EXTERNAL_COMPARISON.md +434 -0
  90. package/docs/reports/FULL_HARNESS_REVIEW_2026-09-10.md +200 -0
  91. package/docs/reports/GA_BUDGET_REMEDIATION_2026-09-19.md +38 -0
  92. package/docs/reports/GA_REPAIR_RECOVERY_2026-09-19.md +700 -0
  93. package/docs/reports/GA_VALIDATION_2026-09-19.md +111 -0
  94. package/docs/reports/HARNESS_ARCHITECTURE_REVIEW_2026-09-09.md +202 -0
  95. package/docs/reports/HARNESS_AUDIT_2026-09-09.md +306 -0
  96. package/docs/reports/HARNESS_REMEDIATION_2026-09-09.md +74 -0
  97. package/docs/reports/HARNESS_REMEDIATION_2026-09-10.md +90 -0
  98. package/docs/reports/MODEL_COMPARISON_2026-09-09.md +184 -0
  99. package/docs/reports/QWEN_REMEDIATION_COMPARISON_2026-09-09.md +142 -0
  100. package/docs/reports/RC14_VALIDATION_2026-09-10.md +45 -0
  101. package/docs/reports/README.md +26 -0
  102. package/docs/reports/evidence/ga-budget-remediation-2026-09-19.json +133 -0
  103. package/docs/reports/evidence/ga-catalog-ablation-2026-09-19.json +699 -0
  104. package/docs/reports/evidence/ga-catalog-ablation-plan-2026-09-19.json +40 -0
  105. package/docs/reports/evidence/ga-catalog-observed-2026-09-19.json +985 -0
  106. package/docs/reports/evidence/ga-catalog-observed-plan-2026-09-19.json +28 -0
  107. package/docs/reports/evidence/ga-catalog-observed-status-2026-09-19.json +15 -0
  108. package/docs/reports/evidence/ga-complete-local-check-2026-09-19.json +47 -0
  109. package/docs/reports/evidence/ga-completion-reminder-2026-09-19.json +22 -0
  110. package/docs/reports/evidence/ga-fresh-cohort-plan-2026-09-19.json +54 -0
  111. package/docs/reports/evidence/ga-fresh-cohort-preparation-2026-09-19.json +242 -0
  112. package/docs/reports/evidence/ga-fresh-cohort-result-2026-09-19.json +4318 -0
  113. package/docs/reports/evidence/ga-local-validation-followup-2026-09-19.json +66 -0
  114. package/docs/reports/evidence/ga-node-consumers-2026-09-19.json +31 -0
  115. package/docs/reports/evidence/ga-package-content-validation-2026-09-19.json +22 -0
  116. package/docs/reports/evidence/ga-plan-completion-2026-09-19.json +39 -0
  117. package/docs/reports/evidence/ga-plan-reserve-2026-09-19.json +573 -0
  118. package/docs/reports/evidence/ga-plan-reserve-plan-2026-09-19.json +24 -0
  119. package/docs/reports/evidence/ga-plan-transition-2026-09-19.json +19 -0
  120. package/docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json +24 -0
  121. package/docs/reports/evidence/ga-private-verifier-root-cause-2026-09-19.json +41 -0
  122. package/docs/reports/evidence/ga-published-sdk-2026-09-19.json +35 -0
  123. package/docs/reports/evidence/ga-python-checkout-imports-2026-09-19.json +95 -0
  124. package/docs/reports/evidence/ga-python-origin-delivery-2026-09-19.json +211 -0
  125. package/docs/reports/evidence/ga-python-origin-delivery-plan-2026-09-19.json +27 -0
  126. package/docs/reports/evidence/ga-qwen-patched-package-2026-09-19.json +1092 -0
  127. package/docs/reports/evidence/ga-qwen-patched-package-plan-2026-09-19.json +18 -0
  128. package/docs/reports/evidence/ga-qwen-published-2026-09-19.json +571 -0
  129. package/docs/reports/evidence/ga-qwen-published-plan-2026-09-19.json +18 -0
  130. package/docs/reports/evidence/ga-qwen-tool-choice-2026-09-19.json +85 -0
  131. package/docs/reports/evidence/ga-qwen-tool-choice-patched-2026-09-19.json +85 -0
  132. package/docs/reports/evidence/ga-recovery-development-2026-09-19.json +1066 -0
  133. package/docs/reports/evidence/ga-recovery-graded-development-2026-09-19.json +1313 -0
  134. package/docs/reports/evidence/ga-recovery-graded-v2-2026-09-19.json +1151 -0
  135. package/docs/reports/evidence/ga-recovery-v3-2026-09-19.json +3943 -0
  136. package/docs/reports/evidence/ga-recovery-v4-2026-09-19.json +1055 -0
  137. package/docs/reports/evidence/ga-recovery-v5-2026-09-19.json +1245 -0
  138. package/docs/reports/evidence/ga-recovery-v6-2026-09-19.json +699 -0
  139. package/docs/reports/evidence/ga-recovery-v7-2026-09-19.json +854 -0
  140. package/docs/reports/evidence/ga-recovery-v8-2026-09-19.json +596 -0
  141. package/docs/reports/evidence/ga-required-delivery-2026-09-19.json +25 -0
  142. package/docs/reports/evidence/ga-required-delivery-live-2026-09-19.json +1337 -0
  143. package/docs/reports/evidence/ga-required-delivery-live-plan-2026-09-19.json +25 -0
  144. package/docs/reports/evidence/ga-retained-candidate-grading-2026-09-19.json +30 -0
  145. package/docs/reports/evidence/ga-tool-selection-live-2026-09-19.json +559 -0
  146. package/docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json +28 -0
  147. package/docs/reports/evidence/ga-tool-selection-recovery-2026-09-19.json +27 -0
  148. package/docs/reports/evidence/ga-validation-2026-09-19.json +236 -0
  149. package/docs/reports/evidence/ga-verifier-discovery-live-2026-09-19.json +1369 -0
  150. package/docs/reports/evidence/ga-verifier-discovery-plan-2026-09-19.json +29 -0
  151. package/docs/reports/evidence/ga-verifier-failure-identification-2026-09-19.json +30 -0
  152. package/docs/reports/evidence/ga-verifier-feedback-2026-09-19.json +65 -0
  153. package/docs/reports/evidence/ga-verifier-hints-live-2026-09-19.json +1297 -0
  154. package/docs/reports/evidence/ga-verifier-hints-plan-2026-09-19.json +27 -0
  155. package/docs/reports/evidence/ga-work-boundary-context-analysis-2026-09-19.json +113 -0
  156. package/docs/reports/evidence/ga-work-boundary-live-2026-09-19.json +301 -0
  157. package/docs/reports/evidence/ga-work-boundary-live-plan-2026-09-19.json +84 -0
  158. package/docs/reports/evidence/ga-work-budget-planning-2026-09-19.json +32 -0
  159. package/docs/reports/evidence/ga-working-context-2026-09-19.json +22 -0
  160. package/docs/reports/evidence/rc14-delegated-security-review-2026-09-20.json +81 -0
  161. package/docs/reports/evidence/rc14-ga-gate-2026-09-20.json +18 -0
  162. package/docs/reports/evidence/rc14-representative-2026-09-20.json +509 -0
  163. package/docs/reports/evidence/rc14-security-regressions-2026-09-20.json +64 -0
  164. package/docs/reports/evidence/security-current-2026-09-19.json +91 -0
  165. package/docs/reports/evidence/security-remediation-2026-09-19.json +97 -0
  166. package/evaluations/repair-controller-experiment.json +24 -0
  167. package/evaluations/representative-assembly-matrix.json +63 -0
  168. package/evaluations/representative-matrix.json +2219 -2
  169. package/fixtures/contracts/v1/init.json +19 -0
  170. package/package.json +25 -16
@@ -0,0 +1,1245 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "kind": "development-comparison",
4
+ "classification": "Known development task, one repetition per provider and agent; not holdout",
5
+ "providers": {
6
+ "openai": {
7
+ "model": "gpt-5.6-luna",
8
+ "limits": {
9
+ "steps": 24,
10
+ "outputPerTurn": 2048,
11
+ "inputTokens": 100000,
12
+ "outputTokens": 16000,
13
+ "timeoutSeconds": 300,
14
+ "memoryMb": 2048,
15
+ "cpus": 2
16
+ },
17
+ "implementation": {
18
+ "gitCommit": "41bd1cb4ab8481a7413f096a0817f612fe9806fe",
19
+ "sources": {
20
+ "bun.lock": "e370459197ce52c3961770f3c5ccaab7ff35ddbe641ef72e8bd61f87a8b8e3fe",
21
+ "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
22
+ "package.json": "a4a2507e3ad876d72cad49a16e149192d33521fb4175bb40a3eb077ccfdf43a1",
23
+ "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
24
+ "scripts/swebench/candidate.py": "f5e3410b91b950acfbe50fc7511891816a8fb1c223082397e2c6300cf1e866cf",
25
+ "scripts/swebench/candidate.ts": "bf102becfcce0c9a3f0df0a8f1ce0e5aaea7db2d252f6d9454a8370c24e53982",
26
+ "scripts/swebench/common.py": "ad790227dac3bec1f592c458c993efff4f9d3de8f0df39b33e3238b58b3f0b3b",
27
+ "scripts/swebench/context-metrics.ts": "7b84da6789413aa634d997c5963ee6796584e06c3eafe5f68f424fb7e27815e9",
28
+ "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
29
+ "scripts/swebench/model-budget.ts": "e46dd406ee5d58a051a15c7d7133513666292303d8c6534f861a6db0e711f47c",
30
+ "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
31
+ "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
32
+ "scripts/swebench/report.py": "d0b8d45f32c48e0ba2cb977f50838065a497a0ecaf3b4243ed94139393ce4d6b",
33
+ "scripts/swebench/run.py": "94dccd55dfe6f70d8f8f2da70331700ec85eddd958547e0c3afe3dc416c9802c",
34
+ "scripts/swebench/telemetry.ts": "743c9baf273e9240d704d098b6bd5eea9e76ef3e49ae636aaafa9bfd7a85b45b",
35
+ "scripts/swebench/test_candidate.py": "e77d16379b748baa2daf184a45ea62587d938b6a7d48dcf9f6061f8af21fafd9",
36
+ "scripts/swebench/test_contracts.py": "902e966d0fbf8501dbb15e011849f160331c033cd2a5480f9920d0f365a08d03",
37
+ "scripts/swebench/zhivex-driver.ts": "b17e937484a28cf7fc7728595b51900efa5015dfa37cf6c1d4f9322051fe1ece",
38
+ "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
39
+ "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
40
+ "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
41
+ "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
42
+ "src/cli-options.ts": "f48d9be132fed38f2ddbcfecbd8222c664a0df2f20edc28ce8d1b9b1a1900863",
43
+ "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
44
+ "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
45
+ "src/cli.ts": "5a4acb4695a968a58fbeac05c4bb89848a9653d6189590f0ff90e430ffa98d14",
46
+ "src/compaction.ts": "9e06a0abeeef155377f40c160b21c56256bca95ec18c5ee897681a5fc73db169",
47
+ "src/config.ts": "b1d3991e84add58603fb85b2a604b9b3eb56ae879d0fa30b0cb40545fa44edc1",
48
+ "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
49
+ "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
50
+ "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
51
+ "src/context-metrics.ts": "3eb30f9db808f918ed8d73d4f8ac19a13acb09ed5c1ee70fdae05201714b9198",
52
+ "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
53
+ "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
54
+ "src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9",
55
+ "src/file-security.ts": "b8c9f4988c4339c73a60ab4fa8d394b3eecc19117a0c61a8d19ab6483ca253df",
56
+ "src/harness.ts": "866115b64d90c0b2ffa3e07da6a015210afae0babe46052912d0cc288e23bc4b",
57
+ "src/index.ts": "4bbfcd11376d48c4f1c1ec17d6b5c9f51baa9331fa999432be178bae94530a5e",
58
+ "src/json-contracts.ts": "0780145636b47829a1aa6087c574d5654eb31b7534ea7d9c38c188d648c90272",
59
+ "src/mcp.ts": "7722de031921c620ec03b9e42a6596ef8415aa8e4bb5f93ea84d2427c466e554",
60
+ "src/model-budget.ts": "e25eca12f998998acb2e03f808ed0d84ba69b97711fc577716f230244c62aa80",
61
+ "src/operations.ts": "bcccaf4d1d0ec117c2ed35df92185564d33399e5fdc5c12225ac61f14fd0b4b1",
62
+ "src/orchestration.ts": "b56c8c7d526a99b745cc17769e0a1b7b500b25920e77f34aa0dd01c9758de319",
63
+ "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
64
+ "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
65
+ "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
66
+ "src/repair-controller.ts": "280aa4d33918ab5b38b170059603eb1368a28e910f2ec2a39afa854af8082cc8",
67
+ "src/repair-progress.ts": "3550509a2a6c1e4a2e091c1012d5fb63ad8512a06f09b41e6b3a915a9a2b31a3",
68
+ "src/repair-verifier.ts": "b70fb11feee8cdc7933947ccac0b78f8bb5c0eaccc3b29ad527197d77f40ffc9",
69
+ "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
70
+ "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
71
+ "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
72
+ "src/runtime-checkpoints.ts": "e3b75a0b86fb5d21992b1a126b35446110ba5fcaf4317ed5005b1a44555118dc",
73
+ "src/runtime-diagnostics.ts": "e1e5a3a558fa2aaeb477335dc336551f022c1234fe5e742eefc6599c8803e364",
74
+ "src/runtime-policy.ts": "8356acf0102f0e73839b81ed628a5c0d9a89d9c21b0ca416efedde8f1b3414e2",
75
+ "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
76
+ "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
77
+ "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
78
+ "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
79
+ "src/task-memory.ts": "5b62c863be5a23c33608633cd7aac0464f5536d01cba930aca58b11870170731",
80
+ "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
81
+ "src/terminal-ui.ts": "4da1878a76ba608a2517bf5d5c992a9a88702aba3fd800a9980129ba93e157bc",
82
+ "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
83
+ "src/tool-registry.ts": "7e90fff11940e525883c9ec0d2adbea2e6616f8bd9bc4ec77e1b7277e4217f2c",
84
+ "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
85
+ "src/workspace-mutation-lock.ts": "ca201242db16173bd3a4d86c5d8bb0c2b6f3c43bfed3e4fa4ff5f06a75b19e80",
86
+ "src/workspace.ts": "809f8bbfcf72fed10d860c48585089ca65bb3ef7c672ee6df3c918907a968302"
87
+ }
88
+ },
89
+ "report": {
90
+ "schemaVersion": 1,
91
+ "kind": "swebench-comparison",
92
+ "manifestSha256": "198ac9eb7e214ba667f121fc7155a0a3e985bdccd1688b473895edecd15a49a8",
93
+ "complete": true,
94
+ "candidates": {
95
+ "zhivex": {
96
+ "planned": 1,
97
+ "recorded": 1,
98
+ "resolved": 1,
99
+ "candidateCaptured": 1,
100
+ "candidateResolved": 0,
101
+ "candidateGraded": 1,
102
+ "resolutionRate": 1.0,
103
+ "missing": 0,
104
+ "gradingFailures": 0,
105
+ "usageComplete": true,
106
+ "totalCostUsd": null,
107
+ "costPerResolvedUsd": null,
108
+ "resolvedP50Ms": 76190.25887496537
109
+ },
110
+ "mini-swe-agent": {
111
+ "planned": 1,
112
+ "recorded": 1,
113
+ "resolved": 1,
114
+ "candidateCaptured": 1,
115
+ "candidateResolved": 1,
116
+ "candidateGraded": 1,
117
+ "resolutionRate": 1.0,
118
+ "missing": 0,
119
+ "gradingFailures": 0,
120
+ "usageComplete": true,
121
+ "totalCostUsd": null,
122
+ "costPerResolvedUsd": null,
123
+ "resolvedP50Ms": 45844.67733296333
124
+ }
125
+ },
126
+ "pairedTaskResolutionDifference": 0.0,
127
+ "pairedTaskBootstrap95": null,
128
+ "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
129
+ },
130
+ "samples": [
131
+ {
132
+ "candidate": "zhivex",
133
+ "instanceId": "django__django-15851",
134
+ "repetition": 0,
135
+ "resolved": true,
136
+ "gradingStatus": "completed",
137
+ "usageComplete": true,
138
+ "costUsd": null,
139
+ "candidateCaptureFailure": null,
140
+ "modelTimings": [
141
+ {
142
+ "durationMs": 1849.6875419999997,
143
+ "firstTokenMs": 1847.9273750000002,
144
+ "completed": true
145
+ },
146
+ {
147
+ "durationMs": 2198.1834580000004,
148
+ "firstTokenMs": 2197.9787080000006,
149
+ "completed": true
150
+ },
151
+ {
152
+ "durationMs": 4292.538541000001,
153
+ "firstTokenMs": 4292.125666000001,
154
+ "completed": true
155
+ },
156
+ {
157
+ "durationMs": 2761.5964159999985,
158
+ "firstTokenMs": 2695.658540999999,
159
+ "completed": true
160
+ },
161
+ {
162
+ "durationMs": 2379.7958750000016,
163
+ "firstTokenMs": 2360.6923750000005,
164
+ "completed": true
165
+ }
166
+ ],
167
+ "stageTimings": [
168
+ {
169
+ "event": "edit-requested",
170
+ "elapsedMs": 17290.478875
171
+ },
172
+ {
173
+ "event": "verification-requested",
174
+ "elapsedMs": 19068.900917
175
+ },
176
+ {
177
+ "event": "governed-edit-completed",
178
+ "elapsedMs": 19076.435707999997
179
+ },
180
+ {
181
+ "event": "verified-import-completed",
182
+ "elapsedMs": 35410.575333
183
+ }
184
+ ],
185
+ "approvalTimings": [
186
+ {
187
+ "durationMs": 0.16533300000082818,
188
+ "resolved": true
189
+ },
190
+ {
191
+ "durationMs": 0.04420800000298186,
192
+ "resolved": true
193
+ }
194
+ ],
195
+ "toolTimings": [
196
+ {
197
+ "name": "list_files",
198
+ "calls": 1,
199
+ "errors": 0,
200
+ "totalMs": 349.51416700000027,
201
+ "maxMs": 349.51416700000027
202
+ },
203
+ {
204
+ "name": "search_files",
205
+ "calls": 5,
206
+ "errors": 0,
207
+ "totalMs": 571.2362929999981,
208
+ "maxMs": 395.7137499999999
209
+ },
210
+ {
211
+ "name": "read_files",
212
+ "calls": 1,
213
+ "errors": 0,
214
+ "totalMs": 1.33916599999975,
215
+ "maxMs": 1.33916599999975
216
+ },
217
+ {
218
+ "name": "repair_plan",
219
+ "calls": 1,
220
+ "errors": 0,
221
+ "totalMs": 0.09883400000035181,
222
+ "maxMs": 0.09883400000035181
223
+ },
224
+ {
225
+ "name": "apply_reviewed_replacement",
226
+ "calls": 1,
227
+ "errors": 0,
228
+ "totalMs": 9.801250000000437,
229
+ "maxMs": 9.801250000000437
230
+ },
231
+ {
232
+ "name": "verify_and_apply_environment_patch",
233
+ "calls": 1,
234
+ "errors": 0,
235
+ "totalMs": 16106.744416000001,
236
+ "maxMs": 16106.744416000001
237
+ }
238
+ ],
239
+ "schemaVersion": 1,
240
+ "status": "completed",
241
+ "failure": null,
242
+ "phase": "agent",
243
+ "durationMs": 40222.662625,
244
+ "approvals": 2,
245
+ "turns": [
246
+ {
247
+ "index": 1,
248
+ "status": "completed",
249
+ "finishReason": "tool-calls",
250
+ "requestedTools": [
251
+ "list_files",
252
+ "search_files"
253
+ ],
254
+ "inputTokens": 3037,
255
+ "outputTokens": 90,
256
+ "cachedInputTokens": 0,
257
+ "requestCharacters": 7258,
258
+ "tools": [
259
+ {
260
+ "name": "list_files",
261
+ "isError": false,
262
+ "characters": 9769
263
+ },
264
+ {
265
+ "name": "search_files",
266
+ "isError": false,
267
+ "characters": 2329
268
+ }
269
+ ]
270
+ },
271
+ {
272
+ "index": 2,
273
+ "status": "completed",
274
+ "finishReason": "tool-calls",
275
+ "requestedTools": [
276
+ "search_files",
277
+ "search_files",
278
+ "search_files"
279
+ ],
280
+ "inputTokens": 6826,
281
+ "outputTokens": 141,
282
+ "cachedInputTokens": 2498,
283
+ "requestCharacters": 23744,
284
+ "tools": [
285
+ {
286
+ "name": "search_files",
287
+ "isError": false,
288
+ "characters": 351
289
+ },
290
+ {
291
+ "name": "search_files",
292
+ "isError": false,
293
+ "characters": 342
294
+ },
295
+ {
296
+ "name": "search_files",
297
+ "isError": false,
298
+ "characters": 2528
299
+ }
300
+ ]
301
+ },
302
+ {
303
+ "index": 3,
304
+ "status": "completed",
305
+ "finishReason": "tool-calls",
306
+ "requestedTools": [
307
+ "read_files",
308
+ "search_files"
309
+ ],
310
+ "inputTokens": 8153,
311
+ "outputTokens": 129,
312
+ "cachedInputTokens": 2498,
313
+ "requestCharacters": 32312,
314
+ "tools": [
315
+ {
316
+ "name": "read_files",
317
+ "isError": false,
318
+ "characters": 10466
319
+ },
320
+ {
321
+ "name": "search_files",
322
+ "isError": false,
323
+ "characters": 2558
324
+ }
325
+ ]
326
+ },
327
+ {
328
+ "index": 4,
329
+ "status": "completed",
330
+ "finishReason": "tool-calls",
331
+ "requestedTools": [
332
+ "repair_plan"
333
+ ],
334
+ "inputTokens": 7727,
335
+ "outputTokens": 330,
336
+ "cachedInputTokens": 2498,
337
+ "requestCharacters": 27124,
338
+ "tools": [
339
+ {
340
+ "name": "repair_plan",
341
+ "isError": false,
342
+ "characters": 1244
343
+ }
344
+ ]
345
+ },
346
+ {
347
+ "index": 5,
348
+ "status": "waiting_approval",
349
+ "finishReason": "tool-calls",
350
+ "requestedTools": [
351
+ "apply_reviewed_replacement"
352
+ ],
353
+ "inputTokens": 8422,
354
+ "outputTokens": 168,
355
+ "cachedInputTokens": 2379,
356
+ "requestCharacters": 35217,
357
+ "tools": []
358
+ },
359
+ {
360
+ "index": 6,
361
+ "status": "completed",
362
+ "finishReason": "tool-calls",
363
+ "requestedTools": [
364
+ "verify_and_apply_environment_patch"
365
+ ],
366
+ "inputTokens": 0,
367
+ "outputTokens": 0,
368
+ "cachedInputTokens": 0,
369
+ "requestCharacters": 40354,
370
+ "tools": [
371
+ {
372
+ "name": "apply_reviewed_replacement",
373
+ "isError": false,
374
+ "characters": 483
375
+ },
376
+ {
377
+ "name": "verify_and_apply_environment_patch",
378
+ "isError": false,
379
+ "characters": 1343,
380
+ "phaseLatencies": {
381
+ "commandAndAttestationMs": 2442.160958,
382
+ "workspaceExportMs": 6614.687625,
383
+ "totalMs": 11389.767625,
384
+ "sessionCreationMs": 2332.262667
385
+ },
386
+ "exitCode": 0,
387
+ "timedOut": false,
388
+ "verificationOutcome": "passed"
389
+ }
390
+ ]
391
+ }
392
+ ],
393
+ "inputTokens": 34165,
394
+ "outputTokens": 858,
395
+ "cachedInputTokens": 9873,
396
+ "modelCalls": 5,
397
+ "compactions": [
398
+ {
399
+ "beforeStep": 4,
400
+ "estimatedTokensBefore": 12160,
401
+ "estimatedTokensAfter": 6422
402
+ }
403
+ ],
404
+ "terminalDiagnostic": null,
405
+ "thrownDiagnostic": null,
406
+ "guardrails": [],
407
+ "repairProgress": {
408
+ "repeatedResults": 0,
409
+ "suppressedResults": 0,
410
+ "blockedBroadCalls": 0,
411
+ "enteredClosure": false,
412
+ "phase": "verify",
413
+ "blockedPhaseCalls": 0,
414
+ "planRecorded": true
415
+ },
416
+ "contextMetrics": [
417
+ {
418
+ "systemCharacters": 4851,
419
+ "userCharacters": 2403,
420
+ "assistantCharacters": 0,
421
+ "toolResultCharacters": 0,
422
+ "otherMessageCharacters": 0,
423
+ "toolDefinitionCharacters": 9375,
424
+ "unmeasuredToolDefinitions": 0
425
+ },
426
+ {
427
+ "systemCharacters": 4851,
428
+ "userCharacters": 2403,
429
+ "assistantCharacters": 4067,
430
+ "toolResultCharacters": 12416,
431
+ "otherMessageCharacters": 0,
432
+ "toolDefinitionCharacters": 9375,
433
+ "unmeasuredToolDefinitions": 0
434
+ },
435
+ {
436
+ "systemCharacters": 4851,
437
+ "userCharacters": 2403,
438
+ "assistantCharacters": 8930,
439
+ "toolResultCharacters": 16117,
440
+ "otherMessageCharacters": 0,
441
+ "toolDefinitionCharacters": 9375,
442
+ "unmeasuredToolDefinitions": 0
443
+ },
444
+ {
445
+ "systemCharacters": 4851,
446
+ "userCharacters": 1437,
447
+ "assistantCharacters": 7487,
448
+ "toolResultCharacters": 13342,
449
+ "otherMessageCharacters": 0,
450
+ "toolDefinitionCharacters": 9375,
451
+ "unmeasuredToolDefinitions": 0
452
+ },
453
+ {
454
+ "systemCharacters": 4851,
455
+ "userCharacters": 1816,
456
+ "assistantCharacters": 13796,
457
+ "toolResultCharacters": 14745,
458
+ "otherMessageCharacters": 0,
459
+ "toolDefinitionCharacters": 9375,
460
+ "unmeasuredToolDefinitions": 0
461
+ }
462
+ ],
463
+ "omittedContextMeasurements": 0,
464
+ "toolCalls": 10,
465
+ "patchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
466
+ "protectedFilesChanged": [],
467
+ "officialResolved": true,
468
+ "candidateCaptured": true,
469
+ "candidatePatchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
470
+ "candidateGradingStatus": "completed",
471
+ "candidateResolved": false,
472
+ "repairStages": {
473
+ "candidatePresent": false,
474
+ "importedPatchPresent": true,
475
+ "verificationAttempted": true
476
+ },
477
+ "gradingDurationMs": 24984.549375018105,
478
+ "totalDurationMs": 76190.25887496537
479
+ },
480
+ {
481
+ "candidate": "mini-swe-agent",
482
+ "instanceId": "django__django-15851",
483
+ "repetition": 0,
484
+ "resolved": true,
485
+ "gradingStatus": "completed",
486
+ "usageComplete": true,
487
+ "costUsd": null,
488
+ "schemaVersion": 1,
489
+ "status": "Submitted",
490
+ "failure": null,
491
+ "phase": "agent",
492
+ "durationMs": 20709.29804199841,
493
+ "approvals": 0,
494
+ "toolCalls": 9,
495
+ "inputTokens": 27952,
496
+ "outputTokens": 1153,
497
+ "cachedInputTokens": 23129,
498
+ "modelCalls": 7,
499
+ "patchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
500
+ "protectedFilesChanged": [],
501
+ "officialResolved": true,
502
+ "candidateCaptured": true,
503
+ "candidatePatchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
504
+ "candidateGradingStatus": "completed",
505
+ "candidateResolved": true,
506
+ "repairStages": {
507
+ "candidatePresent": true,
508
+ "importedPatchPresent": null,
509
+ "verificationAttempted": null
510
+ },
511
+ "gradingDurationMs": 22967.088915989734,
512
+ "totalDurationMs": 45844.67733296333
513
+ }
514
+ ]
515
+ },
516
+ "qwen": {
517
+ "model": "qwen3.8-flash",
518
+ "limits": {
519
+ "steps": 24,
520
+ "outputPerTurn": 2048,
521
+ "inputTokens": 100000,
522
+ "outputTokens": 16000,
523
+ "timeoutSeconds": 300,
524
+ "memoryMb": 2048,
525
+ "cpus": 2
526
+ },
527
+ "implementation": {
528
+ "gitCommit": "41bd1cb4ab8481a7413f096a0817f612fe9806fe",
529
+ "sources": {
530
+ "bun.lock": "e370459197ce52c3961770f3c5ccaab7ff35ddbe641ef72e8bd61f87a8b8e3fe",
531
+ "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
532
+ "package.json": "a4a2507e3ad876d72cad49a16e149192d33521fb4175bb40a3eb077ccfdf43a1",
533
+ "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
534
+ "scripts/swebench/candidate.py": "f5e3410b91b950acfbe50fc7511891816a8fb1c223082397e2c6300cf1e866cf",
535
+ "scripts/swebench/candidate.ts": "bf102becfcce0c9a3f0df0a8f1ce0e5aaea7db2d252f6d9454a8370c24e53982",
536
+ "scripts/swebench/common.py": "ad790227dac3bec1f592c458c993efff4f9d3de8f0df39b33e3238b58b3f0b3b",
537
+ "scripts/swebench/context-metrics.ts": "7b84da6789413aa634d997c5963ee6796584e06c3eafe5f68f424fb7e27815e9",
538
+ "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
539
+ "scripts/swebench/model-budget.ts": "e46dd406ee5d58a051a15c7d7133513666292303d8c6534f861a6db0e711f47c",
540
+ "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
541
+ "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
542
+ "scripts/swebench/report.py": "d0b8d45f32c48e0ba2cb977f50838065a497a0ecaf3b4243ed94139393ce4d6b",
543
+ "scripts/swebench/run.py": "94dccd55dfe6f70d8f8f2da70331700ec85eddd958547e0c3afe3dc416c9802c",
544
+ "scripts/swebench/telemetry.ts": "743c9baf273e9240d704d098b6bd5eea9e76ef3e49ae636aaafa9bfd7a85b45b",
545
+ "scripts/swebench/test_candidate.py": "e77d16379b748baa2daf184a45ea62587d938b6a7d48dcf9f6061f8af21fafd9",
546
+ "scripts/swebench/test_contracts.py": "902e966d0fbf8501dbb15e011849f160331c033cd2a5480f9920d0f365a08d03",
547
+ "scripts/swebench/zhivex-driver.ts": "b17e937484a28cf7fc7728595b51900efa5015dfa37cf6c1d4f9322051fe1ece",
548
+ "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
549
+ "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
550
+ "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
551
+ "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
552
+ "src/cli-options.ts": "f48d9be132fed38f2ddbcfecbd8222c664a0df2f20edc28ce8d1b9b1a1900863",
553
+ "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
554
+ "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
555
+ "src/cli.ts": "5a4acb4695a968a58fbeac05c4bb89848a9653d6189590f0ff90e430ffa98d14",
556
+ "src/compaction.ts": "9e06a0abeeef155377f40c160b21c56256bca95ec18c5ee897681a5fc73db169",
557
+ "src/config.ts": "b1d3991e84add58603fb85b2a604b9b3eb56ae879d0fa30b0cb40545fa44edc1",
558
+ "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
559
+ "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
560
+ "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
561
+ "src/context-metrics.ts": "3eb30f9db808f918ed8d73d4f8ac19a13acb09ed5c1ee70fdae05201714b9198",
562
+ "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
563
+ "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
564
+ "src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9",
565
+ "src/file-security.ts": "b8c9f4988c4339c73a60ab4fa8d394b3eecc19117a0c61a8d19ab6483ca253df",
566
+ "src/harness.ts": "866115b64d90c0b2ffa3e07da6a015210afae0babe46052912d0cc288e23bc4b",
567
+ "src/index.ts": "4bbfcd11376d48c4f1c1ec17d6b5c9f51baa9331fa999432be178bae94530a5e",
568
+ "src/json-contracts.ts": "0780145636b47829a1aa6087c574d5654eb31b7534ea7d9c38c188d648c90272",
569
+ "src/mcp.ts": "7722de031921c620ec03b9e42a6596ef8415aa8e4bb5f93ea84d2427c466e554",
570
+ "src/model-budget.ts": "e25eca12f998998acb2e03f808ed0d84ba69b97711fc577716f230244c62aa80",
571
+ "src/operations.ts": "bcccaf4d1d0ec117c2ed35df92185564d33399e5fdc5c12225ac61f14fd0b4b1",
572
+ "src/orchestration.ts": "b56c8c7d526a99b745cc17769e0a1b7b500b25920e77f34aa0dd01c9758de319",
573
+ "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
574
+ "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
575
+ "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
576
+ "src/repair-controller.ts": "280aa4d33918ab5b38b170059603eb1368a28e910f2ec2a39afa854af8082cc8",
577
+ "src/repair-progress.ts": "3550509a2a6c1e4a2e091c1012d5fb63ad8512a06f09b41e6b3a915a9a2b31a3",
578
+ "src/repair-verifier.ts": "b70fb11feee8cdc7933947ccac0b78f8bb5c0eaccc3b29ad527197d77f40ffc9",
579
+ "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
580
+ "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
581
+ "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
582
+ "src/runtime-checkpoints.ts": "e3b75a0b86fb5d21992b1a126b35446110ba5fcaf4317ed5005b1a44555118dc",
583
+ "src/runtime-diagnostics.ts": "e1e5a3a558fa2aaeb477335dc336551f022c1234fe5e742eefc6599c8803e364",
584
+ "src/runtime-policy.ts": "8356acf0102f0e73839b81ed628a5c0d9a89d9c21b0ca416efedde8f1b3414e2",
585
+ "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
586
+ "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
587
+ "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
588
+ "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
589
+ "src/task-memory.ts": "5b62c863be5a23c33608633cd7aac0464f5536d01cba930aca58b11870170731",
590
+ "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
591
+ "src/terminal-ui.ts": "4da1878a76ba608a2517bf5d5c992a9a88702aba3fd800a9980129ba93e157bc",
592
+ "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
593
+ "src/tool-registry.ts": "7e90fff11940e525883c9ec0d2adbea2e6616f8bd9bc4ec77e1b7277e4217f2c",
594
+ "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
595
+ "src/workspace-mutation-lock.ts": "ca201242db16173bd3a4d86c5d8bb0c2b6f3c43bfed3e4fa4ff5f06a75b19e80",
596
+ "src/workspace.ts": "809f8bbfcf72fed10d860c48585089ca65bb3ef7c672ee6df3c918907a968302"
597
+ }
598
+ },
599
+ "report": {
600
+ "schemaVersion": 1,
601
+ "kind": "swebench-comparison",
602
+ "manifestSha256": "331ea93fbfb74a631cacd7d7c126a8814595f09f2549c93907c7c9be79e4fecd",
603
+ "complete": true,
604
+ "candidates": {
605
+ "zhivex": {
606
+ "planned": 1,
607
+ "recorded": 1,
608
+ "resolved": 0,
609
+ "candidateCaptured": 1,
610
+ "candidateResolved": 0,
611
+ "candidateGraded": 1,
612
+ "resolutionRate": 0.0,
613
+ "missing": 0,
614
+ "gradingFailures": 0,
615
+ "usageComplete": true,
616
+ "totalCostUsd": null,
617
+ "costPerResolvedUsd": null,
618
+ "resolvedP50Ms": null
619
+ },
620
+ "mini-swe-agent": {
621
+ "planned": 1,
622
+ "recorded": 1,
623
+ "resolved": 1,
624
+ "candidateCaptured": 1,
625
+ "candidateResolved": 1,
626
+ "candidateGraded": 1,
627
+ "resolutionRate": 1.0,
628
+ "missing": 0,
629
+ "gradingFailures": 0,
630
+ "usageComplete": true,
631
+ "totalCostUsd": null,
632
+ "costPerResolvedUsd": null,
633
+ "resolvedP50Ms": 50293.124082963914
634
+ }
635
+ },
636
+ "pairedTaskResolutionDifference": -1.0,
637
+ "pairedTaskBootstrap95": null,
638
+ "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
639
+ },
640
+ "samples": [
641
+ {
642
+ "candidate": "zhivex",
643
+ "instanceId": "django__django-15851",
644
+ "repetition": 0,
645
+ "resolved": false,
646
+ "gradingStatus": "completed",
647
+ "usageComplete": true,
648
+ "costUsd": null,
649
+ "candidateCaptureFailure": null,
650
+ "modelTimings": [
651
+ {
652
+ "durationMs": 48835.026291,
653
+ "firstTokenMs": 47222.434833,
654
+ "completed": true
655
+ },
656
+ {
657
+ "durationMs": 1737.1307080000042,
658
+ "firstTokenMs": 1736.2095830000035,
659
+ "completed": true
660
+ },
661
+ {
662
+ "durationMs": 5377.519709,
663
+ "firstTokenMs": 1118.0824999999968,
664
+ "completed": true
665
+ },
666
+ {
667
+ "durationMs": 1724.487957999998,
668
+ "firstTokenMs": 1724.2807919999977,
669
+ "completed": true
670
+ },
671
+ {
672
+ "durationMs": 1809.7376250000016,
673
+ "firstTokenMs": 1795.096292000002,
674
+ "completed": true
675
+ },
676
+ {
677
+ "durationMs": 1926.1222500000003,
678
+ "firstTokenMs": 1926.0149999999994,
679
+ "completed": true
680
+ },
681
+ {
682
+ "durationMs": 1574.1296659999934,
683
+ "firstTokenMs": 1574.0151659999974,
684
+ "completed": true
685
+ },
686
+ {
687
+ "durationMs": 1819.671000000002,
688
+ "firstTokenMs": 1819.585500000001,
689
+ "completed": true
690
+ },
691
+ {
692
+ "durationMs": 4230.956542,
693
+ "firstTokenMs": 4230.676166999998,
694
+ "completed": true
695
+ },
696
+ {
697
+ "durationMs": 2831.794167,
698
+ "firstTokenMs": 2829.474125000008,
699
+ "completed": true
700
+ },
701
+ {
702
+ "durationMs": 5494.217833000002,
703
+ "firstTokenMs": null,
704
+ "completed": true
705
+ }
706
+ ],
707
+ "stageTimings": [
708
+ {
709
+ "event": "edit-requested",
710
+ "elapsedMs": 72615.767708
711
+ },
712
+ {
713
+ "event": "verification-requested",
714
+ "elapsedMs": 74446.437833
715
+ },
716
+ {
717
+ "event": "governed-edit-completed",
718
+ "elapsedMs": 74452.12225
719
+ },
720
+ {
721
+ "event": "command-requested",
722
+ "elapsedMs": 82857.248083
723
+ }
724
+ ],
725
+ "approvalTimings": [
726
+ {
727
+ "durationMs": 0.12895800000114832,
728
+ "resolved": true
729
+ },
730
+ {
731
+ "durationMs": 0.11733400000957772,
732
+ "resolved": true
733
+ },
734
+ {
735
+ "durationMs": 0.02383400000690017,
736
+ "resolved": true
737
+ }
738
+ ],
739
+ "toolTimings": [
740
+ {
741
+ "name": "search_files",
742
+ "calls": 4,
743
+ "errors": 1,
744
+ "totalMs": 472.0544590000063,
745
+ "maxMs": 390.1018750000003
746
+ },
747
+ {
748
+ "name": "read_files",
749
+ "calls": 4,
750
+ "errors": 0,
751
+ "totalMs": 4.181332999985898,
752
+ "maxMs": 1.452041000004101
753
+ },
754
+ {
755
+ "name": "repair_plan",
756
+ "calls": 1,
757
+ "errors": 0,
758
+ "totalMs": 0.08670899999560788,
759
+ "maxMs": 0.08670899999560788
760
+ },
761
+ {
762
+ "name": "apply_reviewed_replacement",
763
+ "calls": 1,
764
+ "errors": 0,
765
+ "totalMs": 6.189165999996476,
766
+ "maxMs": 6.189165999996476
767
+ },
768
+ {
769
+ "name": "verify_and_apply_environment_patch",
770
+ "calls": 1,
771
+ "errors": 1,
772
+ "totalMs": 4402.687459000008,
773
+ "maxMs": 4402.687459000008
774
+ },
775
+ {
776
+ "name": "run_environment_command",
777
+ "calls": 1,
778
+ "errors": 0,
779
+ "totalMs": 11553.121500000008,
780
+ "maxMs": 11553.121500000008
781
+ }
782
+ ],
783
+ "schemaVersion": 1,
784
+ "status": "failed",
785
+ "failure": "AGENT_FAILED",
786
+ "phase": "agent",
787
+ "durationMs": 106168.325708,
788
+ "approvals": 3,
789
+ "turns": [
790
+ {
791
+ "index": 1,
792
+ "status": "completed",
793
+ "finishReason": "tool-calls",
794
+ "requestedTools": [
795
+ "search_files",
796
+ "search_files"
797
+ ],
798
+ "inputTokens": 4580,
799
+ "outputTokens": 152,
800
+ "cachedInputTokens": 0,
801
+ "requestCharacters": 7258,
802
+ "tools": [
803
+ {
804
+ "name": "search_files",
805
+ "isError": false,
806
+ "characters": 714
807
+ },
808
+ {
809
+ "name": "search_files",
810
+ "isError": false,
811
+ "characters": 158
812
+ }
813
+ ]
814
+ },
815
+ {
816
+ "index": 2,
817
+ "status": "completed",
818
+ "finishReason": "tool-calls",
819
+ "requestedTools": [
820
+ "read_files"
821
+ ],
822
+ "inputTokens": 5123,
823
+ "outputTokens": 53,
824
+ "cachedInputTokens": 4096,
825
+ "requestCharacters": 9004,
826
+ "tools": [
827
+ {
828
+ "name": "read_files",
829
+ "isError": false,
830
+ "characters": 2673
831
+ }
832
+ ]
833
+ },
834
+ {
835
+ "index": 3,
836
+ "status": "completed",
837
+ "finishReason": "tool-calls",
838
+ "requestedTools": [
839
+ "repair_plan"
840
+ ],
841
+ "inputTokens": 6058,
842
+ "outputTokens": 281,
843
+ "cachedInputTokens": 4096,
844
+ "requestCharacters": 12056,
845
+ "tools": [
846
+ {
847
+ "name": "repair_plan",
848
+ "isError": false,
849
+ "characters": 897
850
+ }
851
+ ]
852
+ },
853
+ {
854
+ "index": 4,
855
+ "status": "completed",
856
+ "finishReason": "tool-calls",
857
+ "requestedTools": [
858
+ "search_files"
859
+ ],
860
+ "inputTokens": 6636,
861
+ "outputTokens": 66,
862
+ "cachedInputTokens": 5120,
863
+ "requestCharacters": 14573,
864
+ "tools": [
865
+ {
866
+ "name": "search_files",
867
+ "isError": true,
868
+ "characters": 4,
869
+ "diagnostic": {
870
+ "code": "EXECUTION_FAILED",
871
+ "category": "execution",
872
+ "failureType": "execution",
873
+ "retryable": false,
874
+ "chain": [
875
+ {
876
+ "validationCode": null,
877
+ "providerReason": null,
878
+ "validationIssues": [],
879
+ "status": null,
880
+ "errorClass": "Object",
881
+ "providerHints": [],
882
+ "fingerprint": "6f7194f25cfa44d78d03006c1b5da35b4d95b2e39a676c7d881dd9c039203944",
883
+ "reason": "enoent"
884
+ }
885
+ ]
886
+ }
887
+ }
888
+ ]
889
+ },
890
+ {
891
+ "index": 5,
892
+ "status": "completed",
893
+ "finishReason": "tool-calls",
894
+ "requestedTools": [
895
+ "search_files"
896
+ ],
897
+ "inputTokens": 6816,
898
+ "outputTokens": 62,
899
+ "cachedInputTokens": 6144,
900
+ "requestCharacters": 15180,
901
+ "tools": [
902
+ {
903
+ "name": "search_files",
904
+ "isError": false,
905
+ "characters": 4511
906
+ }
907
+ ]
908
+ },
909
+ {
910
+ "index": 6,
911
+ "status": "completed",
912
+ "finishReason": "tool-calls",
913
+ "requestedTools": [
914
+ "read_files"
915
+ ],
916
+ "inputTokens": 9044,
917
+ "outputTokens": 50,
918
+ "cachedInputTokens": 6144,
919
+ "requestCharacters": 20069,
920
+ "tools": [
921
+ {
922
+ "name": "read_files",
923
+ "isError": false,
924
+ "characters": 4790
925
+ }
926
+ ]
927
+ },
928
+ {
929
+ "index": 7,
930
+ "status": "completed",
931
+ "finishReason": "tool-calls",
932
+ "requestedTools": [
933
+ "read_files"
934
+ ],
935
+ "inputTokens": 10546,
936
+ "outputTokens": 52,
937
+ "cachedInputTokens": 8320,
938
+ "requestCharacters": 25232,
939
+ "tools": [
940
+ {
941
+ "name": "read_files",
942
+ "isError": false,
943
+ "characters": 3399
944
+ }
945
+ ]
946
+ },
947
+ {
948
+ "index": 8,
949
+ "status": "completed",
950
+ "finishReason": "tool-calls",
951
+ "requestedTools": [
952
+ "read_files"
953
+ ],
954
+ "inputTokens": 6849,
955
+ "outputTokens": 50,
956
+ "cachedInputTokens": 3328,
957
+ "requestCharacters": 14496,
958
+ "tools": [
959
+ {
960
+ "name": "read_files",
961
+ "isError": false,
962
+ "characters": 2480
963
+ }
964
+ ]
965
+ },
966
+ {
967
+ "index": 9,
968
+ "status": "waiting_approval",
969
+ "finishReason": "tool-calls",
970
+ "requestedTools": [
971
+ "apply_reviewed_replacement"
972
+ ],
973
+ "inputTokens": 7733,
974
+ "outputTokens": 233,
975
+ "cachedInputTokens": 6144,
976
+ "requestCharacters": 17355,
977
+ "tools": []
978
+ },
979
+ {
980
+ "index": 10,
981
+ "status": "completed",
982
+ "finishReason": "tool-calls",
983
+ "requestedTools": [
984
+ "verify_and_apply_environment_patch"
985
+ ],
986
+ "inputTokens": 0,
987
+ "outputTokens": 0,
988
+ "cachedInputTokens": 0,
989
+ "requestCharacters": 18842,
990
+ "tools": [
991
+ {
992
+ "name": "apply_reviewed_replacement",
993
+ "isError": false,
994
+ "characters": 483
995
+ },
996
+ {
997
+ "name": "verify_and_apply_environment_patch",
998
+ "isError": true,
999
+ "characters": 437,
1000
+ "exitCode": 1,
1001
+ "timedOut": false,
1002
+ "verificationOutcome": "failed",
1003
+ "diagnostic": {
1004
+ "code": "EXECUTION_FAILED",
1005
+ "category": "execution",
1006
+ "failureType": "execution",
1007
+ "retryable": false,
1008
+ "chain": [
1009
+ {
1010
+ "validationCode": null,
1011
+ "providerReason": null,
1012
+ "validationIssues": [],
1013
+ "status": null,
1014
+ "errorClass": "Object",
1015
+ "providerHints": [],
1016
+ "fingerprint": "d22955ecb1810cd7e960e732636eec570e04ab3b19eab276ebf23e2096a3c941",
1017
+ "reason": null
1018
+ }
1019
+ ]
1020
+ }
1021
+ }
1022
+ ]
1023
+ },
1024
+ {
1025
+ "index": 11,
1026
+ "status": "waiting_approval",
1027
+ "finishReason": "tool-calls",
1028
+ "requestedTools": [
1029
+ "run_environment_command"
1030
+ ],
1031
+ "inputTokens": 8713,
1032
+ "outputTokens": 135,
1033
+ "cachedInputTokens": 7168,
1034
+ "requestCharacters": 20327,
1035
+ "tools": []
1036
+ },
1037
+ {
1038
+ "index": 12,
1039
+ "status": "completed",
1040
+ "finishReason": "stop",
1041
+ "requestedTools": [],
1042
+ "inputTokens": 6978,
1043
+ "outputTokens": 340,
1044
+ "cachedInputTokens": 0,
1045
+ "requestCharacters": 22196,
1046
+ "tools": []
1047
+ }
1048
+ ],
1049
+ "inputTokens": 79076,
1050
+ "outputTokens": 1474,
1051
+ "cachedInputTokens": 50560,
1052
+ "modelCalls": 11,
1053
+ "compactions": [
1054
+ {
1055
+ "beforeStep": 8,
1056
+ "estimatedTokensBefore": 6810,
1057
+ "estimatedTokensAfter": 3182
1058
+ }
1059
+ ],
1060
+ "terminalDiagnostic": {
1061
+ "code": "EXECUTION_FAILED",
1062
+ "category": "execution",
1063
+ "failureType": "execution",
1064
+ "retryable": false,
1065
+ "chain": [
1066
+ {
1067
+ "validationCode": null,
1068
+ "providerReason": null,
1069
+ "validationIssues": [],
1070
+ "status": null,
1071
+ "errorClass": "Object",
1072
+ "providerHints": [],
1073
+ "fingerprint": "df4ad18f75f381d4abf1c3a9a9cb1ce99246204330b132a8d70938de039a0be4",
1074
+ "reason": null
1075
+ }
1076
+ ]
1077
+ },
1078
+ "thrownDiagnostic": null,
1079
+ "guardrails": [],
1080
+ "repairProgress": {
1081
+ "repeatedResults": 0,
1082
+ "suppressedResults": 0,
1083
+ "blockedBroadCalls": 0,
1084
+ "enteredClosure": true,
1085
+ "phase": "verify",
1086
+ "blockedPhaseCalls": 0,
1087
+ "planRecorded": true
1088
+ },
1089
+ "contextMetrics": [
1090
+ {
1091
+ "systemCharacters": 4851,
1092
+ "userCharacters": 2403,
1093
+ "assistantCharacters": 0,
1094
+ "toolResultCharacters": 0,
1095
+ "otherMessageCharacters": 0,
1096
+ "toolDefinitionCharacters": 9375,
1097
+ "unmeasuredToolDefinitions": 0
1098
+ },
1099
+ {
1100
+ "systemCharacters": 4851,
1101
+ "userCharacters": 2403,
1102
+ "assistantCharacters": 551,
1103
+ "toolResultCharacters": 1192,
1104
+ "otherMessageCharacters": 0,
1105
+ "toolDefinitionCharacters": 9375,
1106
+ "unmeasuredToolDefinitions": 0
1107
+ },
1108
+ {
1109
+ "systemCharacters": 4851,
1110
+ "userCharacters": 2403,
1111
+ "assistantCharacters": 770,
1112
+ "toolResultCharacters": 4023,
1113
+ "otherMessageCharacters": 0,
1114
+ "toolDefinitionCharacters": 9375,
1115
+ "unmeasuredToolDefinitions": 0
1116
+ },
1117
+ {
1118
+ "systemCharacters": 4851,
1119
+ "userCharacters": 2733,
1120
+ "assistantCharacters": 1899,
1121
+ "toolResultCharacters": 5079,
1122
+ "otherMessageCharacters": 0,
1123
+ "toolDefinitionCharacters": 9375,
1124
+ "unmeasuredToolDefinitions": 0
1125
+ },
1126
+ {
1127
+ "systemCharacters": 4851,
1128
+ "userCharacters": 2733,
1129
+ "assistantCharacters": 2133,
1130
+ "toolResultCharacters": 5450,
1131
+ "otherMessageCharacters": 0,
1132
+ "toolDefinitionCharacters": 9375,
1133
+ "unmeasuredToolDefinitions": 0
1134
+ },
1135
+ {
1136
+ "systemCharacters": 4851,
1137
+ "userCharacters": 2733,
1138
+ "assistantCharacters": 2349,
1139
+ "toolResultCharacters": 10121,
1140
+ "otherMessageCharacters": 0,
1141
+ "toolDefinitionCharacters": 9375,
1142
+ "unmeasuredToolDefinitions": 0
1143
+ },
1144
+ {
1145
+ "systemCharacters": 4851,
1146
+ "userCharacters": 2733,
1147
+ "assistantCharacters": 2562,
1148
+ "toolResultCharacters": 15069,
1149
+ "otherMessageCharacters": 0,
1150
+ "toolDefinitionCharacters": 9375,
1151
+ "unmeasuredToolDefinitions": 0
1152
+ },
1153
+ {
1154
+ "systemCharacters": 4851,
1155
+ "userCharacters": 1767,
1156
+ "assistantCharacters": 4315,
1157
+ "toolResultCharacters": 3557,
1158
+ "otherMessageCharacters": 0,
1159
+ "toolDefinitionCharacters": 9375,
1160
+ "unmeasuredToolDefinitions": 0
1161
+ },
1162
+ {
1163
+ "systemCharacters": 4851,
1164
+ "userCharacters": 1767,
1165
+ "assistantCharacters": 4534,
1166
+ "toolResultCharacters": 6195,
1167
+ "otherMessageCharacters": 0,
1168
+ "toolDefinitionCharacters": 9375,
1169
+ "unmeasuredToolDefinitions": 0
1170
+ },
1171
+ {
1172
+ "systemCharacters": 4851,
1173
+ "userCharacters": 2237,
1174
+ "assistantCharacters": 5634,
1175
+ "toolResultCharacters": 7593,
1176
+ "otherMessageCharacters": 0,
1177
+ "toolDefinitionCharacters": 9375,
1178
+ "unmeasuredToolDefinitions": 0
1179
+ },
1180
+ {
1181
+ "systemCharacters": 4851,
1182
+ "userCharacters": 2630,
1183
+ "assistantCharacters": 6150,
1184
+ "toolResultCharacters": 8551,
1185
+ "otherMessageCharacters": 0,
1186
+ "toolDefinitionCharacters": 1635,
1187
+ "unmeasuredToolDefinitions": 0
1188
+ }
1189
+ ],
1190
+ "omittedContextMeasurements": 0,
1191
+ "toolCalls": 12,
1192
+ "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
1193
+ "protectedFilesChanged": [],
1194
+ "gradingMethod": "empty-patch-no-solution",
1195
+ "officialResolved": false,
1196
+ "candidateCaptured": true,
1197
+ "candidatePatchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
1198
+ "candidateGradingStatus": "completed",
1199
+ "candidateResolved": false,
1200
+ "repairStages": {
1201
+ "candidatePresent": false,
1202
+ "importedPatchPresent": false,
1203
+ "verificationAttempted": true
1204
+ },
1205
+ "gradingDurationMs": 0.20687503274530172,
1206
+ "totalDurationMs": 117300.89512496488
1207
+ },
1208
+ {
1209
+ "candidate": "mini-swe-agent",
1210
+ "instanceId": "django__django-15851",
1211
+ "repetition": 0,
1212
+ "resolved": true,
1213
+ "gradingStatus": "completed",
1214
+ "usageComplete": true,
1215
+ "costUsd": null,
1216
+ "schemaVersion": 1,
1217
+ "status": "Submitted",
1218
+ "failure": null,
1219
+ "phase": "agent",
1220
+ "durationMs": 23792.422500031535,
1221
+ "approvals": 0,
1222
+ "toolCalls": 9,
1223
+ "inputTokens": 23391,
1224
+ "outputTokens": 710,
1225
+ "cachedInputTokens": 15360,
1226
+ "modelCalls": 9,
1227
+ "patchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
1228
+ "protectedFilesChanged": [],
1229
+ "officialResolved": true,
1230
+ "candidateCaptured": true,
1231
+ "candidatePatchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
1232
+ "candidateGradingStatus": "completed",
1233
+ "candidateResolved": true,
1234
+ "repairStages": {
1235
+ "candidatePresent": true,
1236
+ "importedPatchPresent": null,
1237
+ "verificationAttempted": null
1238
+ },
1239
+ "gradingDurationMs": 24378.981000045314,
1240
+ "totalDurationMs": 50293.124082963914
1241
+ }
1242
+ ]
1243
+ }
1244
+ }
1245
+ }