@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +47 -62
  3. package/ROADMAP.md +15 -6
  4. package/SECURITY.md +1 -1
  5. package/benchmarks/README.md +48 -0
  6. package/benchmarks/baselines/context-navigation-2026-09-09.json +94 -0
  7. package/benchmarks/baselines/efficiency-2026-09-08.json +233 -0
  8. package/benchmarks/baselines/external-final-2026-09-08.json +588 -0
  9. package/benchmarks/baselines/external-intermediate-2026-09-08.json +116 -0
  10. package/benchmarks/baselines/external-matched-openai-2026-09-08.json +1922 -0
  11. package/benchmarks/baselines/external-matched-qwen-2026-09-08.json +1706 -0
  12. package/benchmarks/baselines/external-pilot-2026-09-08.json +439 -0
  13. package/benchmarks/baselines/external-qwen-flash-2026-09-08.json +1570 -0
  14. package/benchmarks/baselines/external-qwen-remediation-2026-09-09.json +1509 -0
  15. package/benchmarks/baselines/external-qwen-sdk-114-progress-2026-09-09.json +2509 -0
  16. package/benchmarks/baselines/external-sdk-upgrade-2026-09-08.json +564 -0
  17. package/benchmarks/baselines/external-verifier-recovery-2026-09-09.json +794 -0
  18. package/benchmarks/baselines/full-harness-review-2026-09-10.json +134 -0
  19. package/benchmarks/baselines/harness-audit-2026-09-09.json +136 -0
  20. package/benchmarks/baselines/harness-remediation-2026-09-09.json +160 -0
  21. package/benchmarks/baselines/oci-remediation-2026-09-10.json +535 -0
  22. package/benchmarks/baselines/rc14-holdout-2026-09-10.json +854 -0
  23. package/benchmarks/baselines/repair-architecture-audit-2026-09-09.json +75 -0
  24. package/benchmarks/baselines/time-to-safe-fix-live-expanded-2026-08-21.json +186 -0
  25. package/benchmarks/baselines/time-to-safe-fix-live-smoke-2026-08-21.json +186 -0
  26. package/contracts/public-api.json +25 -0
  27. package/contracts/security-controls.json +3 -2
  28. package/contracts/stable-api-signatures.json +58 -54
  29. package/dist/bounded-reads.d.ts +4 -0
  30. package/dist/change-envelope.d.ts +4 -4
  31. package/dist/cli-options.d.ts +2 -1
  32. package/dist/cli-profiles.d.ts +39 -0
  33. package/dist/cli-stream.d.ts +40 -40
  34. package/dist/cli.d.ts +38 -35
  35. package/dist/cli.js +6556 -4030
  36. package/dist/compaction.d.ts +12 -0
  37. package/dist/config.d.ts +6 -1
  38. package/dist/console-context.d.ts +26 -0
  39. package/dist/console-input.d.ts +28 -0
  40. package/dist/context-engineering.d.ts +5 -5
  41. package/dist/context-metrics.d.ts +11 -0
  42. package/dist/edit-contracts.d.ts +2 -2
  43. package/dist/execution-environment.d.ts +9 -5
  44. package/dist/harness.d.ts +52 -18
  45. package/dist/hostile-repository-demo.js +2942 -1088
  46. package/dist/index.d.ts +3 -3
  47. package/dist/index.js +4316 -2435
  48. package/dist/json-contracts.d.ts +292 -236
  49. package/dist/model-budget.d.ts +61 -0
  50. package/dist/operations.d.ts +82 -20
  51. package/dist/orchestration.d.ts +1 -0
  52. package/dist/providers.d.ts +5 -2
  53. package/dist/repair-controller.d.ts +72 -0
  54. package/dist/repair-progress.d.ts +40 -0
  55. package/dist/repair-verifier.d.ts +6 -0
  56. package/dist/replacement-edits.d.ts +9 -0
  57. package/dist/routing.d.ts +1 -1
  58. package/dist/run-interruption.d.ts +4 -0
  59. package/dist/runtime-checkpoints.d.ts +7 -0
  60. package/dist/runtime-diagnostics.d.ts +63 -0
  61. package/dist/runtime-policy.d.ts +36 -0
  62. package/dist/sessions.d.ts +1 -1
  63. package/dist/state-backup.d.ts +2 -2
  64. package/dist/task-memory.d.ts +36 -0
  65. package/dist/terminal-markdown.d.ts +12 -0
  66. package/dist/time-to-safe-fix.d.ts +19 -19
  67. package/dist/tool-registry.d.ts +4 -0
  68. package/dist/verify-historical-migrations.js +6987 -5140
  69. package/dist/workspace-mutation-lock.d.ts +5 -0
  70. package/dist/workspace.d.ts +39 -4
  71. package/dist/zhx.js +6556 -4030
  72. package/docs/CLI.md +89 -3
  73. package/docs/CONTEXT_ENGINEERING.md +160 -0
  74. package/docs/EXECUTION_ENVIRONMENTS.md +21 -0
  75. package/docs/EXTENSIBILITY.md +2 -2
  76. package/docs/GA_READINESS.md +13 -7
  77. package/docs/LIVE_CERTIFICATION.md +12 -2
  78. package/docs/RC13_SECURITY_REVIEW.md +28 -0
  79. package/docs/RC14_DELEGATED_SECURITY_DECISION.md +80 -0
  80. package/docs/RC14_SECURITY_REVIEW.md +40 -0
  81. package/docs/README.md +33 -0
  82. package/docs/RELEASE.md +4 -2
  83. package/docs/REPOSITORY_EDITING.md +47 -0
  84. package/docs/SECURITY_REVIEW_EVIDENCE.md +5 -3
  85. package/docs/THREAT_MODEL.md +3 -3
  86. package/docs/TIME_TO_SAFE_FIX.md +4 -4
  87. package/docs/ga-readiness.json +147 -16
  88. package/docs/reports/EFFICIENCY_REVIEW.md +156 -0
  89. package/docs/reports/EXTERNAL_COMPARISON.md +434 -0
  90. package/docs/reports/FULL_HARNESS_REVIEW_2026-09-10.md +200 -0
  91. package/docs/reports/GA_BUDGET_REMEDIATION_2026-09-19.md +38 -0
  92. package/docs/reports/GA_REPAIR_RECOVERY_2026-09-19.md +700 -0
  93. package/docs/reports/GA_VALIDATION_2026-09-19.md +111 -0
  94. package/docs/reports/HARNESS_ARCHITECTURE_REVIEW_2026-09-09.md +202 -0
  95. package/docs/reports/HARNESS_AUDIT_2026-09-09.md +306 -0
  96. package/docs/reports/HARNESS_REMEDIATION_2026-09-09.md +74 -0
  97. package/docs/reports/HARNESS_REMEDIATION_2026-09-10.md +90 -0
  98. package/docs/reports/MODEL_COMPARISON_2026-09-09.md +184 -0
  99. package/docs/reports/QWEN_REMEDIATION_COMPARISON_2026-09-09.md +142 -0
  100. package/docs/reports/RC14_VALIDATION_2026-09-10.md +45 -0
  101. package/docs/reports/README.md +26 -0
  102. package/docs/reports/evidence/ga-budget-remediation-2026-09-19.json +133 -0
  103. package/docs/reports/evidence/ga-catalog-ablation-2026-09-19.json +699 -0
  104. package/docs/reports/evidence/ga-catalog-ablation-plan-2026-09-19.json +40 -0
  105. package/docs/reports/evidence/ga-catalog-observed-2026-09-19.json +985 -0
  106. package/docs/reports/evidence/ga-catalog-observed-plan-2026-09-19.json +28 -0
  107. package/docs/reports/evidence/ga-catalog-observed-status-2026-09-19.json +15 -0
  108. package/docs/reports/evidence/ga-complete-local-check-2026-09-19.json +47 -0
  109. package/docs/reports/evidence/ga-completion-reminder-2026-09-19.json +22 -0
  110. package/docs/reports/evidence/ga-fresh-cohort-plan-2026-09-19.json +54 -0
  111. package/docs/reports/evidence/ga-fresh-cohort-preparation-2026-09-19.json +242 -0
  112. package/docs/reports/evidence/ga-fresh-cohort-result-2026-09-19.json +4318 -0
  113. package/docs/reports/evidence/ga-local-validation-followup-2026-09-19.json +66 -0
  114. package/docs/reports/evidence/ga-node-consumers-2026-09-19.json +31 -0
  115. package/docs/reports/evidence/ga-package-content-validation-2026-09-19.json +22 -0
  116. package/docs/reports/evidence/ga-plan-completion-2026-09-19.json +39 -0
  117. package/docs/reports/evidence/ga-plan-reserve-2026-09-19.json +573 -0
  118. package/docs/reports/evidence/ga-plan-reserve-plan-2026-09-19.json +24 -0
  119. package/docs/reports/evidence/ga-plan-transition-2026-09-19.json +19 -0
  120. package/docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json +24 -0
  121. package/docs/reports/evidence/ga-private-verifier-root-cause-2026-09-19.json +41 -0
  122. package/docs/reports/evidence/ga-published-sdk-2026-09-19.json +35 -0
  123. package/docs/reports/evidence/ga-python-checkout-imports-2026-09-19.json +95 -0
  124. package/docs/reports/evidence/ga-python-origin-delivery-2026-09-19.json +211 -0
  125. package/docs/reports/evidence/ga-python-origin-delivery-plan-2026-09-19.json +27 -0
  126. package/docs/reports/evidence/ga-qwen-patched-package-2026-09-19.json +1092 -0
  127. package/docs/reports/evidence/ga-qwen-patched-package-plan-2026-09-19.json +18 -0
  128. package/docs/reports/evidence/ga-qwen-published-2026-09-19.json +571 -0
  129. package/docs/reports/evidence/ga-qwen-published-plan-2026-09-19.json +18 -0
  130. package/docs/reports/evidence/ga-qwen-tool-choice-2026-09-19.json +85 -0
  131. package/docs/reports/evidence/ga-qwen-tool-choice-patched-2026-09-19.json +85 -0
  132. package/docs/reports/evidence/ga-recovery-development-2026-09-19.json +1066 -0
  133. package/docs/reports/evidence/ga-recovery-graded-development-2026-09-19.json +1313 -0
  134. package/docs/reports/evidence/ga-recovery-graded-v2-2026-09-19.json +1151 -0
  135. package/docs/reports/evidence/ga-recovery-v3-2026-09-19.json +3943 -0
  136. package/docs/reports/evidence/ga-recovery-v4-2026-09-19.json +1055 -0
  137. package/docs/reports/evidence/ga-recovery-v5-2026-09-19.json +1245 -0
  138. package/docs/reports/evidence/ga-recovery-v6-2026-09-19.json +699 -0
  139. package/docs/reports/evidence/ga-recovery-v7-2026-09-19.json +854 -0
  140. package/docs/reports/evidence/ga-recovery-v8-2026-09-19.json +596 -0
  141. package/docs/reports/evidence/ga-required-delivery-2026-09-19.json +25 -0
  142. package/docs/reports/evidence/ga-required-delivery-live-2026-09-19.json +1337 -0
  143. package/docs/reports/evidence/ga-required-delivery-live-plan-2026-09-19.json +25 -0
  144. package/docs/reports/evidence/ga-retained-candidate-grading-2026-09-19.json +30 -0
  145. package/docs/reports/evidence/ga-tool-selection-live-2026-09-19.json +559 -0
  146. package/docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json +28 -0
  147. package/docs/reports/evidence/ga-tool-selection-recovery-2026-09-19.json +27 -0
  148. package/docs/reports/evidence/ga-validation-2026-09-19.json +236 -0
  149. package/docs/reports/evidence/ga-verifier-discovery-live-2026-09-19.json +1369 -0
  150. package/docs/reports/evidence/ga-verifier-discovery-plan-2026-09-19.json +29 -0
  151. package/docs/reports/evidence/ga-verifier-failure-identification-2026-09-19.json +30 -0
  152. package/docs/reports/evidence/ga-verifier-feedback-2026-09-19.json +65 -0
  153. package/docs/reports/evidence/ga-verifier-hints-live-2026-09-19.json +1297 -0
  154. package/docs/reports/evidence/ga-verifier-hints-plan-2026-09-19.json +27 -0
  155. package/docs/reports/evidence/ga-work-boundary-context-analysis-2026-09-19.json +113 -0
  156. package/docs/reports/evidence/ga-work-boundary-live-2026-09-19.json +301 -0
  157. package/docs/reports/evidence/ga-work-boundary-live-plan-2026-09-19.json +84 -0
  158. package/docs/reports/evidence/ga-work-budget-planning-2026-09-19.json +32 -0
  159. package/docs/reports/evidence/ga-working-context-2026-09-19.json +22 -0
  160. package/docs/reports/evidence/rc14-delegated-security-review-2026-09-20.json +81 -0
  161. package/docs/reports/evidence/rc14-ga-gate-2026-09-20.json +18 -0
  162. package/docs/reports/evidence/rc14-representative-2026-09-20.json +509 -0
  163. package/docs/reports/evidence/rc14-security-regressions-2026-09-20.json +64 -0
  164. package/docs/reports/evidence/security-current-2026-09-19.json +91 -0
  165. package/docs/reports/evidence/security-remediation-2026-09-19.json +97 -0
  166. package/evaluations/repair-controller-experiment.json +24 -0
  167. package/evaluations/representative-assembly-matrix.json +63 -0
  168. package/evaluations/representative-matrix.json +2219 -2
  169. package/fixtures/contracts/v1/init.json +19 -0
  170. package/package.json +25 -16
@@ -0,0 +1,699 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "kind": "diagnostic-tool-catalog-ablation",
4
+ "classification": "Known development task; experimental driver, not product or GA acceptance",
5
+ "plan": "ga-catalog-ablation-plan-2026-09-19.json",
6
+ "implementation": {
7
+ "gitCommit": "c2b2bb917d0471e61afc5f731a61c00b44fa3731",
8
+ "sources": {
9
+ "bun.lock": "e370459197ce52c3961770f3c5ccaab7ff35ddbe641ef72e8bd61f87a8b8e3fe",
10
+ "evaluations/external/requirements.lock": "42e90a08f10f821ebb958af79256f707a9fb5cf365277c4f9e2715d612028179",
11
+ "package.json": "a4a2507e3ad876d72cad49a16e149192d33521fb4175bb40a3eb077ccfdf43a1",
12
+ "scripts/benchmark-swebench.ts": "ed17063e1847017ba2e72f14732b6ff8f5b1af7cd5253b91302af75f2a9e8476",
13
+ "scripts/swebench/candidate.py": "f5e3410b91b950acfbe50fc7511891816a8fb1c223082397e2c6300cf1e866cf",
14
+ "scripts/swebench/candidate.ts": "bf102becfcce0c9a3f0df0a8f1ce0e5aaea7db2d252f6d9454a8370c24e53982",
15
+ "scripts/swebench/common.py": "ad790227dac3bec1f592c458c993efff4f9d3de8f0df39b33e3238b58b3f0b3b",
16
+ "scripts/swebench/context-metrics.ts": "7b84da6789413aa634d997c5963ee6796584e06c3eafe5f68f424fb7e27815e9",
17
+ "scripts/swebench/mini_driver.py": "bd0a77221878370ec44760122a51f07fb56d969612afa9a7f171248b2958baa6",
18
+ "scripts/swebench/model-budget.ts": "e46dd406ee5d58a051a15c7d7133513666292303d8c6534f861a6db0e711f47c",
19
+ "scripts/swebench/preflight.py": "aed092a75dd52f504fba6d14814b69bd627ff17d0e311a1f1279f8b613b69675",
20
+ "scripts/swebench/prepare.py": "bb7edd3fba5dbbe26597c97d744c60b94dbb7bcdfd7740a4a058c8583bd27e4c",
21
+ "scripts/swebench/report.py": "d0b8d45f32c48e0ba2cb977f50838065a497a0ecaf3b4243ed94139393ce4d6b",
22
+ "scripts/swebench/run.py": "94dccd55dfe6f70d8f8f2da70331700ec85eddd958547e0c3afe3dc416c9802c",
23
+ "scripts/swebench/telemetry.ts": "743c9baf273e9240d704d098b6bd5eea9e76ef3e49ae636aaafa9bfd7a85b45b",
24
+ "scripts/swebench/test_candidate.py": "e77d16379b748baa2daf184a45ea62587d938b6a7d48dcf9f6061f8af21fafd9",
25
+ "scripts/swebench/test_contracts.py": "902e966d0fbf8501dbb15e011849f160331c033cd2a5480f9920d0f365a08d03",
26
+ "scripts/swebench/zhivex-driver.ts": "8d68286b42979a603dcded8860133e134d351f25aedb5dd98ed2ef6f810cbd74",
27
+ "scripts/time-to-safe-fix-efficiency.ts": "dd20fd8839280332ae4ca80d6846e98e07ff15d79e200b206fa2ba6be2724d2f",
28
+ "src/bounded-reads.ts": "9807bb460894be3279c5425af19863d1c6e419a1582ba6910ec7e7fc35703ed6",
29
+ "src/capabilities.ts": "56fdb60a2f6e1d33b2dc26d38b6d2709807e491ba1268d47b97876c2c0591d65",
30
+ "src/change-envelope.ts": "9047ec978c9e5117965b8093ec961d855f97bbc2c3accdc79b74cfe6a0e4f01e",
31
+ "src/cli-options.ts": "f48d9be132fed38f2ddbcfecbd8222c664a0df2f20edc28ce8d1b9b1a1900863",
32
+ "src/cli-profiles.ts": "4fe451f1645e95f4957597a8724fa989b3c661ae1693252bfd6e74a51d20fc74",
33
+ "src/cli-stream.ts": "5ab8a4b1b8cf0a3dfa7b5b1f9e4e0facf8f6e7f6c341bdf700a41dc8927c825e",
34
+ "src/cli.ts": "5a4acb4695a968a58fbeac05c4bb89848a9653d6189590f0ff90e430ffa98d14",
35
+ "src/compaction.ts": "9e06a0abeeef155377f40c160b21c56256bca95ec18c5ee897681a5fc73db169",
36
+ "src/config.ts": "b1d3991e84add58603fb85b2a604b9b3eb56ae879d0fa30b0cb40545fa44edc1",
37
+ "src/console-context.ts": "0ad7e4559504b9ed09ab500d615dea09af1075d357f02ec3b77f003d55d951db",
38
+ "src/console-input.ts": "dc8ea0cdbba1364db288d5156c5ff9819362c46b37ef35a98b0475b6c445b1f8",
39
+ "src/context-engineering.ts": "1995e564f6b91974e75bf66dd421ddcbea29b1f8eee59b76eb4e34c9d6d8f47e",
40
+ "src/context-metrics.ts": "3eb30f9db808f918ed8d73d4f8ac19a13acb09ed5c1ee70fdae05201714b9198",
41
+ "src/edit-contracts.ts": "47cc3132e27f8749d54a32bbcc33bb174cf4448f0d0171b2578cdb559cf2176b",
42
+ "src/errors.ts": "e02f4eb80562a83762f908bda56c950ec6a2b78725c0c8a49f41fee0057ab903",
43
+ "src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9",
44
+ "src/file-security.ts": "b8c9f4988c4339c73a60ab4fa8d394b3eecc19117a0c61a8d19ab6483ca253df",
45
+ "src/harness.ts": "195983cfbacb6097a5c8c2a106e59c3fae2437f053299da1c0170dd914879c8d",
46
+ "src/index.ts": "4bbfcd11376d48c4f1c1ec17d6b5c9f51baa9331fa999432be178bae94530a5e",
47
+ "src/json-contracts.ts": "0780145636b47829a1aa6087c574d5654eb31b7534ea7d9c38c188d648c90272",
48
+ "src/mcp.ts": "7722de031921c620ec03b9e42a6596ef8415aa8e4bb5f93ea84d2427c466e554",
49
+ "src/model-budget.ts": "97c00019d7e737fc7acdd04cdac0811dca8d85c3e858de519ebba91888cf1ee7",
50
+ "src/operations.ts": "bcccaf4d1d0ec117c2ed35df92185564d33399e5fdc5c12225ac61f14fd0b4b1",
51
+ "src/orchestration.ts": "b56c8c7d526a99b745cc17769e0a1b7b500b25920e77f34aa0dd01c9758de319",
52
+ "src/package-manager.ts": "cec4666bbab2f42e0a269709631f79f9f87c2c4dbb3cafeffacfaff761069e2c",
53
+ "src/process-runtime.ts": "b7286731a0b5c96dcad0b8e3a4bf1155d0539d73303cd7ebe77ffabf858f1729",
54
+ "src/providers.ts": "70ccfc7f315cd7238e83d69b719ce4908f3b620c564cbb4f00f93e102e025602",
55
+ "src/repair-controller.ts": "2c05370e93978a5bf106b2cee293478a77b47920cec7470f536b31757198f41c",
56
+ "src/repair-progress.ts": "cf30117ab69564f2b36ff8f0e86dcb876e6c031647340a7b1bef648d1e0366b9",
57
+ "src/repair-verifier.ts": "b70fb11feee8cdc7933947ccac0b78f8bb5c0eaccc3b29ad527197d77f40ffc9",
58
+ "src/replacement-edits.ts": "8a0efdd3a7a9fc3e5a5475916cbd17ef882265881bb61076db58673e80b72957",
59
+ "src/routing.ts": "8e7c9f5968b4bd638cbbc86b19847ca183f3418b9a298783b96a88cffa568e22",
60
+ "src/run-interruption.ts": "df99129cf2dd6b938eca629cb897c4350a7d57fe6bf0ce437f5f3f30271617df",
61
+ "src/runtime-checkpoints.ts": "e3b75a0b86fb5d21992b1a126b35446110ba5fcaf4317ed5005b1a44555118dc",
62
+ "src/runtime-diagnostics.ts": "e1e5a3a558fa2aaeb477335dc336551f022c1234fe5e742eefc6599c8803e364",
63
+ "src/runtime-policy.ts": "8356acf0102f0e73839b81ed628a5c0d9a89d9c21b0ca416efedde8f1b3414e2",
64
+ "src/sessions.ts": "5167c8cf36bd4b8845ee278883394700321b65397f49c1dda0cf0d0de09dd06e",
65
+ "src/sqlite-database.ts": "18765d1ca0c2f2ad87bfd82d311312dfe8a42006556f27bc21da5892ed3b044e",
66
+ "src/state-backup.ts": "b6f8f1aedeb5dce31b65c23c7ffc50030870655b58a5aaaef9989bafb9da5c68",
67
+ "src/state-directory.ts": "3711550991951e6b738a08deaa0e06da84d01864c03a9602600f592963fa3211",
68
+ "src/task-memory.ts": "5b62c863be5a23c33608633cd7aac0464f5536d01cba930aca58b11870170731",
69
+ "src/terminal-markdown.ts": "8cffb6ebaf25108cb2bee04956d9b2973635b010f5d7ce6be21b3ceffe3e8bfd",
70
+ "src/terminal-ui.ts": "4da1878a76ba608a2517bf5d5c992a9a88702aba3fd800a9980129ba93e157bc",
71
+ "src/time-to-safe-fix.ts": "4add1d49c0ea209cc496c66fa2b2012dc7bcab87e678a58c8591b3c2cdc8df8a",
72
+ "src/tool-registry.ts": "7e90fff11940e525883c9ec0d2adbea2e6616f8bd9bc4ec77e1b7277e4217f2c",
73
+ "src/version.ts": "b2c2f80a40adc08e8258b81e82434273f0bdfa787528b2221f14efe7f06aa747",
74
+ "src/workspace-mutation-lock.ts": "ca201242db16173bd3a4d86c5d8bb0c2b6f3c43bfed3e4fa4ff5f06a75b19e80",
75
+ "src/workspace.ts": "809f8bbfcf72fed10d860c48585089ca65bb3ef7c672ee6df3c918907a968302"
76
+ }
77
+ },
78
+ "model": "qwen3.8-flash",
79
+ "limits": {
80
+ "steps": 24,
81
+ "outputPerTurn": 2048,
82
+ "inputTokens": 100000,
83
+ "outputTokens": 16000,
84
+ "timeoutSeconds": 300,
85
+ "memoryMb": 2048,
86
+ "cpus": 2
87
+ },
88
+ "samples": [
89
+ {
90
+ "candidate": "zhivex",
91
+ "instanceId": "django__django-15851",
92
+ "repetition": 0,
93
+ "resolved": false,
94
+ "gradingStatus": "completed",
95
+ "usageComplete": true,
96
+ "costUsd": null,
97
+ "candidateCaptureFailure": null,
98
+ "modelTimings": [
99
+ {
100
+ "durationMs": 2857.5545,
101
+ "firstTokenMs": 2856.44675,
102
+ "completed": true
103
+ },
104
+ {
105
+ "durationMs": 1765.3069590000014,
106
+ "firstTokenMs": 1764.9363750000011,
107
+ "completed": true
108
+ },
109
+ {
110
+ "durationMs": 1971.8102500000023,
111
+ "firstTokenMs": 1971.582500000004,
112
+ "completed": true
113
+ },
114
+ {
115
+ "durationMs": 1423.2325410000049,
116
+ "firstTokenMs": 1423.1229160000003,
117
+ "completed": true
118
+ },
119
+ {
120
+ "durationMs": 1604.3790000000008,
121
+ "firstTokenMs": 1604.1530829999974,
122
+ "completed": true
123
+ },
124
+ {
125
+ "durationMs": 1363.2940420000086,
126
+ "firstTokenMs": 1362.9413330000098,
127
+ "completed": true
128
+ },
129
+ {
130
+ "durationMs": 3342.460540999993,
131
+ "firstTokenMs": 3341.9722499999916,
132
+ "completed": true
133
+ },
134
+ {
135
+ "durationMs": 1671.2688750000088,
136
+ "firstTokenMs": 1671.0233750000043,
137
+ "completed": true
138
+ },
139
+ {
140
+ "durationMs": 13546.226541000011,
141
+ "firstTokenMs": 13545.892208000005,
142
+ "completed": true
143
+ }
144
+ ],
145
+ "stageTimings": [
146
+ {
147
+ "event": "command-requested",
148
+ "elapsedMs": 6210.767709
149
+ }
150
+ ],
151
+ "approvalTimings": [
152
+ {
153
+ "durationMs": 0.08533399999942048,
154
+ "resolved": true
155
+ },
156
+ {
157
+ "durationMs": 0.06737500000235741,
158
+ "resolved": true
159
+ },
160
+ {
161
+ "durationMs": 0.01600000000325963,
162
+ "resolved": true
163
+ },
164
+ {
165
+ "durationMs": 0.022415999999793712,
166
+ "resolved": true
167
+ },
168
+ {
169
+ "durationMs": 0.02399999999033753,
170
+ "resolved": true
171
+ },
172
+ {
173
+ "durationMs": 0.02983299999323208,
174
+ "resolved": true
175
+ },
176
+ {
177
+ "durationMs": 0.03674999999930151,
178
+ "resolved": true
179
+ }
180
+ ],
181
+ "toolTimings": [
182
+ {
183
+ "name": "run_environment_shell",
184
+ "calls": 7,
185
+ "errors": 0,
186
+ "totalMs": 89482.442086,
187
+ "maxMs": 13939.199584000002
188
+ },
189
+ {
190
+ "name": "repair_plan",
191
+ "calls": 1,
192
+ "errors": 0,
193
+ "totalMs": 0.08249999998952262,
194
+ "maxMs": 0.08249999998952262
195
+ }
196
+ ],
197
+ "schemaVersion": 1,
198
+ "status": "failed",
199
+ "failure": "EXECUTION_FAILED",
200
+ "phase": "agent",
201
+ "durationMs": 141427.978167,
202
+ "approvals": 7,
203
+ "turns": [
204
+ {
205
+ "index": 1,
206
+ "status": "waiting_approval",
207
+ "finishReason": "tool-calls",
208
+ "requestedTools": [
209
+ "run_environment_shell"
210
+ ],
211
+ "inputTokens": 2628,
212
+ "outputTokens": 59,
213
+ "cachedInputTokens": 0,
214
+ "requestCharacters": 6814,
215
+ "tools": []
216
+ },
217
+ {
218
+ "index": 2,
219
+ "status": "waiting_approval",
220
+ "finishReason": "tool-calls",
221
+ "requestedTools": [
222
+ "run_environment_shell"
223
+ ],
224
+ "inputTokens": 3004,
225
+ "outputTokens": 61,
226
+ "cachedInputTokens": 2048,
227
+ "requestCharacters": 8045,
228
+ "tools": [
229
+ {
230
+ "name": "run_environment_shell",
231
+ "isError": false,
232
+ "characters": 527,
233
+ "phaseLatencies": {
234
+ "commandAndAttestationMs": 2852.733541,
235
+ "workspaceExportMs": 6749.016,
236
+ "totalMs": 13525.643375,
237
+ "sessionCreationMs": 3923.315709
238
+ },
239
+ "exitCode": 0,
240
+ "timedOut": false
241
+ }
242
+ ]
243
+ },
244
+ {
245
+ "index": 3,
246
+ "status": "waiting_approval",
247
+ "finishReason": "tool-calls",
248
+ "requestedTools": [
249
+ "run_environment_shell"
250
+ ],
251
+ "inputTokens": 4115,
252
+ "outputTokens": 80,
253
+ "cachedInputTokens": 2048,
254
+ "requestCharacters": 12232,
255
+ "tools": [
256
+ {
257
+ "name": "run_environment_shell",
258
+ "isError": false,
259
+ "characters": 3462,
260
+ "phaseLatencies": {
261
+ "commandAndAttestationMs": 2308.533958,
262
+ "workspaceExportMs": 7081.781625,
263
+ "totalMs": 11503.703292,
264
+ "sessionCreationMs": 2113.314875
265
+ },
266
+ "exitCode": 0,
267
+ "timedOut": false
268
+ }
269
+ ]
270
+ },
271
+ {
272
+ "index": 4,
273
+ "status": "waiting_approval",
274
+ "finishReason": "tool-calls",
275
+ "requestedTools": [
276
+ "run_environment_shell"
277
+ ],
278
+ "inputTokens": 4415,
279
+ "outputTokens": 57,
280
+ "cachedInputTokens": 3328,
281
+ "requestCharacters": 13344,
282
+ "tools": [
283
+ {
284
+ "name": "run_environment_shell",
285
+ "isError": false,
286
+ "characters": 628,
287
+ "phaseLatencies": {
288
+ "commandAndAttestationMs": 2551.948125,
289
+ "workspaceExportMs": 8370.394875,
290
+ "totalMs": 13939.041666,
291
+ "sessionCreationMs": 3016.637958
292
+ },
293
+ "exitCode": 0,
294
+ "timedOut": false
295
+ }
296
+ ]
297
+ },
298
+ {
299
+ "index": 5,
300
+ "status": "waiting_approval",
301
+ "finishReason": "tool-calls",
302
+ "requestedTools": [
303
+ "run_environment_shell"
304
+ ],
305
+ "inputTokens": 4802,
306
+ "outputTokens": 63,
307
+ "cachedInputTokens": 3328,
308
+ "requestCharacters": 14769,
309
+ "tools": [
310
+ {
311
+ "name": "run_environment_shell",
312
+ "isError": false,
313
+ "characters": 989,
314
+ "phaseLatencies": {
315
+ "commandAndAttestationMs": 3493.600166,
316
+ "workspaceExportMs": 8051.391792,
317
+ "totalMs": 13708.315958,
318
+ "sessionCreationMs": 2163.173
319
+ },
320
+ "exitCode": 0,
321
+ "timedOut": false
322
+ }
323
+ ]
324
+ },
325
+ {
326
+ "index": 6,
327
+ "status": "waiting_approval",
328
+ "finishReason": "tool-calls",
329
+ "requestedTools": [
330
+ "run_environment_shell"
331
+ ],
332
+ "inputTokens": 5089,
333
+ "outputTokens": 34,
334
+ "cachedInputTokens": 4096,
335
+ "requestCharacters": 15822,
336
+ "tools": [
337
+ {
338
+ "name": "run_environment_shell",
339
+ "isError": false,
340
+ "characters": 601,
341
+ "phaseLatencies": {
342
+ "commandAndAttestationMs": 2686.813583,
343
+ "workspaceExportMs": 7644.495041,
344
+ "totalMs": 12726.732584,
345
+ "sessionCreationMs": 2395.366833
346
+ },
347
+ "exitCode": 0,
348
+ "timedOut": false
349
+ }
350
+ ]
351
+ },
352
+ {
353
+ "index": 7,
354
+ "status": "completed",
355
+ "finishReason": "tool-calls",
356
+ "requestedTools": [
357
+ "repair_plan"
358
+ ],
359
+ "inputTokens": 6848,
360
+ "outputTokens": 224,
361
+ "cachedInputTokens": 4096,
362
+ "requestCharacters": 23276,
363
+ "tools": [
364
+ {
365
+ "name": "run_environment_shell",
366
+ "isError": false,
367
+ "characters": 7093,
368
+ "phaseLatencies": {
369
+ "commandAndAttestationMs": 2419.42325,
370
+ "workspaceExportMs": 7370.358916,
371
+ "totalMs": 12047.248042,
372
+ "sessionCreationMs": 2257.312958
373
+ },
374
+ "exitCode": 0,
375
+ "timedOut": false
376
+ }
377
+ ]
378
+ },
379
+ {
380
+ "index": 8,
381
+ "status": "waiting_approval",
382
+ "finishReason": "tool-calls",
383
+ "requestedTools": [
384
+ "run_environment_shell"
385
+ ],
386
+ "inputTokens": 7332,
387
+ "outputTokens": 48,
388
+ "cachedInputTokens": 6144,
389
+ "requestCharacters": 25450,
390
+ "tools": [
391
+ {
392
+ "name": "repair_plan",
393
+ "isError": false,
394
+ "characters": 801
395
+ },
396
+ {
397
+ "name": "run_environment_shell",
398
+ "isError": false,
399
+ "characters": 463,
400
+ "phaseLatencies": {
401
+ "commandAndAttestationMs": 2746.627709,
402
+ "workspaceExportMs": 7016.995125,
403
+ "totalMs": 12029.79325,
404
+ "sessionCreationMs": 2266.033208
405
+ },
406
+ "exitCode": 0,
407
+ "timedOut": false
408
+ }
409
+ ]
410
+ },
411
+ {
412
+ "index": 9,
413
+ "status": "failed",
414
+ "finishReason": "tool-calls",
415
+ "requestedTools": [
416
+ "other-tool"
417
+ ],
418
+ "inputTokens": 3387,
419
+ "outputTokens": 54,
420
+ "cachedInputTokens": 1024,
421
+ "requestCharacters": 2472,
422
+ "tools": [
423
+ {
424
+ "name": "other-tool",
425
+ "isError": true,
426
+ "characters": 4,
427
+ "diagnostic": {
428
+ "code": "EXECUTION_FAILED",
429
+ "category": "execution",
430
+ "failureType": "execution",
431
+ "retryable": false,
432
+ "chain": [
433
+ {
434
+ "validationCode": null,
435
+ "providerReason": null,
436
+ "validationIssues": [],
437
+ "status": null,
438
+ "errorClass": "Object",
439
+ "providerHints": [],
440
+ "fingerprint": "6d0579d910635f92f565c11d29a0704f48dc1b126b5cff8674e01de04ef58a64",
441
+ "reason": "not registered"
442
+ }
443
+ ]
444
+ }
445
+ }
446
+ ]
447
+ }
448
+ ],
449
+ "inputTokens": 41620,
450
+ "outputTokens": 680,
451
+ "cachedInputTokens": 26112,
452
+ "modelCalls": 9,
453
+ "compactions": [
454
+ {
455
+ "beforeStep": 9,
456
+ "estimatedTokensBefore": 5972,
457
+ "estimatedTokensAfter": 1850
458
+ }
459
+ ],
460
+ "terminalDiagnostic": {
461
+ "code": "EXECUTION_FAILED",
462
+ "category": "execution",
463
+ "failureType": "execution",
464
+ "retryable": false,
465
+ "chain": [
466
+ {
467
+ "validationCode": null,
468
+ "providerReason": null,
469
+ "validationIssues": [],
470
+ "status": null,
471
+ "errorClass": "Object",
472
+ "providerHints": [],
473
+ "fingerprint": "4781c6db9c61c01642ad93cb9a3b069cb6ef16d947024b4852fa71e010ef7fed",
474
+ "reason": "not registered"
475
+ }
476
+ ]
477
+ },
478
+ "thrownDiagnostic": {
479
+ "code": "EXECUTION_FAILED",
480
+ "category": "execution",
481
+ "failureType": "execution",
482
+ "retryable": false,
483
+ "chain": [
484
+ {
485
+ "validationCode": null,
486
+ "providerReason": null,
487
+ "validationIssues": [],
488
+ "status": null,
489
+ "errorClass": "HarnessExecutionError",
490
+ "providerHints": [],
491
+ "fingerprint": "4781c6db9c61c01642ad93cb9a3b069cb6ef16d947024b4852fa71e010ef7fed",
492
+ "reason": "not registered"
493
+ },
494
+ {
495
+ "validationCode": null,
496
+ "providerReason": null,
497
+ "validationIssues": [],
498
+ "status": null,
499
+ "errorClass": null,
500
+ "providerHints": [],
501
+ "fingerprint": "4781c6db9c61c01642ad93cb9a3b069cb6ef16d947024b4852fa71e010ef7fed",
502
+ "reason": "not registered"
503
+ }
504
+ ]
505
+ },
506
+ "guardrails": [],
507
+ "repairProgress": {
508
+ "repeatedResults": 0,
509
+ "suppressedResults": 0,
510
+ "blockedBroadCalls": 0,
511
+ "enteredClosure": false,
512
+ "phase": "explore",
513
+ "blockedPhaseCalls": 0,
514
+ "planRecorded": true
515
+ },
516
+ "contextMetrics": [
517
+ {
518
+ "systemCharacters": 4262,
519
+ "userCharacters": 2548,
520
+ "assistantCharacters": 0,
521
+ "toolResultCharacters": 0,
522
+ "otherMessageCharacters": 0,
523
+ "toolDefinitionCharacters": 3357,
524
+ "unmeasuredToolDefinitions": 0
525
+ },
526
+ {
527
+ "systemCharacters": 4262,
528
+ "userCharacters": 2815,
529
+ "assistantCharacters": 266,
530
+ "toolResultCharacters": 696,
531
+ "otherMessageCharacters": 0,
532
+ "toolDefinitionCharacters": 3357,
533
+ "unmeasuredToolDefinitions": 0
534
+ },
535
+ {
536
+ "systemCharacters": 4262,
537
+ "userCharacters": 3081,
538
+ "assistantCharacters": 554,
539
+ "toolResultCharacters": 4327,
540
+ "otherMessageCharacters": 0,
541
+ "toolDefinitionCharacters": 3357,
542
+ "unmeasuredToolDefinitions": 0
543
+ },
544
+ {
545
+ "systemCharacters": 4262,
546
+ "userCharacters": 3081,
547
+ "assistantCharacters": 867,
548
+ "toolResultCharacters": 5124,
549
+ "otherMessageCharacters": 0,
550
+ "toolDefinitionCharacters": 3357,
551
+ "unmeasuredToolDefinitions": 0
552
+ },
553
+ {
554
+ "systemCharacters": 4262,
555
+ "userCharacters": 3081,
556
+ "assistantCharacters": 1132,
557
+ "toolResultCharacters": 6282,
558
+ "otherMessageCharacters": 0,
559
+ "toolDefinitionCharacters": 3357,
560
+ "unmeasuredToolDefinitions": 0
561
+ },
562
+ {
563
+ "systemCharacters": 4262,
564
+ "userCharacters": 3081,
565
+ "assistantCharacters": 1413,
566
+ "toolResultCharacters": 7052,
567
+ "otherMessageCharacters": 0,
568
+ "toolDefinitionCharacters": 3357,
569
+ "unmeasuredToolDefinitions": 0
570
+ },
571
+ {
572
+ "systemCharacters": 4262,
573
+ "userCharacters": 3081,
574
+ "assistantCharacters": 1603,
575
+ "toolResultCharacters": 14314,
576
+ "otherMessageCharacters": 0,
577
+ "toolDefinitionCharacters": 3357,
578
+ "unmeasuredToolDefinitions": 0
579
+ },
580
+ {
581
+ "systemCharacters": 4262,
582
+ "userCharacters": 3399,
583
+ "assistantCharacters": 2497,
584
+ "toolResultCharacters": 15274,
585
+ "otherMessageCharacters": 0,
586
+ "toolDefinitionCharacters": 3357,
587
+ "unmeasuredToolDefinitions": 0
588
+ },
589
+ {
590
+ "systemCharacters": 4262,
591
+ "userCharacters": 2470,
592
+ "assistantCharacters": 2501,
593
+ "toolResultCharacters": 632,
594
+ "otherMessageCharacters": 0,
595
+ "toolDefinitionCharacters": 3357,
596
+ "unmeasuredToolDefinitions": 0
597
+ }
598
+ ],
599
+ "omittedContextMeasurements": 0,
600
+ "toolCalls": 8,
601
+ "patchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
602
+ "protectedFilesChanged": [],
603
+ "gradingMethod": "empty-patch-no-solution",
604
+ "officialResolved": false,
605
+ "candidateCaptured": true,
606
+ "candidatePatchSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
607
+ "candidateGradingStatus": "completed",
608
+ "candidateResolved": false,
609
+ "repairStages": {
610
+ "candidatePresent": false,
611
+ "importedPatchPresent": false,
612
+ "verificationAttempted": false
613
+ },
614
+ "gradingDurationMs": 0.16387499636039138,
615
+ "totalDurationMs": 153439.36329201097
616
+ },
617
+ {
618
+ "candidate": "mini-swe-agent",
619
+ "instanceId": "django__django-15851",
620
+ "repetition": 0,
621
+ "resolved": true,
622
+ "gradingStatus": "completed",
623
+ "usageComplete": true,
624
+ "costUsd": null,
625
+ "schemaVersion": 1,
626
+ "status": "Submitted",
627
+ "failure": null,
628
+ "phase": "agent",
629
+ "durationMs": 22133.75095796073,
630
+ "approvals": 0,
631
+ "toolCalls": 8,
632
+ "inputTokens": 22969,
633
+ "outputTokens": 636,
634
+ "cachedInputTokens": 15872,
635
+ "modelCalls": 8,
636
+ "patchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
637
+ "protectedFilesChanged": [],
638
+ "officialResolved": true,
639
+ "candidateCaptured": true,
640
+ "candidatePatchSha256": "873ee0e2d09108303f5ebe1fa4d0aec08ee9e4ce52b93941c70b4c8c91c21db4",
641
+ "candidateGradingStatus": "completed",
642
+ "candidateResolved": true,
643
+ "repairStages": {
644
+ "candidatePresent": true,
645
+ "importedPatchPresent": null,
646
+ "verificationAttempted": null
647
+ },
648
+ "gradingDurationMs": 24649.95683298912,
649
+ "totalDurationMs": 49208.5399160278
650
+ }
651
+ ],
652
+ "report": {
653
+ "schemaVersion": 1,
654
+ "kind": "swebench-comparison",
655
+ "manifestSha256": "69eec29ad77147b4e43408679277faacc89bb04680b711b15217cfa9fe5f34ed",
656
+ "complete": true,
657
+ "candidates": {
658
+ "zhivex": {
659
+ "planned": 1,
660
+ "recorded": 1,
661
+ "resolved": 0,
662
+ "candidateCaptured": 1,
663
+ "candidateResolved": 0,
664
+ "candidateGraded": 1,
665
+ "resolutionRate": 0.0,
666
+ "missing": 0,
667
+ "gradingFailures": 0,
668
+ "usageComplete": true,
669
+ "totalCostUsd": null,
670
+ "costPerResolvedUsd": null,
671
+ "resolvedP50Ms": null
672
+ },
673
+ "mini-swe-agent": {
674
+ "planned": 1,
675
+ "recorded": 1,
676
+ "resolved": 1,
677
+ "candidateCaptured": 1,
678
+ "candidateResolved": 1,
679
+ "candidateGraded": 1,
680
+ "resolutionRate": 1.0,
681
+ "missing": 0,
682
+ "gradingFailures": 0,
683
+ "usageComplete": true,
684
+ "totalCostUsd": null,
685
+ "costPerResolvedUsd": null,
686
+ "resolvedP50Ms": 49208.5399160278
687
+ }
688
+ },
689
+ "pairedTaskResolutionDifference": -1.0,
690
+ "pairedTaskBootstrap95": null,
691
+ "evidenceBoundary": "Local matched-model comparison. Missing and failed runs remain in the denominator. Small-sample intervals are exploratory. No safety superiority is inferred from SWE-bench correctness."
692
+ },
693
+ "limitations": [
694
+ "One repetition; no causal or general reliability claim",
695
+ "Catalog and rendered instructions changed together",
696
+ "Existing shell route uses different existing quotas from native reads",
697
+ "No runtime policy or product profile changed"
698
+ ]
699
+ }