@zhivex-ai/harness 1.0.0 → 1.1.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/CHANGELOG.md +97 -1
  2. package/README.md +52 -306
  3. package/SECURITY.md +1 -1
  4. package/SUPPORT.md +7 -3
  5. package/benchmarks/README.md +3 -3
  6. package/contracts/client-protocol-v1.schema.json +341 -0
  7. package/contracts/public-api.json +100 -1
  8. package/contracts/stable-api-signatures.json +14 -14
  9. package/dist/approval-diff.d.ts +45 -0
  10. package/dist/approval-history.d.ts +38 -0
  11. package/dist/approval-preview.d.ts +26 -0
  12. package/dist/cli-credentials.d.ts +38 -0
  13. package/dist/cli-help.d.ts +3 -0
  14. package/dist/cli-options.d.ts +1 -1
  15. package/dist/cli-stream.d.ts +3 -3
  16. package/dist/cli.d.ts +15 -69
  17. package/dist/cli.js +13796 -10032
  18. package/dist/client-contract.d.ts +208 -0
  19. package/dist/console-commands.d.ts +6 -0
  20. package/dist/console-context.d.ts +5 -1
  21. package/dist/console-input.d.ts +44 -4
  22. package/dist/console-model-catalog.d.ts +25 -0
  23. package/dist/console-navigation.d.ts +27 -0
  24. package/dist/console-presentation.d.ts +18 -0
  25. package/dist/console-welcome.d.ts +13 -0
  26. package/dist/execution-environment.d.ts +4 -0
  27. package/dist/harness.d.ts +3 -0
  28. package/dist/hostile-repository-demo.js +1913 -798
  29. package/dist/index.d.ts +7 -0
  30. package/dist/index.js +7908 -6330
  31. package/dist/local-service.d.ts +34 -0
  32. package/dist/operations.d.ts +23 -1
  33. package/dist/orchestration.d.ts +3 -0
  34. package/dist/persistence-secrets.d.ts +2 -0
  35. package/dist/recorded-workspace.d.ts +4 -0
  36. package/dist/run-document.d.ts +91 -0
  37. package/dist/runtime-policy.d.ts +4 -4
  38. package/dist/service-cli.d.ts +2 -0
  39. package/dist/service-cli.js +10858 -0
  40. package/dist/service-client-cli.d.ts +2 -0
  41. package/dist/service-events.d.ts +50 -0
  42. package/dist/sessions.d.ts +3 -0
  43. package/dist/sqlite-access.d.ts +26 -0
  44. package/dist/sqlite-database.d.ts +2 -0
  45. package/dist/state-backup.d.ts +8 -0
  46. package/dist/task-memory.d.ts +2 -2
  47. package/dist/terminal-continuation.d.ts +3 -0
  48. package/dist/terminal-markdown.d.ts +1 -0
  49. package/dist/terminal-ui.d.ts +5 -0
  50. package/dist/usage-ledger.d.ts +84 -0
  51. package/dist/verify-historical-migrations.js +5699 -5191
  52. package/dist/workspace.d.ts +22 -0
  53. package/dist/zhx.js +13796 -10032
  54. package/docs/CLI.md +136 -30
  55. package/docs/CLIENT_PROTOCOL.md +212 -0
  56. package/docs/CREDENTIALS.md +59 -0
  57. package/docs/DURABLE_OPERATIONS.md +2 -2
  58. package/docs/FIRST_USE.md +82 -0
  59. package/docs/FIRST_USE_EXAMPLE.md +40 -0
  60. package/docs/HOSTILE_REPOSITORY_DEMO.md +1 -1
  61. package/docs/LOCAL_SERVICE.md +142 -0
  62. package/docs/README.md +24 -23
  63. package/docs/THREAT_MODEL.md +1 -1
  64. package/docs/TIME_TO_SAFE_FIX.md +2 -2
  65. package/docs/USAGE.md +181 -0
  66. package/evaluations/daily-cli-cohort.json +33 -0
  67. package/evaluations/representative-assembly-matrix.json +19 -1
  68. package/examples/README.md +26 -0
  69. package/examples/first-use/greeting.test.ts +3 -0
  70. package/examples/first-use/greeting.ts +1 -0
  71. package/examples/first-use/package.json +7 -0
  72. package/package.json +34 -9
  73. package/ROADMAP.md +0 -414
  74. package/benchmarks/baselines/context-navigation-2026-09-09.json +0 -94
  75. package/benchmarks/baselines/efficiency-2026-09-08.json +0 -233
  76. package/benchmarks/baselines/external-final-2026-09-08.json +0 -588
  77. package/benchmarks/baselines/external-intermediate-2026-09-08.json +0 -116
  78. package/benchmarks/baselines/external-matched-openai-2026-09-08.json +0 -1922
  79. package/benchmarks/baselines/external-matched-qwen-2026-09-08.json +0 -1706
  80. package/benchmarks/baselines/external-pilot-2026-09-08.json +0 -439
  81. package/benchmarks/baselines/external-qwen-flash-2026-09-08.json +0 -1570
  82. package/benchmarks/baselines/external-qwen-remediation-2026-09-09.json +0 -1509
  83. package/benchmarks/baselines/external-qwen-sdk-114-progress-2026-09-09.json +0 -2509
  84. package/benchmarks/baselines/external-sdk-upgrade-2026-09-08.json +0 -564
  85. package/benchmarks/baselines/external-verifier-recovery-2026-09-09.json +0 -794
  86. package/benchmarks/baselines/full-harness-review-2026-09-10.json +0 -134
  87. package/benchmarks/baselines/harness-audit-2026-09-09.json +0 -136
  88. package/benchmarks/baselines/harness-remediation-2026-09-09.json +0 -160
  89. package/benchmarks/baselines/oci-remediation-2026-09-10.json +0 -535
  90. package/benchmarks/baselines/rc14-holdout-2026-09-10.json +0 -854
  91. package/benchmarks/baselines/repair-architecture-audit-2026-09-09.json +0 -75
  92. package/benchmarks/baselines/time-to-safe-fix-live-expanded-2026-08-21.json +0 -186
  93. package/benchmarks/baselines/time-to-safe-fix-live-smoke-2026-08-21.json +0 -186
  94. package/docs/GA_READINESS.md +0 -29
  95. package/docs/GEMINI_1_0_DECISION.md +0 -9
  96. package/docs/LIVE_CERTIFICATION.md +0 -107
  97. package/docs/RC13_SECURITY_REVIEW.md +0 -28
  98. package/docs/RC14_DELEGATED_SECURITY_DECISION.md +0 -80
  99. package/docs/RC14_SECURITY_REVIEW.md +0 -40
  100. package/docs/RELEASE.md +0 -96
  101. package/docs/ROLLBACK.md +0 -9
  102. package/docs/SECURITY_REVIEW_EVIDENCE.md +0 -17
  103. package/docs/ga-readiness.json +0 -366
  104. package/docs/reports/EFFICIENCY_REVIEW.md +0 -156
  105. package/docs/reports/EXTERNAL_COMPARISON.md +0 -434
  106. package/docs/reports/FULL_HARNESS_REVIEW_2026-09-10.md +0 -200
  107. package/docs/reports/GA_BUDGET_REMEDIATION_2026-09-19.md +0 -38
  108. package/docs/reports/GA_REPAIR_RECOVERY_2026-09-19.md +0 -700
  109. package/docs/reports/GA_VALIDATION_2026-09-19.md +0 -111
  110. package/docs/reports/HARNESS_ARCHITECTURE_REVIEW_2026-09-09.md +0 -202
  111. package/docs/reports/HARNESS_AUDIT_2026-09-09.md +0 -306
  112. package/docs/reports/HARNESS_REMEDIATION_2026-09-09.md +0 -74
  113. package/docs/reports/HARNESS_REMEDIATION_2026-09-10.md +0 -90
  114. package/docs/reports/MODEL_COMPARISON_2026-09-09.md +0 -184
  115. package/docs/reports/QWEN_REMEDIATION_COMPARISON_2026-09-09.md +0 -142
  116. package/docs/reports/RC14_VALIDATION_2026-09-10.md +0 -45
  117. package/docs/reports/README.md +0 -26
  118. package/docs/reports/evidence/ga-budget-remediation-2026-09-19.json +0 -133
  119. package/docs/reports/evidence/ga-catalog-ablation-2026-09-19.json +0 -699
  120. package/docs/reports/evidence/ga-catalog-ablation-plan-2026-09-19.json +0 -40
  121. package/docs/reports/evidence/ga-catalog-observed-2026-09-19.json +0 -985
  122. package/docs/reports/evidence/ga-catalog-observed-plan-2026-09-19.json +0 -28
  123. package/docs/reports/evidence/ga-catalog-observed-status-2026-09-19.json +0 -15
  124. package/docs/reports/evidence/ga-complete-local-check-2026-09-19.json +0 -47
  125. package/docs/reports/evidence/ga-completion-reminder-2026-09-19.json +0 -22
  126. package/docs/reports/evidence/ga-fresh-cohort-plan-2026-09-19.json +0 -54
  127. package/docs/reports/evidence/ga-fresh-cohort-preparation-2026-09-19.json +0 -242
  128. package/docs/reports/evidence/ga-fresh-cohort-result-2026-09-19.json +0 -4318
  129. package/docs/reports/evidence/ga-local-validation-followup-2026-09-19.json +0 -66
  130. package/docs/reports/evidence/ga-node-consumers-2026-09-19.json +0 -31
  131. package/docs/reports/evidence/ga-package-content-validation-2026-09-19.json +0 -22
  132. package/docs/reports/evidence/ga-plan-completion-2026-09-19.json +0 -39
  133. package/docs/reports/evidence/ga-plan-reserve-2026-09-19.json +0 -573
  134. package/docs/reports/evidence/ga-plan-reserve-plan-2026-09-19.json +0 -24
  135. package/docs/reports/evidence/ga-plan-transition-2026-09-19.json +0 -19
  136. package/docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json +0 -24
  137. package/docs/reports/evidence/ga-private-verifier-root-cause-2026-09-19.json +0 -41
  138. package/docs/reports/evidence/ga-published-sdk-2026-09-19.json +0 -35
  139. package/docs/reports/evidence/ga-python-checkout-imports-2026-09-19.json +0 -95
  140. package/docs/reports/evidence/ga-python-origin-delivery-2026-09-19.json +0 -211
  141. package/docs/reports/evidence/ga-python-origin-delivery-plan-2026-09-19.json +0 -27
  142. package/docs/reports/evidence/ga-qwen-patched-package-2026-09-19.json +0 -1092
  143. package/docs/reports/evidence/ga-qwen-patched-package-plan-2026-09-19.json +0 -18
  144. package/docs/reports/evidence/ga-qwen-published-2026-09-19.json +0 -571
  145. package/docs/reports/evidence/ga-qwen-published-plan-2026-09-19.json +0 -18
  146. package/docs/reports/evidence/ga-qwen-tool-choice-2026-09-19.json +0 -85
  147. package/docs/reports/evidence/ga-qwen-tool-choice-patched-2026-09-19.json +0 -85
  148. package/docs/reports/evidence/ga-recovery-development-2026-09-19.json +0 -1066
  149. package/docs/reports/evidence/ga-recovery-graded-development-2026-09-19.json +0 -1313
  150. package/docs/reports/evidence/ga-recovery-graded-v2-2026-09-19.json +0 -1151
  151. package/docs/reports/evidence/ga-recovery-v3-2026-09-19.json +0 -3943
  152. package/docs/reports/evidence/ga-recovery-v4-2026-09-19.json +0 -1055
  153. package/docs/reports/evidence/ga-recovery-v5-2026-09-19.json +0 -1245
  154. package/docs/reports/evidence/ga-recovery-v6-2026-09-19.json +0 -699
  155. package/docs/reports/evidence/ga-recovery-v7-2026-09-19.json +0 -854
  156. package/docs/reports/evidence/ga-recovery-v8-2026-09-19.json +0 -596
  157. package/docs/reports/evidence/ga-required-delivery-2026-09-19.json +0 -25
  158. package/docs/reports/evidence/ga-required-delivery-live-2026-09-19.json +0 -1337
  159. package/docs/reports/evidence/ga-required-delivery-live-plan-2026-09-19.json +0 -25
  160. package/docs/reports/evidence/ga-retained-candidate-grading-2026-09-19.json +0 -30
  161. package/docs/reports/evidence/ga-tool-selection-live-2026-09-19.json +0 -559
  162. package/docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json +0 -28
  163. package/docs/reports/evidence/ga-tool-selection-recovery-2026-09-19.json +0 -27
  164. package/docs/reports/evidence/ga-validation-2026-09-19.json +0 -236
  165. package/docs/reports/evidence/ga-verifier-discovery-live-2026-09-19.json +0 -1369
  166. package/docs/reports/evidence/ga-verifier-discovery-plan-2026-09-19.json +0 -29
  167. package/docs/reports/evidence/ga-verifier-failure-identification-2026-09-19.json +0 -30
  168. package/docs/reports/evidence/ga-verifier-feedback-2026-09-19.json +0 -65
  169. package/docs/reports/evidence/ga-verifier-hints-live-2026-09-19.json +0 -1297
  170. package/docs/reports/evidence/ga-verifier-hints-plan-2026-09-19.json +0 -27
  171. package/docs/reports/evidence/ga-work-boundary-context-analysis-2026-09-19.json +0 -113
  172. package/docs/reports/evidence/ga-work-boundary-live-2026-09-19.json +0 -301
  173. package/docs/reports/evidence/ga-work-boundary-live-plan-2026-09-19.json +0 -84
  174. package/docs/reports/evidence/ga-work-budget-planning-2026-09-19.json +0 -32
  175. package/docs/reports/evidence/ga-working-context-2026-09-19.json +0 -22
  176. package/docs/reports/evidence/rc14-delegated-security-review-2026-09-20.json +0 -81
  177. package/docs/reports/evidence/rc14-ga-gate-2026-09-20.json +0 -18
  178. package/docs/reports/evidence/rc14-representative-2026-09-20.json +0 -509
  179. package/docs/reports/evidence/rc14-security-regressions-2026-09-20.json +0 -64
  180. package/docs/reports/evidence/security-current-2026-09-19.json +0 -91
  181. package/docs/reports/evidence/security-remediation-2026-09-19.json +0 -97
@@ -1,25 +0,0 @@
1
- {
2
- "kind": "required-delivery-development-validation",
3
- "classification": "known development case; not holdout or GA acceptance",
4
- "baseline": "ga-qwen-published",
5
- "qwenVersion": "0.14.4",
6
- "repetitions": 1,
7
- "control": "unchanged mini-SWE-agent",
8
- "limits": {
9
- "steps": 24,
10
- "outputPerTurn": 2048,
11
- "inputTokens": 100000,
12
- "outputTokens": 16000,
13
- "timeoutSeconds": 300,
14
- "memoryMb": 2048,
15
- "cpus": 2
16
- },
17
- "change": "explicit verified-delivery obligation enabled",
18
- "sources": {
19
- "src/config.ts": "67ed44b55505ff8dd94f60ba89c33c217759cd7c79de45ccda369f248309cdff",
20
- "src/harness.ts": "1d305078e3c56c24bb659c34aac37c96bf11a4e60af6607bb4b9d65964e84791",
21
- "src/repair-controller.ts": "41f1f09032131306f3284edbf105930577504af0bf764ba98a105c4eee70a1e7",
22
- "scripts/swebench/zhivex-driver.ts": "3d1c143b4c2a1438a1249db8cc10c70a80cbd1c9e3ed44317230795b7a9be2b4",
23
- "bun.lock": "dbabdf161b3dba306f475c81f37e5ab1620fb98052127bb3ed2a0a0cb611ede4"
24
- }
25
- }
@@ -1,30 +0,0 @@
1
- {
2
- "schemaVersion": 1,
3
- "kind": "offline-retained-candidate-grading",
4
- "classification": "Diagnostic of prior failed development run; not delivery, holdout or GA acceptance",
5
- "instanceId": "django__django-15851",
6
- "patchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
7
- "snapshotPatchId": "sha256:b6a09cef9555d8c46b7721d28bec8583f21c786d198c20efcff992d6d6f8949e",
8
- "changedPaths": [
9
- "django/db/backends/postgresql/client.py"
10
- ],
11
- "protectedFilesChanged": [],
12
- "evaluatorImage": "sha256:aa80884e053cf1a238df7c4ceec43e4e07929547f658278bd3c3154f0ef8bc90",
13
- "grading": {
14
- "gradingStatus": "completed",
15
- "resolved": true
16
- },
17
- "newModelRequests": 0,
18
- "originalRunStatus": "failed",
19
- "originalImportSucceeded": false,
20
- "sourceRunPlan": "docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json",
21
- "captureSources": {
22
- "scripts/swebench/candidate.ts": "5767ae241c4a58bd731456abe2c3d92f7a57bba764c80e0f0d00748a0f47d02b",
23
- "src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9"
24
- },
25
- "fullLocalSuite": {
26
- "passed": 571,
27
- "failed": 0,
28
- "files": 70
29
- }
30
- }
@@ -1,559 +0,0 @@
1
- {
2
- "schemaVersion": 1,
3
- "kind": "corrected-image-verified-delivery",
4
- "classification": "Single known development diagnostic, not holdout or GA acceptance",
5
- "result": {
6
- "candidateCaptureFailure": null,
7
- "modelTimings": [
8
- {
9
- "durationMs": 3608.6125829999996,
10
- "firstTokenMs": 3608.2372499999997,
11
- "completed": true
12
- },
13
- {
14
- "durationMs": 2005.5725419999999,
15
- "firstTokenMs": 2005.4179999999997,
16
- "completed": true
17
- },
18
- {
19
- "durationMs": 2233.2373750000006,
20
- "firstTokenMs": 2233.012917,
21
- "completed": true
22
- },
23
- {
24
- "durationMs": 2664.5729170000013,
25
- "firstTokenMs": 2663.945167,
26
- "completed": true
27
- },
28
- {
29
- "durationMs": 7502.284083000002,
30
- "firstTokenMs": 7501.927,
31
- "completed": true
32
- },
33
- {
34
- "durationMs": 30397.212916000004,
35
- "firstTokenMs": 30396.758041,
36
- "completed": true
37
- }
38
- ],
39
- "modelCatalog": {
40
- "records": [
41
- {
42
- "call": 1,
43
- "offered": [
44
- "apply_reviewed_edits",
45
- "apply_reviewed_replacement",
46
- "inspect_environment_patch",
47
- "list_files",
48
- "mutation_audit",
49
- "read_files",
50
- "read_task",
51
- "repair_plan",
52
- "run_environment_command",
53
- "run_environment_shell",
54
- "search_files",
55
- "search_many",
56
- "verify_and_apply_environment_patch"
57
- ],
58
- "unknownOffered": 0,
59
- "systemMessages": 1,
60
- "choice": "required",
61
- "returned": [
62
- {
63
- "name": "search_files",
64
- "offered": true,
65
- "registered": true
66
- },
67
- {
68
- "name": "list_files",
69
- "offered": true,
70
- "registered": true
71
- }
72
- ],
73
- "omittedReturned": 0,
74
- "completed": true
75
- },
76
- {
77
- "call": 2,
78
- "offered": [
79
- "apply_reviewed_edits",
80
- "apply_reviewed_replacement",
81
- "inspect_environment_patch",
82
- "list_files",
83
- "mutation_audit",
84
- "read_files",
85
- "read_task",
86
- "repair_plan",
87
- "run_environment_command",
88
- "run_environment_shell",
89
- "search_files",
90
- "search_many",
91
- "verify_and_apply_environment_patch"
92
- ],
93
- "unknownOffered": 0,
94
- "systemMessages": 1,
95
- "choice": "required",
96
- "returned": [
97
- {
98
- "name": "read_files",
99
- "offered": true,
100
- "registered": true
101
- }
102
- ],
103
- "omittedReturned": 0,
104
- "completed": true
105
- },
106
- {
107
- "call": 3,
108
- "offered": [
109
- "apply_reviewed_edits",
110
- "apply_reviewed_replacement",
111
- "inspect_environment_patch",
112
- "list_files",
113
- "mutation_audit",
114
- "read_files",
115
- "read_task",
116
- "repair_plan",
117
- "run_environment_command",
118
- "run_environment_shell",
119
- "search_files",
120
- "search_many",
121
- "verify_and_apply_environment_patch"
122
- ],
123
- "unknownOffered": 0,
124
- "systemMessages": 1,
125
- "choice": "required",
126
- "returned": [
127
- {
128
- "name": "read_files",
129
- "offered": true,
130
- "registered": true
131
- }
132
- ],
133
- "omittedReturned": 0,
134
- "completed": true
135
- },
136
- {
137
- "call": 4,
138
- "offered": [
139
- "apply_reviewed_edits",
140
- "apply_reviewed_replacement",
141
- "inspect_environment_patch",
142
- "list_files",
143
- "mutation_audit",
144
- "read_files",
145
- "read_task",
146
- "repair_plan",
147
- "run_environment_command",
148
- "run_environment_shell",
149
- "search_files",
150
- "search_many",
151
- "verify_and_apply_environment_patch"
152
- ],
153
- "unknownOffered": 0,
154
- "systemMessages": 1,
155
- "choice": "required",
156
- "returned": [
157
- {
158
- "name": "search_files",
159
- "offered": true,
160
- "registered": true
161
- }
162
- ],
163
- "omittedReturned": 0,
164
- "completed": true
165
- },
166
- {
167
- "call": 5,
168
- "offered": [
169
- "apply_reviewed_edits",
170
- "apply_reviewed_replacement",
171
- "inspect_environment_patch",
172
- "list_files",
173
- "mutation_audit",
174
- "read_files",
175
- "read_task",
176
- "repair_plan",
177
- "run_environment_command",
178
- "run_environment_shell",
179
- "search_files",
180
- "search_many",
181
- "verify_and_apply_environment_patch"
182
- ],
183
- "unknownOffered": 0,
184
- "systemMessages": 1,
185
- "choice": "required",
186
- "returned": [
187
- {
188
- "name": "repair_plan",
189
- "offered": true,
190
- "registered": true
191
- }
192
- ],
193
- "omittedReturned": 0,
194
- "completed": true
195
- },
196
- {
197
- "call": 6,
198
- "offered": [
199
- "apply_reviewed_edits",
200
- "apply_reviewed_replacement",
201
- "inspect_environment_patch",
202
- "list_files",
203
- "mutation_audit",
204
- "read_files",
205
- "read_task",
206
- "repair_plan",
207
- "run_environment_command",
208
- "run_environment_shell",
209
- "search_files",
210
- "search_many",
211
- "verify_and_apply_environment_patch"
212
- ],
213
- "unknownOffered": 0,
214
- "systemMessages": 1,
215
- "choice": "required",
216
- "returned": [
217
- {
218
- "name": "apply_reviewed_replacement",
219
- "offered": true,
220
- "registered": true
221
- }
222
- ],
223
- "omittedReturned": 0,
224
- "completed": true
225
- }
226
- ],
227
- "omittedCalls": 0
228
- },
229
- "stageTimings": [
230
- {
231
- "event": "edit-requested",
232
- "elapsedMs": 53170.93125
233
- },
234
- {
235
- "event": "verification-requested",
236
- "elapsedMs": 55683.3165
237
- },
238
- {
239
- "event": "governed-edit-completed",
240
- "elapsedMs": 55689.149709
241
- },
242
- {
243
- "event": "verified-import-completed",
244
- "elapsedMs": 73038.63445900001
245
- }
246
- ],
247
- "approvalTimings": [
248
- {
249
- "durationMs": 0.16820799999550218,
250
- "resolved": true
251
- },
252
- {
253
- "durationMs": 0.026583999999274965,
254
- "resolved": true
255
- }
256
- ],
257
- "toolTimings": [
258
- {
259
- "name": "search_files",
260
- "calls": 2,
261
- "errors": 0,
262
- "totalMs": 1259.881583999998,
263
- "maxMs": 871.3713749999988
264
- },
265
- {
266
- "name": "list_files",
267
- "calls": 1,
268
- "errors": 0,
269
- "totalMs": 19.485791999999492,
270
- "maxMs": 19.485791999999492
271
- },
272
- {
273
- "name": "read_files",
274
- "calls": 2,
275
- "errors": 0,
276
- "totalMs": 3.180249000000913,
277
- "maxMs": 2.609291000000667
278
- },
279
- {
280
- "name": "repair_plan",
281
- "calls": 1,
282
- "errors": 0,
283
- "totalMs": 0.11391699999876437,
284
- "maxMs": 0.11391699999876437
285
- },
286
- {
287
- "name": "apply_reviewed_replacement",
288
- "calls": 1,
289
- "errors": 0,
290
- "totalMs": 11.653957999995328,
291
- "maxMs": 11.653957999995328
292
- },
293
- {
294
- "name": "verify_and_apply_environment_patch",
295
- "calls": 1,
296
- "errors": 0,
297
- "totalMs": 17082.548167,
298
- "maxMs": 17082.548167
299
- }
300
- ],
301
- "schemaVersion": 1,
302
- "candidate": "zhivex",
303
- "instanceId": "django__django-15851",
304
- "status": "completed",
305
- "failure": null,
306
- "phase": "agent",
307
- "durationMs": 74568.26725,
308
- "approvals": 2,
309
- "turns": [
310
- {
311
- "index": 1,
312
- "status": "completed",
313
- "finishReason": "tool-calls",
314
- "requestedTools": [
315
- "search_files",
316
- "list_files"
317
- ],
318
- "inputTokens": 4652,
319
- "outputTokens": 122,
320
- "cachedInputTokens": 4096,
321
- "requestCharacters": 7658,
322
- "tools": [
323
- {
324
- "name": "search_files",
325
- "isError": false,
326
- "characters": 346
327
- },
328
- {
329
- "name": "list_files",
330
- "isError": false,
331
- "characters": 1213
332
- }
333
- ]
334
- },
335
- {
336
- "index": 2,
337
- "status": "completed",
338
- "finishReason": "tool-calls",
339
- "requestedTools": [
340
- "read_files"
341
- ],
342
- "inputTokens": 5612,
343
- "outputTokens": 61,
344
- "cachedInputTokens": 4096,
345
- "requestCharacters": 9939,
346
- "tools": [
347
- {
348
- "name": "read_files",
349
- "isError": false,
350
- "characters": 7433
351
- }
352
- ]
353
- },
354
- {
355
- "index": 3,
356
- "status": "completed",
357
- "finishReason": "tool-calls",
358
- "requestedTools": [
359
- "read_files"
360
- ],
361
- "inputTokens": 7977,
362
- "outputTokens": 52,
363
- "cachedInputTokens": 5120,
364
- "requestCharacters": 17796,
365
- "tools": [
366
- {
367
- "name": "read_files",
368
- "isError": false,
369
- "characters": 3399
370
- }
371
- ]
372
- },
373
- {
374
- "index": 4,
375
- "status": "completed",
376
- "finishReason": "tool-calls",
377
- "requestedTools": [
378
- "search_files"
379
- ],
380
- "inputTokens": 9135,
381
- "outputTokens": 58,
382
- "cachedInputTokens": 7168,
383
- "requestCharacters": 21570,
384
- "tools": [
385
- {
386
- "name": "search_files",
387
- "isError": false,
388
- "characters": 1241
389
- }
390
- ]
391
- },
392
- {
393
- "index": 5,
394
- "status": "completed",
395
- "finishReason": "tool-calls",
396
- "requestedTools": [
397
- "repair_plan"
398
- ],
399
- "inputTokens": 9751,
400
- "outputTokens": 403,
401
- "cachedInputTokens": 8320,
402
- "requestCharacters": 23205,
403
- "tools": [
404
- {
405
- "name": "repair_plan",
406
- "isError": false,
407
- "characters": 1379
408
- }
409
- ]
410
- },
411
- {
412
- "index": 6,
413
- "status": "waiting_approval",
414
- "finishReason": "tool-calls",
415
- "requestedTools": [
416
- "apply_reviewed_replacement"
417
- ],
418
- "inputTokens": 10612,
419
- "outputTokens": 164,
420
- "cachedInputTokens": 9216,
421
- "requestCharacters": 26638,
422
- "tools": []
423
- },
424
- {
425
- "index": 7,
426
- "status": "completed",
427
- "finishReason": "tool-calls",
428
- "requestedTools": [
429
- "verify_and_apply_environment_patch"
430
- ],
431
- "inputTokens": 0,
432
- "outputTokens": 0,
433
- "cachedInputTokens": 0,
434
- "requestCharacters": 27845,
435
- "tools": [
436
- {
437
- "name": "apply_reviewed_replacement",
438
- "isError": false,
439
- "characters": 483
440
- },
441
- {
442
- "name": "verify_and_apply_environment_patch",
443
- "isError": false,
444
- "characters": 1523,
445
- "phaseLatencies": {
446
- "commandAndAttestationMs": 2911.652083,
447
- "workspaceExportMs": 7508.893542,
448
- "totalMs": 12796.734666,
449
- "sessionCreationMs": 2374.661416
450
- },
451
- "exitCode": 0,
452
- "timedOut": false,
453
- "verificationOutcome": "passed"
454
- }
455
- ]
456
- }
457
- ],
458
- "inputTokens": 47739,
459
- "outputTokens": 860,
460
- "cachedInputTokens": 38016,
461
- "modelCalls": 6,
462
- "usageComplete": true,
463
- "compactions": [],
464
- "terminalDiagnostic": null,
465
- "thrownDiagnostic": null,
466
- "guardrails": [],
467
- "repairProgress": {
468
- "repeatedResults": 0,
469
- "suppressedResults": 0,
470
- "blockedBroadCalls": 0,
471
- "enteredClosure": false,
472
- "phase": "verify",
473
- "blockedPhaseCalls": 0,
474
- "planRecorded": true
475
- },
476
- "contextMetrics": [
477
- {
478
- "systemCharacters": 5073,
479
- "userCharacters": 2581,
480
- "assistantCharacters": 0,
481
- "toolResultCharacters": 0,
482
- "otherMessageCharacters": 0,
483
- "toolDefinitionCharacters": 9375,
484
- "unmeasuredToolDefinitions": 0
485
- },
486
- {
487
- "systemCharacters": 5073,
488
- "userCharacters": 2581,
489
- "assistantCharacters": 401,
490
- "toolResultCharacters": 1877,
491
- "otherMessageCharacters": 0,
492
- "toolDefinitionCharacters": 9375,
493
- "unmeasuredToolDefinitions": 0
494
- },
495
- {
496
- "systemCharacters": 5073,
497
- "userCharacters": 2581,
498
- "assistantCharacters": 665,
499
- "toolResultCharacters": 9468,
500
- "otherMessageCharacters": 0,
501
- "toolDefinitionCharacters": 9375,
502
- "unmeasuredToolDefinitions": 0
503
- },
504
- {
505
- "systemCharacters": 5073,
506
- "userCharacters": 2581,
507
- "assistantCharacters": 880,
508
- "toolResultCharacters": 13025,
509
- "otherMessageCharacters": 0,
510
- "toolDefinitionCharacters": 9375,
511
- "unmeasuredToolDefinitions": 0
512
- },
513
- {
514
- "systemCharacters": 5073,
515
- "userCharacters": 2581,
516
- "assistantCharacters": 1112,
517
- "toolResultCharacters": 14426,
518
- "otherMessageCharacters": 0,
519
- "toolDefinitionCharacters": 9375,
520
- "unmeasuredToolDefinitions": 0
521
- },
522
- {
523
- "systemCharacters": 5073,
524
- "userCharacters": 3002,
525
- "assistantCharacters": 2584,
526
- "toolResultCharacters": 15964,
527
- "otherMessageCharacters": 0,
528
- "toolDefinitionCharacters": 9375,
529
- "unmeasuredToolDefinitions": 0
530
- }
531
- ],
532
- "omittedContextMeasurements": 0,
533
- "toolCalls": 8
534
- },
535
- "importedPatchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
536
- "importedPaths": [
537
- "django/db/backends/postgresql/client.py"
538
- ],
539
- "protectedFilesChanged": [],
540
- "candidatePresent": true,
541
- "grading": {
542
- "gradingStatus": "completed",
543
- "resolved": true
544
- },
545
- "model": "qwen3.8-flash",
546
- "image": "sha256:4c8b7a320988313b8881a96f334a189bfd0d6fda47cc0219d648867ffe765b80",
547
- "evaluatorImage": "sha256:aa80884e053cf1a238df7c4ceec43e4e07929547f658278bd3c3154f0ef8bc90",
548
- "limits": {
549
- "steps": 24,
550
- "outputPerTurn": 2048,
551
- "inputTokens": 100000,
552
- "outputTokens": 16000,
553
- "timeoutSeconds": 300,
554
- "memoryMb": 2048,
555
- "cpus": 2
556
- },
557
- "newModelRequestsDuringGrading": 0,
558
- "sourcePlan": "docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json"
559
- }
@@ -1,28 +0,0 @@
1
- {
2
- "schemaVersion": 1,
3
- "kind": "tool-selection-recovery-delivery-diagnostic",
4
- "classification": "Known development task; not comparison, fresh holdout or GA acceptance",
5
- "repetitions": 1,
6
- "model": "qwen3.8-flash",
7
- "image": "sha256:4c8b7a320988313b8881a96f334a189bfd0d6fda47cc0219d648867ffe765b80",
8
- "limits": {
9
- "steps": 24,
10
- "outputPerTurn": 2048,
11
- "inputTokens": 100000,
12
- "outputTokens": 16000,
13
- "timeoutSeconds": 300,
14
- "memoryMb": 2048,
15
- "cpus": 2
16
- },
17
- "change": "Enable bounded unknownToolMode tool-result in repair mode; corrected image and verifier-discovery instruction unchanged",
18
- "sourceHashes": {
19
- "scripts/swebench/zhivex-driver.ts": "67cee044060e78afea9fa663af8a686220aeafdb0db9ec318ef846f31aa9985d",
20
- "scripts/swebench/python-environment.ts": "b83440b5ea095c6eded67745ba009b82e4d3d2c40f9ee3d53bd5b53766a375f8",
21
- "scripts/swebench/verifier-observer.ts": "a90544437473218a83477a1cffb59ee9f645c12abdf8960b787f3e21606035bf",
22
- "src/harness.ts": "f506b8833167255c2bb9df270d8ab9bba540cf9bfeade56152ab1188074d6882",
23
- "src/repair-controller.ts": "41f1f09032131306f3284edbf105930577504af0bf764ba98a105c4eee70a1e7",
24
- "bun.lock": "dbabdf161b3dba306f475c81f37e5ab1620fb98052127bb3ed2a0a0cb611ede4"
25
- },
26
- "stopRule": "One model run; retain bounded private verifier failures; no retry",
27
- "gitCommit": "fbceb4f294109255917d90ba5c28c145dcee17f2"
28
- }
@@ -1,27 +0,0 @@
1
- {
2
- "schemaVersion": 1,
3
- "kind": "bounded-unknown-tool-selection-recovery",
4
- "sources": {
5
- "src/harness.ts": "f506b8833167255c2bb9df270d8ab9bba540cf9bfeade56152ab1188074d6882",
6
- "tests/repair-tool-selection.test.ts": "ffc9612908c4077c8319dc757ce384746405fb0cb4d91229b4e8e3251e333fde"
7
- },
8
- "validation": {
9
- "focusedTests": 5,
10
- "fullSuite": {
11
- "passed": 577,
12
- "failed": 0,
13
- "files": 72
14
- },
15
- "runtimeTypecheck": "passed",
16
- "toolingTypecheck": "passed",
17
- "documentation": "passed",
18
- "preparationContract": "passed; one GA blocker open"
19
- },
20
- "boundaries": [
21
- "Repair profile only; explicit caller overrides retained.",
22
- "Unregistered tools are never resolved or executed.",
23
- "Tool error, step and token limits remain unchanged.",
24
- "Subsequent mutations still require approval.",
25
- "No live effectiveness claim; the observed failure remains unchanged."
26
- ]
27
- }