@zhivex-ai/harness 1.0.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +47 -62
- package/ROADMAP.md +15 -6
- package/SECURITY.md +1 -1
- package/benchmarks/README.md +48 -0
- package/benchmarks/baselines/context-navigation-2026-09-09.json +94 -0
- package/benchmarks/baselines/efficiency-2026-09-08.json +233 -0
- package/benchmarks/baselines/external-final-2026-09-08.json +588 -0
- package/benchmarks/baselines/external-intermediate-2026-09-08.json +116 -0
- package/benchmarks/baselines/external-matched-openai-2026-09-08.json +1922 -0
- package/benchmarks/baselines/external-matched-qwen-2026-09-08.json +1706 -0
- package/benchmarks/baselines/external-pilot-2026-09-08.json +439 -0
- package/benchmarks/baselines/external-qwen-flash-2026-09-08.json +1570 -0
- package/benchmarks/baselines/external-qwen-remediation-2026-09-09.json +1509 -0
- package/benchmarks/baselines/external-qwen-sdk-114-progress-2026-09-09.json +2509 -0
- package/benchmarks/baselines/external-sdk-upgrade-2026-09-08.json +564 -0
- package/benchmarks/baselines/external-verifier-recovery-2026-09-09.json +794 -0
- package/benchmarks/baselines/full-harness-review-2026-09-10.json +134 -0
- package/benchmarks/baselines/harness-audit-2026-09-09.json +136 -0
- package/benchmarks/baselines/harness-remediation-2026-09-09.json +160 -0
- package/benchmarks/baselines/oci-remediation-2026-09-10.json +535 -0
- package/benchmarks/baselines/rc14-holdout-2026-09-10.json +854 -0
- package/benchmarks/baselines/repair-architecture-audit-2026-09-09.json +75 -0
- package/benchmarks/baselines/time-to-safe-fix-live-expanded-2026-08-21.json +186 -0
- package/benchmarks/baselines/time-to-safe-fix-live-smoke-2026-08-21.json +186 -0
- package/contracts/public-api.json +25 -0
- package/contracts/security-controls.json +3 -2
- package/contracts/stable-api-signatures.json +58 -54
- package/dist/bounded-reads.d.ts +4 -0
- package/dist/change-envelope.d.ts +4 -4
- package/dist/cli-options.d.ts +2 -1
- package/dist/cli-profiles.d.ts +39 -0
- package/dist/cli-stream.d.ts +40 -40
- package/dist/cli.d.ts +38 -35
- package/dist/cli.js +6556 -4030
- package/dist/compaction.d.ts +12 -0
- package/dist/config.d.ts +6 -1
- package/dist/console-context.d.ts +26 -0
- package/dist/console-input.d.ts +28 -0
- package/dist/context-engineering.d.ts +5 -5
- package/dist/context-metrics.d.ts +11 -0
- package/dist/edit-contracts.d.ts +2 -2
- package/dist/execution-environment.d.ts +9 -5
- package/dist/harness.d.ts +52 -18
- package/dist/hostile-repository-demo.js +2942 -1088
- package/dist/index.d.ts +3 -3
- package/dist/index.js +4316 -2435
- package/dist/json-contracts.d.ts +292 -236
- package/dist/model-budget.d.ts +61 -0
- package/dist/operations.d.ts +82 -20
- package/dist/orchestration.d.ts +1 -0
- package/dist/providers.d.ts +5 -2
- package/dist/repair-controller.d.ts +72 -0
- package/dist/repair-progress.d.ts +40 -0
- package/dist/repair-verifier.d.ts +6 -0
- package/dist/replacement-edits.d.ts +9 -0
- package/dist/routing.d.ts +1 -1
- package/dist/run-interruption.d.ts +4 -0
- package/dist/runtime-checkpoints.d.ts +7 -0
- package/dist/runtime-diagnostics.d.ts +63 -0
- package/dist/runtime-policy.d.ts +36 -0
- package/dist/sessions.d.ts +1 -1
- package/dist/state-backup.d.ts +2 -2
- package/dist/task-memory.d.ts +36 -0
- package/dist/terminal-markdown.d.ts +12 -0
- package/dist/time-to-safe-fix.d.ts +19 -19
- package/dist/tool-registry.d.ts +4 -0
- package/dist/verify-historical-migrations.js +6987 -5140
- package/dist/workspace-mutation-lock.d.ts +5 -0
- package/dist/workspace.d.ts +39 -4
- package/dist/zhx.js +6556 -4030
- package/docs/CLI.md +89 -3
- package/docs/CONTEXT_ENGINEERING.md +160 -0
- package/docs/EXECUTION_ENVIRONMENTS.md +21 -0
- package/docs/EXTENSIBILITY.md +2 -2
- package/docs/GA_READINESS.md +13 -7
- package/docs/LIVE_CERTIFICATION.md +12 -2
- package/docs/RC13_SECURITY_REVIEW.md +28 -0
- package/docs/RC14_DELEGATED_SECURITY_DECISION.md +80 -0
- package/docs/RC14_SECURITY_REVIEW.md +40 -0
- package/docs/README.md +33 -0
- package/docs/RELEASE.md +4 -2
- package/docs/REPOSITORY_EDITING.md +47 -0
- package/docs/SECURITY_REVIEW_EVIDENCE.md +5 -3
- package/docs/THREAT_MODEL.md +3 -3
- package/docs/TIME_TO_SAFE_FIX.md +4 -4
- package/docs/ga-readiness.json +147 -16
- package/docs/reports/EFFICIENCY_REVIEW.md +156 -0
- package/docs/reports/EXTERNAL_COMPARISON.md +434 -0
- package/docs/reports/FULL_HARNESS_REVIEW_2026-09-10.md +200 -0
- package/docs/reports/GA_BUDGET_REMEDIATION_2026-09-19.md +38 -0
- package/docs/reports/GA_REPAIR_RECOVERY_2026-09-19.md +700 -0
- package/docs/reports/GA_VALIDATION_2026-09-19.md +111 -0
- package/docs/reports/HARNESS_ARCHITECTURE_REVIEW_2026-09-09.md +202 -0
- package/docs/reports/HARNESS_AUDIT_2026-09-09.md +306 -0
- package/docs/reports/HARNESS_REMEDIATION_2026-09-09.md +74 -0
- package/docs/reports/HARNESS_REMEDIATION_2026-09-10.md +90 -0
- package/docs/reports/MODEL_COMPARISON_2026-09-09.md +184 -0
- package/docs/reports/QWEN_REMEDIATION_COMPARISON_2026-09-09.md +142 -0
- package/docs/reports/RC14_VALIDATION_2026-09-10.md +45 -0
- package/docs/reports/README.md +26 -0
- package/docs/reports/evidence/ga-budget-remediation-2026-09-19.json +133 -0
- package/docs/reports/evidence/ga-catalog-ablation-2026-09-19.json +699 -0
- package/docs/reports/evidence/ga-catalog-ablation-plan-2026-09-19.json +40 -0
- package/docs/reports/evidence/ga-catalog-observed-2026-09-19.json +985 -0
- package/docs/reports/evidence/ga-catalog-observed-plan-2026-09-19.json +28 -0
- package/docs/reports/evidence/ga-catalog-observed-status-2026-09-19.json +15 -0
- package/docs/reports/evidence/ga-complete-local-check-2026-09-19.json +47 -0
- package/docs/reports/evidence/ga-completion-reminder-2026-09-19.json +22 -0
- package/docs/reports/evidence/ga-fresh-cohort-plan-2026-09-19.json +54 -0
- package/docs/reports/evidence/ga-fresh-cohort-preparation-2026-09-19.json +242 -0
- package/docs/reports/evidence/ga-fresh-cohort-result-2026-09-19.json +4318 -0
- package/docs/reports/evidence/ga-local-validation-followup-2026-09-19.json +66 -0
- package/docs/reports/evidence/ga-node-consumers-2026-09-19.json +31 -0
- package/docs/reports/evidence/ga-package-content-validation-2026-09-19.json +22 -0
- package/docs/reports/evidence/ga-plan-completion-2026-09-19.json +39 -0
- package/docs/reports/evidence/ga-plan-reserve-2026-09-19.json +573 -0
- package/docs/reports/evidence/ga-plan-reserve-plan-2026-09-19.json +24 -0
- package/docs/reports/evidence/ga-plan-transition-2026-09-19.json +19 -0
- package/docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json +24 -0
- package/docs/reports/evidence/ga-private-verifier-root-cause-2026-09-19.json +41 -0
- package/docs/reports/evidence/ga-published-sdk-2026-09-19.json +35 -0
- package/docs/reports/evidence/ga-python-checkout-imports-2026-09-19.json +95 -0
- package/docs/reports/evidence/ga-python-origin-delivery-2026-09-19.json +211 -0
- package/docs/reports/evidence/ga-python-origin-delivery-plan-2026-09-19.json +27 -0
- package/docs/reports/evidence/ga-qwen-patched-package-2026-09-19.json +1092 -0
- package/docs/reports/evidence/ga-qwen-patched-package-plan-2026-09-19.json +18 -0
- package/docs/reports/evidence/ga-qwen-published-2026-09-19.json +571 -0
- package/docs/reports/evidence/ga-qwen-published-plan-2026-09-19.json +18 -0
- package/docs/reports/evidence/ga-qwen-tool-choice-2026-09-19.json +85 -0
- package/docs/reports/evidence/ga-qwen-tool-choice-patched-2026-09-19.json +85 -0
- package/docs/reports/evidence/ga-recovery-development-2026-09-19.json +1066 -0
- package/docs/reports/evidence/ga-recovery-graded-development-2026-09-19.json +1313 -0
- package/docs/reports/evidence/ga-recovery-graded-v2-2026-09-19.json +1151 -0
- package/docs/reports/evidence/ga-recovery-v3-2026-09-19.json +3943 -0
- package/docs/reports/evidence/ga-recovery-v4-2026-09-19.json +1055 -0
- package/docs/reports/evidence/ga-recovery-v5-2026-09-19.json +1245 -0
- package/docs/reports/evidence/ga-recovery-v6-2026-09-19.json +699 -0
- package/docs/reports/evidence/ga-recovery-v7-2026-09-19.json +854 -0
- package/docs/reports/evidence/ga-recovery-v8-2026-09-19.json +596 -0
- package/docs/reports/evidence/ga-required-delivery-2026-09-19.json +25 -0
- package/docs/reports/evidence/ga-required-delivery-live-2026-09-19.json +1337 -0
- package/docs/reports/evidence/ga-required-delivery-live-plan-2026-09-19.json +25 -0
- package/docs/reports/evidence/ga-retained-candidate-grading-2026-09-19.json +30 -0
- package/docs/reports/evidence/ga-tool-selection-live-2026-09-19.json +559 -0
- package/docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json +28 -0
- package/docs/reports/evidence/ga-tool-selection-recovery-2026-09-19.json +27 -0
- package/docs/reports/evidence/ga-validation-2026-09-19.json +236 -0
- package/docs/reports/evidence/ga-verifier-discovery-live-2026-09-19.json +1369 -0
- package/docs/reports/evidence/ga-verifier-discovery-plan-2026-09-19.json +29 -0
- package/docs/reports/evidence/ga-verifier-failure-identification-2026-09-19.json +30 -0
- package/docs/reports/evidence/ga-verifier-feedback-2026-09-19.json +65 -0
- package/docs/reports/evidence/ga-verifier-hints-live-2026-09-19.json +1297 -0
- package/docs/reports/evidence/ga-verifier-hints-plan-2026-09-19.json +27 -0
- package/docs/reports/evidence/ga-work-boundary-context-analysis-2026-09-19.json +113 -0
- package/docs/reports/evidence/ga-work-boundary-live-2026-09-19.json +301 -0
- package/docs/reports/evidence/ga-work-boundary-live-plan-2026-09-19.json +84 -0
- package/docs/reports/evidence/ga-work-budget-planning-2026-09-19.json +32 -0
- package/docs/reports/evidence/ga-working-context-2026-09-19.json +22 -0
- package/docs/reports/evidence/rc14-delegated-security-review-2026-09-20.json +81 -0
- package/docs/reports/evidence/rc14-ga-gate-2026-09-20.json +18 -0
- package/docs/reports/evidence/rc14-representative-2026-09-20.json +509 -0
- package/docs/reports/evidence/rc14-security-regressions-2026-09-20.json +64 -0
- package/docs/reports/evidence/security-current-2026-09-19.json +91 -0
- package/docs/reports/evidence/security-remediation-2026-09-19.json +97 -0
- package/evaluations/repair-controller-experiment.json +24 -0
- package/evaluations/representative-assembly-matrix.json +63 -0
- package/evaluations/representative-matrix.json +2219 -2
- package/fixtures/contracts/v1/init.json +19 -0
- package/package.json +25 -16
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"kind": "required-delivery-development-validation",
|
|
3
|
+
"classification": "known development case; not holdout or GA acceptance",
|
|
4
|
+
"baseline": "ga-qwen-published",
|
|
5
|
+
"qwenVersion": "0.14.4",
|
|
6
|
+
"repetitions": 1,
|
|
7
|
+
"control": "unchanged mini-SWE-agent",
|
|
8
|
+
"limits": {
|
|
9
|
+
"steps": 24,
|
|
10
|
+
"outputPerTurn": 2048,
|
|
11
|
+
"inputTokens": 100000,
|
|
12
|
+
"outputTokens": 16000,
|
|
13
|
+
"timeoutSeconds": 300,
|
|
14
|
+
"memoryMb": 2048,
|
|
15
|
+
"cpus": 2
|
|
16
|
+
},
|
|
17
|
+
"change": "explicit verified-delivery obligation enabled",
|
|
18
|
+
"sources": {
|
|
19
|
+
"src/config.ts": "67ed44b55505ff8dd94f60ba89c33c217759cd7c79de45ccda369f248309cdff",
|
|
20
|
+
"src/harness.ts": "1d305078e3c56c24bb659c34aac37c96bf11a4e60af6607bb4b9d65964e84791",
|
|
21
|
+
"src/repair-controller.ts": "41f1f09032131306f3284edbf105930577504af0bf764ba98a105c4eee70a1e7",
|
|
22
|
+
"scripts/swebench/zhivex-driver.ts": "3d1c143b4c2a1438a1249db8cc10c70a80cbd1c9e3ed44317230795b7a9be2b4",
|
|
23
|
+
"bun.lock": "dbabdf161b3dba306f475c81f37e5ab1620fb98052127bb3ed2a0a0cb611ede4"
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"kind": "offline-retained-candidate-grading",
|
|
4
|
+
"classification": "Diagnostic of prior failed development run; not delivery, holdout or GA acceptance",
|
|
5
|
+
"instanceId": "django__django-15851",
|
|
6
|
+
"patchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
|
|
7
|
+
"snapshotPatchId": "sha256:b6a09cef9555d8c46b7721d28bec8583f21c786d198c20efcff992d6d6f8949e",
|
|
8
|
+
"changedPaths": [
|
|
9
|
+
"django/db/backends/postgresql/client.py"
|
|
10
|
+
],
|
|
11
|
+
"protectedFilesChanged": [],
|
|
12
|
+
"evaluatorImage": "sha256:aa80884e053cf1a238df7c4ceec43e4e07929547f658278bd3c3154f0ef8bc90",
|
|
13
|
+
"grading": {
|
|
14
|
+
"gradingStatus": "completed",
|
|
15
|
+
"resolved": true
|
|
16
|
+
},
|
|
17
|
+
"newModelRequests": 0,
|
|
18
|
+
"originalRunStatus": "failed",
|
|
19
|
+
"originalImportSucceeded": false,
|
|
20
|
+
"sourceRunPlan": "docs/reports/evidence/ga-private-verifier-plan-2026-09-19.json",
|
|
21
|
+
"captureSources": {
|
|
22
|
+
"scripts/swebench/candidate.ts": "5767ae241c4a58bd731456abe2c3d92f7a57bba764c80e0f0d00748a0f47d02b",
|
|
23
|
+
"src/execution-environment.ts": "9ce6c49afab26527d5296f0c11f106063590f7333300d58dd49d5a1ec1b4b6e9"
|
|
24
|
+
},
|
|
25
|
+
"fullLocalSuite": {
|
|
26
|
+
"passed": 571,
|
|
27
|
+
"failed": 0,
|
|
28
|
+
"files": 70
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"kind": "corrected-image-verified-delivery",
|
|
4
|
+
"classification": "Single known development diagnostic, not holdout or GA acceptance",
|
|
5
|
+
"result": {
|
|
6
|
+
"candidateCaptureFailure": null,
|
|
7
|
+
"modelTimings": [
|
|
8
|
+
{
|
|
9
|
+
"durationMs": 3608.6125829999996,
|
|
10
|
+
"firstTokenMs": 3608.2372499999997,
|
|
11
|
+
"completed": true
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"durationMs": 2005.5725419999999,
|
|
15
|
+
"firstTokenMs": 2005.4179999999997,
|
|
16
|
+
"completed": true
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"durationMs": 2233.2373750000006,
|
|
20
|
+
"firstTokenMs": 2233.012917,
|
|
21
|
+
"completed": true
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"durationMs": 2664.5729170000013,
|
|
25
|
+
"firstTokenMs": 2663.945167,
|
|
26
|
+
"completed": true
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"durationMs": 7502.284083000002,
|
|
30
|
+
"firstTokenMs": 7501.927,
|
|
31
|
+
"completed": true
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"durationMs": 30397.212916000004,
|
|
35
|
+
"firstTokenMs": 30396.758041,
|
|
36
|
+
"completed": true
|
|
37
|
+
}
|
|
38
|
+
],
|
|
39
|
+
"modelCatalog": {
|
|
40
|
+
"records": [
|
|
41
|
+
{
|
|
42
|
+
"call": 1,
|
|
43
|
+
"offered": [
|
|
44
|
+
"apply_reviewed_edits",
|
|
45
|
+
"apply_reviewed_replacement",
|
|
46
|
+
"inspect_environment_patch",
|
|
47
|
+
"list_files",
|
|
48
|
+
"mutation_audit",
|
|
49
|
+
"read_files",
|
|
50
|
+
"read_task",
|
|
51
|
+
"repair_plan",
|
|
52
|
+
"run_environment_command",
|
|
53
|
+
"run_environment_shell",
|
|
54
|
+
"search_files",
|
|
55
|
+
"search_many",
|
|
56
|
+
"verify_and_apply_environment_patch"
|
|
57
|
+
],
|
|
58
|
+
"unknownOffered": 0,
|
|
59
|
+
"systemMessages": 1,
|
|
60
|
+
"choice": "required",
|
|
61
|
+
"returned": [
|
|
62
|
+
{
|
|
63
|
+
"name": "search_files",
|
|
64
|
+
"offered": true,
|
|
65
|
+
"registered": true
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"name": "list_files",
|
|
69
|
+
"offered": true,
|
|
70
|
+
"registered": true
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"omittedReturned": 0,
|
|
74
|
+
"completed": true
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"call": 2,
|
|
78
|
+
"offered": [
|
|
79
|
+
"apply_reviewed_edits",
|
|
80
|
+
"apply_reviewed_replacement",
|
|
81
|
+
"inspect_environment_patch",
|
|
82
|
+
"list_files",
|
|
83
|
+
"mutation_audit",
|
|
84
|
+
"read_files",
|
|
85
|
+
"read_task",
|
|
86
|
+
"repair_plan",
|
|
87
|
+
"run_environment_command",
|
|
88
|
+
"run_environment_shell",
|
|
89
|
+
"search_files",
|
|
90
|
+
"search_many",
|
|
91
|
+
"verify_and_apply_environment_patch"
|
|
92
|
+
],
|
|
93
|
+
"unknownOffered": 0,
|
|
94
|
+
"systemMessages": 1,
|
|
95
|
+
"choice": "required",
|
|
96
|
+
"returned": [
|
|
97
|
+
{
|
|
98
|
+
"name": "read_files",
|
|
99
|
+
"offered": true,
|
|
100
|
+
"registered": true
|
|
101
|
+
}
|
|
102
|
+
],
|
|
103
|
+
"omittedReturned": 0,
|
|
104
|
+
"completed": true
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"call": 3,
|
|
108
|
+
"offered": [
|
|
109
|
+
"apply_reviewed_edits",
|
|
110
|
+
"apply_reviewed_replacement",
|
|
111
|
+
"inspect_environment_patch",
|
|
112
|
+
"list_files",
|
|
113
|
+
"mutation_audit",
|
|
114
|
+
"read_files",
|
|
115
|
+
"read_task",
|
|
116
|
+
"repair_plan",
|
|
117
|
+
"run_environment_command",
|
|
118
|
+
"run_environment_shell",
|
|
119
|
+
"search_files",
|
|
120
|
+
"search_many",
|
|
121
|
+
"verify_and_apply_environment_patch"
|
|
122
|
+
],
|
|
123
|
+
"unknownOffered": 0,
|
|
124
|
+
"systemMessages": 1,
|
|
125
|
+
"choice": "required",
|
|
126
|
+
"returned": [
|
|
127
|
+
{
|
|
128
|
+
"name": "read_files",
|
|
129
|
+
"offered": true,
|
|
130
|
+
"registered": true
|
|
131
|
+
}
|
|
132
|
+
],
|
|
133
|
+
"omittedReturned": 0,
|
|
134
|
+
"completed": true
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"call": 4,
|
|
138
|
+
"offered": [
|
|
139
|
+
"apply_reviewed_edits",
|
|
140
|
+
"apply_reviewed_replacement",
|
|
141
|
+
"inspect_environment_patch",
|
|
142
|
+
"list_files",
|
|
143
|
+
"mutation_audit",
|
|
144
|
+
"read_files",
|
|
145
|
+
"read_task",
|
|
146
|
+
"repair_plan",
|
|
147
|
+
"run_environment_command",
|
|
148
|
+
"run_environment_shell",
|
|
149
|
+
"search_files",
|
|
150
|
+
"search_many",
|
|
151
|
+
"verify_and_apply_environment_patch"
|
|
152
|
+
],
|
|
153
|
+
"unknownOffered": 0,
|
|
154
|
+
"systemMessages": 1,
|
|
155
|
+
"choice": "required",
|
|
156
|
+
"returned": [
|
|
157
|
+
{
|
|
158
|
+
"name": "search_files",
|
|
159
|
+
"offered": true,
|
|
160
|
+
"registered": true
|
|
161
|
+
}
|
|
162
|
+
],
|
|
163
|
+
"omittedReturned": 0,
|
|
164
|
+
"completed": true
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
"call": 5,
|
|
168
|
+
"offered": [
|
|
169
|
+
"apply_reviewed_edits",
|
|
170
|
+
"apply_reviewed_replacement",
|
|
171
|
+
"inspect_environment_patch",
|
|
172
|
+
"list_files",
|
|
173
|
+
"mutation_audit",
|
|
174
|
+
"read_files",
|
|
175
|
+
"read_task",
|
|
176
|
+
"repair_plan",
|
|
177
|
+
"run_environment_command",
|
|
178
|
+
"run_environment_shell",
|
|
179
|
+
"search_files",
|
|
180
|
+
"search_many",
|
|
181
|
+
"verify_and_apply_environment_patch"
|
|
182
|
+
],
|
|
183
|
+
"unknownOffered": 0,
|
|
184
|
+
"systemMessages": 1,
|
|
185
|
+
"choice": "required",
|
|
186
|
+
"returned": [
|
|
187
|
+
{
|
|
188
|
+
"name": "repair_plan",
|
|
189
|
+
"offered": true,
|
|
190
|
+
"registered": true
|
|
191
|
+
}
|
|
192
|
+
],
|
|
193
|
+
"omittedReturned": 0,
|
|
194
|
+
"completed": true
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
"call": 6,
|
|
198
|
+
"offered": [
|
|
199
|
+
"apply_reviewed_edits",
|
|
200
|
+
"apply_reviewed_replacement",
|
|
201
|
+
"inspect_environment_patch",
|
|
202
|
+
"list_files",
|
|
203
|
+
"mutation_audit",
|
|
204
|
+
"read_files",
|
|
205
|
+
"read_task",
|
|
206
|
+
"repair_plan",
|
|
207
|
+
"run_environment_command",
|
|
208
|
+
"run_environment_shell",
|
|
209
|
+
"search_files",
|
|
210
|
+
"search_many",
|
|
211
|
+
"verify_and_apply_environment_patch"
|
|
212
|
+
],
|
|
213
|
+
"unknownOffered": 0,
|
|
214
|
+
"systemMessages": 1,
|
|
215
|
+
"choice": "required",
|
|
216
|
+
"returned": [
|
|
217
|
+
{
|
|
218
|
+
"name": "apply_reviewed_replacement",
|
|
219
|
+
"offered": true,
|
|
220
|
+
"registered": true
|
|
221
|
+
}
|
|
222
|
+
],
|
|
223
|
+
"omittedReturned": 0,
|
|
224
|
+
"completed": true
|
|
225
|
+
}
|
|
226
|
+
],
|
|
227
|
+
"omittedCalls": 0
|
|
228
|
+
},
|
|
229
|
+
"stageTimings": [
|
|
230
|
+
{
|
|
231
|
+
"event": "edit-requested",
|
|
232
|
+
"elapsedMs": 53170.93125
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
"event": "verification-requested",
|
|
236
|
+
"elapsedMs": 55683.3165
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
"event": "governed-edit-completed",
|
|
240
|
+
"elapsedMs": 55689.149709
|
|
241
|
+
},
|
|
242
|
+
{
|
|
243
|
+
"event": "verified-import-completed",
|
|
244
|
+
"elapsedMs": 73038.63445900001
|
|
245
|
+
}
|
|
246
|
+
],
|
|
247
|
+
"approvalTimings": [
|
|
248
|
+
{
|
|
249
|
+
"durationMs": 0.16820799999550218,
|
|
250
|
+
"resolved": true
|
|
251
|
+
},
|
|
252
|
+
{
|
|
253
|
+
"durationMs": 0.026583999999274965,
|
|
254
|
+
"resolved": true
|
|
255
|
+
}
|
|
256
|
+
],
|
|
257
|
+
"toolTimings": [
|
|
258
|
+
{
|
|
259
|
+
"name": "search_files",
|
|
260
|
+
"calls": 2,
|
|
261
|
+
"errors": 0,
|
|
262
|
+
"totalMs": 1259.881583999998,
|
|
263
|
+
"maxMs": 871.3713749999988
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
"name": "list_files",
|
|
267
|
+
"calls": 1,
|
|
268
|
+
"errors": 0,
|
|
269
|
+
"totalMs": 19.485791999999492,
|
|
270
|
+
"maxMs": 19.485791999999492
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
"name": "read_files",
|
|
274
|
+
"calls": 2,
|
|
275
|
+
"errors": 0,
|
|
276
|
+
"totalMs": 3.180249000000913,
|
|
277
|
+
"maxMs": 2.609291000000667
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"name": "repair_plan",
|
|
281
|
+
"calls": 1,
|
|
282
|
+
"errors": 0,
|
|
283
|
+
"totalMs": 0.11391699999876437,
|
|
284
|
+
"maxMs": 0.11391699999876437
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
"name": "apply_reviewed_replacement",
|
|
288
|
+
"calls": 1,
|
|
289
|
+
"errors": 0,
|
|
290
|
+
"totalMs": 11.653957999995328,
|
|
291
|
+
"maxMs": 11.653957999995328
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"name": "verify_and_apply_environment_patch",
|
|
295
|
+
"calls": 1,
|
|
296
|
+
"errors": 0,
|
|
297
|
+
"totalMs": 17082.548167,
|
|
298
|
+
"maxMs": 17082.548167
|
|
299
|
+
}
|
|
300
|
+
],
|
|
301
|
+
"schemaVersion": 1,
|
|
302
|
+
"candidate": "zhivex",
|
|
303
|
+
"instanceId": "django__django-15851",
|
|
304
|
+
"status": "completed",
|
|
305
|
+
"failure": null,
|
|
306
|
+
"phase": "agent",
|
|
307
|
+
"durationMs": 74568.26725,
|
|
308
|
+
"approvals": 2,
|
|
309
|
+
"turns": [
|
|
310
|
+
{
|
|
311
|
+
"index": 1,
|
|
312
|
+
"status": "completed",
|
|
313
|
+
"finishReason": "tool-calls",
|
|
314
|
+
"requestedTools": [
|
|
315
|
+
"search_files",
|
|
316
|
+
"list_files"
|
|
317
|
+
],
|
|
318
|
+
"inputTokens": 4652,
|
|
319
|
+
"outputTokens": 122,
|
|
320
|
+
"cachedInputTokens": 4096,
|
|
321
|
+
"requestCharacters": 7658,
|
|
322
|
+
"tools": [
|
|
323
|
+
{
|
|
324
|
+
"name": "search_files",
|
|
325
|
+
"isError": false,
|
|
326
|
+
"characters": 346
|
|
327
|
+
},
|
|
328
|
+
{
|
|
329
|
+
"name": "list_files",
|
|
330
|
+
"isError": false,
|
|
331
|
+
"characters": 1213
|
|
332
|
+
}
|
|
333
|
+
]
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
"index": 2,
|
|
337
|
+
"status": "completed",
|
|
338
|
+
"finishReason": "tool-calls",
|
|
339
|
+
"requestedTools": [
|
|
340
|
+
"read_files"
|
|
341
|
+
],
|
|
342
|
+
"inputTokens": 5612,
|
|
343
|
+
"outputTokens": 61,
|
|
344
|
+
"cachedInputTokens": 4096,
|
|
345
|
+
"requestCharacters": 9939,
|
|
346
|
+
"tools": [
|
|
347
|
+
{
|
|
348
|
+
"name": "read_files",
|
|
349
|
+
"isError": false,
|
|
350
|
+
"characters": 7433
|
|
351
|
+
}
|
|
352
|
+
]
|
|
353
|
+
},
|
|
354
|
+
{
|
|
355
|
+
"index": 3,
|
|
356
|
+
"status": "completed",
|
|
357
|
+
"finishReason": "tool-calls",
|
|
358
|
+
"requestedTools": [
|
|
359
|
+
"read_files"
|
|
360
|
+
],
|
|
361
|
+
"inputTokens": 7977,
|
|
362
|
+
"outputTokens": 52,
|
|
363
|
+
"cachedInputTokens": 5120,
|
|
364
|
+
"requestCharacters": 17796,
|
|
365
|
+
"tools": [
|
|
366
|
+
{
|
|
367
|
+
"name": "read_files",
|
|
368
|
+
"isError": false,
|
|
369
|
+
"characters": 3399
|
|
370
|
+
}
|
|
371
|
+
]
|
|
372
|
+
},
|
|
373
|
+
{
|
|
374
|
+
"index": 4,
|
|
375
|
+
"status": "completed",
|
|
376
|
+
"finishReason": "tool-calls",
|
|
377
|
+
"requestedTools": [
|
|
378
|
+
"search_files"
|
|
379
|
+
],
|
|
380
|
+
"inputTokens": 9135,
|
|
381
|
+
"outputTokens": 58,
|
|
382
|
+
"cachedInputTokens": 7168,
|
|
383
|
+
"requestCharacters": 21570,
|
|
384
|
+
"tools": [
|
|
385
|
+
{
|
|
386
|
+
"name": "search_files",
|
|
387
|
+
"isError": false,
|
|
388
|
+
"characters": 1241
|
|
389
|
+
}
|
|
390
|
+
]
|
|
391
|
+
},
|
|
392
|
+
{
|
|
393
|
+
"index": 5,
|
|
394
|
+
"status": "completed",
|
|
395
|
+
"finishReason": "tool-calls",
|
|
396
|
+
"requestedTools": [
|
|
397
|
+
"repair_plan"
|
|
398
|
+
],
|
|
399
|
+
"inputTokens": 9751,
|
|
400
|
+
"outputTokens": 403,
|
|
401
|
+
"cachedInputTokens": 8320,
|
|
402
|
+
"requestCharacters": 23205,
|
|
403
|
+
"tools": [
|
|
404
|
+
{
|
|
405
|
+
"name": "repair_plan",
|
|
406
|
+
"isError": false,
|
|
407
|
+
"characters": 1379
|
|
408
|
+
}
|
|
409
|
+
]
|
|
410
|
+
},
|
|
411
|
+
{
|
|
412
|
+
"index": 6,
|
|
413
|
+
"status": "waiting_approval",
|
|
414
|
+
"finishReason": "tool-calls",
|
|
415
|
+
"requestedTools": [
|
|
416
|
+
"apply_reviewed_replacement"
|
|
417
|
+
],
|
|
418
|
+
"inputTokens": 10612,
|
|
419
|
+
"outputTokens": 164,
|
|
420
|
+
"cachedInputTokens": 9216,
|
|
421
|
+
"requestCharacters": 26638,
|
|
422
|
+
"tools": []
|
|
423
|
+
},
|
|
424
|
+
{
|
|
425
|
+
"index": 7,
|
|
426
|
+
"status": "completed",
|
|
427
|
+
"finishReason": "tool-calls",
|
|
428
|
+
"requestedTools": [
|
|
429
|
+
"verify_and_apply_environment_patch"
|
|
430
|
+
],
|
|
431
|
+
"inputTokens": 0,
|
|
432
|
+
"outputTokens": 0,
|
|
433
|
+
"cachedInputTokens": 0,
|
|
434
|
+
"requestCharacters": 27845,
|
|
435
|
+
"tools": [
|
|
436
|
+
{
|
|
437
|
+
"name": "apply_reviewed_replacement",
|
|
438
|
+
"isError": false,
|
|
439
|
+
"characters": 483
|
|
440
|
+
},
|
|
441
|
+
{
|
|
442
|
+
"name": "verify_and_apply_environment_patch",
|
|
443
|
+
"isError": false,
|
|
444
|
+
"characters": 1523,
|
|
445
|
+
"phaseLatencies": {
|
|
446
|
+
"commandAndAttestationMs": 2911.652083,
|
|
447
|
+
"workspaceExportMs": 7508.893542,
|
|
448
|
+
"totalMs": 12796.734666,
|
|
449
|
+
"sessionCreationMs": 2374.661416
|
|
450
|
+
},
|
|
451
|
+
"exitCode": 0,
|
|
452
|
+
"timedOut": false,
|
|
453
|
+
"verificationOutcome": "passed"
|
|
454
|
+
}
|
|
455
|
+
]
|
|
456
|
+
}
|
|
457
|
+
],
|
|
458
|
+
"inputTokens": 47739,
|
|
459
|
+
"outputTokens": 860,
|
|
460
|
+
"cachedInputTokens": 38016,
|
|
461
|
+
"modelCalls": 6,
|
|
462
|
+
"usageComplete": true,
|
|
463
|
+
"compactions": [],
|
|
464
|
+
"terminalDiagnostic": null,
|
|
465
|
+
"thrownDiagnostic": null,
|
|
466
|
+
"guardrails": [],
|
|
467
|
+
"repairProgress": {
|
|
468
|
+
"repeatedResults": 0,
|
|
469
|
+
"suppressedResults": 0,
|
|
470
|
+
"blockedBroadCalls": 0,
|
|
471
|
+
"enteredClosure": false,
|
|
472
|
+
"phase": "verify",
|
|
473
|
+
"blockedPhaseCalls": 0,
|
|
474
|
+
"planRecorded": true
|
|
475
|
+
},
|
|
476
|
+
"contextMetrics": [
|
|
477
|
+
{
|
|
478
|
+
"systemCharacters": 5073,
|
|
479
|
+
"userCharacters": 2581,
|
|
480
|
+
"assistantCharacters": 0,
|
|
481
|
+
"toolResultCharacters": 0,
|
|
482
|
+
"otherMessageCharacters": 0,
|
|
483
|
+
"toolDefinitionCharacters": 9375,
|
|
484
|
+
"unmeasuredToolDefinitions": 0
|
|
485
|
+
},
|
|
486
|
+
{
|
|
487
|
+
"systemCharacters": 5073,
|
|
488
|
+
"userCharacters": 2581,
|
|
489
|
+
"assistantCharacters": 401,
|
|
490
|
+
"toolResultCharacters": 1877,
|
|
491
|
+
"otherMessageCharacters": 0,
|
|
492
|
+
"toolDefinitionCharacters": 9375,
|
|
493
|
+
"unmeasuredToolDefinitions": 0
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
"systemCharacters": 5073,
|
|
497
|
+
"userCharacters": 2581,
|
|
498
|
+
"assistantCharacters": 665,
|
|
499
|
+
"toolResultCharacters": 9468,
|
|
500
|
+
"otherMessageCharacters": 0,
|
|
501
|
+
"toolDefinitionCharacters": 9375,
|
|
502
|
+
"unmeasuredToolDefinitions": 0
|
|
503
|
+
},
|
|
504
|
+
{
|
|
505
|
+
"systemCharacters": 5073,
|
|
506
|
+
"userCharacters": 2581,
|
|
507
|
+
"assistantCharacters": 880,
|
|
508
|
+
"toolResultCharacters": 13025,
|
|
509
|
+
"otherMessageCharacters": 0,
|
|
510
|
+
"toolDefinitionCharacters": 9375,
|
|
511
|
+
"unmeasuredToolDefinitions": 0
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
"systemCharacters": 5073,
|
|
515
|
+
"userCharacters": 2581,
|
|
516
|
+
"assistantCharacters": 1112,
|
|
517
|
+
"toolResultCharacters": 14426,
|
|
518
|
+
"otherMessageCharacters": 0,
|
|
519
|
+
"toolDefinitionCharacters": 9375,
|
|
520
|
+
"unmeasuredToolDefinitions": 0
|
|
521
|
+
},
|
|
522
|
+
{
|
|
523
|
+
"systemCharacters": 5073,
|
|
524
|
+
"userCharacters": 3002,
|
|
525
|
+
"assistantCharacters": 2584,
|
|
526
|
+
"toolResultCharacters": 15964,
|
|
527
|
+
"otherMessageCharacters": 0,
|
|
528
|
+
"toolDefinitionCharacters": 9375,
|
|
529
|
+
"unmeasuredToolDefinitions": 0
|
|
530
|
+
}
|
|
531
|
+
],
|
|
532
|
+
"omittedContextMeasurements": 0,
|
|
533
|
+
"toolCalls": 8
|
|
534
|
+
},
|
|
535
|
+
"importedPatchSha256": "15989da845ac1f766848c5c2e153296d7eef926a752ec658548702ac0e028bc7",
|
|
536
|
+
"importedPaths": [
|
|
537
|
+
"django/db/backends/postgresql/client.py"
|
|
538
|
+
],
|
|
539
|
+
"protectedFilesChanged": [],
|
|
540
|
+
"candidatePresent": true,
|
|
541
|
+
"grading": {
|
|
542
|
+
"gradingStatus": "completed",
|
|
543
|
+
"resolved": true
|
|
544
|
+
},
|
|
545
|
+
"model": "qwen3.8-flash",
|
|
546
|
+
"image": "sha256:4c8b7a320988313b8881a96f334a189bfd0d6fda47cc0219d648867ffe765b80",
|
|
547
|
+
"evaluatorImage": "sha256:aa80884e053cf1a238df7c4ceec43e4e07929547f658278bd3c3154f0ef8bc90",
|
|
548
|
+
"limits": {
|
|
549
|
+
"steps": 24,
|
|
550
|
+
"outputPerTurn": 2048,
|
|
551
|
+
"inputTokens": 100000,
|
|
552
|
+
"outputTokens": 16000,
|
|
553
|
+
"timeoutSeconds": 300,
|
|
554
|
+
"memoryMb": 2048,
|
|
555
|
+
"cpus": 2
|
|
556
|
+
},
|
|
557
|
+
"newModelRequestsDuringGrading": 0,
|
|
558
|
+
"sourcePlan": "docs/reports/evidence/ga-tool-selection-live-plan-2026-09-19.json"
|
|
559
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"kind": "tool-selection-recovery-delivery-diagnostic",
|
|
4
|
+
"classification": "Known development task; not comparison, fresh holdout or GA acceptance",
|
|
5
|
+
"repetitions": 1,
|
|
6
|
+
"model": "qwen3.8-flash",
|
|
7
|
+
"image": "sha256:4c8b7a320988313b8881a96f334a189bfd0d6fda47cc0219d648867ffe765b80",
|
|
8
|
+
"limits": {
|
|
9
|
+
"steps": 24,
|
|
10
|
+
"outputPerTurn": 2048,
|
|
11
|
+
"inputTokens": 100000,
|
|
12
|
+
"outputTokens": 16000,
|
|
13
|
+
"timeoutSeconds": 300,
|
|
14
|
+
"memoryMb": 2048,
|
|
15
|
+
"cpus": 2
|
|
16
|
+
},
|
|
17
|
+
"change": "Enable bounded unknownToolMode tool-result in repair mode; corrected image and verifier-discovery instruction unchanged",
|
|
18
|
+
"sourceHashes": {
|
|
19
|
+
"scripts/swebench/zhivex-driver.ts": "67cee044060e78afea9fa663af8a686220aeafdb0db9ec318ef846f31aa9985d",
|
|
20
|
+
"scripts/swebench/python-environment.ts": "b83440b5ea095c6eded67745ba009b82e4d3d2c40f9ee3d53bd5b53766a375f8",
|
|
21
|
+
"scripts/swebench/verifier-observer.ts": "a90544437473218a83477a1cffb59ee9f645c12abdf8960b787f3e21606035bf",
|
|
22
|
+
"src/harness.ts": "f506b8833167255c2bb9df270d8ab9bba540cf9bfeade56152ab1188074d6882",
|
|
23
|
+
"src/repair-controller.ts": "41f1f09032131306f3284edbf105930577504af0bf764ba98a105c4eee70a1e7",
|
|
24
|
+
"bun.lock": "dbabdf161b3dba306f475c81f37e5ab1620fb98052127bb3ed2a0a0cb611ede4"
|
|
25
|
+
},
|
|
26
|
+
"stopRule": "One model run; retain bounded private verifier failures; no retry",
|
|
27
|
+
"gitCommit": "fbceb4f294109255917d90ba5c28c145dcee17f2"
|
|
28
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"kind": "bounded-unknown-tool-selection-recovery",
|
|
4
|
+
"sources": {
|
|
5
|
+
"src/harness.ts": "f506b8833167255c2bb9df270d8ab9bba540cf9bfeade56152ab1188074d6882",
|
|
6
|
+
"tests/repair-tool-selection.test.ts": "ffc9612908c4077c8319dc757ce384746405fb0cb4d91229b4e8e3251e333fde"
|
|
7
|
+
},
|
|
8
|
+
"validation": {
|
|
9
|
+
"focusedTests": 5,
|
|
10
|
+
"fullSuite": {
|
|
11
|
+
"passed": 577,
|
|
12
|
+
"failed": 0,
|
|
13
|
+
"files": 72
|
|
14
|
+
},
|
|
15
|
+
"runtimeTypecheck": "passed",
|
|
16
|
+
"toolingTypecheck": "passed",
|
|
17
|
+
"documentation": "passed",
|
|
18
|
+
"preparationContract": "passed; one GA blocker open"
|
|
19
|
+
},
|
|
20
|
+
"boundaries": [
|
|
21
|
+
"Repair profile only; explicit caller overrides retained.",
|
|
22
|
+
"Unregistered tools are never resolved or executed.",
|
|
23
|
+
"Tool error, step and token limits remain unchanged.",
|
|
24
|
+
"Subsequent mutations still require approval.",
|
|
25
|
+
"No live effectiveness claim; the observed failure remains unchanged."
|
|
26
|
+
]
|
|
27
|
+
}
|