@openai/codex-security 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. package/README.md +76 -39
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/examples/completed-scan/findings.json +34 -34
  4. package/_bundled_plugin/examples/completed-scan/report.md +22 -31
  5. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +27 -29
  6. package/_bundled_plugin/examples/{completed-scan → legacy-completed-scan}/coverage.json +1 -0
  7. package/_bundled_plugin/examples/legacy-completed-scan/findings.json +57 -0
  8. package/_bundled_plugin/examples/legacy-completed-scan/report.md +98 -0
  9. package/_bundled_plugin/examples/legacy-completed-scan/scan-manifest.json +43 -0
  10. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  11. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  12. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
  13. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  20. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  21. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  22. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  23. package/_bundled_plugin/references/access-advisory.md +11 -0
  24. package/_bundled_plugin/references/core-scan.md +30 -77
  25. package/_bundled_plugin/references/final-report.md +50 -184
  26. package/_bundled_plugin/references/scan-artifacts.md +5 -5
  27. package/_bundled_plugin/references/scan-contract.md +12 -12
  28. package/_bundled_plugin/references/scan-prologue.md +1 -1
  29. package/_bundled_plugin/references/scan-runtime.md +1 -5
  30. package/_bundled_plugin/references/threat-model.md +33 -41
  31. package/_bundled_plugin/schemas/coverage.schema.json +12 -1
  32. package/_bundled_plugin/schemas/scan-manifest.schema.json +37 -11
  33. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
  34. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +5 -5
  35. package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
  36. package/_bundled_plugin/scripts/deep_scan_workbench.py +146 -86
  37. package/_bundled_plugin/scripts/finalize_scan_contract.py +249 -76
  38. package/_bundled_plugin/scripts/report_projection.py +49 -17
  39. package/_bundled_plugin/scripts/review_state.py +68 -0
  40. package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
  41. package/_bundled_plugin/scripts/validate_scan_contract.py +1 -2
  42. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
  43. package/_bundled_plugin/scripts/workbench/retained_coverage.py +3230 -0
  44. package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
  45. package/_bundled_plugin/scripts/workbench/saved_coverage.py +651 -8
  46. package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
  47. package/_bundled_plugin/scripts/workbench/saved_sources.py +730 -0
  48. package/_bundled_plugin/scripts/workbench_cli.py +3 -0
  49. package/_bundled_plugin/scripts/workbench_constants.py +0 -1
  50. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
  51. package/_bundled_plugin/scripts/workbench_db.py +55 -171
  52. package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
  53. package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
  54. package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
  55. package/_bundled_plugin/scripts/workbench_saved_results.py +808 -2389
  56. package/_bundled_plugin/scripts/workbench_scan_history.py +77 -7
  57. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  58. package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
  59. package/_bundled_plugin/scripts/workbench_target.py +142 -17
  60. package/_bundled_plugin/scripts/workbench_validation.py +9 -2
  61. package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
  62. package/_bundled_plugin/shared/workbench-migrations.json +28 -0
  63. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
  64. package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
  65. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
  66. package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
  67. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
  68. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
  69. package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
  70. package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
  71. package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
  72. package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
  73. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
  74. package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -24
  75. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
  76. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
  77. package/_bundled_plugin/skills/security-scan/SKILL.md +29 -17
  78. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
  79. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
  80. package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
  81. package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
  82. package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
  83. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
  84. package/_bundled_plugin/skills/validation/SKILL.md +42 -104
  85. package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
  86. package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
  87. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
  88. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
  89. package/dist/api.d.ts +5 -3
  90. package/dist/api.d.ts.map +1 -1
  91. package/dist/api.js +359 -53
  92. package/dist/api.js.map +1 -1
  93. package/dist/auth.d.ts.map +1 -1
  94. package/dist/auth.js +4 -1
  95. package/dist/auth.js.map +1 -1
  96. package/dist/cli.d.ts +2 -2
  97. package/dist/cli.d.ts.map +1 -1
  98. package/dist/cli.js +696 -430
  99. package/dist/cli.js.map +1 -1
  100. package/dist/cloud-publish.d.ts.map +1 -1
  101. package/dist/cloud-publish.js +2 -23
  102. package/dist/cloud-publish.js.map +1 -1
  103. package/dist/component-scan.d.ts +8 -2
  104. package/dist/component-scan.d.ts.map +1 -1
  105. package/dist/component-scan.js +2 -0
  106. package/dist/component-scan.js.map +1 -1
  107. package/dist/contract.d.ts +5 -2
  108. package/dist/contract.d.ts.map +1 -1
  109. package/dist/contract.js +93 -45
  110. package/dist/contract.js.map +1 -1
  111. package/dist/cost.d.ts +18 -0
  112. package/dist/cost.d.ts.map +1 -1
  113. package/dist/cost.js +107 -12
  114. package/dist/cost.js.map +1 -1
  115. package/dist/coverage-presentation.d.ts +6 -0
  116. package/dist/coverage-presentation.d.ts.map +1 -0
  117. package/dist/coverage-presentation.js +34 -0
  118. package/dist/coverage-presentation.js.map +1 -0
  119. package/dist/custom-publish.js +2 -2
  120. package/dist/custom-publish.js.map +1 -1
  121. package/dist/custom-validation-prompt.js +5 -5
  122. package/dist/custom-validation-prompt.js.map +1 -1
  123. package/dist/custom-validation.d.ts.map +1 -1
  124. package/dist/custom-validation.js +47 -19
  125. package/dist/custom-validation.js.map +1 -1
  126. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  127. package/dist/deduplication/checkpointed-review.js +3 -1
  128. package/dist/deduplication/checkpointed-review.js.map +1 -1
  129. package/dist/deduplication/codex-review.d.ts.map +1 -1
  130. package/dist/deduplication/codex-review.js +2 -2
  131. package/dist/deduplication/codex-review.js.map +1 -1
  132. package/dist/deduplication/deduplication-prompts.d.ts +1 -0
  133. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  134. package/dist/deduplication/deduplication-prompts.js +3 -0
  135. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  136. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  137. package/dist/deduplication/records-protocol.js +21 -1
  138. package/dist/deduplication/records-protocol.js.map +1 -1
  139. package/dist/deduplication/scan.d.ts.map +1 -1
  140. package/dist/deduplication/scan.js +9 -7
  141. package/dist/deduplication/scan.js.map +1 -1
  142. package/dist/endor-findings.d.ts +4 -3
  143. package/dist/endor-findings.d.ts.map +1 -1
  144. package/dist/endor-findings.js +16 -11
  145. package/dist/endor-findings.js.map +1 -1
  146. package/dist/external-findings-import.d.ts.map +1 -1
  147. package/dist/external-findings-import.js +13 -5
  148. package/dist/external-findings-import.js.map +1 -1
  149. package/dist/external-findings-local.d.ts +1 -0
  150. package/dist/external-findings-local.d.ts.map +1 -1
  151. package/dist/external-findings-local.js +14 -3
  152. package/dist/external-findings-local.js.map +1 -1
  153. package/dist/external-findings-publish.d.ts +2 -1
  154. package/dist/external-findings-publish.d.ts.map +1 -1
  155. package/dist/external-findings-publish.js +64 -28
  156. package/dist/external-findings-publish.js.map +1 -1
  157. package/dist/external-findings-read.d.ts +12 -1
  158. package/dist/external-findings-read.d.ts.map +1 -1
  159. package/dist/external-findings-read.js +25 -6
  160. package/dist/external-findings-read.js.map +1 -1
  161. package/dist/external-findings.d.ts +2 -0
  162. package/dist/external-findings.d.ts.map +1 -1
  163. package/dist/external-findings.js.map +1 -1
  164. package/dist/external-import-contract.d.ts +7 -1
  165. package/dist/external-import-contract.d.ts.map +1 -1
  166. package/dist/external-import-contract.js +63 -31
  167. package/dist/external-import-contract.js.map +1 -1
  168. package/dist/external-imports-feature.d.ts +6 -0
  169. package/dist/external-imports-feature.d.ts.map +1 -0
  170. package/dist/external-imports-feature.js +51 -0
  171. package/dist/external-imports-feature.js.map +1 -0
  172. package/dist/finding-workflow.d.ts +24 -5
  173. package/dist/finding-workflow.d.ts.map +1 -1
  174. package/dist/finding-workflow.js +24 -9
  175. package/dist/finding-workflow.js.map +1 -1
  176. package/dist/github-advisories.d.ts.map +1 -1
  177. package/dist/github-advisories.js +3 -1
  178. package/dist/github-advisories.js.map +1 -1
  179. package/dist/github-findings.d.ts +4 -3
  180. package/dist/github-findings.d.ts.map +1 -1
  181. package/dist/github-findings.js +16 -14
  182. package/dist/github-findings.js.map +1 -1
  183. package/dist/host-protocol.d.ts +1 -1
  184. package/dist/host-protocol.d.ts.map +1 -1
  185. package/dist/host-protocol.js +2 -2
  186. package/dist/host-protocol.js.map +1 -1
  187. package/dist/hosted-remediation.d.ts.map +1 -1
  188. package/dist/hosted-remediation.js +19 -2
  189. package/dist/hosted-remediation.js.map +1 -1
  190. package/dist/hosted-scan.d.ts +1 -1
  191. package/dist/hosted-scan.d.ts.map +1 -1
  192. package/dist/hosted-scan.js +0 -6
  193. package/dist/hosted-scan.js.map +1 -1
  194. package/dist/import-scan.js +17 -17
  195. package/dist/import-scan.js.map +1 -1
  196. package/dist/index.d.ts +4 -3
  197. package/dist/index.d.ts.map +1 -1
  198. package/dist/index.js +2 -1
  199. package/dist/index.js.map +1 -1
  200. package/dist/knowledge-base.d.ts +1 -0
  201. package/dist/knowledge-base.d.ts.map +1 -1
  202. package/dist/knowledge-base.js +20 -8
  203. package/dist/knowledge-base.js.map +1 -1
  204. package/dist/mock-scan.d.ts +1 -2
  205. package/dist/mock-scan.d.ts.map +1 -1
  206. package/dist/mock-scan.js +12 -17
  207. package/dist/mock-scan.js.map +1 -1
  208. package/dist/models.d.ts +10 -1
  209. package/dist/models.d.ts.map +1 -1
  210. package/dist/models.js.map +1 -1
  211. package/dist/multiscan.d.ts.map +1 -1
  212. package/dist/multiscan.js +1 -6
  213. package/dist/multiscan.js.map +1 -1
  214. package/dist/native-json-findings.d.ts +30 -0
  215. package/dist/native-json-findings.d.ts.map +1 -0
  216. package/dist/native-json-findings.js +114 -0
  217. package/dist/native-json-findings.js.map +1 -0
  218. package/dist/opengrep-findings.d.ts +10 -0
  219. package/dist/opengrep-findings.d.ts.map +1 -0
  220. package/dist/opengrep-findings.js +198 -0
  221. package/dist/opengrep-findings.js.map +1 -0
  222. package/dist/owner-evidence.d.ts +2 -0
  223. package/dist/owner-evidence.d.ts.map +1 -1
  224. package/dist/owner-evidence.js +2 -1
  225. package/dist/owner-evidence.js.map +1 -1
  226. package/dist/patch-tui.d.ts.map +1 -1
  227. package/dist/patch-tui.js +2 -0
  228. package/dist/patch-tui.js.map +1 -1
  229. package/dist/publication.js +1 -2
  230. package/dist/publication.js.map +1 -1
  231. package/dist/publish.d.ts +2 -0
  232. package/dist/publish.d.ts.map +1 -1
  233. package/dist/publish.js +1 -1
  234. package/dist/publish.js.map +1 -1
  235. package/dist/record.d.ts +1 -0
  236. package/dist/record.d.ts.map +1 -1
  237. package/dist/record.js +10 -0
  238. package/dist/record.js.map +1 -1
  239. package/dist/remediation.d.ts +1 -25
  240. package/dist/remediation.d.ts.map +1 -1
  241. package/dist/remediation.js +3 -24
  242. package/dist/remediation.js.map +1 -1
  243. package/dist/result.d.ts +5 -3
  244. package/dist/result.d.ts.map +1 -1
  245. package/dist/result.js +9 -6
  246. package/dist/result.js.map +1 -1
  247. package/dist/runtime.d.ts.map +1 -1
  248. package/dist/runtime.js +7 -7
  249. package/dist/runtime.js.map +1 -1
  250. package/dist/scan-dashboard.d.ts +5 -3
  251. package/dist/scan-dashboard.d.ts.map +1 -1
  252. package/dist/scan-dashboard.js +79 -26
  253. package/dist/scan-dashboard.js.map +1 -1
  254. package/dist/scan-history-renderer.d.ts.map +1 -1
  255. package/dist/scan-history-renderer.js +9 -3
  256. package/dist/scan-history-renderer.js.map +1 -1
  257. package/dist/scan-owner-suggestions.d.ts +1 -1
  258. package/dist/scan-owner-suggestions.d.ts.map +1 -1
  259. package/dist/scan-owner-suggestions.js +4 -4
  260. package/dist/scan-owner-suggestions.js.map +1 -1
  261. package/dist/scan-usage-display.d.ts +8 -0
  262. package/dist/scan-usage-display.d.ts.map +1 -0
  263. package/dist/scan-usage-display.js +46 -0
  264. package/dist/scan-usage-display.js.map +1 -0
  265. package/dist/scanner-import.d.ts +1 -1
  266. package/dist/scanner-import.d.ts.map +1 -1
  267. package/dist/scanner-import.js +7 -3
  268. package/dist/scanner-import.js.map +1 -1
  269. package/dist/security-policy.d.ts.map +1 -1
  270. package/dist/security-policy.js +17 -22
  271. package/dist/security-policy.js.map +1 -1
  272. package/dist/semgrep-findings.d.ts +4 -3
  273. package/dist/semgrep-findings.d.ts.map +1 -1
  274. package/dist/semgrep-findings.js +22 -89
  275. package/dist/semgrep-findings.js.map +1 -1
  276. package/dist/skill-authentication.d.ts +18 -0
  277. package/dist/skill-authentication.d.ts.map +1 -0
  278. package/dist/skill-authentication.js +82 -0
  279. package/dist/skill-authentication.js.map +1 -0
  280. package/dist/snyk-findings.d.ts +5 -4
  281. package/dist/snyk-findings.d.ts.map +1 -1
  282. package/dist/snyk-findings.js +18 -17
  283. package/dist/snyk-findings.js.map +1 -1
  284. package/dist/suggest-owners.d.ts +4 -1
  285. package/dist/suggest-owners.d.ts.map +1 -1
  286. package/dist/suggest-owners.js +53 -21
  287. package/dist/suggest-owners.js.map +1 -1
  288. package/dist/targets.d.ts.map +1 -1
  289. package/dist/targets.js +4 -1
  290. package/dist/targets.js.map +1 -1
  291. package/dist/wiz-csv.d.ts +2 -1
  292. package/dist/wiz-csv.d.ts.map +1 -1
  293. package/dist/wiz-csv.js +3 -3
  294. package/dist/wiz-csv.js.map +1 -1
  295. package/dist/wiz-findings.d.ts +2 -1
  296. package/dist/wiz-findings.d.ts.map +1 -1
  297. package/dist/wiz-findings.js +27 -23
  298. package/dist/wiz-findings.js.map +1 -1
  299. package/docs/classify-severity-records.md +3 -4
  300. package/docs/cli.md +171 -41
  301. package/docs/dedupe-records.md +15 -1
  302. package/docs/findings-service.md +119 -118
  303. package/docs/hosted-remediation.md +19 -17
  304. package/docs/hosted-scans.md +37 -11
  305. package/package.json +9 -9
  306. package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
  307. package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
  308. package/_bundled_plugin/scripts/rank_preview.py +0 -129
@@ -10,9 +10,8 @@ import json
10
10
  import os
11
11
  import re
12
12
  import sqlite3
13
- import stat
14
13
  import sys
15
- from collections.abc import Callable, Iterator
14
+ from collections.abc import Iterator
16
15
  from contextlib import contextmanager
17
16
  from pathlib import Path
18
17
  from typing import Any
@@ -20,47 +19,128 @@ from typing import Any
20
19
  sys.path.insert(0, str(Path(__file__).resolve().parent))
21
20
  from finalize_scan_contract import (
22
21
  ContractError,
23
- _finding_strength,
22
+ ScanLocalIOError,
24
23
  _populate_unsealed_artifact_envelope,
25
24
  _populate_unsealed_manifest_envelope,
26
25
  _prepare_scan_finalization,
27
- _read_json,
28
26
  _read_saved_threat_model,
29
27
  _read_scan_local_json,
30
28
  _read_scan_local_json_bytes,
31
- _read_scan_local_json_with_metadata,
29
+ _recover_unsealed_coverage,
32
30
  _recover_unsealed_findings,
33
31
  _remove_scan_local_file_if_exists,
34
- _require_portable_relative_path,
35
- _require_str,
36
32
  _validate_completion_binding,
37
- _validate_resolved_deferred,
38
- _validate_schema_node,
39
33
  _write_prepared_scan_finalization,
40
34
  finalize_scan,
41
- finding_candidate_id,
35
+ manifest_is_sealed,
42
36
  open_scan_local_file_descriptor,
43
37
  write_scan_local_bytes,
44
38
  write_threat_model_projection_if_possible,
45
39
  )
40
+ from finalize_scan_contract import finding_candidate_id as finding_candidate_id
41
+ from review_state import checkpoint_review_state, review_state, review_summary, uses_legacy_review
42
+ from workbench.retained_coverage import (
43
+ _append_previous_finding as _append_previous_finding,
44
+ )
45
+ from workbench.retained_coverage import (
46
+ _coverage_rows as _coverage_rows,
47
+ )
48
+ from workbench.retained_coverage import (
49
+ _ensure_finding_identities as _ensure_finding_identities,
50
+ )
51
+ from workbench.retained_coverage import (
52
+ _ensure_finding_identity as _ensure_finding_identity,
53
+ )
54
+ from workbench.retained_coverage import (
55
+ _finding_content as _finding_content,
56
+ )
57
+ from workbench.retained_coverage import (
58
+ _finding_identity as _finding_identity,
59
+ )
60
+ from workbench.retained_coverage import (
61
+ _finding_key as _finding_key,
62
+ )
63
+ from workbench.retained_coverage import (
64
+ _has_candidate_metadata as _has_candidate_metadata,
65
+ )
66
+ from workbench.retained_coverage import (
67
+ _identity_candidate as _identity_candidate,
68
+ )
69
+ from workbench.retained_coverage import (
70
+ _inherit_coverage_sources as _inherit_coverage_sources,
71
+ )
72
+ from workbench.retained_coverage import (
73
+ _retained_findings as _retained_findings,
74
+ )
75
+ from workbench.retained_coverage import (
76
+ _semantic_identity_key as _semantic_identity_key,
77
+ )
78
+ from workbench.retained_coverage import (
79
+ _worker_candidate_key as _worker_candidate_key,
80
+ )
81
+ from workbench.retained_coverage import merge_saved_results as merge_saved_results
82
+ from workbench.saved_coverage import (
83
+ _deferred_rows as _deferred_rows,
84
+ )
85
+ from workbench.saved_coverage import (
86
+ _merge_tied_parent_observations as _merge_tied_parent_observations,
87
+ )
46
88
  from workbench.saved_coverage import (
47
- _prepare_worker_coverage_recovery,
48
- _project_worker_coverage_row,
49
- _saved_coverage_id,
89
+ _resolved_deferred_rows as _resolved_deferred_rows,
90
+ )
91
+ from workbench.saved_coverage import (
92
+ _saved_coverage_id as _saved_coverage_id,
93
+ )
94
+ from workbench.saved_outputs import _restore_published_outputs, _snapshot_published_outputs
95
+ from workbench.saved_sources import (
96
+ _admit_parent_checkpoint,
97
+ _capture_saved_source,
98
+ _checkpoint_paths,
99
+ _CheckpointSnapshotError,
100
+ _encode_retained_sources,
101
+ _encoded,
102
+ _freeze_source_times,
103
+ _frozen_checkpoint_heads,
104
+ _frozen_source_times,
105
+ _is_source_order_snapshot,
106
+ _legacy_checkpoint_observed_at,
107
+ _parent_scan_draft,
108
+ _read_saved_parent_result,
109
+ _read_saved_result,
110
+ _require_current_deep_publication,
111
+ _restore_checkpoint_head,
112
+ _restore_frozen_model_capture,
113
+ _retained_checkpoint_state,
114
+ _retained_source_state,
115
+ _saved_result_paths,
116
+ _source_digests,
117
+ )
118
+ from workbench.saved_sources import (
119
+ _capture_parent_observation as _capture_parent_observation,
120
+ )
121
+ from workbench.saved_sources import (
122
+ _checkpoint_head_directory as _checkpoint_head_directory,
123
+ )
124
+ from workbench.saved_sources import (
125
+ _digest as _digest,
126
+ )
127
+ from workbench.saved_sources import (
128
+ _latest_successful_reducer as _latest_successful_reducer,
129
+ )
130
+ from workbench.saved_sources import (
131
+ _parent_publication_ancestors as _parent_publication_ancestors,
132
+ )
133
+ from workbench.saved_sources import (
134
+ _worker_outputs as _worker_outputs,
50
135
  )
51
136
  from workbench_constants import PHASES
52
- from workbench_target import committed_diff_snapshot_digest
53
- from workbench_validation import path_within_scope
54
-
55
- _PUBLISHED_OUTPUTS = (
56
- "findings.json",
57
- "coverage.json",
58
- "scan-manifest.json",
59
- "report.md",
60
- "threatmodel.md",
61
- "report.html",
62
- "exports/results.sarif",
137
+ from workbench_coverage_receipts import (
138
+ _deferred_candidate_id as _deferred_candidate_id,
139
+ )
140
+ from workbench_coverage_receipts import (
141
+ _generic_surface_updates as _generic_surface_updates,
63
142
  )
143
+
64
144
  _PUBLICATION_FOLLOW_UP_WARNING = (
65
145
  "Saved scan evidence remains on disk; result publication needs follow-up:"
66
146
  )
@@ -124,284 +204,6 @@ def refresh_completed_scan(
124
204
  return db.scan_context(connection, scan["id"])
125
205
 
126
206
 
127
- def _encoded(value: Any) -> bytes:
128
- return json.dumps(
129
- value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
130
- ).encode()
131
-
132
-
133
- def _digest(value: Any) -> str:
134
- return hashlib.sha256(_encoded(value)).hexdigest()
135
-
136
-
137
- def _children(scan_dir: Path, relative: str) -> list[str]:
138
- cursor = scan_dir
139
- for part in Path(relative).parts:
140
- if part in {"..", "."}:
141
- return []
142
- cursor = cursor / part
143
- try:
144
- if not stat.S_ISDIR(cursor.lstat().st_mode):
145
- return []
146
- except FileNotFoundError:
147
- return []
148
- return sorted(child.name for child in cursor.iterdir())
149
-
150
-
151
- def _latest_successful_reducer(workers: list[Any]) -> Any | None:
152
- return max(
153
- (
154
- worker
155
- for worker in workers
156
- if worker["kind"] == "dedup"
157
- and worker["status"] == "succeeded"
158
- and worker["result_manifest_path"]
159
- ),
160
- key=lambda worker: (worker["completed_at"] or "", worker["id"]),
161
- default=None,
162
- )
163
-
164
-
165
- def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
166
- return [
167
- f"{directory}/{name}"
168
- for name in _children(scan_dir, directory)
169
- if re.fullmatch(r"[0-9a-f]{64}\.json", name)
170
- ]
171
-
172
-
173
- def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
174
- output = Path(worker["artifact_dir"]).relative_to(scan_dir)
175
- attempts = (output.parent if output.name == "output" else output) / "attempts"
176
- archived = [
177
- ((attempts / name).as_posix(), int(name.split("-")[1]))
178
- for name in _children(scan_dir, attempts.as_posix())
179
- if re.fullmatch(r"attempt-\d+", name)
180
- ]
181
- attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
182
- if worker["kind"] == "discovery" and not attempt:
183
- attempt = max((attempt for _, attempt in archived), default=0) + 1
184
- return [(output.as_posix(), attempt), *archived]
185
-
186
-
187
- def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
188
- latest_reducer = _latest_successful_reducer(workers)
189
- yield "checkpoint-head.json", None
190
- for directory in ("checkpoint-heads", "checkpoints"):
191
- yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
192
- for worker in workers:
193
- if worker["kind"] not in {"dedup", "discovery"}:
194
- continue
195
- try:
196
- outputs = _worker_outputs(scan_dir, worker)
197
- except (TypeError, ValueError):
198
- continue
199
- for directory, _ in outputs:
200
- if worker["kind"] == "discovery":
201
- yield f"{directory}/checkpoint-head.json", worker["kind"]
202
- yield from (
203
- (path, worker["kind"])
204
- for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
205
- )
206
- checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
207
- if worker["kind"] == "discovery" or checkpoint_paths:
208
- yield f"{directory}/result.json", worker["kind"]
209
- yield from ((path, worker["kind"]) for path in checkpoint_paths)
210
- if worker["result_manifest_path"] and (
211
- worker["kind"] == "discovery"
212
- or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
213
- ):
214
- try:
215
- yield (
216
- Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
217
- worker["kind"],
218
- )
219
- except ValueError:
220
- continue
221
-
222
-
223
- def _checkpoint_head_directory(relative: str) -> Path | None:
224
- path = Path(relative)
225
- if path.name == "checkpoint-head.json":
226
- return path.parent
227
- if path.parent.name == "checkpoint-heads":
228
- return path.parent.parent
229
- return None
230
-
231
-
232
- def _capture_saved_source(
233
- scan_dir: Path,
234
- relative: str,
235
- scan_id: str,
236
- *,
237
- kind: str | None = None,
238
- snapshot_head: bool = True,
239
- write: bool = True,
240
- ) -> dict[str, tuple[str, int]]:
241
- if not snapshot_head or Path(relative).name != "checkpoint-head.json":
242
- _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
243
- return {relative: (digest, observed)}
244
- head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
245
- observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
246
- directory = Path(relative).parent
247
- selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
248
- _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
249
- digest = _digest(observation)
250
- snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
251
- # Capture the selected file even if the worker created it after directory enumeration.
252
- if write and not (scan_dir / snapshot).exists():
253
- write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
254
- return {
255
- snapshot: (digest, int(observation["observedAtNs"])),
256
- selected: (selected_digest, selected_time),
257
- }
258
-
259
-
260
- def _is_source_order_snapshot(relative: str) -> bool:
261
- path = Path(relative)
262
- return path.parent == Path("source-order") and bool(
263
- re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
264
- )
265
-
266
-
267
- def _read_saved_result(
268
- scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
269
- ) -> tuple[dict[str, Any], str, int]:
270
- draft, _, metadata = _read_scan_local_json_with_metadata(
271
- scan_dir, relative, "Saved scan checkpoint"
272
- )
273
- directory = _checkpoint_head_directory(relative)
274
- if directory is not None:
275
- checkpoint = draft.get("checkpoint")
276
- if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
277
- raise ContractError("checkpoint head does not name a saved checkpoint")
278
- _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
279
- if Path(relative).name == "checkpoint-head.json":
280
- return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
281
- observed = draft.get("observedAtNs")
282
- if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
283
- raise ContractError("checkpoint head has no observation time")
284
- return draft, _digest(draft), int(observed)
285
- if draft.get("scanId") != scan_id:
286
- raise ContractError("checkpoint belongs to a different scan")
287
- if not _is_source_order_snapshot(relative) and (
288
- not isinstance(draft.get("findings"), list)
289
- or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
290
- ):
291
- raise ContractError("checkpoint has no semantic findings or coverage")
292
- return draft, _digest(draft), metadata.st_mtime_ns
293
-
294
-
295
- def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
296
- times: dict[str, int] = {}
297
- snapshots = [path for path in sources if _is_source_order_snapshot(path)]
298
- for path in snapshots:
299
- record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
300
- if digest != sources[path] or not isinstance(record.get("sources"), dict):
301
- raise ContractError("saved source ordering changed after the scan stopped")
302
- for relative, observation in record["sources"].items():
303
- if (
304
- not isinstance(observation, dict)
305
- or relative not in sources
306
- or observation.get("digest") != sources[relative]
307
- or not isinstance(observation.get("observedAtNs"), str)
308
- or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
309
- ):
310
- raise ContractError("saved source ordering does not match its frozen sources")
311
- observed = int(observation["observedAtNs"])
312
- if relative in times and times[relative] != observed:
313
- raise ContractError("saved source ordering has conflicting observations")
314
- times[relative] = observed
315
- if snapshots and sources.keys() - set(snapshots) - times.keys():
316
- raise ContractError("saved source ordering is incomplete")
317
- return times
318
-
319
-
320
- def _freeze_source_times(
321
- scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
322
- ) -> None:
323
- # Identical result rewrites must not change the order of frozen review evidence.
324
- observations = {
325
- path: {"digest": digest, "observedAtNs": str(times[path])}
326
- for path, digest in sources.items()
327
- if not _is_source_order_snapshot(path)
328
- }
329
- if not observations:
330
- return
331
- record = {"scanId": scan_id, "sources": observations}
332
- digest = _digest(record)
333
- path = f"source-order/{digest}.json"
334
- if not (scan_dir / path).exists():
335
- write_scan_local_bytes(scan_dir, path, _encoded(record))
336
- if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
337
- raise ContractError("saved source ordering does not match its digest")
338
- sources[path] = digest
339
-
340
-
341
- def _parent_scan_draft(
342
- scan_id: str,
343
- parent_scan: dict[str, Any],
344
- findings: dict[str, Any],
345
- coverage: dict[str, Any],
346
- ) -> dict[str, Any]:
347
- parent = {
348
- "scanId": scan_id,
349
- "findings": findings.get("findings"),
350
- "coverage": coverage,
351
- **{
352
- key: parent_scan[key]
353
- for key in ("scope", "threatModel", "complete")
354
- if key in parent_scan
355
- },
356
- }
357
- if not isinstance(parent["findings"], list):
358
- raise ContractError("Saved parent draft has no findings array")
359
- return parent
360
-
361
-
362
- def _read_saved_parent_result(
363
- scan_dir: Path, scan_id: str
364
- ) -> tuple[dict[str, Any], dict[str, Any]]:
365
- manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
366
- findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
367
- coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
368
- parent_scan = manifest.get("scan")
369
- if not isinstance(parent_scan, dict):
370
- raise ContractError("Saved parent manifest has no scan object")
371
- if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
372
- parent_scan.get("id", scan_id) != scan_id
373
- or findings.get("scanId", scan_id) != scan_id
374
- or coverage.get("scanId", scan_id) != scan_id
375
- ):
376
- raise ContractError("Saved parent documents belong to a different scan")
377
- return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
378
-
379
-
380
- def _source_digests(value: Any, error: str) -> dict[str, str]:
381
- if not isinstance(value, dict) or not all(
382
- isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
383
- ):
384
- raise ContractError(error)
385
- return value
386
-
387
-
388
- def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
389
- if isinstance(value, dict) and isinstance(value.get("sources"), dict):
390
- sources = _source_digests(
391
- value["sources"], "Saved stopped-scan source digests are malformed."
392
- )
393
- model_source = value.get("threatModelSource")
394
- if not isinstance(model_source, str) or model_source not in sources:
395
- raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
396
- return sources, model_source
397
- return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
398
-
399
-
400
- def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
401
- state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
402
- return json.dumps(state, sort_keys=True)
403
-
404
-
405
207
  def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
406
208
  try:
407
209
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
@@ -446,6 +248,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
446
248
  manifest_scan.get("preservedSources", {}),
447
249
  "Published scan source digests are malformed.",
448
250
  )
251
+ _, pending_recovery = _retained_checkpoint_state(scan)
252
+ if pending_recovery:
253
+ return True
254
+ published_heads = manifest_scan.get("preservedCheckpointHeads")
255
+ if published_heads is None:
256
+ published_heads = _frozen_checkpoint_heads(
257
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
258
+ )
259
+ elif "." not in published_heads:
260
+ # Older publications recorded root observations without selecting a root head.
261
+ legacy_heads = _frozen_checkpoint_heads(
262
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
263
+ )
264
+ if "." in legacy_heads:
265
+ published_heads = {**published_heads, ".": legacy_heads["."]}
266
+ checkpoint_heads = dict(published_heads)
449
267
  current_sources = dict(published_sources)
450
268
  paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
451
269
  for path in paths:
@@ -457,23 +275,53 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
457
275
  kind=paths[path],
458
276
  snapshot_head=path not in published_sources,
459
277
  write=False,
278
+ checkpoint_heads=checkpoint_heads,
460
279
  )
461
280
  current_sources.update({path: value[0] for path, value in captured.items()})
281
+ directory = Path(path).parent.as_posix()
282
+ previous_head = published_heads.get(directory)
283
+ if (
284
+ Path(path).name == "checkpoint-head.json"
285
+ and previous_head in captured
286
+ and Path(previous_head).parent.name == "checkpoints"
287
+ ):
288
+ observed = captured[checkpoint_heads[directory]][1]
289
+ if observed == _legacy_checkpoint_observed_at(
290
+ scan_dir, scan["id"], published_sources, previous_head
291
+ ):
292
+ checkpoint_heads[directory] = previous_head
462
293
  except (ContractError, OSError, ValueError):
463
- continue
464
- return current_sources != published_sources
294
+ _restore_checkpoint_head(checkpoint_heads, published_heads, published_sources, path)
295
+ return current_sources != published_sources or checkpoint_heads != published_heads
465
296
  except (ContractError, OSError, SystemExit, ValueError):
466
297
  return False
467
298
 
468
299
 
469
- def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]:
300
+ def _recovery_source_digests(
301
+ db: Any, connection: Any, scan: Any
302
+ ) -> tuple[dict[str, str], bool, dict[str, str | None]]:
470
303
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
471
304
  frozen_sources: dict[str, str] | None = None
472
305
  include_parent = True
473
306
  raw_frozen_sources = scan["retained_source_digests_json"]
474
307
  if raw_frozen_sources is not None:
475
- frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
308
+ retained = json.loads(raw_frozen_sources)
309
+ frozen_sources, model_source = _retained_source_state(retained)
310
+ if isinstance(retained.get("sources"), dict) and retained.get("sourceTimes") is not None:
311
+ _restore_frozen_model_capture(
312
+ scan_dir, scan["id"], frozen_sources, retained["sourceTimes"], model_source
313
+ )
314
+ _freeze_source_times(
315
+ scan_dir,
316
+ scan["id"],
317
+ frozen_sources,
318
+ retained["sourceTimes"],
319
+ retained.get("parentContext"),
320
+ )
476
321
  include_parent = False
322
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
323
+ if pending_recovery:
324
+ return frozen_sources, False, checkpoint_heads
477
325
 
478
326
  manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
479
327
  if manifest_path is not None:
@@ -485,9 +333,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
485
333
  manifest_scan = manifest.get("scan")
486
334
  if not isinstance(manifest_scan, dict):
487
335
  raise ContractError("Saved scan manifest has no scan object")
488
- if scan["seal_manifest_digest"] is not None or (
489
- manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
490
- ):
336
+ if scan["seal_manifest_digest"] is not None or manifest_is_sealed(manifest_scan):
491
337
  if "preservedSources" in manifest_scan:
492
338
  published_sources = _source_digests(
493
339
  manifest_scan["preservedSources"],
@@ -510,9 +356,14 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
510
356
  "FROM deep_scan_workers WHERE scan_id = ?",
511
357
  (scan["id"],),
512
358
  ).fetchall()
359
+ previous_heads = checkpoint_heads
360
+ if previous_heads is None:
361
+ previous_heads = _frozen_checkpoint_heads(scan_dir, scan["id"], frozen_sources or {})
362
+ checkpoint_heads = dict(previous_heads)
513
363
  paths = dict(_saved_result_paths(scan_dir, workers))
514
364
  recovery_sources = dict(frozen_sources or {})
515
- source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
365
+ parent_context: dict[str, str] = {}
366
+ source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources, parent_context)
516
367
  for relative, expected_digest in recovery_sources.items():
517
368
  try:
518
369
  _, digest, observed = _read_saved_result(
@@ -525,18 +376,52 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
525
376
  if not _is_source_order_snapshot(relative):
526
377
  source_times.setdefault(relative, observed)
527
378
 
379
+ snapshot_error = None
380
+ captured_records = {}
528
381
  for relative in paths.keys() - recovery_sources.keys():
529
382
  try:
530
- captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
383
+ captured = _capture_saved_source(
384
+ scan_dir,
385
+ relative,
386
+ scan["id"],
387
+ kind=paths[relative],
388
+ checkpoint_heads=checkpoint_heads,
389
+ )
390
+ except _CheckpointSnapshotError as exc:
391
+ snapshot_error = snapshot_error or exc
392
+ captured = exc.captured
393
+ captured_records.update(exc.records)
531
394
  except (ContractError, OSError, ValueError):
395
+ _restore_checkpoint_head(
396
+ checkpoint_heads, previous_heads, frozen_sources or {}, relative
397
+ )
532
398
  continue
399
+ if relative == "checkpoint-head.json":
400
+ parent_context["head"] = next(iter(captured))
533
401
  for path, (digest, observed) in captured.items():
534
402
  if path in recovery_sources and recovery_sources[path] != digest:
535
403
  raise ContractError("checkpoint changed after the scan stopped")
536
404
  recovery_sources[path] = digest
537
405
  source_times.setdefault(path, observed)
538
- _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
539
- return recovery_sources, include_parent
406
+ try:
407
+ if snapshot_error is not None:
408
+ raise snapshot_error
409
+ _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
410
+ except (ContractError, OSError) as exc:
411
+ failure = _CheckpointSnapshotError(
412
+ str(exc),
413
+ {
414
+ path: (digest, source_times[path])
415
+ for path, digest in recovery_sources.items()
416
+ if not _is_source_order_snapshot(path)
417
+ },
418
+ )
419
+ failure.records = captured_records
420
+ failure.heads = checkpoint_heads
421
+ failure.include_parent = include_parent
422
+ failure.parent_context = parent_context
423
+ raise failure from exc
424
+ return recovery_sources, include_parent, checkpoint_heads
540
425
 
541
426
 
542
427
  def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
@@ -556,1847 +441,44 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
556
441
  return True
557
442
  return _saved_results_changed(db, connection, scan)
558
443
 
559
-
560
- def _finding_key(finding: dict[str, Any]) -> str:
561
- # Wording and evidence may improve between checkpoints; distinct source locations
562
- # must not collide merely because two workers chose the same semantic identity.
563
- provenance = finding.get("provenance")
564
- identity = (
565
- provenance.get("preservedIdentity", finding.get("identity"))
566
- if isinstance(provenance, dict)
567
- else finding.get("identity")
568
- )
569
- if not isinstance(identity, dict):
570
- normalized = dict(finding)
571
- normalized.pop("identity", None)
572
- _ensure_finding_identity(normalized)
573
- identity = normalized["identity"]
574
- locations = finding.get("locations", [])
575
- if not isinstance(locations, list):
576
- locations = []
577
- return _digest(
578
- [
579
- finding.get("ruleId"),
580
- identity,
581
- sorted(
582
- (
583
- (
584
- location.get("path"),
585
- location.get("startLine"),
586
- location.get("endLine", location.get("startLine")),
587
- )
588
- for location in locations
589
- if isinstance(location, dict)
590
- ),
591
- key=_encoded,
592
- ),
593
- ]
594
- )
595
-
596
-
597
- def _worker_candidate_key(
598
- worker_id: str, candidate_id: str, finding: dict[str, Any]
599
- ) -> tuple[str, str, Any, Any, Any]:
600
- """Identify one worker-local candidate without merging unrelated locations."""
601
- provenance = finding.get("provenance")
602
- identity = (
603
- provenance.get("preservedIdentity", finding.get("identity"))
604
- if isinstance(provenance, dict)
605
- else finding.get("identity")
606
- )
607
- if not isinstance(identity, dict):
608
- normalized = dict(finding)
609
- _ensure_finding_identity(normalized)
610
- identity = normalized.get("identity")
611
- anchor = identity.get("anchor") if isinstance(identity, dict) else None
612
- instance = identity.get("instance") if isinstance(identity, dict) else None
613
- return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
614
-
615
-
616
- def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
617
- """Return substantive finding content without generated identity or provenance."""
618
- return {
619
- key: value
620
- for key, value in finding.items()
621
- if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
622
- }
623
-
624
-
625
- def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:
626
- if not isinstance(finding, dict) or "identity" in finding:
627
- return
628
- if candidate_only and not finding_candidate_id(finding):
629
- return
630
- extensions = finding.get("extensions")
631
- source = str(
632
- (extensions.get("candidateId") if isinstance(extensions, dict) else None)
633
- or finding.get("title")
634
- or "finding"
635
- )
636
- anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
637
- finding["identity"] = {"anchor": anchor}
638
-
639
-
640
- def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
641
- """Yield canonical and historical findings without trusting candidate IDs."""
642
- pending = [finding]
643
- seen: set[int] = set()
644
- while pending:
645
- current = pending.pop()
646
- marker = id(current)
647
- if marker in seen:
648
- continue
649
- seen.add(marker)
650
- yield current
651
- provenance = current.get("provenance")
652
- if not isinstance(provenance, dict):
653
- continue
654
- previous = provenance.get("previousFindings")
655
- if isinstance(previous, list):
656
- pending.extend(item for item in reversed(previous) if isinstance(item, dict))
657
- sources = provenance.get("sourceFindings")
658
- if isinstance(sources, list):
659
- pending.extend(
660
- source["finding"]
661
- for source in reversed(sources)
662
- if isinstance(source, dict) and isinstance(source.get("finding"), dict)
663
- )
664
-
665
-
666
- def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
667
- rows = coverage.get("deferred", [])
668
- return rows if isinstance(rows, list) else []
669
-
670
-
671
- def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
672
- if draft.get("complete") is False:
673
- return []
674
- coverage = draft["coverage"]
675
- rows = coverage.get("resolvedDeferred", [])
676
- try:
677
- # Invalid closure metadata cannot discard the evidence it names.
678
- _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
679
- _validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
680
- except ContractError:
681
- return []
682
- return rows
683
-
684
-
685
- def _merge_tied_parent_observations(
686
- current: dict[str, Any], previous: dict[str, Any]
687
- ) -> dict[str, Any]:
688
- merged = copy.deepcopy(current)
689
- for finding in previous["findings"]:
690
- if finding not in merged["findings"]:
691
- merged["findings"].append(copy.deepcopy(finding))
692
- coverage = merged["coverage"]
693
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
694
- rows = previous["coverage"].get(field, [])
695
- output = coverage.setdefault(field, [])
696
- if isinstance(rows, list) and isinstance(output, list):
697
- for row in rows:
698
- if row not in output:
699
- output.append(copy.deepcopy(row))
700
- pending_ids = {
701
- identity
702
- for row in _deferred_rows(coverage)
703
- if isinstance(row, dict)
704
- for identity in (row.get("id"), row.get("candidateId"))
705
- if isinstance(identity, str)
706
- }
707
- closure_schema = _read_json(
708
- Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
709
- )["properties"]["resolvedDeferred"]
710
- closures = {}
711
- for observed in (current, previous):
712
- for row in _resolved_deferred_rows(observed, closure_schema):
713
- if row["id"] not in pending_ids:
714
- closures.setdefault(row["id"], copy.deepcopy(row))
715
- coverage.pop("resolvedDeferred", None)
716
- if closures:
717
- coverage["resolvedDeferred"] = list(closures.values())
718
- if current.get("complete") is False or previous.get("complete") is False:
719
- merged["complete"] = False
720
- coverage.pop("completeness", None)
721
- return merged
722
-
723
-
724
- def _deferred_candidate_id(
725
- row: dict[str, Any],
726
- owner: str | None,
727
- ambiguous_deferred: set[tuple[str | None, str]],
728
- ) -> str | None:
729
- identity = row.get("candidateId") or row.get("id")
730
- if not isinstance(identity, str):
731
- return None
732
- if (owner, identity) in ambiguous_deferred and not any(
733
- key in row for key in ("candidateId", "candidate", "finding")
734
- ):
735
- return None
736
- return identity
737
-
738
-
739
- def _generic_surface_updates(
740
- sources: list[tuple[str, dict[str, Any], str | None]],
741
- source_order: dict[str, tuple[int, int]],
742
- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
743
- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
744
- resolved_candidates: dict[tuple[str | None, str], str],
745
- reopened_generic: set[tuple[str | None, str]],
746
- surface_schema: dict[str, Any],
747
- deferred_rows: dict[str, list[Any]],
748
- ambiguous_deferred: set[tuple[str | None, str]],
749
- *,
750
- surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
751
- eligible_receipt_sources: set[str],
752
- receipt_root: Path | None = None,
753
- receipt_warnings: list[str] | None = None,
754
- fallback_surfaces: list[dict[str, Any]] | None = None,
755
- ) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
756
- if not closed_deferred and not reopened_generic and receipt_root is None:
757
- return set(), []
758
-
759
- def receipt_error(ref: Any) -> ContractError | None:
760
- if receipt_root is None or not isinstance(ref, str):
761
- return ContractError("Saved coverage receipt is unavailable.")
762
- try:
763
- descriptor = open_scan_local_file_descriptor(
764
- receipt_root, ref, "Saved coverage receipt"
765
- )
766
- os.close(descriptor)
767
- except ContractError as exc:
768
- return exc
769
- return None
770
-
771
- def linked(row: dict[str, Any], identity: str | None) -> bool:
772
- surface_ids = row.get("surfaceIds", [])
773
- return identity is not None and (
774
- row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
775
- )
776
-
777
- saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
778
- for relative, draft, owner in sources:
779
- rows = draft["coverage"].get("surfaces", [])
780
- for row in rows if isinstance(rows, list) else []:
781
- if isinstance(row, dict) and isinstance(row.get("id"), str):
782
- saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
783
- replaced: set[int] = set()
784
- updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
785
- # Current selections across all surfaces take precedence over historical refs.
786
- receipt_paths: dict[str, str] = {}
787
- retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
788
- if receipt_root is not None:
789
- receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
790
- for key, matches in saved_surfaces.items():
791
- retained = [
792
- match for match in matches if surface_is_retained(match[0], key[0], match[1])
793
- ]
794
- if not retained:
795
- continue
796
- latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
797
- retained_surfaces[key] = latest_surface
798
- receipt_surfaces.extend(
799
- (key[0], row)
800
- for relative, row in retained
801
- if source_order[relative] == source_order[latest_source]
802
- )
803
- for relative, draft, owner in sources:
804
- rows = draft["coverage"].get("surfaces", [])
805
- for row in rows if isinstance(rows, list) else []:
806
- if (
807
- isinstance(row, dict)
808
- and "id" not in row
809
- and surface_is_retained(relative, owner, row)
810
- ):
811
- receipt_surfaces.append((owner, row))
812
- receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
813
- for owner, latest_surface in receipt_surfaces:
814
- # Recovery repairs disposition and receipts before validating row
815
- # metadata. A row discarded there cannot reserve a receipt spelling.
816
- metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
817
- metadata.setdefault("id", _saved_coverage_id(metadata))
818
- metadata.update(disposition="needs_follow_up", receiptRefs=[])
819
- try:
820
- _require_str(metadata, "id", "coverage.surfaces")
821
- _validate_schema_node(metadata, surface_schema, "coverage.surfaces")
822
- except ContractError:
823
- continue
824
- refs = latest_surface.get("receiptRefs", [])
825
- for ref in refs if isinstance(refs, list) else []:
826
- if receipt_error(ref) is None:
827
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
828
- receipt_paths.setdefault(normalized.lower(), normalized)
829
- for relative, draft, owner in sources:
830
- closed_ids = {
831
- identity
832
- for (saved_owner, identity), (_, _, source) in closed_deferred.items()
833
- if saved_owner == owner and source == relative
834
- }
835
- reopened_ids = {
836
- identity
837
- for (saved_owner, identity), (_, _, source) in active_deferred.items()
838
- if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
839
- }
840
- if not closed_ids and not reopened_ids and receipt_root is None:
841
- continue
842
- current = draft["coverage"].get("surfaces", [])
843
- for surface in current if isinstance(current, list) else []:
844
- if not isinstance(surface, dict):
845
- continue
846
- reopening = surface.get("disposition") == "needs_follow_up"
847
- work_ids = reopened_ids if reopening else closed_ids
848
- repair_only = not work_ids
849
- if repair_only and receipt_root is None:
850
- continue
851
- identity = surface.get("id")
852
- if not isinstance(identity, str):
853
- continue
854
- matches = saved_surfaces[(owner, identity)]
855
- # Receipt repair alone cannot close or reopen a saved surface.
856
- if repair_only and any(
857
- row.get("disposition") != surface.get("disposition") for _, row in matches
858
- ):
859
- continue
860
- # Older writers could assign one ID to distinct surfaces in a draft.
861
- # A closure cannot identify which of those observations it replaces.
862
- by_source = dict(matches)
863
- if any(row != by_source[saved_path] for saved_path, row in matches):
864
- continue
865
- if any(
866
- "candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
867
- ):
868
- continue
869
- if any(
870
- row is not surface
871
- and source_order[saved_path] >= source_order[relative]
872
- and row.get("disposition") != surface.get("disposition")
873
- for saved_path, row in matches
874
- ):
875
- continue
876
-
877
- if not reopening and any(
878
- saved_owner == owner
879
- and (
880
- not isinstance(row.get("id") or row.get("candidateId"), str)
881
- or (
882
- isinstance(row.get("id"), str)
883
- and (owner, row["id"]) in ambiguous_deferred
884
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
885
- )
886
- )
887
- and linked(row, identity)
888
- for saved_path, _, saved_owner in sources
889
- for row in deferred_rows[saved_path]
890
- if isinstance(row, dict)
891
- ):
892
- continue
893
- if not reopening and any(
894
- saved_owner == owner
895
- and (owner, deferred_id) not in closed_deferred
896
- and (
897
- (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
898
- or (owner, candidate_id) not in resolved_candidates
899
- )
900
- and linked(row, identity)
901
- for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
902
- ):
903
- continue
904
- # An accepted checkpoint can update a saved surface by ID without
905
- # optional surfaceIds links on its generic task.
906
- receipt_matches = [
907
- (saved_path, row)
908
- for saved_path, row in matches
909
- if receipt_root is None
910
- or saved_path in eligible_receipt_sources
911
- or surface_is_retained(saved_path, owner, row)
912
- ]
913
- if not receipt_matches:
914
- continue
915
- latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
916
- target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
917
- if target is None:
918
- continue
919
- update = copy.deepcopy(target)
920
- # Historical evidence can repair a retained row without promoting
921
- # an unselected observation's label, notes, or other metadata.
922
- refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
923
- update["receiptRefs"] = refs
924
- replaced_receipt = False
925
- replacement_paths = dict(receipt_paths)
926
- if isinstance(refs, list):
927
- repaired = (
928
- not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
929
- )
930
- if repaired:
931
- for ref in refs:
932
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
933
- key = normalized.lower()
934
- if key in replacement_paths and replacement_paths[key] != normalized:
935
- repaired = False
936
- break
937
- replacement_paths[key] = normalized
938
- for _, row in receipt_matches:
939
- previous_refs = row.get("receiptRefs", [])
940
- for ref in previous_refs if isinstance(previous_refs, list) else []:
941
- if ref not in refs:
942
- failure = receipt_error(ref) if repaired else None
943
- replaceable = failure is not None and isinstance(
944
- failure.__cause__, FileNotFoundError
945
- )
946
- if repaired and isinstance(ref, str):
947
- try:
948
- normalized = _require_portable_relative_path(
949
- ref, "Saved coverage receipt"
950
- )
951
- except ContractError as error:
952
- failure = error
953
- replaceable = True
954
- else:
955
- key = normalized.lower()
956
- previous = replacement_paths.get(key)
957
- if previous is not None and previous != normalized:
958
- failure = ContractError(
959
- f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
960
- )
961
- replaceable = True
962
- elif not replaceable:
963
- replacement_paths[key] = normalized
964
- if replaceable:
965
- replaced_receipt = True
966
- if receipt_warnings is not None:
967
- receipt_warnings.append(
968
- f"Replaced invalid saved coverage receipt {ref}: {failure}"
969
- )
970
- continue
971
- refs.append(ref)
972
- if repair_only and not replaced_receipt:
973
- continue
974
- try:
975
- _validate_schema_node(update, surface_schema, "coverage.surfaces")
976
- except ContractError:
977
- continue
978
- replaced_sources = {
979
- id(row)
980
- for _, row in matches
981
- if reopening
982
- or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
983
- }
984
- replaced.update(replaced_sources)
985
- existing = next(
986
- (entry for entry in updates if entry[0] == owner and entry[1] == update), None
987
- )
988
- if existing is None:
989
- receipt_paths.update(replacement_paths)
990
- updates.append((owner, update, replaced_sources))
991
- else:
992
- existing[2].update(replaced_sources)
993
- return replaced, updates
994
-
995
-
996
- @contextmanager
997
- def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
998
- """Keep rejected completion attempts from becoming accepted parent observations."""
999
- head_path = scan_dir / "checkpoint-head.json"
1000
- previous = None
1001
- try:
1002
- head_path.lstat()
1003
- except FileNotFoundError:
1004
- pass
1005
- else:
1006
- descriptor = open_scan_local_file_descriptor(
1007
- scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
1008
- )
1009
- with os.fdopen(descriptor, "rb") as handle:
1010
- metadata = os.fstat(handle.fileno())
1011
- previous = (handle.read(), metadata)
1012
- directories = ("checkpoints", "checkpoint-heads")
1013
- previous_files = {
1014
- relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
1015
- }
1016
- try:
1017
- yield
1018
- except BaseException:
1019
- if previous is None:
1020
- _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
1021
- else:
1022
- payload, metadata = previous
1023
- write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
1024
- os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
1025
- current_files = {
1026
- relative
1027
- for directory in directories
1028
- for relative in _checkpoint_paths(scan_dir, directory)
1029
- }
1030
- for relative in current_files - previous_files:
1031
- _remove_scan_local_file_if_exists(scan_dir, relative)
1032
- raise
1033
-
1034
-
1035
- def merge_saved_results(
1036
- scan_dir: Path,
1037
- scan_id: str,
1038
- binding: dict[str, Any],
1039
- workers: list[Any],
1040
- warnings: list[str],
1041
- *,
1042
- stopped: bool,
1043
- reason: str,
1044
- frozen_source_digests: dict[str, str] | None = None,
1045
- allow_frozen_legacy_parent: bool = False,
1046
- frozen_model_source: str | None = None,
1047
- selected_model_source: list[str] | None = None,
1048
- project_worker_coverage: bool = False,
1049
- ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
1050
- """Read only bound parent/worker files; return an unsealed loss-preserving union."""
1051
- initial_warnings = set(warnings)
1052
- try:
1053
- source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
1054
- except (ContractError, OSError, ValueError) as exc:
1055
- raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
1056
- parent: dict[str, Any] | None = None
1057
- parent_manifest: dict[str, Any] | None = None
1058
- parent_is_canonical = False
1059
- parent_modified = 0
1060
- if frozen_source_digests is None or allow_frozen_legacy_parent:
1061
- try:
1062
- parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
1063
- # Without an accepted head, file-authored coverage is a full replacement.
1064
- parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
1065
- parent_is_canonical = True
1066
- except (ContractError, OSError, ValueError) as exc:
1067
- if not stopped:
1068
- raise
1069
- if (scan_dir / "scan-manifest.json").exists():
1070
- warnings.append(f"Could not read the saved parent draft: {exc}")
1071
- parent_manifest = None
1072
- parent = None
1073
- if parent_manifest is not None and parent is not None:
1074
- parent_scan = parent_manifest["scan"]
1075
- try:
1076
- previous_head, _, head_modified = _read_saved_result(
1077
- scan_dir, "checkpoint-head.json", scan_id
1078
- )
1079
- except (ContractError, OSError, ValueError):
1080
- head_modified = None
1081
- if head_modified is not None:
1082
- # A partial tool publication must not outrank its accepted head.
1083
- parent_modified = min(
1084
- (scan_dir / name).lstat().st_mtime_ns
1085
- for name in ("findings.json", "coverage.json", "scan-manifest.json")
1086
- )
1087
- if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
1088
- head_path = scan_dir / "checkpoint-head.json"
1089
- tied_observations = False
1090
- if head_modified == parent_modified:
1091
- previous_parent, _, _ = _read_saved_result(
1092
- scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
1093
- )
1094
- if previous_parent != parent:
1095
- # Tied observations cannot decide which pending work came last.
1096
- parent = _merge_tied_parent_observations(parent, previous_parent)
1097
- parent_is_canonical = False
1098
- tied_observations = True
1099
- payload = _encoded(parent)
1100
- parent_digest = hashlib.sha256(payload).hexdigest()
1101
- parent_checkpoint = f"checkpoints/{parent_digest}.json"
1102
- checkpoint_path = scan_dir / parent_checkpoint
1103
- if not checkpoint_path.exists():
1104
- write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
1105
- # A recovery copy must not appear newer than the review it copies.
1106
- os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
1107
- if head_modified is None or head_modified < parent_modified or tied_observations:
1108
- write_scan_local_bytes(
1109
- scan_dir,
1110
- "checkpoint-head.json",
1111
- _encoded({"checkpoint": checkpoint_path.name}),
1112
- )
1113
- os.utime(head_path, ns=(parent_modified, parent_modified))
1114
- if frozen_source_digests is not None:
1115
- captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
1116
- frozen_source_digests = {
1117
- **frozen_source_digests,
1118
- parent_checkpoint: parent_digest,
1119
- **{path: value[0] for path, value in captured.items()},
1120
- }
1121
-
1122
- sources: list[tuple[str, dict[str, Any], str | None]] = []
1123
- parent_preserved_sources: dict[str, str] = {}
1124
- source_digests: dict[str, str] = {}
1125
- source_order: dict[str, tuple[int, int]] = {}
1126
- worker_attempts: dict[str, tuple[str, int]] = {}
1127
- saved_heads: dict[str, str] = {}
1128
- if parent_manifest:
1129
- recorded = parent_manifest["scan"].get("preservedSources", {})
1130
- if isinstance(recorded, dict):
1131
- parent_preserved_sources = recorded
1132
- source_digests.update(parent_preserved_sources)
1133
- if frozen_source_digests is None:
1134
- source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
1135
- paths: dict[str, str | None] = {}
1136
- reducer_paths: set[str] = set()
1137
- current_results: set[str] = set()
1138
- reducer_outputs: list[tuple[Any, str, list[str], int]] = []
1139
- reducer = _latest_successful_reducer(workers)
1140
- latest_reducer: str | None = None
1141
- if reducer is not None:
1142
- try:
1143
- latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
1144
- paths[latest_reducer] = None
1145
- reducer_paths.add(latest_reducer)
1146
- except ValueError:
1147
- warnings.append("Skipped a reducer result outside the scan directory.")
1148
-
1149
- def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
1150
- if worker_id is not None:
1151
- root = Path(directory).parent.as_posix()
1152
- worker_attempts[root] = (worker_id, attempt)
1153
- paths[f"{root}/checkpoint-head.json"] = worker_id
1154
- head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
1155
- for saved_directory in (directory, head_snapshots):
1156
- paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
1157
-
1158
- paths["checkpoint-head.json"] = None
1159
- checkpoints("checkpoints", None)
1160
- for worker in workers:
1161
- try:
1162
- outputs = _worker_outputs(scan_dir, worker)
1163
- except (TypeError, ValueError):
1164
- warnings.append("Skipped a worker checkpoint outside the scan directory.")
1165
- continue
1166
- if worker["kind"] == "dedup":
1167
- for directory, attempt in outputs:
1168
- checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
1169
- if not checkpoint_paths:
1170
- continue
1171
- result_path = f"{directory}/result.json"
1172
- retained_paths = [result_path, *checkpoint_paths]
1173
- paths.update(dict.fromkeys(retained_paths))
1174
- reducer_paths.update(retained_paths)
1175
- reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
1176
- continue
1177
- if worker["kind"] != "discovery":
1178
- continue
1179
- output, attempt = outputs[0]
1180
- paths[f"{output}/result.json"] = worker["id"]
1181
- current_results.add(f"{output}/result.json")
1182
- for archived, archived_attempt in outputs[1:]:
1183
- paths[f"{archived}/result.json"] = worker["id"]
1184
- checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
1185
- checkpoints(f"{output}/checkpoints", worker["id"], attempt)
1186
- if worker["result_manifest_path"]:
1187
- try:
1188
- current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
1189
- paths[current_path] = worker["id"]
1190
- current_results.add(current_path)
1191
- except ValueError:
1192
- warnings.append("Skipped a worker result outside the scan directory.")
1193
-
1194
- if frozen_source_digests is None:
1195
- for relative, worker_id in list(paths.items()):
1196
- if Path(relative).name != "checkpoint-head.json":
1197
- continue
1198
- del paths[relative]
1199
- try:
1200
- captured = _capture_saved_source(scan_dir, relative, scan_id)
1201
- paths.update({path: worker_id for path in captured})
1202
- except (ContractError, OSError, ValueError) as exc:
1203
- if (scan_dir / relative).exists():
1204
- warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1205
-
1206
- if frozen_source_digests is not None:
1207
- source_digests.update(
1208
- {
1209
- path: digest
1210
- for path, digest in frozen_source_digests.items()
1211
- if _is_source_order_snapshot(path)
1212
- }
1213
- )
1214
- paths = {
1215
- relative: worker_id
1216
- for relative, worker_id in paths.items()
1217
- if relative in frozen_source_digests
1218
- }
1219
- current_results.intersection_update(frozen_source_digests)
1220
- if latest_reducer not in frozen_source_digests:
1221
- latest_reducer = None
1222
-
1223
- for relative, worker_id in paths.items():
1224
- try:
1225
- draft, digest, observed = _read_saved_result(
1226
- scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
1227
- )
1228
- if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
1229
- raise ContractError("checkpoint changed after the scan stopped")
1230
- source_digests[relative] = digest
1231
- source_times.setdefault(relative, observed)
1232
- source_order[relative] = (0, source_times[relative])
1233
- if _checkpoint_head_directory(relative) is not None:
1234
- saved_heads[relative] = draft["checkpoint"]
1235
- continue
1236
- # Recovery expects coverage, but reducer results only contain findings
1237
- # and context. Add an empty value after hashing the original result.
1238
- sources.append((relative, {"coverage": {}, **draft}, worker_id))
1239
- except (ContractError, OSError, ValueError) as exc:
1240
- if (scan_dir / relative).exists():
1241
- warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1242
- if frozen_source_digests is not None:
1243
- if frozen_source_digests.keys() - source_digests.keys():
1244
- raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
1245
-
1246
- # Frozen observations retain checkpoint selection even if a worker moves its head.
1247
- headed_workers = {
1248
- worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
1249
- for head in saved_heads
1250
- if _checkpoint_head_directory(head) != Path(".")
1251
- }
1252
- for relative, _, worker_id in sources:
1253
- if worker_id not in headed_workers:
1254
- continue
1255
- directory = Path(relative).parent
1256
- if directory.name == "checkpoints":
1257
- directory = directory.parent
1258
- _, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
1259
- source_order[relative] = (attempt, source_order[relative][1])
1260
- selected_observations: dict[str, tuple[int, int]] = {}
1261
- parent_heads: list[tuple[int, str]] = []
1262
- for head, checkpoint in saved_heads.items():
1263
- directory = _checkpoint_head_directory(head)
1264
- selected = (directory / "checkpoints" / checkpoint).as_posix()
1265
- if selected not in source_order:
1266
- raise ContractError("Checkpoint head is outside the saved source set.")
1267
- observed = source_order[head][1]
1268
- if directory == Path("."):
1269
- order = (0, max(source_order[selected][1], observed))
1270
- parent_heads.append((observed, selected))
1271
- else:
1272
- _, attempt = worker_attempts[directory.as_posix()]
1273
- order = (attempt, observed)
1274
- selected_observations[selected] = max(selected_observations.get(selected, order), order)
1275
- source_order.update(selected_observations)
1276
-
1277
- drafts_by_path = {relative: draft for relative, draft, _ in sources}
1278
- latest_reducer_key = (
1279
- (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
1280
- if reducer is not None and latest_reducer in drafts_by_path
1281
- else None
1282
- )
1283
- if latest_reducer_key is None:
1284
- latest_reducer = None
1285
- for worker, result_path, checkpoint_paths, attempt in reducer_outputs:
1286
- result = drafts_by_path.get(result_path)
1287
- if result is None or not any(
1288
- drafts_by_path.get(checkpoint_path) == result for checkpoint_path in checkpoint_paths
1289
- ):
1290
- continue
1291
- current_results.add(result_path)
1292
- candidate_key = (worker["completed_at"] or "", worker["id"], attempt)
1293
- if latest_reducer_key is None or candidate_key > latest_reducer_key:
1294
- latest_reducer_key = candidate_key
1295
- latest_reducer = result_path
1296
-
1297
- if parent_heads:
1298
- latest_observation = max(observed for observed, _ in parent_heads)
1299
- for observed, parent_path in parent_heads:
1300
- if observed != latest_observation:
1301
- continue
1302
- draft = drafts_by_path[parent_path]
1303
- modified = source_order[parent_path][1]
1304
- if parent is None or modified > parent_modified:
1305
- parent = draft
1306
- parent_modified = modified
1307
- parent_is_canonical = False
1308
- elif modified == parent_modified and draft != parent:
1309
- parent = _merge_tied_parent_observations(parent, draft)
1310
- parent_is_canonical = False
1311
- if parent is None and latest_reducer is not None:
1312
- parent = drafts_by_path[latest_reducer]
1313
-
1314
- project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
1315
- sources, parent, source_order, enabled=project_worker_coverage
1316
- )
1317
- # Keep the actual retained row even when recovery adds previous findings.
1318
- linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
1319
- linked_surface_outputs: dict[int, dict[str, Any]] = {}
1320
- worker_row_owners: dict[int, str] = {}
1321
- worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
1322
-
1323
- def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
1324
- # Parent aggregates can carry worker metadata without originating in
1325
- # that worker's retained source. Track the actual source of output rows.
1326
- if project_worker_coverage and owner is not None and isinstance(retained, dict):
1327
- worker_row_owners[id(retained)] = owner
1328
- if field == "surfaces" and isinstance(item.get("id"), str):
1329
- worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
1330
-
1331
- def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
1332
- retained = _project_worker_coverage_row(
1333
- item, owner if project_worker_coverage else None, field
1334
- )
1335
- if retained not in output:
1336
- output.append(retained)
1337
- retained = output[output.index(retained)]
1338
- record_coverage_owner(retained, item, owner, field)
1339
- return retained
1340
-
1341
- def linked_tasks(
1342
- task: dict[str, Any],
1343
- published_id: str,
1344
- source_task: dict[str, Any] | None,
1345
- candidates: list[tuple[str | None, Any]],
1346
- ) -> list[Any]:
1347
- expected = (
1348
- _project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
1349
- if source_task is not None
1350
- else None
1351
- )
1352
- return [
1353
- retained
1354
- for owner, retained in candidates
1355
- if retained == task
1356
- or (
1357
- expected is not None
1358
- and owner == expected["workerId"]
1359
- and isinstance(retained, dict)
1360
- and retained.get("id") == expected.get("id")
1361
- and isinstance(retained.get("surfaceIds"), list)
1362
- and published_id in retained["surfaceIds"]
1363
- )
1364
- ]
1365
-
1366
- def linked_surface(surface, outputs, named_outputs):
1367
- retained = outputs.get(id(surface))
1368
- if (
1369
- retained is None
1370
- and isinstance(owner := surface.get("workerId"), str)
1371
- and isinstance(identity := surface.get("id"), str)
1372
- ):
1373
- matches = named_outputs.get((owner, identity), {})
1374
- if len(matches) == 1:
1375
- retained = next(iter(matches.values()))
1376
- return retained
1377
-
1378
- all_sources = ([("parent", parent, None)] if parent else []) + sources
1379
- # Older checkpoints can omit IDs already assigned in their published output.
1380
- for field in ("deferred", "surfaces"):
1381
- named_rows: dict[str | None, list[dict[str, Any]]] = {}
1382
- for _, draft, owner in all_sources:
1383
- rows = draft["coverage"].get(field, [])
1384
- named_rows.setdefault(owner, []).extend(
1385
- row
1386
- for row in (rows if isinstance(rows, list) else [])
1387
- if isinstance(row, dict) and isinstance(row.get("id"), str)
1388
- )
1389
- for _, draft, owner in all_sources:
1390
- rows = draft["coverage"].get(field, [])
1391
- if not isinstance(rows, list):
1392
- continue
1393
- reserved = {
1394
- row["id"]
1395
- for row in rows
1396
- if isinstance(row, dict) and isinstance(row.get("id"), str)
1397
- }
1398
- for row in rows:
1399
- if not isinstance(row, dict) or "id" in row:
1400
- continue
1401
- if field == "deferred" and any(
1402
- key in row for key in ("candidateId", "candidate", "finding")
1403
- ):
1404
- continue
1405
- content = {"receiptRefs": [], **row} if field == "surfaces" else row
1406
- identity = next(
1407
- (
1408
- named["id"]
1409
- for named in named_rows[owner]
1410
- if named["id"] not in reserved
1411
- and {
1412
- **({"receiptRefs": []} if field == "surfaces" else {}),
1413
- **{key: value for key, value in named.items() if key != "id"},
1414
- }
1415
- == content
1416
- ),
1417
- None,
1418
- )
1419
- if identity is not None:
1420
- row["id"] = identity
1421
- reserved.add(identity)
1422
-
1423
- if parent is None and not sources:
1424
- return None
1425
- if (
1426
- parent_manifest
1427
- and parent_manifest["scan"].get("sealedAt")
1428
- and parent_manifest["scan"].get("status") == binding["status"]
1429
- and parent_manifest["scan"].get("preservedSources") == source_digests
1430
- and all(warning in initial_warnings for warning in warnings)
1431
- ):
1432
- return None
1433
-
1434
- if (
1435
- frozen_source_digests is None
1436
- or any(_is_source_order_snapshot(path) for path in source_digests)
1437
- or allow_frozen_legacy_parent
1438
- ):
1439
- _freeze_source_times(scan_dir, scan_id, source_digests, source_times)
1440
-
1441
- target_kind = binding["allowedTargetKinds"][0]
1442
- if (
1443
- target_kind == "git_worktree"
1444
- and "snapshotDigest" not in binding["target"]
1445
- and "git_revision" in binding["allowedTargetKinds"]
1446
- ):
1447
- target_kind = "git_revision"
1448
- target = {"kind": target_kind, **binding["target"]}
1449
- if target["kind"] == "git_diff" and "snapshotDigest" not in target:
1450
- diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
1451
- target["snapshotDigest"] = committed_diff_snapshot_digest(
1452
- diff_kind, target["baseRevision"], target["headRevision"]
1453
- )
1454
- manifest = (
1455
- copy.deepcopy(parent_manifest)
1456
- if parent_manifest
1457
- else {"scan": {"target": target, "scope": binding["scope"]}}
1458
- )
1459
- for key in ("sealedAt", "artifacts"):
1460
- manifest["scan"].pop(key, None)
1461
- manifest["scan"]["preservedSources"] = source_digests
1462
- # Completion follows the selected parent, including legacy terminal drafts
1463
- # that omit the optional marker, rather than an older canonical manifest.
1464
- if parent is not None:
1465
- manifest["scan"].pop("complete", None)
1466
- if "complete" in parent:
1467
- manifest["scan"]["complete"] = parent["complete"]
1468
- for key in ("scope", "threatModel"):
1469
- if isinstance(parent.get(key), dict):
1470
- manifest["scan"][key] = copy.deepcopy(parent[key])
1471
- if isinstance(manifest["scan"].get("scope"), dict):
1472
- manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
1473
- if frozen_model_source is not None:
1474
- # Publication retries retain the choice made with the frozen source set.
1475
- model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
1476
- if not isinstance(model, dict):
1477
- raise ContractError("Frozen stopped-scan model source is unavailable.")
1478
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
1479
- if paths[frozen_model_source] is not None:
1480
- manifest["scan"]["threatModel"]["origin"] = "recovered"
1481
- if selected_model_source is not None:
1482
- selected_model_source[:] = [frozen_model_source]
1483
- coverage = (
1484
- copy.deepcopy(parent["coverage"])
1485
- if parent and parent["coverage"]
1486
- else {
1487
- "mode": binding["coverageMode"],
1488
- "inventoryStrategy": "diff"
1489
- if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
1490
- else "scoped_path"
1491
- if binding["coverageMode"] == "scoped_path"
1492
- else "repository",
1493
- **binding["scope"],
1494
- "surfaces": [],
1495
- "explicitExclusions": [],
1496
- "deferred": [],
1497
- }
1498
- )
1499
- if isinstance(coverage.get("openQuestions"), list):
1500
- coverage["openQuestions"] = [
1501
- {"question": item.strip()} if isinstance(item, str) else item
1502
- for item in coverage["openQuestions"]
1503
- ]
1504
- canonical_rows = (
1505
- {
1506
- id(item)
1507
- for field in ("surfaces", "explicitExclusions", "deferred")
1508
- for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
1509
- }
1510
- if parent_is_canonical
1511
- else set()
1512
- )
1513
- findings: list[dict[str, Any]] = []
1514
- finding_positions: dict[str, int] = {}
1515
- represented: dict[str, str | None] = {}
1516
- represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}
1517
- represented_history: dict[str, set[str]] = {}
1518
- represented_candidate_history: dict[tuple[str, str, Any, Any, Any], set[str]] = {}
1519
- rejected_history: dict[tuple[str, str], list[dict[str, Any]]] = {}
1520
- stopped_parent_seal = bool(
1521
- stopped and parent_manifest and parent_manifest["scan"].get("sealedAt")
1522
- )
1523
-
1524
- def valid_finding(value: Any) -> bool:
1525
- # Use the finalizer's own per-record recovery before a draft can suppress
1526
- # an earlier checkpoint. Invalid latest records must not hide valid history.
1527
- document = {"scanId": scan_id, "findings": [copy.deepcopy(value)]}
1528
- if isinstance(document["findings"][0], dict):
1529
- _ensure_finding_identity(document["findings"][0], candidate_only=True)
1530
- _recover_unsealed_findings(
1531
- {"scan": {"id": scan_id, "target": binding["target"]}},
1532
- document,
1533
- Path(__file__).resolve().parent.parent / "schemas",
1534
- scan_dir,
1535
- [],
1536
- )
1537
- return bool(document["findings"])
1538
-
1539
- source_order["parent"] = (0, parent_modified)
1540
- deferred_rows = {
1541
- relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
1542
- }
1543
- # A legacy source can contain independent tasks with the same explicit ID.
1544
- # Later rewrites cannot make an ID-only closure identify one of those tasks.
1545
- ambiguous_deferred: set[tuple[str | None, str]] = set()
1546
- for relative, _, owner in all_sources:
1547
- by_id: dict[str, dict[str, Any]] = {}
1548
- candidate_aliases = {
1549
- identity
1550
- for row in deferred_rows[relative]
1551
- if isinstance(row, dict)
1552
- and any(key in row for key in ("candidateId", "candidate", "finding"))
1553
- for identity in (row.get("id"), row.get("candidateId"))
1554
- if isinstance(identity, str)
1555
- }
1556
- for row in deferred_rows[relative]:
1557
- if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
1558
- continue
1559
- if any(key in row for key in ("candidateId", "candidate", "finding")):
1560
- continue
1561
- if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
1562
- ambiguous_deferred.add((owner, identity))
1563
- by_id[identity] = row
1564
- current_drafts = ([("parent", parent, None)] if parent else []) + [
1565
- source for source in sources if source[0] in current_results | selected_observations.keys()
1566
- ]
1567
- # Generic closures belong to one logical scan or worker, just like candidates.
1568
- # Keep them when recovering a terminal checkpoint without its canonical write.
1569
- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1570
-
1571
- candidate_ids: set[tuple[str | None, str]] = set()
1572
- accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
1573
- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1574
- coverage_schema = _read_json(
1575
- Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
1576
- )["properties"]
1577
- for relative, draft, owner in all_sources:
1578
- order = source_order[relative]
1579
- for item in deferred_rows[relative]:
1580
- if not isinstance(item, dict):
1581
- continue
1582
- if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
1583
- candidate_ids.update(
1584
- (owner, identity)
1585
- for identity in (item.get("id"), item.get("candidateId"))
1586
- if isinstance(identity, str)
1587
- )
1588
- identity = item.get("id") or item.get("candidateId")
1589
- if isinstance(identity, str):
1590
- key = (owner, identity)
1591
- previous = active_deferred.get(key)
1592
- if previous is None or order > previous[0]:
1593
- active_deferred[key] = (order, item, relative)
1594
- if (
1595
- relative == "parent"
1596
- or relative in current_results
1597
- or relative in selected_observations
1598
- ):
1599
- try:
1600
- # A malformed update cannot discard valid saved evidence.
1601
- _validate_schema_node(
1602
- item, coverage_schema["deferred"]["items"], "coverage.deferred"
1603
- )
1604
- except ContractError:
1605
- continue
1606
- accepted_deferred_orders[key] = max(
1607
- accepted_deferred_orders.get(key, order), order
1608
- )
1609
- for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
1610
- key = (owner, closure["id"])
1611
- previous = closed_deferred.get(key)
1612
- if previous is None or order > previous[0]:
1613
- # The parent comes first, preserving its reason on equal timestamps.
1614
- closed_deferred[key] = (order, closure, relative)
1615
- reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
1616
- reopened_generic: set[tuple[str | None, str]] = set()
1617
- for key, (order, _, _) in list(closed_deferred.items()):
1618
- # Equal timestamps cannot distinguish closure from reopened work.
1619
- reopened = [
1620
- active
1621
- for (owner, identity), active in active_deferred.items()
1622
- if owner == key[0]
1623
- and (identity == key[1] or active[1].get("candidateId") == key[1])
1624
- and active[0] >= order
1625
- ]
1626
- if key in candidate_ids or key in ambiguous_deferred or reopened:
1627
- del closed_deferred[key]
1628
- for _, item, _ in reopened:
1629
- reopened_rows.append((key[0], item))
1630
- if key not in candidate_ids:
1631
- reopened_generic.add(key)
1632
- # Retain every distinct ambiguous task even when a terminal result supersedes
1633
- # its source. Existing output ID normalization keeps each row distinct.
1634
- for relative, _, owner in all_sources:
1635
- for row in deferred_rows[relative]:
1636
- if (
1637
- isinstance(row, dict)
1638
- and isinstance(identity := row.get("id"), str)
1639
- and (owner, identity) in ambiguous_deferred
1640
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1641
- and (owner, row) not in reopened_rows
1642
- ):
1643
- reopened_rows.append((owner, row))
1644
- parent_closures = [
1645
- closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
1646
- ]
1647
- coverage.pop("resolvedDeferred", None)
1648
- if parent_closures:
1649
- coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
1650
- if parent:
1651
- coverage["deferred"] = [
1652
- row
1653
- for row in deferred_rows["parent"]
1654
- if not isinstance(row, dict)
1655
- or not isinstance(row.get("id"), str)
1656
- or (None, row["id"]) not in closed_deferred
1657
- ]
1658
- resolved: dict[tuple[str | None, str], str] = {}
1659
-
1660
- ordered_candidates = {
1661
- (owner, identity)
1662
- for owner, row in reopened_rows
1663
- if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
1664
- and (owner, identity) in candidate_ids
1665
- }
1666
- ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
1667
-
1668
- outcomes: list[tuple[str, str | None, str, str]] = []
1669
- for relative, draft, owner in current_drafts:
1670
- for finding in draft["findings"]:
1671
- if (
1672
- isinstance(finding, dict)
1673
- and valid_finding(finding)
1674
- and (candidate_id := finding_candidate_id(finding))
1675
- ):
1676
- outcomes.append((relative, owner, candidate_id, "reported"))
1677
- for field in ("surfaces", "explicitExclusions"):
1678
- items = draft["coverage"].get(field, [])
1679
- for item in items if isinstance(items, list) else []:
1680
- if (
1681
- isinstance(item, dict)
1682
- and isinstance(item.get("candidateId"), str)
1683
- and item.get("disposition") in {"rejected", "not_applicable"}
1684
- ):
1685
- outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
1686
- ordered_candidates.update(
1687
- (owner, candidate_id)
1688
- for relative, owner, candidate_id, _ in outcomes
1689
- if owner is not None and relative in selected_observations
1690
- )
1691
- # Reopened work and selected checkpoint outcomes follow the saved source order.
1692
- for relative, owner, candidate_id, disposition in outcomes:
1693
- key = (owner, candidate_id)
1694
- if key not in ordered_candidates:
1695
- if relative == "parent" or relative in current_results:
1696
- resolved.setdefault(key, disposition)
1697
- continue
1698
- order = source_order[relative]
1699
- if any(
1700
- saved_owner == owner
1701
- and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
1702
- and modified >= order
1703
- for (saved_owner, _), (modified, row, _) in active_deferred.items()
1704
- ):
1705
- continue
1706
- if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
1707
- resolved[key] = disposition
1708
- ordered_outcomes[key] = (order, relative)
1709
- # Only the current parent may claim that another worker finding was absorbed.
1710
- # A superseded checkpoint must not suppress a newer independent result.
1711
- if parent:
1712
- for finding in parent["findings"]:
1713
- if valid_finding(finding):
1714
- canonical_key = _finding_key(finding)
1715
- for retained in _retained_findings(finding):
1716
- retained_key = _finding_key(retained)
1717
- if retained is not finding:
1718
- represented_history.setdefault(retained_key, set()).add(
1719
- _digest(_finding_content(retained))
1720
- )
1721
- previous_key = represented.get(retained_key)
1722
- if retained_key not in represented:
1723
- represented[retained_key] = canonical_key
1724
- elif previous_key != canonical_key:
1725
- # Ambiguous history cannot suppress an independent source.
1726
- represented[retained_key] = None
1727
- originals = finding["provenance"].get("sourceFindings", [])
1728
- for original in originals if isinstance(originals, list) else []:
1729
- if isinstance(original, dict) and isinstance(original.get("finding"), dict):
1730
- source_id = original.get("id")
1731
- candidate_id = finding_candidate_id(original["finding"])
1732
- if isinstance(source_id, str) and ":" in source_id and candidate_id:
1733
- candidate_key = _worker_candidate_key(
1734
- source_id.rsplit(":", 1)[0],
1735
- candidate_id,
1736
- original["finding"],
1737
- )
1738
- previous_key = represented_candidates.get(candidate_key)
1739
- if candidate_key not in represented_candidates:
1740
- represented_candidates[candidate_key] = canonical_key
1741
- elif previous_key != canonical_key:
1742
- # Candidate ids are only authoritative within one
1743
- # logical worker. Multiple canonical owners make
1744
- # that worker-local identity ambiguous.
1745
- represented_candidates[candidate_key] = None
1746
- represented_candidate_history.setdefault(candidate_key, set()).add(
1747
- _digest(_finding_content(original["finding"]))
1748
- )
1749
- selected_terminal_orders: dict[str, tuple[int, int]] = {}
1750
- for relative, draft, worker_id in sources:
1751
- if (
1752
- worker_id is not None
1753
- and relative in selected_observations
1754
- and draft.get("complete") is not False
1755
- ):
1756
- order = source_order[relative]
1757
- selected_terminal_orders[worker_id] = max(
1758
- selected_terminal_orders.get(worker_id, order), order
1759
- )
1760
- terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
1761
- for relative, draft, worker_id in sources:
1762
- if relative in current_results and draft.get("complete") is not False:
1763
- order = source_order[relative]
1764
- terminal_worker_orders[worker_id] = max(
1765
- terminal_worker_orders.get(worker_id, order), order
1766
- )
1767
-
1768
- def source_coverage_selection(
1769
- relative: str,
1770
- worker_id: str | None,
1771
- ) -> tuple[bool, set[str], bool, bool]:
1772
- worker_result_order = terminal_worker_orders.get(worker_id)
1773
- selected_coverage_superseded = worker_id in selected_terminal_orders and (
1774
- source_order[relative] < selected_terminal_orders[worker_id]
1775
- or (
1776
- relative in current_results
1777
- and source_order[relative] <= selected_terminal_orders[worker_id]
1778
- )
1779
- )
1780
- selected_candidates = {
1781
- candidate_id
1782
- for (owner, candidate_id), (_, source) in ordered_outcomes.items()
1783
- if owner == worker_id and source == relative
1784
- }
1785
- superseded = (
1786
- worker_id is None
1787
- and parent is not None
1788
- and parent.get("complete") is not False
1789
- and relative != "parent"
1790
- and source_order[relative] <= (0, parent_modified)
1791
- and (not stopped_parent_seal or relative in parent_preserved_sources)
1792
- ) or (
1793
- relative not in current_results
1794
- and worker_result_order is not None
1795
- and (
1796
- relative not in selected_observations
1797
- or source_order[relative] < worker_result_order
1798
- )
1799
- )
1800
- # A failed result write can leave pending work outside the accepted result.
1801
- accepted_order = (
1802
- (0, parent_modified)
1803
- if worker_id is None and parent is not None
1804
- else worker_result_order
1805
- )
1806
- retain_pending = (
1807
- superseded
1808
- and (worker_id in headed_workers or (worker_id is None and parent_heads))
1809
- and accepted_order is not None
1810
- and source_order[relative] >= accepted_order
1811
- and any(isinstance(row, dict) for row in deferred_rows[relative])
1812
- )
1813
- return selected_coverage_superseded, selected_candidates, superseded, retain_pending
1814
-
1815
- def coverage_row_is_retained(
1816
- field: str,
1817
- item: Any,
1818
- worker_id: str | None,
1819
- selection: tuple[bool, set[str], bool, bool],
1820
- ) -> bool:
1821
- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
1822
- # A selected terminal checkpoint replaces ordinary coverage, while
1823
- # candidate evidence and pending tasks retain their own reconciliation.
1824
- if (
1825
- selected_coverage_superseded
1826
- and field != "deferred"
1827
- and not (
1828
- isinstance(item, dict)
1829
- and any(key in item for key in ("candidateId", "candidate", "finding"))
1830
- )
1831
- ):
1832
- return False
1833
- # A selected outcome must retain its evidence even if its result write failed.
1834
- if superseded and not (
1835
- isinstance(item, dict)
1836
- and (
1837
- (field == "deferred" and retain_pending)
1838
- or (
1839
- isinstance(item.get("candidateId"), str)
1840
- and item["candidateId"] in selected_candidates
1841
- and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
1842
- )
1843
- )
1844
- ):
1845
- return False
1846
- if (
1847
- isinstance(item, dict)
1848
- and isinstance(
1849
- identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
1850
- if field == "deferred"
1851
- else item.get("candidateId"),
1852
- str,
1853
- )
1854
- and (worker_id, identity) in resolved
1855
- and (field == "deferred" or item.get("disposition") == "needs_follow_up")
1856
- ):
1857
- return False
1858
- return True
1859
-
1860
- ambiguous_surface_ids = {
1861
- (owner, identity)
1862
- for owner, row in reopened_rows
1863
- if isinstance(row.get("id"), str)
1864
- and (owner, row["id"]) in ambiguous_deferred
1865
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1866
- for identity in [
1867
- row["id"],
1868
- *(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
1869
- ]
1870
- if isinstance(identity, str)
1871
- }
1872
- for _, draft, owner in all_sources:
1873
- surfaces = draft["coverage"].get("surfaces", [])
1874
- by_id: dict[str, dict[str, Any]] = {}
1875
- for surface in surfaces if isinstance(surfaces, list) else []:
1876
- if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
1877
- continue
1878
- if identity in by_id and surface != by_id[identity]:
1879
- ambiguous_surface_ids.add((owner, identity))
1880
- by_id[identity] = surface
1881
-
1882
- replaced_deferred = {
1883
- id(row)
1884
- for relative, _, owner in all_sources
1885
- for row in deferred_rows[relative]
1886
- if isinstance(row, dict)
1887
- and isinstance(identity := row.get("id"), str)
1888
- and (key := (owner, identity)) not in candidate_ids
1889
- and key not in ambiguous_deferred
1890
- and (updated := accepted_deferred_orders.get(key)) is not None
1891
- and updated > source_order[relative]
1892
- }
1893
- fallback_surfaces = []
1894
- if surface_links and isinstance(coverage.get("surfaces"), list):
1895
- # Plan the surviving associations before historical receipts can claim
1896
- # a spelling needed by a surface that must be restored for its task.
1897
- prospective_tasks = []
1898
- prospective_surfaces = {}
1899
- prospective_named = {}
1900
- for relative, draft, owner in all_sources:
1901
- selection = source_coverage_selection(relative, owner)
1902
- for row in deferred_rows[relative]:
1903
- if (
1904
- coverage_row_is_retained("deferred", row, owner, selection)
1905
- and id(row) not in replaced_deferred
1906
- and not (
1907
- isinstance(row, dict)
1908
- and isinstance(row.get("id"), str)
1909
- and (owner, row["id"]) in closed_deferred
1910
- )
1911
- ):
1912
- prospective_tasks.append(
1913
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
1914
- )
1915
- rows = draft["coverage"].get("surfaces", [])
1916
- for row in rows if isinstance(rows, list) else []:
1917
- if owner is None or not isinstance(row, dict):
1918
- continue
1919
- emitted = coverage_row_is_retained("surfaces", row, owner, selection)
1920
- ambiguous = (
1921
- isinstance(row.get("id"), str)
1922
- and (owner, row["id"]) in ambiguous_surface_ids
1923
- and row.get("disposition") == "needs_follow_up"
1924
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1925
- )
1926
- if not emitted and not ambiguous:
1927
- continue
1928
- projected = _project_worker_coverage_row(row, owner, "surfaces")
1929
- if isinstance(row.get("id"), str):
1930
- named = prospective_named.setdefault((owner, row["id"]), {})
1931
- projected = next(
1932
- (existing for existing in named.values() if existing == projected),
1933
- projected,
1934
- )
1935
- named[id(projected)] = projected
1936
- if emitted:
1937
- prospective_surfaces[id(row)] = projected
1938
- prospective_tasks.extend(
1939
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
1940
- for owner, row in reopened_rows
1941
- if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
1942
- )
1943
- bound = {
1944
- (id(retained), published_id)
1945
- for task, published_id, surface, _, task_source in surface_links
1946
- if linked_surface(surface, prospective_surfaces, prospective_named) is not None
1947
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
1948
- }
1949
- for task, published_id, _, parent_surface, task_source in surface_links:
1950
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
1951
- key = (id(retained), published_id)
1952
- if key not in bound:
1953
- fallback_surfaces.append(parent_surface)
1954
- bound.add(key)
1955
-
1956
- replaced_surfaces, surface_updates = _generic_surface_updates(
1957
- all_sources,
1958
- source_order,
1959
- closed_deferred,
1960
- active_deferred,
1961
- resolved,
1962
- reopened_generic,
1963
- coverage_schema["surfaces"]["items"],
1964
- deferred_rows,
1965
- ambiguous_deferred,
1966
- receipt_root=scan_dir if project_worker_coverage else None,
1967
- receipt_warnings=warnings,
1968
- fallback_surfaces=fallback_surfaces,
1969
- surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
1970
- "surfaces", row, owner, source_coverage_selection(relative, owner)
1971
- ),
1972
- eligible_receipt_sources={
1973
- relative
1974
- for relative, draft, _ in all_sources
1975
- if draft.get("complete") is not False or relative in selected_observations
1976
- },
1977
- )
1978
- replaced_rows = {
1979
- "surfaces": replaced_surfaces,
1980
- "deferred": replaced_deferred,
1981
- }
1982
- for field, replaced in replaced_rows.items():
1983
- if parent and isinstance(coverage.get(field), list):
1984
- previous = parent["coverage"].get(field, [])
1985
- if isinstance(previous, list):
1986
- removed_parent = [row for row in previous if id(row) in replaced]
1987
- coverage[field] = [row for row in coverage[field] if row not in removed_parent]
1988
- if isinstance(coverage.get("surfaces"), list):
1989
- for owner, surface, replaced_sources in surface_updates:
1990
- retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
1991
- for source_id in replaced_sources & linked_surface_sources:
1992
- linked_surface_outputs[source_id] = retained
1993
- for surface in coverage["surfaces"]:
1994
- if isinstance(surface, dict):
1995
- surface.setdefault("receiptRefs", [])
1996
-
1997
- # Reopened work survives a superseded checkpoint, but current candidate
1998
- # outcomes still apply. Parent closures cannot remove another worker's row.
1999
- for owner, item in reopened_rows:
2000
- if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
2001
- owner,
2002
- identity,
2003
- ) in resolved:
2004
- continue
2005
- pending = coverage.setdefault("deferred", [])
2006
- if isinstance(pending, list):
2007
- append_coverage_row(pending, item, owner, "deferred")
2008
- for _, draft, owner in all_sources:
2009
- surfaces = draft["coverage"].get("surfaces", [])
2010
- for surface in surfaces if isinstance(surfaces, list) else []:
2011
- if (
2012
- isinstance(surface, dict)
2013
- and isinstance(surface.get("id"), str)
2014
- and (owner, surface["id"]) in ambiguous_surface_ids
2015
- and surface.get("disposition") == "needs_follow_up"
2016
- and not any(key in surface for key in ("candidateId", "candidate", "finding"))
2017
- and isinstance(coverage.get("surfaces"), list)
2018
- ):
2019
- append_coverage_row(
2020
- coverage["surfaces"],
2021
- {**surface, "receiptRefs": surface.get("receiptRefs", [])},
2022
- owner,
2023
- "surfaces",
2024
- )
2025
- for relative, draft, worker_id in all_sources:
2026
- selection = source_coverage_selection(relative, worker_id)
2027
- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
2028
- skip_superseded_findings = (
2029
- superseded
2030
- and not stopped
2031
- and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
2032
- )
2033
- if skip_superseded_findings and not selected_candidates and not retain_pending:
2034
- continue
2035
- if (
2036
- not skip_superseded_findings
2037
- and "threatModel" not in manifest["scan"]
2038
- and isinstance(draft.get("threatModel"), dict)
2039
- ):
2040
- model = draft["threatModel"]
2041
- model_path = relative
2042
- checkpoint_dir = Path(relative).parent
2043
- prefer_worker_head = worker_id is not None and (
2044
- checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
2045
- )
2046
- if worker_id is not None:
2047
- if checkpoint_dir.name != "checkpoints":
2048
- checkpoint_dir /= "checkpoints"
2049
- selected_models = [
2050
- path
2051
- for path in selected_observations
2052
- if Path(path).parent == checkpoint_dir
2053
- and isinstance(drafts_by_path[path].get("threatModel"), dict)
2054
- ]
2055
- if selected_models:
2056
- head_path = max(selected_models, key=source_order.__getitem__)
2057
- current = drafts_by_path[head_path]
2058
- # A terminal checkpoint is committed before result.json is replaced.
2059
- # Use the admitted observation, including its frozen ordering on retries.
2060
- if not prefer_worker_head and current.get("complete") is not False:
2061
- prefer_worker_head = source_order[head_path] >= source_order[relative]
2062
- if prefer_worker_head:
2063
- model = current["threatModel"]
2064
- model_path = head_path
2065
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
2066
- if worker_id is not None:
2067
- manifest["scan"]["threatModel"]["origin"] = "recovered"
2068
- if selected_model_source is not None and worker_id is not None:
2069
- selected_model_source[:] = [model_path]
2070
- for value in draft["findings"]:
2071
- if skip_superseded_findings and not (
2072
- isinstance(value, dict)
2073
- and (candidate_id := finding_candidate_id(value)) in selected_candidates
2074
- and resolved.get((worker_id, candidate_id)) == "reported"
2075
- ):
2076
- continue
2077
- if relative == "parent" and parent_is_canonical:
2078
- finding = copy.deepcopy(value)
2079
- _ensure_finding_identity(finding, candidate_only=True)
2080
- provenance = finding.get("provenance") if isinstance(finding, dict) else None
2081
- owner = provenance.get("workerId") if isinstance(provenance, dict) else None
2082
- candidate_id = finding_candidate_id(finding) if isinstance(finding, dict) else None
2083
- if (
2084
- stopped_parent_seal
2085
- and isinstance(owner, str)
2086
- and candidate_id
2087
- and resolved.get((owner, candidate_id)) in {"rejected", "not_applicable"}
2088
- ):
2089
- rejected_history.setdefault((owner, candidate_id), []).append(finding)
2090
- continue
2091
- if valid_finding(finding):
2092
- finding_positions.setdefault(_finding_key(finding), len(findings))
2093
- findings.append(finding)
2094
- continue
2095
- if relative != "parent" and parent and value in parent["findings"]:
2096
- continue
2097
- if not isinstance(value, dict):
2098
- warnings.append(f"Retained malformed finding evidence in {relative}.")
2099
- continue
2100
- finding = copy.deepcopy(value)
2101
- candidate_id = finding_candidate_id(finding)
2102
- if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
2103
- "rejected",
2104
- "not_applicable",
2105
- }:
2106
- surfaces = coverage.get("surfaces")
2107
- for item in surfaces if isinstance(surfaces, list) else []:
2108
- if isinstance(item, dict) and item.get("candidateId") == candidate_id:
2109
- if not isinstance(item.get("previousFindings"), list):
2110
- item["previousFindings"] = []
2111
- history = item["previousFindings"]
2112
- if not any(
2113
- isinstance(previous, dict)
2114
- and _finding_key(previous) == _finding_key(finding)
2115
- and _finding_content(previous) == _finding_content(finding)
2116
- for previous in history
2117
- ):
2118
- history.append(finding)
2119
- continue
2120
- for key in ("findingId", "occurrenceId", "fingerprints"):
2121
- finding.pop(key, None)
2122
- locations = finding.get("locations", [])
2123
- if not isinstance(locations, list) or not any(
2124
- isinstance(location, dict)
2125
- and isinstance(location.get("path"), str)
2126
- and any(
2127
- path_within_scope(location["path"], path)
2128
- for path in binding["scope"]["includePaths"]
2129
- )
2130
- for location in locations
2131
- ):
2132
- warnings.append(f"Skipped out-of-scope finding from {relative}.")
2133
- continue
2134
- provenance = finding.setdefault("provenance", {"source": "local_plugin"})
2135
- if not isinstance(provenance, dict):
2136
- findings.append(finding)
2137
- continue
2138
- if worker_id:
2139
- provenance.setdefault("workerId", worker_id)
2140
- _ensure_finding_identity(finding)
2141
- if not valid_finding(finding):
2142
- findings.append(finding)
2143
- continue
2144
- key = _finding_key(finding)
2145
- represented_by_parent = False
2146
- if relative != "parent":
2147
- if key in represented:
2148
- mapped_key = represented[key]
2149
- historical_contents = represented_history.get(key, set())
2150
- elif worker_id and candidate_id:
2151
- candidate_key = _worker_candidate_key(worker_id, candidate_id, finding)
2152
- if candidate_key not in represented_candidates:
2153
- represented_candidates[candidate_key] = key
2154
- mapped_key = represented_candidates[candidate_key]
2155
- historical_contents = represented_candidate_history.get(candidate_key, set())
2156
- else:
2157
- mapped_key = None
2158
- historical_contents = set()
2159
- if mapped_key is not None:
2160
- key = mapped_key
2161
- represented_by_parent = _digest(_finding_content(value)) in historical_contents
2162
- if key in finding_positions:
2163
- retained = findings[finding_positions[key]]
2164
- if finding != retained:
2165
- if not represented_by_parent and _finding_strength(finding) > _finding_strength(
2166
- retained
2167
- ):
2168
- previous = copy.deepcopy(retained)
2169
- previous_history = previous["provenance"].pop("previousFindings", [])
2170
- retained = finding
2171
- findings[finding_positions[key]] = retained
2172
- else:
2173
- previous = copy.deepcopy(value)
2174
- previous_history = previous.get("provenance", {}).pop(
2175
- "previousFindings", []
2176
- )
2177
- retained_provenance = retained["provenance"]
2178
- retained_history = retained_provenance.get("previousFindings")
2179
- history = (
2180
- [item for item in retained_history if isinstance(item, dict)]
2181
- if isinstance(retained_history, list)
2182
- else []
2183
- )
2184
- retained_provenance["previousFindings"] = history
2185
- for original in [
2186
- *(previous_history if isinstance(previous_history, list) else []),
2187
- previous,
2188
- ]:
2189
- if not isinstance(original, dict):
2190
- continue
2191
- source_key = _finding_key(original)
2192
- source_content = _finding_content(original)
2193
- already_retained = any(
2194
- source_key == _finding_key(historical)
2195
- and source_content == _finding_content(historical)
2196
- for historical in _retained_findings(retained)
2197
- )
2198
- if (
2199
- not already_retained
2200
- and original not in history
2201
- and original != retained
2202
- ):
2203
- history.append(original)
2204
- continue
2205
- finding_positions[key] = len(findings)
2206
- findings.append(finding)
2207
- if superseded and not selected_candidates and not retain_pending:
2208
- continue
2209
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
2210
- if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
2211
- continue
2212
- items = draft["coverage"].get(field, [])
2213
- if field == "deferred":
2214
- items = deferred_rows[relative]
2215
- if not isinstance(items, list):
2216
- continue
2217
- output = coverage.setdefault(field, [])
2218
- if not isinstance(output, list):
2219
- # Keep malformed canonical collections for the existing finalizer's
2220
- # recovery and warnings rather than silently changing its contract.
2221
- continue
2222
- for item in items:
2223
- source_item = item
2224
- if not coverage_row_is_retained(field, item, worker_id, selection):
2225
- continue
2226
- if id(item) in replaced_rows.get(field, ()):
2227
- continue
2228
- if (
2229
- field == "deferred"
2230
- and isinstance(item, dict)
2231
- and isinstance(item.get("id"), str)
2232
- and (worker_id, item["id"]) in closed_deferred
2233
- ):
2234
- continue
2235
- if field == "openQuestions" and isinstance(item, str):
2236
- item = {"question": item.strip()}
2237
- if (
2238
- field == "surfaces"
2239
- and isinstance(item, dict)
2240
- and item.get("disposition") in {"rejected", "not_applicable"}
2241
- and isinstance(item.get("candidateId"), str)
2242
- and (history_findings := rejected_history.get((worker_id, item["candidateId"])))
2243
- ):
2244
- item = copy.deepcopy(item)
2245
- if not isinstance(item.get("previousFindings"), list):
2246
- item["previousFindings"] = []
2247
- history = item["previousFindings"]
2248
- for finding in history_findings:
2249
- if not any(
2250
- isinstance(previous, dict)
2251
- and _finding_key(previous) == _finding_key(finding)
2252
- and _finding_content(previous) == _finding_content(finding)
2253
- for previous in history
2254
- ):
2255
- history.append(copy.deepcopy(finding))
2256
- if field == "surfaces" and isinstance(item, dict):
2257
- item = {**item, "receiptRefs": item.get("receiptRefs", [])}
2258
- if isinstance(item, dict) and "id" not in item:
2259
- semantic_item = dict(item)
2260
- existing = next(
2261
- (
2262
- row
2263
- for row in output
2264
- if isinstance(row, dict)
2265
- and {key: value for key, value in row.items() if key != "id"}
2266
- == semantic_item
2267
- ),
2268
- None,
2269
- )
2270
- if existing is not None:
2271
- if project_worker_coverage and worker_id is not None:
2272
- worker_row_owners[id(existing)] = worker_id
2273
- if field == "surfaces" and id(source_item) in linked_surface_sources:
2274
- linked_surface_outputs[id(source_item)] = existing
2275
- continue
2276
- retained = append_coverage_row(output, item, worker_id, field)
2277
- if field == "surfaces" and id(source_item) in linked_surface_sources:
2278
- linked_surface_outputs[id(source_item)] = retained
2279
-
2280
- identities: dict[str, str] = {}
2281
- for finding in findings:
2282
- if not valid_finding(finding):
2283
- continue
2284
- identity = finding.get("identity")
2285
- if not isinstance(identity, dict):
2286
- continue
2287
- key = _encoded([finding.get("ruleId"), identity]).decode()
2288
- variant = _finding_key(finding)
2289
- if key in identities and identities[key] != variant:
2290
- finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)
2291
- identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"
2292
- identities[key] = variant
2293
- # Bind links to retained output objects before their IDs become unique.
2294
- # Candidate metadata can give an anonymous row the same preliminary ID as
2295
- # another surface, so only the final surface ID can be used for its links.
2296
- retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
2297
- output_tasks = coverage.get("deferred", [])
2298
- task_candidates = [
2299
- (worker_row_owners.get(id(row)), row)
2300
- for row in (output_tasks if isinstance(output_tasks, list) else [])
2301
- ]
2302
- for task, published_id, surface, _, task_source in surface_links:
2303
- retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
2304
- if retained_surface is None or not isinstance(retained_surface.get("id"), str):
2305
- continue
2306
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
2307
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
2308
- links.setdefault(published_id, retained_surface)
2309
- # A newer worker checkpoint may omit a surface still needed by a retained
2310
- # task. Restore its parent copy only when no worker replacement survived.
2311
- for task, published_id, surface, parent_surface, task_source in surface_links:
2312
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
2313
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
2314
- if published_id not in links and isinstance(coverage.get("surfaces"), list):
2315
- retained_surface = append_coverage_row(
2316
- coverage["surfaces"], parent_surface, None, "surfaces"
2317
- )
2318
- record_coverage_owner(
2319
- retained_surface, surface, surface.get("workerId"), "surfaces"
2320
- )
2321
- links[published_id] = retained_surface
2322
- if project_worker_coverage:
2323
- worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
2324
- named_surface_outputs = {
2325
- identity for outputs in worker_surface_outputs.values() for identity in outputs
2326
- }
2327
- surfaces = coverage.get("surfaces", [])
2328
- for surface in surfaces if isinstance(surfaces, list) else []:
2329
- if not isinstance(surface, dict):
2330
- continue
2331
- owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
2332
- if isinstance(owner, str) and isinstance(identity, str):
2333
- key = (owner, identity)
2334
- existing = worker_surfaces.get(key)
2335
- # An anonymous candidate ID can collide with an explicit surface ID.
2336
- # Bind a named worker link to the explicitly named source.
2337
- if existing is None or (
2338
- id(surface) in named_surface_outputs
2339
- and id(existing) not in named_surface_outputs
2340
- ):
2341
- worker_surfaces[key] = surface
2342
- for task in output_tasks if isinstance(output_tasks, list) else []:
2343
- if (
2344
- not isinstance(task, dict)
2345
- or (owner := worker_row_owners.get(id(task))) is None
2346
- or not isinstance(task.get("surfaceIds"), list)
2347
- ):
2348
- continue
2349
- for identity in task["surfaceIds"]:
2350
- if (
2351
- isinstance(identity, str)
2352
- and (surface := worker_surfaces.get((owner, identity))) is not None
2353
- ):
2354
- _, links = retained_surface_links.setdefault(id(task), (task, {}))
2355
- links.setdefault(identity, surface)
2356
- for field in ("surfaces", "explicitExclusions", "deferred"):
2357
- used: set[str] = set()
2358
- items = coverage.setdefault(field, [])
2359
- reserved = {
2360
- row["id"]
2361
- for row in (items if isinstance(items, list) else [])
2362
- if isinstance(row, dict) and isinstance(row.get("id"), str)
444
+
445
+ @contextmanager
446
+ def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
447
+ """Keep rejected completion attempts from becoming accepted parent observations."""
448
+ head_path = scan_dir / "checkpoint-head.json"
449
+ previous = None
450
+ try:
451
+ head_path.lstat()
452
+ except FileNotFoundError:
453
+ pass
454
+ else:
455
+ descriptor = open_scan_local_file_descriptor(
456
+ scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
457
+ )
458
+ with os.fdopen(descriptor, "rb") as handle:
459
+ metadata = os.fstat(handle.fileno())
460
+ previous = (handle.read(), metadata)
461
+ directories = ("checkpoints", "checkpoint-heads")
462
+ previous_files = {
463
+ relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
464
+ }
465
+ try:
466
+ yield
467
+ except BaseException:
468
+ if previous is None:
469
+ _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
470
+ else:
471
+ payload, metadata = previous
472
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
473
+ os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
474
+ current_files = {
475
+ relative
476
+ for directory in directories
477
+ for relative in _checkpoint_paths(scan_dir, directory)
2363
478
  }
2364
- for item in items if isinstance(items, list) else []:
2365
- if not isinstance(item, dict):
2366
- continue
2367
- if id(item) in canonical_rows:
2368
- if isinstance(item.get("id"), str):
2369
- used.add(item["id"])
2370
- continue
2371
- item.setdefault("id", _saved_coverage_id(item))
2372
- if not isinstance(item["id"], str):
2373
- # Preserve malformed rows for per-record recovery, including frozen replay.
2374
- continue
2375
- if item["id"] in used:
2376
- if project_worker_coverage:
2377
- base = item["id"]
2378
- suffix = 2
2379
- while item["id"] in used or item["id"] in reserved:
2380
- item["id"] = f"{base}-{suffix}"
2381
- suffix += 1
2382
- else:
2383
- item["id"] = f"{item['id']}-{_digest(item)[:16]}"
2384
- used.add(item["id"])
2385
- if field == "surfaces":
2386
- item.setdefault("receiptRefs", [])
2387
- for task, links in retained_surface_links.values():
2388
- task["surfaceIds"] = [
2389
- links[identity]["id"] if isinstance(identity, str) and identity in links else identity
2390
- for identity in task["surfaceIds"]
2391
- ]
2392
- coverage.pop("completeness", None)
2393
- if stopped:
2394
- if not isinstance(coverage.get("deferred"), list):
2395
- coverage["deferred"] = []
2396
- item = {"id": "scan-stopped", "reason": reason}
2397
- if item not in coverage["deferred"]:
2398
- coverage["deferred"].append(item)
2399
- return manifest, {"findings": findings}, coverage
479
+ for relative in current_files - previous_files:
480
+ _remove_scan_local_file_if_exists(scan_dir, relative)
481
+ raise
2400
482
 
2401
483
 
2402
484
  def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
@@ -2413,107 +495,142 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
2413
495
  return prepared.coverage
2414
496
 
2415
497
 
2416
- def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]:
2417
- snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {}
2418
- for relative in _PUBLISHED_OUTPUTS:
2419
- descriptor = -1
2420
- try:
2421
- descriptor = open_scan_local_file_descriptor(
2422
- scan_dir, relative, "Published scan output"
2423
- )
2424
- with os.fdopen(descriptor, "rb") as handle:
2425
- descriptor = -1
2426
- metadata = os.fstat(handle.fileno())
2427
- snapshots[relative] = (handle.read(), metadata)
2428
- except ContractError:
2429
- path = scan_dir / relative
2430
- if path.exists() or path.is_symlink():
2431
- if relative == "threatmodel.md":
2432
- # This optional projection will not replace an unsafe destination.
2433
- continue
2434
- raise
2435
- snapshots[relative] = None
2436
- finally:
2437
- if descriptor >= 0:
2438
- os.close(descriptor)
2439
- return snapshots
2440
-
2441
-
2442
- def _restore_published_outputs(
2443
- scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
2444
- ) -> None:
2445
- for relative, snapshot in snapshots.items():
2446
- if snapshot is None:
2447
- path = scan_dir / relative
2448
- if path.exists() or path.is_symlink():
2449
- _remove_scan_local_file_if_exists(scan_dir, relative)
2450
- else:
2451
- contents, metadata = snapshot
2452
- write_scan_local_bytes(scan_dir, relative, contents)
2453
- # Saved-result selection uses mtime; rollback must preserve its ordering.
2454
- os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
2455
-
2456
-
2457
- @contextmanager
2458
- def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
2459
- """Recover a settled, zero-accepted Deep deadline without sealing the parent."""
498
+ def preserve_deep_deadline_draft(
499
+ db: Any,
500
+ connection: Any,
501
+ scan: Any,
502
+ *,
503
+ preserve_reduction: bool = False,
504
+ allow_empty: bool = False,
505
+ ) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:
506
+ """Recover unfinished reviews at a settled Deep deadline without sealing the parent."""
2460
507
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2461
- snapshots = _snapshot_published_outputs(scan_dir)
2462
508
  warnings = json.loads(scan["completion_warnings_json"])
2463
509
  binding = db.workbench_completion_binding(scan, db.now())
2464
- with preserve_parent_head_on_error(scan_dir):
2465
- try:
2466
- documents = merge_saved_results(
2467
- scan_dir,
2468
- scan["id"],
2469
- binding,
2470
- connection.execute(
2471
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
2472
- (scan["id"],),
2473
- ).fetchall(),
2474
- warnings,
2475
- stopped=True,
2476
- reason=(
2477
- "The configured discovery time limit elapsed before any source review completed. "
2478
- "Saved findings and pending review were preserved without a completed reduction."
2479
- ),
2480
- project_worker_coverage=True,
2481
- )
2482
- if documents is None:
510
+ workers = connection.execute(
511
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
512
+ (scan["id"],),
513
+ ).fetchall()
514
+ if preserve_reduction:
515
+ # Completed reviews already went through reduction. Only add evidence
516
+ # from unfinished reviews; raw accepted drafts must not undo that result.
517
+ workers = [
518
+ worker
519
+ for worker in workers
520
+ if worker["kind"] == "discovery" and worker["status"] == "canceled"
521
+ ]
522
+ if not workers:
523
+ return None
524
+ _prepare_scan_finalization(
525
+ scan_dir,
526
+ expected_coverage_mode=db.expected_coverage_mode(scan),
527
+ completion_binding=binding,
528
+ allow_missing_writeups=True,
529
+ )
530
+ snapshots = _snapshot_published_outputs(
531
+ scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json")
532
+ )
533
+ created_paths: set[str] = set()
534
+ try:
535
+ documents = merge_saved_results(
536
+ scan_dir,
537
+ scan["id"],
538
+ binding,
539
+ workers,
540
+ warnings,
541
+ stopped=not preserve_reduction,
542
+ preserve_canonical_parent=preserve_reduction,
543
+ project_worker_coverage=True,
544
+ # Without reduction, the coordinator's empty draft cannot replace worker evidence.
545
+ scan_mode="deep" if preserve_reduction else None,
546
+ finding_scope=db.requested_finding_scope(scan),
547
+ created_paths=created_paths,
548
+ current_contract=True,
549
+ reason=(
550
+ "The configured discovery time limit elapsed before any source review completed. "
551
+ "Saved findings and pending review were preserved without a completed reduction."
552
+ ),
553
+ )
554
+ if documents is None:
555
+ if not allow_empty:
2483
556
  raise ContractError("Deep Scan deadline has no saved parent draft.")
2484
- documents[0]["scan"].pop("complete", None)
2485
- # Validate recovered evidence against the current finding contract,
2486
- # scope, source lineage, and receipts before admitting this draft.
2487
- prepared = _prepare_scan_finalization(
2488
- scan_dir,
2489
- expected_coverage_mode=db.expected_coverage_mode(scan),
2490
- completion_binding=binding,
2491
- recover_unsealed=True,
2492
- draft_documents=documents,
2493
- )
2494
- warnings.extend(prepared.warnings)
2495
- manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
2496
- manifest["scan"].pop("sealedAt", None)
2497
- manifest["scan"].pop("artifacts", None)
2498
- checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
2499
- contents = _encoded(checkpoint)
2500
- name = f"{hashlib.sha256(contents).hexdigest()}.json"
2501
- write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
2502
- write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
2503
- for filename, document in (
2504
- ("findings.json", findings),
2505
- ("coverage.json", coverage),
2506
- ("scan-manifest.json", manifest),
2507
- ):
2508
- write_scan_local_bytes(scan_dir, filename, _encoded(document))
557
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
2509
558
  connection.execute(
2510
559
  "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
2511
560
  (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
2512
561
  )
2513
- yield
2514
- except BaseException:
2515
- _restore_published_outputs(scan_dir, snapshots)
2516
- raise
562
+ return None
563
+ if preserve_reduction:
564
+ marker = {
565
+ "id": "scan-deadline",
566
+ "reason": (
567
+ "The configured discovery time limit elapsed. Saved findings and "
568
+ "pending work from unfinished source reviews were preserved."
569
+ ),
570
+ }
571
+ if not any(
572
+ marker["id"] in (row.get("id"), row.get("candidateId"))
573
+ for key in ("deferred", "resolvedDeferred")
574
+ for row in documents[2].get(key, [])
575
+ ):
576
+ documents[2]["deferred"].append(marker)
577
+ documents[0]["scan"].pop("complete", None)
578
+ expected_coverage_mode = db.expected_coverage_mode(scan)
579
+ if not preserve_reduction:
580
+ # A capped deadline can retain interrupted records even though its
581
+ # completion binding requires strict validation of completed findings.
582
+ _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)
583
+ _populate_unsealed_artifact_envelope(*documents, binding)
584
+ if expected_coverage_mode == "deep_repository":
585
+ documents[2]["inventoryStrategy"] = "repository"
586
+ elif expected_coverage_mode == "scoped_path":
587
+ documents[2]["inventoryStrategy"] = "scoped_path"
588
+ schema_dir = Path(__file__).resolve().parent.parent / "schemas"
589
+ discarded_findings = _recover_unsealed_findings(
590
+ documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True
591
+ )
592
+ _recover_unsealed_coverage(
593
+ documents[2], schema_dir, scan_dir, warnings, discarded_findings
594
+ )
595
+ # Reuse stopped-result recovery's per-record validation, scope checks,
596
+ # source lineage, and receipt validation before admitting this draft.
597
+ prepared = _prepare_scan_finalization(
598
+ scan_dir,
599
+ expected_coverage_mode=expected_coverage_mode,
600
+ completion_binding=binding,
601
+ recover_unsealed=not preserve_reduction,
602
+ allow_missing_writeups=preserve_reduction,
603
+ draft_documents=documents,
604
+ finding_scope=db.requested_finding_scope(scan),
605
+ )
606
+ warnings.extend(prepared.warnings)
607
+ manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
608
+ manifest["scan"].pop("sealedAt", None)
609
+ manifest["scan"].pop("artifacts", None)
610
+ manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
611
+ checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
612
+ contents = _encoded(checkpoint)
613
+ name = f"{hashlib.sha256(contents).hexdigest()}.json"
614
+ checkpoint_path = f"checkpoints/{name}"
615
+ # This draft is recoverable parent evidence too. Restore its prior state
616
+ # on local publication failure or a later caller transaction rollback.
617
+ snapshots.update(_snapshot_published_outputs(scan_dir, (checkpoint_path,)))
618
+ write_scan_local_bytes(scan_dir, checkpoint_path, contents)
619
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
620
+ for filename, document in (
621
+ ("findings.json", findings),
622
+ *([("coverage.json", coverage)] if "coverageRef" in manifest["scan"] else []),
623
+ ("scan-manifest.json", manifest),
624
+ ):
625
+ write_scan_local_bytes(scan_dir, filename, _encoded(document))
626
+ connection.execute(
627
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
628
+ (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
629
+ )
630
+ except BaseException:
631
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
632
+ raise
633
+ return scan_dir, snapshots, created_paths
2517
634
 
2518
635
 
2519
636
  def preserve_scan_results_locked(
@@ -2522,25 +639,44 @@ def preserve_scan_results_locked(
2522
639
  scan_id: str,
2523
640
  *,
2524
641
  recovery_source_digests: dict[str, str] | None = None,
642
+ recovery_checkpoint_heads: dict[str, str | None] | None = None,
2525
643
  include_parent_with_recovery: bool = False,
644
+ recovery_snapshot_error: _CheckpointSnapshotError | None = None,
2526
645
  ) -> bool:
2527
646
  """Publish or verify retained terminal results through the workbench host."""
2528
647
  scan = db.require_scan(connection, scan_id)
2529
648
  if scan["status"] != "failed":
2530
649
  return False
2531
650
  frozen_source_digests: dict[str, str] | None = None
651
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
2532
652
  model_source: list[str] = []
2533
653
  saved_model_source: str | None = None
2534
654
  raw_frozen_sources = scan["retained_source_digests_json"]
655
+ frozen_source_times = None
2535
656
  if raw_frozen_sources is not None:
2536
- frozen_source_digests, saved_model_source = _retained_source_state(
2537
- json.loads(raw_frozen_sources)
2538
- )
657
+ retained = json.loads(raw_frozen_sources)
658
+ frozen_source_digests, saved_model_source = _retained_source_state(retained)
659
+ if isinstance(retained.get("sources"), dict):
660
+ frozen_source_times = retained.get("sourceTimes")
2539
661
  if saved_model_source is not None:
2540
662
  model_source.append(saved_model_source)
663
+ canonical_sources = frozen_source_digests
2541
664
  if recovery_source_digests is not None:
2542
665
  frozen_source_digests = recovery_source_digests
666
+ checkpoint_heads = recovery_checkpoint_heads
2543
667
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
668
+ if frozen_source_digests is not None:
669
+ _restore_frozen_model_capture(
670
+ scan_dir, scan_id, frozen_source_digests, frozen_source_times, saved_model_source
671
+ )
672
+ if frozen_source_times is not None and recovery_source_digests is None:
673
+ _freeze_source_times(
674
+ scan_dir,
675
+ scan_id,
676
+ frozen_source_digests,
677
+ frozen_source_times,
678
+ retained.get("parentContext"),
679
+ )
2544
680
  deep_run = connection.execute(
2545
681
  "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
2546
682
  ).fetchone()
@@ -2591,11 +727,15 @@ def preserve_scan_results_locked(
2591
727
  db.index_findings(connection, scan_id, findings, scan["completed_at"])
2592
728
  connection.execute(
2593
729
  "UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
730
+ "retained_checkpoint_heads_json = ?, "
2594
731
  "completion_warnings_json = ?, "
2595
732
  "updated_at = ? WHERE id = ? AND status = 'failed'",
2596
733
  (
2597
734
  digest,
2598
735
  _encode_retained_sources(retained_sources, model_source),
736
+ json.dumps(
737
+ manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True
738
+ ),
2599
739
  json.dumps(list(dict.fromkeys(warnings))),
2600
740
  timestamp,
2601
741
  scan_id,
@@ -2611,10 +751,7 @@ def preserve_scan_results_locked(
2611
751
  existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
2612
752
  existing = None
2613
753
  if scan["seal_manifest_digest"] is not None or (
2614
- isinstance(existing_scan, dict)
2615
- and (
2616
- existing_scan.get("sealedAt") is not None or existing_scan.get("artifacts") is not None
2617
- )
754
+ isinstance(existing_scan, dict) and manifest_is_sealed(existing_scan)
2618
755
  ):
2619
756
  db.require_recorded_manifest_digest(scan, scan_dir)
2620
757
  existing, existing_findings, _ = finalize_scan(
@@ -2625,11 +762,23 @@ def preserve_scan_results_locked(
2625
762
  db.verify_manifest_binding(scan, existing)
2626
763
  if existing_scan.get("status") == outcome:
2627
764
  existing_sources = existing_scan.get("preservedSources")
765
+ existing_heads = existing_scan.get("preservedCheckpointHeads")
2628
766
  if frozen_source_digests is None:
2629
767
  frozen_source_digests = _source_digests(
2630
768
  existing_sources, "Stopped scan source digests could not be frozen."
2631
769
  )
2632
- if existing_sources == frozen_source_digests:
770
+ checkpoint_heads = existing_heads
771
+ unchanged = existing_sources == frozen_source_digests and (
772
+ existing_heads == checkpoint_heads
773
+ or (
774
+ "preservedCheckpointHeads" not in existing_scan
775
+ and recovery_source_digests is None
776
+ and not pending_recovery
777
+ )
778
+ )
779
+ if unchanged and not any(
780
+ Path(path).name == "checkpoint-head.json" for path in frozen_source_digests
781
+ ):
2633
782
  if (
2634
783
  raw_frozen_sources is not None
2635
784
  and scan["seal_manifest_digest"] is not None
@@ -2639,43 +788,116 @@ def preserve_scan_results_locked(
2639
788
  return True
2640
789
  record_publication(existing, existing_findings)
2641
790
  return True
2642
- if recovery_source_digests is None:
791
+ if not unchanged and recovery_source_digests is None and not pending_recovery:
2643
792
  raise ContractError("Stopped scan sources changed after terminal publication.")
2644
793
  binding = {
2645
794
  **db.workbench_completion_binding(scan, scan["completed_at"], existing),
2646
795
  "status": outcome,
2647
796
  }
2648
- if recovery_source_digests is not None:
797
+ canonical_model_source = saved_model_source
798
+ canonical_model = existing_scan.get("threatModel") if isinstance(existing_scan, dict) else None
799
+ if existing is None and frozen_source_digests is not None:
800
+ # An unsealed live draft is a fallback only when its complete identity
801
+ # belongs to the frozen set. Otherwise recover from the captured sources.
802
+ canonical_model = None
803
+ try:
804
+ _, canonical_draft = _read_saved_parent_result(scan_dir, scan_id)
805
+ digest = _digest(canonical_draft)
806
+ path = f"checkpoints/{digest}.json"
807
+ if frozen_source_digests.get(path) == digest:
808
+ canonical_model_source = path
809
+ canonical_model = canonical_draft.get("threatModel")
810
+ except (ContractError, OSError, ValueError):
811
+ # This live fallback is optional.
812
+ pass
813
+ if recovery_source_digests is not None and not pending_recovery:
2649
814
  model_source.clear()
2650
- documents = merge_saved_results(
2651
- scan_dir,
2652
- scan_id,
2653
- binding,
2654
- connection.execute(
2655
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
2656
- (scan_id,),
2657
- ).fetchall(),
2658
- warnings,
2659
- stopped=True,
2660
- reason=(
2661
- f"Scan {outcome}; saved findings and pending review were preserved. "
2662
- f"{scan['failure_message'] or ''}"
2663
- ).strip(),
2664
- frozen_source_digests=frozen_source_digests,
2665
- frozen_model_source=model_source[0] if model_source else None,
2666
- selected_model_source=model_source,
2667
- project_worker_coverage=scan["mode"] == "deep",
2668
- allow_frozen_legacy_parent=(
2669
- include_parent_with_recovery
2670
- or (
2671
- recovery_source_digests is None
2672
- and frozen_source_digests == {}
2673
- and isinstance(existing_scan, dict)
2674
- and existing_scan.get("sealedAt") is not None
2675
- and "preservedSources" not in existing_scan
2676
- )
2677
- ),
815
+ # Older pending recoveries retained the parent union without recording the mode.
816
+ legacy_parent = (
817
+ include_parent_with_recovery
818
+ or (pending_recovery is not None and pending_recovery.get("legacyParent", True) is True)
819
+ or (
820
+ recovery_source_digests is None
821
+ and frozen_source_digests == {}
822
+ and isinstance(existing_scan, dict)
823
+ and existing_scan.get("sealedAt") is not None
824
+ and "preservedSources" not in existing_scan
825
+ )
2678
826
  )
827
+
828
+ def pending_selection(heads: dict[str, str | None]) -> dict[str, Any]:
829
+ return {
830
+ "heads": heads,
831
+ "recoveryBaseDigest": scan["seal_manifest_digest"],
832
+ "legacyParent": legacy_parent,
833
+ }
834
+
835
+ try:
836
+ documents = merge_saved_results(
837
+ scan_dir,
838
+ scan_id,
839
+ binding,
840
+ connection.execute(
841
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
842
+ (scan_id,),
843
+ ).fetchall(),
844
+ warnings,
845
+ stopped=True,
846
+ scan_mode=scan["mode"],
847
+ finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
848
+ reason=(
849
+ f"Scan {outcome}; saved findings and pending review were preserved. "
850
+ f"{scan['failure_message'] or ''}"
851
+ ).strip(),
852
+ frozen_source_digests=frozen_source_digests,
853
+ checkpoint_heads=checkpoint_heads,
854
+ frozen_model_source=model_source[0] if model_source else None,
855
+ selected_model_source=model_source,
856
+ project_worker_coverage=scan["mode"] == "deep",
857
+ canonical_model=canonical_model,
858
+ canonical_model_source=canonical_model_source,
859
+ canonical_sources=canonical_sources,
860
+ recovery_snapshot_error=recovery_snapshot_error,
861
+ allow_frozen_legacy_parent=legacy_parent,
862
+ parent_already_captured=pending_recovery is not None,
863
+ )
864
+ except _CheckpointSnapshotError as exc:
865
+ if frozen_source_digests is None or recovery_snapshot_error is not None:
866
+ # The stop has already committed. Keep the source set and times in
867
+ # the database even when auxiliary snapshot files cannot be written.
868
+ # Changed raw heads remain rejected on retry.
869
+ captured = exc.captured
870
+ retained_sources = {path: value[0] for path, value in captured.items()}
871
+ source_times = {path: value[1] for path, value in captured.items()}
872
+ retained_heads = exc.heads
873
+ if recovery_snapshot_error is not None:
874
+ _prepare_scan_finalization(
875
+ scan_dir,
876
+ expected_coverage_mode=db.expected_coverage_mode(scan),
877
+ completion_binding=binding,
878
+ recover_unsealed=True,
879
+ draft_documents=exc.documents,
880
+ finding_scope=db.requested_finding_scope(scan)
881
+ if scan["mode"] == "deep"
882
+ else None,
883
+ )
884
+ retained_heads = pending_selection(retained_heads)
885
+ with connection:
886
+ connection.execute(
887
+ "UPDATE scans SET retained_source_digests_json = ?, "
888
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
889
+ "WHERE id = ? AND retained_source_digests_json IS ?",
890
+ (
891
+ _encode_retained_sources(
892
+ retained_sources, exc.model_source, source_times, exc.parent_context
893
+ ),
894
+ json.dumps(retained_heads, sort_keys=True),
895
+ json.dumps(list(dict.fromkeys(warnings))),
896
+ scan_id,
897
+ raw_frozen_sources,
898
+ ),
899
+ )
900
+ raise
2679
901
  if documents is None:
2680
902
  unpublished_warnings = list(dict.fromkeys([*warnings, *publication_follow_up_warnings]))
2681
903
  if unpublished_warnings != stored_warnings:
@@ -2686,23 +908,46 @@ def preserve_scan_results_locked(
2686
908
  (json.dumps(unpublished_warnings), db.now(), scan_id),
2687
909
  )
2688
910
  return False
2689
- if frozen_source_digests is None or (
2690
- recovery_source_digests is None and saved_model_source is None and model_source
911
+ retained_state = None
912
+ if (
913
+ frozen_source_digests is None
914
+ or scan["retained_checkpoint_heads_json"] is None
915
+ or recovery_source_digests is not None
916
+ or documents[0]["scan"]["preservedSources"] != frozen_source_digests
917
+ or model_source != ([saved_model_source] if saved_model_source is not None else [])
2691
918
  ):
2692
919
  retained_sources = _source_digests(
2693
920
  documents[0].get("scan", {}).get("preservedSources"),
2694
921
  "Stopped scan source digests could not be frozen.",
2695
922
  )
2696
- with connection:
2697
- connection.execute(
2698
- "UPDATE scans SET retained_source_digests_json = ? "
2699
- "WHERE id = ? AND retained_source_digests_json IS ?",
2700
- (
2701
- _encode_retained_sources(retained_sources, model_source),
2702
- scan_id,
2703
- raw_frozen_sources,
2704
- ),
2705
- )
923
+ retained_heads = documents[0]["scan"]["preservedCheckpointHeads"]
924
+ if (
925
+ recovery_source_digests is not None
926
+ or pending_recovery
927
+ or (existing is not None and existing_scan.get("status") == outcome)
928
+ ):
929
+ retained_heads = pending_selection(retained_heads)
930
+ retained_state = (
931
+ _encode_retained_sources(retained_sources, model_source),
932
+ json.dumps(retained_heads, sort_keys=True),
933
+ json.dumps(list(dict.fromkeys(warnings))),
934
+ scan_id,
935
+ raw_frozen_sources,
936
+ )
937
+
938
+ def retain_sources() -> None:
939
+ if retained_state is not None:
940
+ with connection:
941
+ connection.execute(
942
+ "UPDATE scans SET retained_source_digests_json = ?, "
943
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
944
+ "WHERE id = ? AND retained_source_digests_json IS ?",
945
+ retained_state,
946
+ )
947
+
948
+ first_preservation = recovery_source_digests is None and not pending_recovery
949
+ if first_preservation:
950
+ retain_sources()
2706
951
  prepared = _prepare_scan_finalization(
2707
952
  scan_dir,
2708
953
  expected_coverage_mode=db.expected_coverage_mode(scan),
@@ -2711,6 +956,8 @@ def preserve_scan_results_locked(
2711
956
  draft_documents=documents,
2712
957
  finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
2713
958
  )
959
+ if not first_preservation:
960
+ retain_sources()
2714
961
  warnings.extend(prepared.warnings)
2715
962
  snapshots = _snapshot_published_outputs(scan_dir)
2716
963
  try:
@@ -2733,13 +980,24 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2733
980
  raise SystemExit("Only a stopped scan can recover terminal results.")
2734
981
  if scan["canceled_at"] is not None:
2735
982
  raise SystemExit("Canceled scans cannot recover terminal results.")
2736
- recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan)
983
+ recovery_snapshot_error = None
984
+ try:
985
+ recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests(
986
+ db, connection, scan
987
+ )
988
+ except _CheckpointSnapshotError as exc:
989
+ recovery_snapshot_error = exc
990
+ recovery_source_digests = {path: value[0] for path, value in exc.captured.items()}
991
+ include_parent = exc.include_parent
992
+ checkpoint_heads = exc.heads
2737
993
  if not preserve_scan_results_locked(
2738
994
  db,
2739
995
  connection,
2740
996
  scan_id,
2741
997
  recovery_source_digests=recovery_source_digests,
998
+ recovery_checkpoint_heads=checkpoint_heads,
2742
999
  include_parent_with_recovery=include_parent,
1000
+ recovery_snapshot_error=recovery_snapshot_error,
2743
1001
  ):
2744
1002
  raise SystemExit("No saved stopped-scan results were available to recover.")
2745
1003
  db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
@@ -2807,7 +1065,7 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2807
1065
  manifest_path = db.artifact_path(scan_dir, "scan-manifest.json", required=False)
2808
1066
  if manifest_path is not None:
2809
1067
  manifest = db.read_json_object(manifest_path).get("scan", {})
2810
- if manifest.get("sealedAt") is not None or manifest.get("artifacts") is not None:
1068
+ if manifest_is_sealed(manifest):
2811
1069
  raise SystemExit("The scan is sealed; its artifacts cannot be modified.")
2812
1070
  output = args.artifact_path
2813
1071
  key = output.lower()
@@ -2823,66 +1081,6 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2823
1081
  return {"scanId": scan_id, "path": str(scan_dir / output)}
2824
1082
 
2825
1083
 
2826
- def _require_current_deep_publication(
2827
- db: Any, connection: Any, scan: Any, draft: dict[str, Any]
2828
- ) -> None:
2829
- scan_id = scan["id"]
2830
- run = connection.execute(
2831
- "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
2832
- ).fetchone()
2833
- publication = draft.get("deepScanPublication")
2834
- if (
2835
- publication is None
2836
- and draft["manifest"]["scan"].get("complete") is False
2837
- and (run is None or run["status"] == "running")
2838
- ):
2839
- return
2840
- if run is None:
2841
- run = db.deep_scan.require_deep_scan_run(connection, scan_id)
2842
- # Finished discovery-only runs hand remaining publication back to the parent.
2843
- if (
2844
- publication is None
2845
- and run["status"] != "running"
2846
- and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
2847
- ):
2848
- return
2849
- db.deep_scan.require_current_coordinator(
2850
- run,
2851
- argparse.Namespace(
2852
- coordinator_generation=publication.get("coordinatorGeneration") if publication else None
2853
- ),
2854
- )
2855
- # Once finish selects the canonical parent, only completion may replay it.
2856
- if publication is None:
2857
- if run["status"] != "running" and run["manifest_path"] == str(
2858
- Path(scan["scan_dir"]) / "scan-manifest.json"
2859
- ):
2860
- raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
2861
- return
2862
-
2863
- # Match the durable reducer sequence used by coordinator recovery.
2864
- def reducer_order(worker: Any) -> tuple[int, str]:
2865
- match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
2866
- return (int(match[1]) if match else 0, worker["id"])
2867
-
2868
- reducer = max(
2869
- (
2870
- worker
2871
- for worker in connection.execute(
2872
- "SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
2873
- )
2874
- if worker["kind"] == "dedup"
2875
- and worker["status"] == "succeeded"
2876
- and worker["result_manifest_path"]
2877
- ),
2878
- key=reducer_order,
2879
- default=None,
2880
- )
2881
- selected_result = reducer["result_manifest_path"] if reducer is not None else None
2882
- if publication["resultPath"] != selected_result:
2883
- raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
2884
-
2885
-
2886
1084
  def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2887
1085
  scan_id = db.require_uuid(args.scan_id, "scan-id")
2888
1086
  with db.scan_completion_lock(scan_id):
@@ -2895,13 +1093,6 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2895
1093
  "The scan stopped; its saved checkpoint was retained without replacing sealed results."
2896
1094
  )
2897
1095
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2898
- if (
2899
- args.expected_draft_digest is not None
2900
- and args.expected_draft_digest != _scan_draft_digest(scan_dir)
2901
- ):
2902
- raise SystemExit(
2903
- "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
2904
- )
2905
1096
  try:
2906
1097
  relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
2907
1098
  except ValueError as exc:
@@ -2911,12 +1102,24 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2911
1102
  if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
2912
1103
  raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
2913
1104
  draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
2914
- manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
1105
+ accepted_input = copy.deepcopy(draft)
1106
+ accepted_checkpoint = None
1107
+ manifest, findings, coverage = (
1108
+ draft["manifest"],
1109
+ draft["findings"],
1110
+ draft.get("reviewState", draft.get("coverage")),
1111
+ )
1112
+ manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
1113
+ manifest["scan"].pop("coverageRef", None)
2915
1114
  binding = db.workbench_completion_binding(scan, db.now())
2916
1115
  # Save scan IDs without sealing the draft.
2917
1116
  _populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
2918
1117
  _populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
2919
1118
  _validate_completion_binding(manifest, findings, coverage, binding)
1119
+ manifest["scan"]["mode"] = coverage["mode"]
1120
+ manifest["scan"]["review"] = review_summary(coverage)
1121
+ projected = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
1122
+ admitted_input = None
2920
1123
  checkpoint = manifest["scan"]
2921
1124
  checkpoint_contents = None
2922
1125
  if args.checkpoint_path is not None:
@@ -2933,8 +1136,13 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2933
1136
  checkpoint, checkpoint_contents = _read_scan_local_json_bytes(
2934
1137
  scan_dir, checkpoint_relative, "Staged scan checkpoint"
2935
1138
  )
1139
+ accepted_checkpoint = checkpoint
2936
1140
  if checkpoint.get("scanId") != scan_id:
2937
1141
  raise SystemExit("Staged scan checkpoint belongs to another scan.")
1142
+ if scan["mode"] == "deep":
1143
+ checkpoint_contents = _admit_parent_checkpoint(
1144
+ scan_dir, scan_id, checkpoint, projected
1145
+ )
2938
1146
  checkpoint_only = (
2939
1147
  scan["mode"] == "deep"
2940
1148
  and draft.get("deepScanPublication") is None
@@ -2948,8 +1156,40 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2948
1156
  )
2949
1157
  if scan["mode"] == "deep" and not checkpoint_only:
2950
1158
  _require_current_deep_publication(db, connection, scan, draft)
1159
+ acceptance_input = {**accepted_input, "checkpoint": accepted_checkpoint}
1160
+ acceptance_relative = Path(relative).with_suffix(".accepted.json").as_posix()
1161
+ acknowledge = (
1162
+ scan["mode"] == "deep"
1163
+ and draft.get("deepScanPublication") is not None
1164
+ and db.deep_scan.require_deep_scan_run(connection, scan_id)["workflow_version"]
1165
+ in {"deep-scan-mcp/v1", "deep-security-scan/v1"}
1166
+ )
1167
+ if acknowledge and (scan_dir / acceptance_relative).exists():
1168
+ accepted = _read_scan_local_json(scan_dir, acceptance_relative, "Accepted scan draft")
1169
+ if (
1170
+ accepted.get("status") == "draft_written"
1171
+ and accepted.get("input") == acceptance_input
1172
+ ):
1173
+ if accepted.get("draftDigest") != _scan_draft_digest(scan_dir):
1174
+ raise SystemExit(
1175
+ "scan_draft_conflict: canonical scan results changed after publication."
1176
+ )
1177
+ return {
1178
+ "scanId": scan_id,
1179
+ "status": "draft_written",
1180
+ **({"warnings": accepted["warnings"]} if accepted.get("warnings") else {}),
1181
+ }
1182
+ if (
1183
+ args.expected_draft_digest is not None
1184
+ and args.expected_draft_digest != _scan_draft_digest(scan_dir)
1185
+ ):
1186
+ raise SystemExit(
1187
+ "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
1188
+ )
2951
1189
  if checkpoint_contents is not None:
2952
1190
  checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
1191
+ if scan["mode"] == "deep":
1192
+ admitted_input = f"{checkpoint_digest}.json"
2953
1193
  write_scan_local_bytes(
2954
1194
  scan_dir,
2955
1195
  f"checkpoints/{checkpoint_digest}.json",
@@ -2957,27 +1197,27 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2957
1197
  )
2958
1198
  if checkpoint_only:
2959
1199
  return {"scanId": scan_id, "status": "draft_written"}
2960
- checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
2961
- checkpoint_contents = _encoded(checkpoint)
2962
- checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
1200
+ projected_contents = _encoded(projected)
1201
+ checkpoint_name = f"{hashlib.sha256(projected_contents).hexdigest()}.json"
2963
1202
  checkpoint_relative = f"checkpoints/{checkpoint_name}"
2964
1203
  if not (scan_dir / checkpoint_relative).exists():
2965
- write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
1204
+ write_scan_local_bytes(scan_dir, checkpoint_relative, projected_contents)
2966
1205
  if checkpoint_only:
2967
1206
  return {"scanId": scan_id, "status": "draft_written"}
2968
- write_scan_local_bytes(
2969
- scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
2970
- )
2971
- for filename, document in (
2972
- ("findings.json", findings),
2973
- ("coverage.json", coverage),
2974
- ("scan-manifest.json", manifest),
2975
- ):
2976
- write_scan_local_bytes(
2977
- scan_dir,
2978
- filename,
2979
- (json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
1207
+ head = {"checkpoint": checkpoint_name}
1208
+ if admitted_input is not None:
1209
+ head["inputCheckpoint"] = admitted_input
1210
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded(head))
1211
+ documents = {
1212
+ filename: (json.dumps(document, allow_nan=False, indent=2) + "\n").encode()
1213
+ for filename, document in (
1214
+ ("findings.json", findings),
1215
+ ("scan-manifest.json", manifest),
2980
1216
  )
1217
+ }
1218
+ for filename, contents in documents.items():
1219
+ write_scan_local_bytes(scan_dir, filename, contents)
1220
+ _remove_scan_local_file_if_exists(scan_dir, "coverage.json")
2981
1221
  if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
2982
1222
  # Select the accepted final publication before releasing the write lock.
2983
1223
  with connection:
@@ -2987,6 +1227,19 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2987
1227
  (str(scan_dir / "scan-manifest.json"), scan_id),
2988
1228
  )
2989
1229
  model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
1230
+ if acknowledge:
1231
+ # Optional recovery evidence is written only after the publication succeeds.
1232
+ try:
1233
+ acceptance = {
1234
+ "status": "draft_written",
1235
+ "input": acceptance_input,
1236
+ "draftDigest": _scan_draft_contents_digest(documents),
1237
+ **({"warnings": [model_warning]} if model_warning else {}),
1238
+ }
1239
+ write_scan_local_bytes(scan_dir, acceptance_relative, _encoded(acceptance))
1240
+ except (OSError, ScanLocalIOError):
1241
+ # A missing receipt prevents recovery but does not undo accepted results.
1242
+ pass
2990
1243
  # Accepted Standard drafts are evidence of review or report assembly,
2991
1244
  # even when the parent omitted its explicit progress call.
2992
1245
  model_only_checkpoint = (
@@ -3025,20 +1278,31 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
3025
1278
 
3026
1279
 
3027
1280
  def _scan_draft_digest(scan_dir: Path) -> str:
3028
- digest = hashlib.sha256()
1281
+ documents: dict[str, bytes | None] = {}
3029
1282
  for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
3030
- digest.update(filename.encode())
3031
- digest.update(b"\0")
3032
1283
  try:
3033
1284
  (scan_dir / filename).lstat()
3034
1285
  except FileNotFoundError:
3035
- digest.update(b"missing\0")
1286
+ documents[filename] = None
3036
1287
  continue
3037
- digest.update(b"present\0")
3038
1288
  descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
3039
1289
  with os.fdopen(descriptor, "rb") as handle:
3040
- digest.update(handle.read())
1290
+ documents[filename] = handle.read()
1291
+ return _scan_draft_contents_digest(documents)
1292
+
1293
+
1294
+ def _scan_draft_contents_digest(documents: dict[str, bytes | None]) -> str:
1295
+ digest = hashlib.sha256()
1296
+ for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
1297
+ digest.update(filename.encode())
3041
1298
  digest.update(b"\0")
1299
+ contents = documents.get(filename)
1300
+ if contents is None:
1301
+ digest.update(b"missing\0")
1302
+ else:
1303
+ digest.update(b"present\0")
1304
+ digest.update(contents)
1305
+ digest.update(b"\0")
3042
1306
  return digest.hexdigest()
3043
1307
 
3044
1308
 
@@ -3064,7 +1328,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
3064
1328
  args.claim_token,
3065
1329
  error_message="Scan failure is owned by another continuation.",
3066
1330
  )
3067
- message = db.optional_text(args.message, maximum=2400)
1331
+ message = db.optional_text(args.message)
3068
1332
  updated = connection.execute(
3069
1333
  """
3070
1334
  UPDATE scans
@@ -3148,3 +1412,158 @@ def preserve_stopped_results_after_transition(db: Any, connection: Any, scan_id:
3148
1412
 
3149
1413
  if __name__ == "__main__":
3150
1414
  argparse.ArgumentParser(description=__doc__).parse_args()
1415
+
1416
+
1417
+ def budget_exhausted_draft(
1418
+ db: Any,
1419
+ scan: sqlite3.Row,
1420
+ scan_dir: Path,
1421
+ candidates: list[dict[str, Any]],
1422
+ warning: str,
1423
+ ) -> None:
1424
+ documents: dict[str, dict[str, Any]] = {}
1425
+ for name in ("scan-manifest.json", "findings.json"):
1426
+ path = db.artifact_path(scan_dir, name, required=False)
1427
+ if path is not None:
1428
+ documents[name] = db.read_json_object(path)
1429
+ if documents and len(documents) != 2:
1430
+ raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.")
1431
+
1432
+ if documents:
1433
+ manifest = documents["scan-manifest.json"]
1434
+ findings = documents["findings.json"]
1435
+ coverage = (
1436
+ db.read_json_object(db.artifact_path(scan_dir, "coverage.json", required=True))
1437
+ if uses_legacy_review(manifest["scan"])
1438
+ else review_state(manifest["scan"])
1439
+ )
1440
+ if not isinstance(manifest.get("scan"), dict) or not isinstance(
1441
+ findings.get("findings"), list
1442
+ ):
1443
+ raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.")
1444
+ for key in ("surfaces", "explicitExclusions", "deferred"):
1445
+ if not isinstance(coverage.get(key), list):
1446
+ raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.")
1447
+ if manifest_is_sealed(manifest["scan"]):
1448
+ raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.")
1449
+ else:
1450
+ contract = db.scan_contract(scan)
1451
+ target_contract = contract["target"]
1452
+ target: dict[str, Any] = {
1453
+ "kind": target_contract["allowedKinds"][0],
1454
+ "targetId": target_contract["targetId"],
1455
+ "displayName": target_contract["displayName"],
1456
+ }
1457
+ if scan["target_revision"] != "unversioned":
1458
+ target["revision"] = scan["target_revision"]
1459
+ if "requiredSnapshotDigest" in target_contract:
1460
+ target["snapshotDigest"] = target_contract["requiredSnapshotDigest"]
1461
+ manifest = {
1462
+ "scan": {
1463
+ "target": target,
1464
+ "scope": {"limitations": [warning]},
1465
+ }
1466
+ }
1467
+ findings = {"findings": []}
1468
+ coverage = {
1469
+ "inventoryStrategy": (
1470
+ "scoped_path" if db.expected_coverage_mode(scan) == "scoped_path" else "repository"
1471
+ ),
1472
+ "surfaces": [],
1473
+ "explicitExclusions": [],
1474
+ "deferred": [],
1475
+ }
1476
+
1477
+ findings_by_candidate = {
1478
+ candidate_id
1479
+ for finding in findings["findings"]
1480
+ if isinstance(finding, dict)
1481
+ and isinstance(candidate_id := finding_candidate_id(finding), str)
1482
+ }
1483
+ existing_deferred = {
1484
+ item.get("candidateId", item.get("id"))
1485
+ for item in coverage["deferred"]
1486
+ if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str)
1487
+ }
1488
+ existing_surfaces = {
1489
+ item.get("id")
1490
+ for item in coverage["surfaces"]
1491
+ if isinstance(item, dict) and isinstance(item.get("id"), str)
1492
+ }
1493
+ for candidate in candidates:
1494
+ candidate_id = candidate["candidate_id"]
1495
+ if candidate_id in findings_by_candidate or candidate_id in existing_deferred:
1496
+ continue
1497
+ paths = list(dict.fromkeys(location["path"] for location in candidate["locations"]))
1498
+ surface_id = f"candidate-{candidate_id}"
1499
+ validation = candidate.get("validation")
1500
+ validation = validation.get("disposition") if isinstance(validation, dict) else None
1501
+ attack = candidate.get("attack_path")
1502
+ attack = attack.get("decision") if isinstance(attack, dict) else None
1503
+ disposition = (
1504
+ "needs_follow_up"
1505
+ if validation == "deferred" or attack == "deferred"
1506
+ else "not_applicable"
1507
+ if validation == "not_applicable"
1508
+ else "rejected"
1509
+ if validation == "suppressed" or attack == "ignore"
1510
+ else "needs_follow_up"
1511
+ )
1512
+ if surface_id not in existing_surfaces:
1513
+ coverage["surfaces"].append(
1514
+ {
1515
+ "id": surface_id,
1516
+ "label": candidate["summary"],
1517
+ "disposition": disposition,
1518
+ "notes": candidate["evidence"],
1519
+ "receiptRefs": [],
1520
+ }
1521
+ )
1522
+ existing_surfaces.add(surface_id)
1523
+ if disposition != "needs_follow_up":
1524
+ continue
1525
+ coverage["deferred"].append(
1526
+ {
1527
+ "id": candidate_id,
1528
+ "candidateId": candidate_id,
1529
+ "reason": (
1530
+ "Validation was deferred because the scan reached its cost limit: "
1531
+ f"{candidate['summary']}. Evidence: {candidate['evidence']}"
1532
+ ),
1533
+ "paths": paths,
1534
+ "surfaceIds": [surface_id],
1535
+ }
1536
+ )
1537
+ if not any(
1538
+ isinstance(item, dict)
1539
+ and isinstance(reason := item.get("reason"), str)
1540
+ and (
1541
+ reason == "Validation was deferred because the scan reached its cost limit."
1542
+ or reason.startswith(
1543
+ "Validation was deferred because the scan reached its cost limit: "
1544
+ )
1545
+ )
1546
+ for item in coverage["deferred"]
1547
+ ):
1548
+ coverage["deferred"].append(
1549
+ {
1550
+ "id": "scan-cost-limit",
1551
+ "reason": "Validation was deferred because the scan reached its cost limit.",
1552
+ }
1553
+ )
1554
+ coverage.pop("completeness", None)
1555
+ manifest["scan"].pop("coverageRef", None)
1556
+ manifest["scan"]["mode"] = db.expected_coverage_mode(scan)
1557
+ manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
1558
+ for name, payload in (
1559
+ ("findings.json", findings),
1560
+ ("scan-manifest.json", manifest),
1561
+ ):
1562
+ try:
1563
+ write_scan_local_bytes(
1564
+ scan_dir,
1565
+ name,
1566
+ (json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(),
1567
+ )
1568
+ except (ContractError, OSError, TypeError, ValueError) as exc:
1569
+ raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc