@openai/codex-security 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +68 -37
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  5. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
  6. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  7. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  8. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  9. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  10. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  11. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  12. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  13. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  14. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  15. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  16. package/_bundled_plugin/references/access-advisory.md +11 -0
  17. package/_bundled_plugin/references/core-scan.md +30 -77
  18. package/_bundled_plugin/references/final-report.md +45 -189
  19. package/_bundled_plugin/references/scan-contract.md +2 -0
  20. package/_bundled_plugin/references/scan-prologue.md +1 -1
  21. package/_bundled_plugin/references/scan-runtime.md +1 -5
  22. package/_bundled_plugin/references/threat-model.md +33 -41
  23. package/_bundled_plugin/schemas/coverage.schema.json +12 -1
  24. package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
  25. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
  26. package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
  27. package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
  28. package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
  29. package/_bundled_plugin/scripts/report_projection.py +44 -15
  30. package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
  31. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
  32. package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
  33. package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
  34. package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
  35. package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
  36. package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
  37. package/_bundled_plugin/scripts/workbench_cli.py +3 -0
  38. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
  39. package/_bundled_plugin/scripts/workbench_db.py +51 -22
  40. package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
  41. package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
  42. package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
  43. package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
  44. package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
  45. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  46. package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
  47. package/_bundled_plugin/scripts/workbench_target.py +142 -17
  48. package/_bundled_plugin/scripts/workbench_validation.py +9 -2
  49. package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
  50. package/_bundled_plugin/shared/workbench-migrations.json +28 -0
  51. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
  52. package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
  53. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
  54. package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
  55. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
  56. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
  57. package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
  58. package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
  59. package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
  60. package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
  61. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
  62. package/_bundled_plugin/skills/publish-findings/SKILL.md +28 -24
  63. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
  64. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
  65. package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
  66. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
  67. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
  68. package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
  69. package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
  70. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
  71. package/_bundled_plugin/skills/validation/SKILL.md +42 -104
  72. package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
  73. package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
  74. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
  75. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
  76. package/dist/api.d.ts +5 -3
  77. package/dist/api.d.ts.map +1 -1
  78. package/dist/api.js +341 -37
  79. package/dist/api.js.map +1 -1
  80. package/dist/auth.d.ts.map +1 -1
  81. package/dist/auth.js +4 -1
  82. package/dist/auth.js.map +1 -1
  83. package/dist/cli.d.ts +2 -2
  84. package/dist/cli.d.ts.map +1 -1
  85. package/dist/cli.js +674 -419
  86. package/dist/cli.js.map +1 -1
  87. package/dist/component-scan.d.ts +8 -2
  88. package/dist/component-scan.d.ts.map +1 -1
  89. package/dist/component-scan.js +2 -0
  90. package/dist/component-scan.js.map +1 -1
  91. package/dist/contract.d.ts +2 -0
  92. package/dist/contract.d.ts.map +1 -1
  93. package/dist/contract.js +67 -33
  94. package/dist/contract.js.map +1 -1
  95. package/dist/cost.d.ts +18 -0
  96. package/dist/cost.d.ts.map +1 -1
  97. package/dist/cost.js +107 -12
  98. package/dist/cost.js.map +1 -1
  99. package/dist/coverage-presentation.d.ts +6 -0
  100. package/dist/coverage-presentation.d.ts.map +1 -0
  101. package/dist/coverage-presentation.js +34 -0
  102. package/dist/coverage-presentation.js.map +1 -0
  103. package/dist/custom-validation-prompt.js +3 -3
  104. package/dist/custom-validation-prompt.js.map +1 -1
  105. package/dist/custom-validation.d.ts.map +1 -1
  106. package/dist/custom-validation.js +30 -4
  107. package/dist/custom-validation.js.map +1 -1
  108. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  109. package/dist/deduplication/checkpointed-review.js +3 -1
  110. package/dist/deduplication/checkpointed-review.js.map +1 -1
  111. package/dist/deduplication/scan.d.ts.map +1 -1
  112. package/dist/deduplication/scan.js +7 -5
  113. package/dist/deduplication/scan.js.map +1 -1
  114. package/dist/endor-findings.d.ts +4 -3
  115. package/dist/endor-findings.d.ts.map +1 -1
  116. package/dist/endor-findings.js +16 -11
  117. package/dist/endor-findings.js.map +1 -1
  118. package/dist/external-findings-import.d.ts.map +1 -1
  119. package/dist/external-findings-import.js +13 -5
  120. package/dist/external-findings-import.js.map +1 -1
  121. package/dist/external-findings-local.d.ts.map +1 -1
  122. package/dist/external-findings-local.js +8 -1
  123. package/dist/external-findings-local.js.map +1 -1
  124. package/dist/external-findings-publish.d.ts +1 -1
  125. package/dist/external-findings-publish.d.ts.map +1 -1
  126. package/dist/external-findings-publish.js +11 -3
  127. package/dist/external-findings-publish.js.map +1 -1
  128. package/dist/external-findings-read.d.ts +11 -1
  129. package/dist/external-findings-read.d.ts.map +1 -1
  130. package/dist/external-findings-read.js +15 -6
  131. package/dist/external-findings-read.js.map +1 -1
  132. package/dist/external-import-contract.d.ts +3 -1
  133. package/dist/external-import-contract.d.ts.map +1 -1
  134. package/dist/external-import-contract.js +33 -25
  135. package/dist/external-import-contract.js.map +1 -1
  136. package/dist/external-imports-feature.d.ts +6 -0
  137. package/dist/external-imports-feature.d.ts.map +1 -0
  138. package/dist/external-imports-feature.js +51 -0
  139. package/dist/external-imports-feature.js.map +1 -0
  140. package/dist/finding-workflow.d.ts +22 -5
  141. package/dist/finding-workflow.d.ts.map +1 -1
  142. package/dist/finding-workflow.js +21 -9
  143. package/dist/finding-workflow.js.map +1 -1
  144. package/dist/github-advisories.d.ts.map +1 -1
  145. package/dist/github-advisories.js +3 -1
  146. package/dist/github-advisories.js.map +1 -1
  147. package/dist/github-findings.d.ts +4 -3
  148. package/dist/github-findings.d.ts.map +1 -1
  149. package/dist/github-findings.js +16 -14
  150. package/dist/github-findings.js.map +1 -1
  151. package/dist/hosted-remediation.d.ts.map +1 -1
  152. package/dist/hosted-remediation.js +19 -2
  153. package/dist/hosted-remediation.js.map +1 -1
  154. package/dist/hosted-scan.d.ts +1 -1
  155. package/dist/hosted-scan.d.ts.map +1 -1
  156. package/dist/hosted-scan.js +0 -5
  157. package/dist/hosted-scan.js.map +1 -1
  158. package/dist/import-scan.js +1 -1
  159. package/dist/import-scan.js.map +1 -1
  160. package/dist/index.d.ts +3 -3
  161. package/dist/index.d.ts.map +1 -1
  162. package/dist/index.js +1 -1
  163. package/dist/index.js.map +1 -1
  164. package/dist/knowledge-base.d.ts +1 -0
  165. package/dist/knowledge-base.d.ts.map +1 -1
  166. package/dist/knowledge-base.js +17 -8
  167. package/dist/knowledge-base.js.map +1 -1
  168. package/dist/models.d.ts +1 -0
  169. package/dist/models.d.ts.map +1 -1
  170. package/dist/models.js.map +1 -1
  171. package/dist/owner-evidence.d.ts +2 -0
  172. package/dist/owner-evidence.d.ts.map +1 -1
  173. package/dist/owner-evidence.js +2 -1
  174. package/dist/owner-evidence.js.map +1 -1
  175. package/dist/publish.d.ts +2 -0
  176. package/dist/publish.d.ts.map +1 -1
  177. package/dist/publish.js +1 -1
  178. package/dist/publish.js.map +1 -1
  179. package/dist/remediation.d.ts +1 -25
  180. package/dist/remediation.d.ts.map +1 -1
  181. package/dist/remediation.js +3 -24
  182. package/dist/remediation.js.map +1 -1
  183. package/dist/runtime.d.ts.map +1 -1
  184. package/dist/runtime.js +7 -7
  185. package/dist/runtime.js.map +1 -1
  186. package/dist/scan-dashboard.d.ts +5 -3
  187. package/dist/scan-dashboard.d.ts.map +1 -1
  188. package/dist/scan-dashboard.js +79 -26
  189. package/dist/scan-dashboard.js.map +1 -1
  190. package/dist/scan-history-renderer.d.ts.map +1 -1
  191. package/dist/scan-history-renderer.js +8 -2
  192. package/dist/scan-history-renderer.js.map +1 -1
  193. package/dist/scan-owner-suggestions.d.ts +1 -1
  194. package/dist/scan-owner-suggestions.d.ts.map +1 -1
  195. package/dist/scan-owner-suggestions.js +4 -4
  196. package/dist/scan-owner-suggestions.js.map +1 -1
  197. package/dist/scan-usage-display.d.ts +8 -0
  198. package/dist/scan-usage-display.d.ts.map +1 -0
  199. package/dist/scan-usage-display.js +46 -0
  200. package/dist/scan-usage-display.js.map +1 -0
  201. package/dist/scanner-import.d.ts +1 -1
  202. package/dist/scanner-import.d.ts.map +1 -1
  203. package/dist/scanner-import.js +7 -3
  204. package/dist/scanner-import.js.map +1 -1
  205. package/dist/security-policy.d.ts.map +1 -1
  206. package/dist/security-policy.js +1 -20
  207. package/dist/security-policy.js.map +1 -1
  208. package/dist/semgrep-findings.d.ts +4 -3
  209. package/dist/semgrep-findings.d.ts.map +1 -1
  210. package/dist/semgrep-findings.js +9 -9
  211. package/dist/semgrep-findings.js.map +1 -1
  212. package/dist/skill-authentication.d.ts +18 -0
  213. package/dist/skill-authentication.d.ts.map +1 -0
  214. package/dist/skill-authentication.js +82 -0
  215. package/dist/skill-authentication.js.map +1 -0
  216. package/dist/snyk-findings.d.ts +5 -4
  217. package/dist/snyk-findings.d.ts.map +1 -1
  218. package/dist/snyk-findings.js +18 -17
  219. package/dist/snyk-findings.js.map +1 -1
  220. package/dist/suggest-owners.d.ts +4 -1
  221. package/dist/suggest-owners.d.ts.map +1 -1
  222. package/dist/suggest-owners.js +53 -21
  223. package/dist/suggest-owners.js.map +1 -1
  224. package/dist/targets.d.ts.map +1 -1
  225. package/dist/targets.js +4 -1
  226. package/dist/targets.js.map +1 -1
  227. package/dist/wiz-csv.d.ts +2 -1
  228. package/dist/wiz-csv.d.ts.map +1 -1
  229. package/dist/wiz-csv.js +3 -3
  230. package/dist/wiz-csv.js.map +1 -1
  231. package/dist/wiz-findings.d.ts +2 -1
  232. package/dist/wiz-findings.d.ts.map +1 -1
  233. package/dist/wiz-findings.js +27 -23
  234. package/dist/wiz-findings.js.map +1 -1
  235. package/docs/classify-severity-records.md +3 -4
  236. package/docs/cli.md +155 -38
  237. package/docs/findings-service.md +119 -118
  238. package/docs/hosted-remediation.md +19 -17
  239. package/docs/hosted-scans.md +37 -11
  240. package/package.json +8 -8
  241. package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
  242. package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
  243. package/_bundled_plugin/scripts/rank_preview.py +0 -129
@@ -10,9 +10,8 @@ import json
10
10
  import os
11
11
  import re
12
12
  import sqlite3
13
- import stat
14
13
  import sys
15
- from collections.abc import Callable, Iterator
14
+ from collections.abc import Iterator
16
15
  from contextlib import contextmanager
17
16
  from pathlib import Path
18
17
  from typing import Any
@@ -20,47 +19,127 @@ from typing import Any
20
19
  sys.path.insert(0, str(Path(__file__).resolve().parent))
21
20
  from finalize_scan_contract import (
22
21
  ContractError,
23
- _finding_strength,
22
+ ScanLocalIOError,
24
23
  _populate_unsealed_artifact_envelope,
25
24
  _populate_unsealed_manifest_envelope,
26
25
  _prepare_scan_finalization,
27
- _read_json,
28
26
  _read_saved_threat_model,
29
27
  _read_scan_local_json,
30
28
  _read_scan_local_json_bytes,
31
- _read_scan_local_json_with_metadata,
29
+ _recover_unsealed_coverage,
32
30
  _recover_unsealed_findings,
33
31
  _remove_scan_local_file_if_exists,
34
- _require_portable_relative_path,
35
- _require_str,
36
32
  _validate_completion_binding,
37
- _validate_resolved_deferred,
38
- _validate_schema_node,
39
33
  _write_prepared_scan_finalization,
40
34
  finalize_scan,
41
- finding_candidate_id,
35
+ manifest_is_sealed,
42
36
  open_scan_local_file_descriptor,
43
37
  write_scan_local_bytes,
44
38
  write_threat_model_projection_if_possible,
45
39
  )
40
+ from finalize_scan_contract import finding_candidate_id as finding_candidate_id
41
+ from workbench.retained_coverage import (
42
+ _append_previous_finding as _append_previous_finding,
43
+ )
44
+ from workbench.retained_coverage import (
45
+ _coverage_rows as _coverage_rows,
46
+ )
47
+ from workbench.retained_coverage import (
48
+ _ensure_finding_identities as _ensure_finding_identities,
49
+ )
50
+ from workbench.retained_coverage import (
51
+ _ensure_finding_identity as _ensure_finding_identity,
52
+ )
53
+ from workbench.retained_coverage import (
54
+ _finding_content as _finding_content,
55
+ )
56
+ from workbench.retained_coverage import (
57
+ _finding_identity as _finding_identity,
58
+ )
59
+ from workbench.retained_coverage import (
60
+ _finding_key as _finding_key,
61
+ )
62
+ from workbench.retained_coverage import (
63
+ _has_candidate_metadata as _has_candidate_metadata,
64
+ )
65
+ from workbench.retained_coverage import (
66
+ _identity_candidate as _identity_candidate,
67
+ )
68
+ from workbench.retained_coverage import (
69
+ _inherit_coverage_sources as _inherit_coverage_sources,
70
+ )
71
+ from workbench.retained_coverage import (
72
+ _retained_findings as _retained_findings,
73
+ )
74
+ from workbench.retained_coverage import (
75
+ _semantic_identity_key as _semantic_identity_key,
76
+ )
77
+ from workbench.retained_coverage import (
78
+ _worker_candidate_key as _worker_candidate_key,
79
+ )
80
+ from workbench.retained_coverage import merge_saved_results as merge_saved_results
81
+ from workbench.saved_coverage import (
82
+ _deferred_rows as _deferred_rows,
83
+ )
84
+ from workbench.saved_coverage import (
85
+ _merge_tied_parent_observations as _merge_tied_parent_observations,
86
+ )
87
+ from workbench.saved_coverage import (
88
+ _resolved_deferred_rows as _resolved_deferred_rows,
89
+ )
46
90
  from workbench.saved_coverage import (
47
- _prepare_worker_coverage_recovery,
48
- _project_worker_coverage_row,
49
- _saved_coverage_id,
91
+ _saved_coverage_id as _saved_coverage_id,
92
+ )
93
+ from workbench.saved_outputs import _restore_published_outputs, _snapshot_published_outputs
94
+ from workbench.saved_sources import (
95
+ _admit_parent_checkpoint,
96
+ _capture_saved_source,
97
+ _checkpoint_paths,
98
+ _CheckpointSnapshotError,
99
+ _encode_retained_sources,
100
+ _encoded,
101
+ _freeze_source_times,
102
+ _frozen_checkpoint_heads,
103
+ _frozen_source_times,
104
+ _is_source_order_snapshot,
105
+ _legacy_checkpoint_observed_at,
106
+ _parent_scan_draft,
107
+ _read_saved_parent_result,
108
+ _read_saved_result,
109
+ _require_current_deep_publication,
110
+ _restore_checkpoint_head,
111
+ _restore_frozen_model_capture,
112
+ _retained_checkpoint_state,
113
+ _retained_source_state,
114
+ _saved_result_paths,
115
+ _source_digests,
116
+ )
117
+ from workbench.saved_sources import (
118
+ _capture_parent_observation as _capture_parent_observation,
119
+ )
120
+ from workbench.saved_sources import (
121
+ _checkpoint_head_directory as _checkpoint_head_directory,
122
+ )
123
+ from workbench.saved_sources import (
124
+ _digest as _digest,
125
+ )
126
+ from workbench.saved_sources import (
127
+ _latest_successful_reducer as _latest_successful_reducer,
128
+ )
129
+ from workbench.saved_sources import (
130
+ _parent_publication_ancestors as _parent_publication_ancestors,
131
+ )
132
+ from workbench.saved_sources import (
133
+ _worker_outputs as _worker_outputs,
50
134
  )
51
135
  from workbench_constants import PHASES
52
- from workbench_target import committed_diff_snapshot_digest
53
- from workbench_validation import path_within_scope
54
-
55
- _PUBLISHED_OUTPUTS = (
56
- "findings.json",
57
- "coverage.json",
58
- "scan-manifest.json",
59
- "report.md",
60
- "threatmodel.md",
61
- "report.html",
62
- "exports/results.sarif",
136
+ from workbench_coverage_receipts import (
137
+ _deferred_candidate_id as _deferred_candidate_id,
63
138
  )
139
+ from workbench_coverage_receipts import (
140
+ _generic_surface_updates as _generic_surface_updates,
141
+ )
142
+
64
143
  _PUBLICATION_FOLLOW_UP_WARNING = (
65
144
  "Saved scan evidence remains on disk; result publication needs follow-up:"
66
145
  )
@@ -124,284 +203,6 @@ def refresh_completed_scan(
124
203
  return db.scan_context(connection, scan["id"])
125
204
 
126
205
 
127
- def _encoded(value: Any) -> bytes:
128
- return json.dumps(
129
- value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
130
- ).encode()
131
-
132
-
133
- def _digest(value: Any) -> str:
134
- return hashlib.sha256(_encoded(value)).hexdigest()
135
-
136
-
137
- def _children(scan_dir: Path, relative: str) -> list[str]:
138
- cursor = scan_dir
139
- for part in Path(relative).parts:
140
- if part in {"..", "."}:
141
- return []
142
- cursor = cursor / part
143
- try:
144
- if not stat.S_ISDIR(cursor.lstat().st_mode):
145
- return []
146
- except FileNotFoundError:
147
- return []
148
- return sorted(child.name for child in cursor.iterdir())
149
-
150
-
151
- def _latest_successful_reducer(workers: list[Any]) -> Any | None:
152
- return max(
153
- (
154
- worker
155
- for worker in workers
156
- if worker["kind"] == "dedup"
157
- and worker["status"] == "succeeded"
158
- and worker["result_manifest_path"]
159
- ),
160
- key=lambda worker: (worker["completed_at"] or "", worker["id"]),
161
- default=None,
162
- )
163
-
164
-
165
- def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
166
- return [
167
- f"{directory}/{name}"
168
- for name in _children(scan_dir, directory)
169
- if re.fullmatch(r"[0-9a-f]{64}\.json", name)
170
- ]
171
-
172
-
173
- def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
174
- output = Path(worker["artifact_dir"]).relative_to(scan_dir)
175
- attempts = (output.parent if output.name == "output" else output) / "attempts"
176
- archived = [
177
- ((attempts / name).as_posix(), int(name.split("-")[1]))
178
- for name in _children(scan_dir, attempts.as_posix())
179
- if re.fullmatch(r"attempt-\d+", name)
180
- ]
181
- attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
182
- if worker["kind"] == "discovery" and not attempt:
183
- attempt = max((attempt for _, attempt in archived), default=0) + 1
184
- return [(output.as_posix(), attempt), *archived]
185
-
186
-
187
- def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
188
- latest_reducer = _latest_successful_reducer(workers)
189
- yield "checkpoint-head.json", None
190
- for directory in ("checkpoint-heads", "checkpoints"):
191
- yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
192
- for worker in workers:
193
- if worker["kind"] not in {"dedup", "discovery"}:
194
- continue
195
- try:
196
- outputs = _worker_outputs(scan_dir, worker)
197
- except (TypeError, ValueError):
198
- continue
199
- for directory, _ in outputs:
200
- if worker["kind"] == "discovery":
201
- yield f"{directory}/checkpoint-head.json", worker["kind"]
202
- yield from (
203
- (path, worker["kind"])
204
- for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
205
- )
206
- checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
207
- if worker["kind"] == "discovery" or checkpoint_paths:
208
- yield f"{directory}/result.json", worker["kind"]
209
- yield from ((path, worker["kind"]) for path in checkpoint_paths)
210
- if worker["result_manifest_path"] and (
211
- worker["kind"] == "discovery"
212
- or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
213
- ):
214
- try:
215
- yield (
216
- Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
217
- worker["kind"],
218
- )
219
- except ValueError:
220
- continue
221
-
222
-
223
- def _checkpoint_head_directory(relative: str) -> Path | None:
224
- path = Path(relative)
225
- if path.name == "checkpoint-head.json":
226
- return path.parent
227
- if path.parent.name == "checkpoint-heads":
228
- return path.parent.parent
229
- return None
230
-
231
-
232
- def _capture_saved_source(
233
- scan_dir: Path,
234
- relative: str,
235
- scan_id: str,
236
- *,
237
- kind: str | None = None,
238
- snapshot_head: bool = True,
239
- write: bool = True,
240
- ) -> dict[str, tuple[str, int]]:
241
- if not snapshot_head or Path(relative).name != "checkpoint-head.json":
242
- _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
243
- return {relative: (digest, observed)}
244
- head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
245
- observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
246
- directory = Path(relative).parent
247
- selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
248
- _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
249
- digest = _digest(observation)
250
- snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
251
- # Capture the selected file even if the worker created it after directory enumeration.
252
- if write and not (scan_dir / snapshot).exists():
253
- write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
254
- return {
255
- snapshot: (digest, int(observation["observedAtNs"])),
256
- selected: (selected_digest, selected_time),
257
- }
258
-
259
-
260
- def _is_source_order_snapshot(relative: str) -> bool:
261
- path = Path(relative)
262
- return path.parent == Path("source-order") and bool(
263
- re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
264
- )
265
-
266
-
267
- def _read_saved_result(
268
- scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
269
- ) -> tuple[dict[str, Any], str, int]:
270
- draft, _, metadata = _read_scan_local_json_with_metadata(
271
- scan_dir, relative, "Saved scan checkpoint"
272
- )
273
- directory = _checkpoint_head_directory(relative)
274
- if directory is not None:
275
- checkpoint = draft.get("checkpoint")
276
- if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
277
- raise ContractError("checkpoint head does not name a saved checkpoint")
278
- _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
279
- if Path(relative).name == "checkpoint-head.json":
280
- return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
281
- observed = draft.get("observedAtNs")
282
- if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
283
- raise ContractError("checkpoint head has no observation time")
284
- return draft, _digest(draft), int(observed)
285
- if draft.get("scanId") != scan_id:
286
- raise ContractError("checkpoint belongs to a different scan")
287
- if not _is_source_order_snapshot(relative) and (
288
- not isinstance(draft.get("findings"), list)
289
- or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
290
- ):
291
- raise ContractError("checkpoint has no semantic findings or coverage")
292
- return draft, _digest(draft), metadata.st_mtime_ns
293
-
294
-
295
- def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
296
- times: dict[str, int] = {}
297
- snapshots = [path for path in sources if _is_source_order_snapshot(path)]
298
- for path in snapshots:
299
- record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
300
- if digest != sources[path] or not isinstance(record.get("sources"), dict):
301
- raise ContractError("saved source ordering changed after the scan stopped")
302
- for relative, observation in record["sources"].items():
303
- if (
304
- not isinstance(observation, dict)
305
- or relative not in sources
306
- or observation.get("digest") != sources[relative]
307
- or not isinstance(observation.get("observedAtNs"), str)
308
- or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
309
- ):
310
- raise ContractError("saved source ordering does not match its frozen sources")
311
- observed = int(observation["observedAtNs"])
312
- if relative in times and times[relative] != observed:
313
- raise ContractError("saved source ordering has conflicting observations")
314
- times[relative] = observed
315
- if snapshots and sources.keys() - set(snapshots) - times.keys():
316
- raise ContractError("saved source ordering is incomplete")
317
- return times
318
-
319
-
320
- def _freeze_source_times(
321
- scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
322
- ) -> None:
323
- # Identical result rewrites must not change the order of frozen review evidence.
324
- observations = {
325
- path: {"digest": digest, "observedAtNs": str(times[path])}
326
- for path, digest in sources.items()
327
- if not _is_source_order_snapshot(path)
328
- }
329
- if not observations:
330
- return
331
- record = {"scanId": scan_id, "sources": observations}
332
- digest = _digest(record)
333
- path = f"source-order/{digest}.json"
334
- if not (scan_dir / path).exists():
335
- write_scan_local_bytes(scan_dir, path, _encoded(record))
336
- if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
337
- raise ContractError("saved source ordering does not match its digest")
338
- sources[path] = digest
339
-
340
-
341
- def _parent_scan_draft(
342
- scan_id: str,
343
- parent_scan: dict[str, Any],
344
- findings: dict[str, Any],
345
- coverage: dict[str, Any],
346
- ) -> dict[str, Any]:
347
- parent = {
348
- "scanId": scan_id,
349
- "findings": findings.get("findings"),
350
- "coverage": coverage,
351
- **{
352
- key: parent_scan[key]
353
- for key in ("scope", "threatModel", "complete")
354
- if key in parent_scan
355
- },
356
- }
357
- if not isinstance(parent["findings"], list):
358
- raise ContractError("Saved parent draft has no findings array")
359
- return parent
360
-
361
-
362
- def _read_saved_parent_result(
363
- scan_dir: Path, scan_id: str
364
- ) -> tuple[dict[str, Any], dict[str, Any]]:
365
- manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
366
- findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
367
- coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
368
- parent_scan = manifest.get("scan")
369
- if not isinstance(parent_scan, dict):
370
- raise ContractError("Saved parent manifest has no scan object")
371
- if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
372
- parent_scan.get("id", scan_id) != scan_id
373
- or findings.get("scanId", scan_id) != scan_id
374
- or coverage.get("scanId", scan_id) != scan_id
375
- ):
376
- raise ContractError("Saved parent documents belong to a different scan")
377
- return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
378
-
379
-
380
- def _source_digests(value: Any, error: str) -> dict[str, str]:
381
- if not isinstance(value, dict) or not all(
382
- isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
383
- ):
384
- raise ContractError(error)
385
- return value
386
-
387
-
388
- def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
389
- if isinstance(value, dict) and isinstance(value.get("sources"), dict):
390
- sources = _source_digests(
391
- value["sources"], "Saved stopped-scan source digests are malformed."
392
- )
393
- model_source = value.get("threatModelSource")
394
- if not isinstance(model_source, str) or model_source not in sources:
395
- raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
396
- return sources, model_source
397
- return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
398
-
399
-
400
- def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
401
- state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
402
- return json.dumps(state, sort_keys=True)
403
-
404
-
405
206
  def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
406
207
  try:
407
208
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
@@ -446,6 +247,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
446
247
  manifest_scan.get("preservedSources", {}),
447
248
  "Published scan source digests are malformed.",
448
249
  )
250
+ _, pending_recovery = _retained_checkpoint_state(scan)
251
+ if pending_recovery:
252
+ return True
253
+ published_heads = manifest_scan.get("preservedCheckpointHeads")
254
+ if published_heads is None:
255
+ published_heads = _frozen_checkpoint_heads(
256
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
257
+ )
258
+ elif "." not in published_heads:
259
+ # Older publications recorded root observations without selecting a root head.
260
+ legacy_heads = _frozen_checkpoint_heads(
261
+ scan_dir, scan["id"], published_sources, skip_unreadable=True
262
+ )
263
+ if "." in legacy_heads:
264
+ published_heads = {**published_heads, ".": legacy_heads["."]}
265
+ checkpoint_heads = dict(published_heads)
449
266
  current_sources = dict(published_sources)
450
267
  paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
451
268
  for path in paths:
@@ -457,23 +274,53 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
457
274
  kind=paths[path],
458
275
  snapshot_head=path not in published_sources,
459
276
  write=False,
277
+ checkpoint_heads=checkpoint_heads,
460
278
  )
461
279
  current_sources.update({path: value[0] for path, value in captured.items()})
280
+ directory = Path(path).parent.as_posix()
281
+ previous_head = published_heads.get(directory)
282
+ if (
283
+ Path(path).name == "checkpoint-head.json"
284
+ and previous_head in captured
285
+ and Path(previous_head).parent.name == "checkpoints"
286
+ ):
287
+ observed = captured[checkpoint_heads[directory]][1]
288
+ if observed == _legacy_checkpoint_observed_at(
289
+ scan_dir, scan["id"], published_sources, previous_head
290
+ ):
291
+ checkpoint_heads[directory] = previous_head
462
292
  except (ContractError, OSError, ValueError):
463
- continue
464
- return current_sources != published_sources
293
+ _restore_checkpoint_head(checkpoint_heads, published_heads, published_sources, path)
294
+ return current_sources != published_sources or checkpoint_heads != published_heads
465
295
  except (ContractError, OSError, SystemExit, ValueError):
466
296
  return False
467
297
 
468
298
 
469
- def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]:
299
+ def _recovery_source_digests(
300
+ db: Any, connection: Any, scan: Any
301
+ ) -> tuple[dict[str, str], bool, dict[str, str | None]]:
470
302
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
471
303
  frozen_sources: dict[str, str] | None = None
472
304
  include_parent = True
473
305
  raw_frozen_sources = scan["retained_source_digests_json"]
474
306
  if raw_frozen_sources is not None:
475
- frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
307
+ retained = json.loads(raw_frozen_sources)
308
+ frozen_sources, model_source = _retained_source_state(retained)
309
+ if isinstance(retained.get("sources"), dict) and retained.get("sourceTimes") is not None:
310
+ _restore_frozen_model_capture(
311
+ scan_dir, scan["id"], frozen_sources, retained["sourceTimes"], model_source
312
+ )
313
+ _freeze_source_times(
314
+ scan_dir,
315
+ scan["id"],
316
+ frozen_sources,
317
+ retained["sourceTimes"],
318
+ retained.get("parentContext"),
319
+ )
476
320
  include_parent = False
321
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
322
+ if pending_recovery:
323
+ return frozen_sources, False, checkpoint_heads
477
324
 
478
325
  manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
479
326
  if manifest_path is not None:
@@ -485,9 +332,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
485
332
  manifest_scan = manifest.get("scan")
486
333
  if not isinstance(manifest_scan, dict):
487
334
  raise ContractError("Saved scan manifest has no scan object")
488
- if scan["seal_manifest_digest"] is not None or (
489
- manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
490
- ):
335
+ if scan["seal_manifest_digest"] is not None or manifest_is_sealed(manifest_scan):
491
336
  if "preservedSources" in manifest_scan:
492
337
  published_sources = _source_digests(
493
338
  manifest_scan["preservedSources"],
@@ -510,9 +355,14 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
510
355
  "FROM deep_scan_workers WHERE scan_id = ?",
511
356
  (scan["id"],),
512
357
  ).fetchall()
358
+ previous_heads = checkpoint_heads
359
+ if previous_heads is None:
360
+ previous_heads = _frozen_checkpoint_heads(scan_dir, scan["id"], frozen_sources or {})
361
+ checkpoint_heads = dict(previous_heads)
513
362
  paths = dict(_saved_result_paths(scan_dir, workers))
514
363
  recovery_sources = dict(frozen_sources or {})
515
- source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
364
+ parent_context: dict[str, str] = {}
365
+ source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources, parent_context)
516
366
  for relative, expected_digest in recovery_sources.items():
517
367
  try:
518
368
  _, digest, observed = _read_saved_result(
@@ -525,18 +375,52 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
525
375
  if not _is_source_order_snapshot(relative):
526
376
  source_times.setdefault(relative, observed)
527
377
 
378
+ snapshot_error = None
379
+ captured_records = {}
528
380
  for relative in paths.keys() - recovery_sources.keys():
529
381
  try:
530
- captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
382
+ captured = _capture_saved_source(
383
+ scan_dir,
384
+ relative,
385
+ scan["id"],
386
+ kind=paths[relative],
387
+ checkpoint_heads=checkpoint_heads,
388
+ )
389
+ except _CheckpointSnapshotError as exc:
390
+ snapshot_error = snapshot_error or exc
391
+ captured = exc.captured
392
+ captured_records.update(exc.records)
531
393
  except (ContractError, OSError, ValueError):
394
+ _restore_checkpoint_head(
395
+ checkpoint_heads, previous_heads, frozen_sources or {}, relative
396
+ )
532
397
  continue
398
+ if relative == "checkpoint-head.json":
399
+ parent_context["head"] = next(iter(captured))
533
400
  for path, (digest, observed) in captured.items():
534
401
  if path in recovery_sources and recovery_sources[path] != digest:
535
402
  raise ContractError("checkpoint changed after the scan stopped")
536
403
  recovery_sources[path] = digest
537
404
  source_times.setdefault(path, observed)
538
- _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
539
- return recovery_sources, include_parent
405
+ try:
406
+ if snapshot_error is not None:
407
+ raise snapshot_error
408
+ _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
409
+ except (ContractError, OSError) as exc:
410
+ failure = _CheckpointSnapshotError(
411
+ str(exc),
412
+ {
413
+ path: (digest, source_times[path])
414
+ for path, digest in recovery_sources.items()
415
+ if not _is_source_order_snapshot(path)
416
+ },
417
+ )
418
+ failure.records = captured_records
419
+ failure.heads = checkpoint_heads
420
+ failure.include_parent = include_parent
421
+ failure.parent_context = parent_context
422
+ raise failure from exc
423
+ return recovery_sources, include_parent, checkpoint_heads
540
424
 
541
425
 
542
426
  def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
@@ -554,1849 +438,46 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
554
438
  ).fetchone()
555
439
  if publication_error is not None and publication_error["publication_error_message"]:
556
440
  return True
557
- return _saved_results_changed(db, connection, scan)
558
-
559
-
560
- def _finding_key(finding: dict[str, Any]) -> str:
561
- # Wording and evidence may improve between checkpoints; distinct source locations
562
- # must not collide merely because two workers chose the same semantic identity.
563
- provenance = finding.get("provenance")
564
- identity = (
565
- provenance.get("preservedIdentity", finding.get("identity"))
566
- if isinstance(provenance, dict)
567
- else finding.get("identity")
568
- )
569
- if not isinstance(identity, dict):
570
- normalized = dict(finding)
571
- normalized.pop("identity", None)
572
- _ensure_finding_identity(normalized)
573
- identity = normalized["identity"]
574
- locations = finding.get("locations", [])
575
- if not isinstance(locations, list):
576
- locations = []
577
- return _digest(
578
- [
579
- finding.get("ruleId"),
580
- identity,
581
- sorted(
582
- (
583
- (
584
- location.get("path"),
585
- location.get("startLine"),
586
- location.get("endLine", location.get("startLine")),
587
- )
588
- for location in locations
589
- if isinstance(location, dict)
590
- ),
591
- key=_encoded,
592
- ),
593
- ]
594
- )
595
-
596
-
597
- def _worker_candidate_key(
598
- worker_id: str, candidate_id: str, finding: dict[str, Any]
599
- ) -> tuple[str, str, Any, Any, Any]:
600
- """Identify one worker-local candidate without merging unrelated locations."""
601
- provenance = finding.get("provenance")
602
- identity = (
603
- provenance.get("preservedIdentity", finding.get("identity"))
604
- if isinstance(provenance, dict)
605
- else finding.get("identity")
606
- )
607
- if not isinstance(identity, dict):
608
- normalized = dict(finding)
609
- _ensure_finding_identity(normalized)
610
- identity = normalized.get("identity")
611
- anchor = identity.get("anchor") if isinstance(identity, dict) else None
612
- instance = identity.get("instance") if isinstance(identity, dict) else None
613
- return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
614
-
615
-
616
- def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
617
- """Return substantive finding content without generated identity or provenance."""
618
- return {
619
- key: value
620
- for key, value in finding.items()
621
- if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
622
- }
623
-
624
-
625
- def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:
626
- if not isinstance(finding, dict) or "identity" in finding:
627
- return
628
- if candidate_only and not finding_candidate_id(finding):
629
- return
630
- extensions = finding.get("extensions")
631
- source = str(
632
- (extensions.get("candidateId") if isinstance(extensions, dict) else None)
633
- or finding.get("title")
634
- or "finding"
635
- )
636
- anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
637
- finding["identity"] = {"anchor": anchor}
638
-
639
-
640
- def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
641
- """Yield canonical and historical findings without trusting candidate IDs."""
642
- pending = [finding]
643
- seen: set[int] = set()
644
- while pending:
645
- current = pending.pop()
646
- marker = id(current)
647
- if marker in seen:
648
- continue
649
- seen.add(marker)
650
- yield current
651
- provenance = current.get("provenance")
652
- if not isinstance(provenance, dict):
653
- continue
654
- previous = provenance.get("previousFindings")
655
- if isinstance(previous, list):
656
- pending.extend(item for item in reversed(previous) if isinstance(item, dict))
657
- sources = provenance.get("sourceFindings")
658
- if isinstance(sources, list):
659
- pending.extend(
660
- source["finding"]
661
- for source in reversed(sources)
662
- if isinstance(source, dict) and isinstance(source.get("finding"), dict)
663
- )
664
-
665
-
666
- def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
667
- rows = coverage.get("deferred", [])
668
- return rows if isinstance(rows, list) else []
669
-
670
-
671
- def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
672
- if draft.get("complete") is False:
673
- return []
674
- coverage = draft["coverage"]
675
- rows = coverage.get("resolvedDeferred", [])
676
- try:
677
- # Invalid closure metadata cannot discard the evidence it names.
678
- _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
679
- _validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
680
- except ContractError:
681
- return []
682
- return rows
683
-
684
-
685
- def _merge_tied_parent_observations(
686
- current: dict[str, Any], previous: dict[str, Any]
687
- ) -> dict[str, Any]:
688
- merged = copy.deepcopy(current)
689
- for finding in previous["findings"]:
690
- if finding not in merged["findings"]:
691
- merged["findings"].append(copy.deepcopy(finding))
692
- coverage = merged["coverage"]
693
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
694
- rows = previous["coverage"].get(field, [])
695
- output = coverage.setdefault(field, [])
696
- if isinstance(rows, list) and isinstance(output, list):
697
- for row in rows:
698
- if row not in output:
699
- output.append(copy.deepcopy(row))
700
- pending_ids = {
701
- identity
702
- for row in _deferred_rows(coverage)
703
- if isinstance(row, dict)
704
- for identity in (row.get("id"), row.get("candidateId"))
705
- if isinstance(identity, str)
706
- }
707
- closure_schema = _read_json(
708
- Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
709
- )["properties"]["resolvedDeferred"]
710
- closures = {}
711
- for observed in (current, previous):
712
- for row in _resolved_deferred_rows(observed, closure_schema):
713
- if row["id"] not in pending_ids:
714
- closures.setdefault(row["id"], copy.deepcopy(row))
715
- coverage.pop("resolvedDeferred", None)
716
- if closures:
717
- coverage["resolvedDeferred"] = list(closures.values())
718
- if current.get("complete") is False or previous.get("complete") is False:
719
- merged["complete"] = False
720
- coverage.pop("completeness", None)
721
- return merged
722
-
723
-
724
- def _deferred_candidate_id(
725
- row: dict[str, Any],
726
- owner: str | None,
727
- ambiguous_deferred: set[tuple[str | None, str]],
728
- ) -> str | None:
729
- identity = row.get("candidateId") or row.get("id")
730
- if not isinstance(identity, str):
731
- return None
732
- if (owner, identity) in ambiguous_deferred and not any(
733
- key in row for key in ("candidateId", "candidate", "finding")
734
- ):
735
- return None
736
- return identity
737
-
738
-
739
- def _generic_surface_updates(
740
- sources: list[tuple[str, dict[str, Any], str | None]],
741
- source_order: dict[str, tuple[int, int]],
742
- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
743
- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
744
- resolved_candidates: dict[tuple[str | None, str], str],
745
- reopened_generic: set[tuple[str | None, str]],
746
- surface_schema: dict[str, Any],
747
- deferred_rows: dict[str, list[Any]],
748
- ambiguous_deferred: set[tuple[str | None, str]],
749
- *,
750
- surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
751
- eligible_receipt_sources: set[str],
752
- receipt_root: Path | None = None,
753
- receipt_warnings: list[str] | None = None,
754
- fallback_surfaces: list[dict[str, Any]] | None = None,
755
- ) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
756
- if not closed_deferred and not reopened_generic and receipt_root is None:
757
- return set(), []
758
-
759
- def receipt_error(ref: Any) -> ContractError | None:
760
- if receipt_root is None or not isinstance(ref, str):
761
- return ContractError("Saved coverage receipt is unavailable.")
762
- try:
763
- descriptor = open_scan_local_file_descriptor(
764
- receipt_root, ref, "Saved coverage receipt"
765
- )
766
- os.close(descriptor)
767
- except ContractError as exc:
768
- return exc
769
- return None
770
-
771
- def linked(row: dict[str, Any], identity: str | None) -> bool:
772
- surface_ids = row.get("surfaceIds", [])
773
- return identity is not None and (
774
- row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
775
- )
776
-
777
- saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
778
- for relative, draft, owner in sources:
779
- rows = draft["coverage"].get("surfaces", [])
780
- for row in rows if isinstance(rows, list) else []:
781
- if isinstance(row, dict) and isinstance(row.get("id"), str):
782
- saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
783
- replaced: set[int] = set()
784
- updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
785
- # Current selections across all surfaces take precedence over historical refs.
786
- receipt_paths: dict[str, str] = {}
787
- retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
788
- if receipt_root is not None:
789
- receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
790
- for key, matches in saved_surfaces.items():
791
- retained = [
792
- match for match in matches if surface_is_retained(match[0], key[0], match[1])
793
- ]
794
- if not retained:
795
- continue
796
- latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
797
- retained_surfaces[key] = latest_surface
798
- receipt_surfaces.extend(
799
- (key[0], row)
800
- for relative, row in retained
801
- if source_order[relative] == source_order[latest_source]
802
- )
803
- for relative, draft, owner in sources:
804
- rows = draft["coverage"].get("surfaces", [])
805
- for row in rows if isinstance(rows, list) else []:
806
- if (
807
- isinstance(row, dict)
808
- and "id" not in row
809
- and surface_is_retained(relative, owner, row)
810
- ):
811
- receipt_surfaces.append((owner, row))
812
- receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
813
- for owner, latest_surface in receipt_surfaces:
814
- # Recovery repairs disposition and receipts before validating row
815
- # metadata. A row discarded there cannot reserve a receipt spelling.
816
- metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
817
- metadata.setdefault("id", _saved_coverage_id(metadata))
818
- metadata.update(disposition="needs_follow_up", receiptRefs=[])
819
- try:
820
- _require_str(metadata, "id", "coverage.surfaces")
821
- _validate_schema_node(metadata, surface_schema, "coverage.surfaces")
822
- except ContractError:
823
- continue
824
- refs = latest_surface.get("receiptRefs", [])
825
- for ref in refs if isinstance(refs, list) else []:
826
- if receipt_error(ref) is None:
827
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
828
- receipt_paths.setdefault(normalized.lower(), normalized)
829
- for relative, draft, owner in sources:
830
- closed_ids = {
831
- identity
832
- for (saved_owner, identity), (_, _, source) in closed_deferred.items()
833
- if saved_owner == owner and source == relative
834
- }
835
- reopened_ids = {
836
- identity
837
- for (saved_owner, identity), (_, _, source) in active_deferred.items()
838
- if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
839
- }
840
- if not closed_ids and not reopened_ids and receipt_root is None:
841
- continue
842
- current = draft["coverage"].get("surfaces", [])
843
- for surface in current if isinstance(current, list) else []:
844
- if not isinstance(surface, dict):
845
- continue
846
- reopening = surface.get("disposition") == "needs_follow_up"
847
- work_ids = reopened_ids if reopening else closed_ids
848
- repair_only = not work_ids
849
- if repair_only and receipt_root is None:
850
- continue
851
- identity = surface.get("id")
852
- if not isinstance(identity, str):
853
- continue
854
- matches = saved_surfaces[(owner, identity)]
855
- # Receipt repair alone cannot close or reopen a saved surface.
856
- if repair_only and any(
857
- row.get("disposition") != surface.get("disposition") for _, row in matches
858
- ):
859
- continue
860
- # Older writers could assign one ID to distinct surfaces in a draft.
861
- # A closure cannot identify which of those observations it replaces.
862
- by_source = dict(matches)
863
- if any(row != by_source[saved_path] for saved_path, row in matches):
864
- continue
865
- if any(
866
- "candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
867
- ):
868
- continue
869
- if any(
870
- row is not surface
871
- and source_order[saved_path] >= source_order[relative]
872
- and row.get("disposition") != surface.get("disposition")
873
- for saved_path, row in matches
874
- ):
875
- continue
876
-
877
- if not reopening and any(
878
- saved_owner == owner
879
- and (
880
- not isinstance(row.get("id") or row.get("candidateId"), str)
881
- or (
882
- isinstance(row.get("id"), str)
883
- and (owner, row["id"]) in ambiguous_deferred
884
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
885
- )
886
- )
887
- and linked(row, identity)
888
- for saved_path, _, saved_owner in sources
889
- for row in deferred_rows[saved_path]
890
- if isinstance(row, dict)
891
- ):
892
- continue
893
- if not reopening and any(
894
- saved_owner == owner
895
- and (owner, deferred_id) not in closed_deferred
896
- and (
897
- (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
898
- or (owner, candidate_id) not in resolved_candidates
899
- )
900
- and linked(row, identity)
901
- for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
902
- ):
903
- continue
904
- # An accepted checkpoint can update a saved surface by ID without
905
- # optional surfaceIds links on its generic task.
906
- receipt_matches = [
907
- (saved_path, row)
908
- for saved_path, row in matches
909
- if receipt_root is None
910
- or saved_path in eligible_receipt_sources
911
- or surface_is_retained(saved_path, owner, row)
912
- ]
913
- if not receipt_matches:
914
- continue
915
- latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
916
- target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
917
- if target is None:
918
- continue
919
- update = copy.deepcopy(target)
920
- # Historical evidence can repair a retained row without promoting
921
- # an unselected observation's label, notes, or other metadata.
922
- refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
923
- update["receiptRefs"] = refs
924
- replaced_receipt = False
925
- replacement_paths = dict(receipt_paths)
926
- if isinstance(refs, list):
927
- repaired = (
928
- not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
929
- )
930
- if repaired:
931
- for ref in refs:
932
- normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
933
- key = normalized.lower()
934
- if key in replacement_paths and replacement_paths[key] != normalized:
935
- repaired = False
936
- break
937
- replacement_paths[key] = normalized
938
- for _, row in receipt_matches:
939
- previous_refs = row.get("receiptRefs", [])
940
- for ref in previous_refs if isinstance(previous_refs, list) else []:
941
- if ref not in refs:
942
- failure = receipt_error(ref) if repaired else None
943
- replaceable = failure is not None and isinstance(
944
- failure.__cause__, FileNotFoundError
945
- )
946
- if repaired and isinstance(ref, str):
947
- try:
948
- normalized = _require_portable_relative_path(
949
- ref, "Saved coverage receipt"
950
- )
951
- except ContractError as error:
952
- failure = error
953
- replaceable = True
954
- else:
955
- key = normalized.lower()
956
- previous = replacement_paths.get(key)
957
- if previous is not None and previous != normalized:
958
- failure = ContractError(
959
- f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
960
- )
961
- replaceable = True
962
- elif not replaceable:
963
- replacement_paths[key] = normalized
964
- if replaceable:
965
- replaced_receipt = True
966
- if receipt_warnings is not None:
967
- receipt_warnings.append(
968
- f"Replaced invalid saved coverage receipt {ref}: {failure}"
969
- )
970
- continue
971
- refs.append(ref)
972
- if repair_only and not replaced_receipt:
973
- continue
974
- try:
975
- _validate_schema_node(update, surface_schema, "coverage.surfaces")
976
- except ContractError:
977
- continue
978
- replaced_sources = {
979
- id(row)
980
- for _, row in matches
981
- if reopening
982
- or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
983
- }
984
- replaced.update(replaced_sources)
985
- existing = next(
986
- (entry for entry in updates if entry[0] == owner and entry[1] == update), None
987
- )
988
- if existing is None:
989
- receipt_paths.update(replacement_paths)
990
- updates.append((owner, update, replaced_sources))
991
- else:
992
- existing[2].update(replaced_sources)
993
- return replaced, updates
994
-
995
-
996
- @contextmanager
997
- def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
998
- """Keep rejected completion attempts from becoming accepted parent observations."""
999
- head_path = scan_dir / "checkpoint-head.json"
1000
- previous = None
1001
- try:
1002
- head_path.lstat()
1003
- except FileNotFoundError:
1004
- pass
1005
- else:
1006
- descriptor = open_scan_local_file_descriptor(
1007
- scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
1008
- )
1009
- with os.fdopen(descriptor, "rb") as handle:
1010
- metadata = os.fstat(handle.fileno())
1011
- previous = (handle.read(), metadata)
1012
- directories = ("checkpoints", "checkpoint-heads")
1013
- previous_files = {
1014
- relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
1015
- }
1016
- try:
1017
- yield
1018
- except BaseException:
1019
- if previous is None:
1020
- _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
1021
- else:
1022
- payload, metadata = previous
1023
- write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
1024
- os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
1025
- current_files = {
1026
- relative
1027
- for directory in directories
1028
- for relative in _checkpoint_paths(scan_dir, directory)
1029
- }
1030
- for relative in current_files - previous_files:
1031
- _remove_scan_local_file_if_exists(scan_dir, relative)
1032
- raise
1033
-
1034
-
1035
- def merge_saved_results(
1036
- scan_dir: Path,
1037
- scan_id: str,
1038
- binding: dict[str, Any],
1039
- workers: list[Any],
1040
- warnings: list[str],
1041
- *,
1042
- stopped: bool,
1043
- reason: str,
1044
- frozen_source_digests: dict[str, str] | None = None,
1045
- allow_frozen_legacy_parent: bool = False,
1046
- frozen_model_source: str | None = None,
1047
- selected_model_source: list[str] | None = None,
1048
- project_worker_coverage: bool = False,
1049
- ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
1050
- """Read only bound parent/worker files; return an unsealed loss-preserving union."""
1051
- initial_warnings = set(warnings)
1052
- try:
1053
- source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
1054
- except (ContractError, OSError, ValueError) as exc:
1055
- raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
1056
- parent: dict[str, Any] | None = None
1057
- parent_manifest: dict[str, Any] | None = None
1058
- parent_is_canonical = False
1059
- parent_modified = 0
1060
- if frozen_source_digests is None or allow_frozen_legacy_parent:
1061
- try:
1062
- parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
1063
- # Without an accepted head, file-authored coverage is a full replacement.
1064
- parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
1065
- parent_is_canonical = True
1066
- except (ContractError, OSError, ValueError) as exc:
1067
- if not stopped:
1068
- raise
1069
- if (scan_dir / "scan-manifest.json").exists():
1070
- warnings.append(f"Could not read the saved parent draft: {exc}")
1071
- parent_manifest = None
1072
- parent = None
1073
- if parent_manifest is not None and parent is not None:
1074
- parent_scan = parent_manifest["scan"]
1075
- try:
1076
- previous_head, _, head_modified = _read_saved_result(
1077
- scan_dir, "checkpoint-head.json", scan_id
1078
- )
1079
- except (ContractError, OSError, ValueError):
1080
- head_modified = None
1081
- if head_modified is not None:
1082
- # A partial tool publication must not outrank its accepted head.
1083
- parent_modified = min(
1084
- (scan_dir / name).lstat().st_mtime_ns
1085
- for name in ("findings.json", "coverage.json", "scan-manifest.json")
1086
- )
1087
- if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
1088
- head_path = scan_dir / "checkpoint-head.json"
1089
- tied_observations = False
1090
- if head_modified == parent_modified:
1091
- previous_parent, _, _ = _read_saved_result(
1092
- scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
1093
- )
1094
- if previous_parent != parent:
1095
- # Tied observations cannot decide which pending work came last.
1096
- parent = _merge_tied_parent_observations(parent, previous_parent)
1097
- parent_is_canonical = False
1098
- tied_observations = True
1099
- payload = _encoded(parent)
1100
- parent_digest = hashlib.sha256(payload).hexdigest()
1101
- parent_checkpoint = f"checkpoints/{parent_digest}.json"
1102
- checkpoint_path = scan_dir / parent_checkpoint
1103
- if not checkpoint_path.exists():
1104
- write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
1105
- # A recovery copy must not appear newer than the review it copies.
1106
- os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
1107
- if head_modified is None or head_modified < parent_modified or tied_observations:
1108
- write_scan_local_bytes(
1109
- scan_dir,
1110
- "checkpoint-head.json",
1111
- _encoded({"checkpoint": checkpoint_path.name}),
1112
- )
1113
- os.utime(head_path, ns=(parent_modified, parent_modified))
1114
- if frozen_source_digests is not None:
1115
- captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
1116
- frozen_source_digests = {
1117
- **frozen_source_digests,
1118
- parent_checkpoint: parent_digest,
1119
- **{path: value[0] for path, value in captured.items()},
1120
- }
1121
-
1122
- sources: list[tuple[str, dict[str, Any], str | None]] = []
1123
- parent_preserved_sources: dict[str, str] = {}
1124
- source_digests: dict[str, str] = {}
1125
- source_order: dict[str, tuple[int, int]] = {}
1126
- worker_attempts: dict[str, tuple[str, int]] = {}
1127
- saved_heads: dict[str, str] = {}
1128
- if parent_manifest:
1129
- recorded = parent_manifest["scan"].get("preservedSources", {})
1130
- if isinstance(recorded, dict):
1131
- parent_preserved_sources = recorded
1132
- source_digests.update(parent_preserved_sources)
1133
- if frozen_source_digests is None:
1134
- source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
1135
- paths: dict[str, str | None] = {}
1136
- reducer_paths: set[str] = set()
1137
- current_results: set[str] = set()
1138
- reducer_outputs: list[tuple[Any, str, list[str], int]] = []
1139
- reducer = _latest_successful_reducer(workers)
1140
- latest_reducer: str | None = None
1141
- if reducer is not None:
1142
- try:
1143
- latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
1144
- paths[latest_reducer] = None
1145
- reducer_paths.add(latest_reducer)
1146
- except ValueError:
1147
- warnings.append("Skipped a reducer result outside the scan directory.")
1148
-
1149
- def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
1150
- if worker_id is not None:
1151
- root = Path(directory).parent.as_posix()
1152
- worker_attempts[root] = (worker_id, attempt)
1153
- paths[f"{root}/checkpoint-head.json"] = worker_id
1154
- head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
1155
- for saved_directory in (directory, head_snapshots):
1156
- paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
1157
-
1158
- paths["checkpoint-head.json"] = None
1159
- checkpoints("checkpoints", None)
1160
- for worker in workers:
1161
- try:
1162
- outputs = _worker_outputs(scan_dir, worker)
1163
- except (TypeError, ValueError):
1164
- warnings.append("Skipped a worker checkpoint outside the scan directory.")
1165
- continue
1166
- if worker["kind"] == "dedup":
1167
- for directory, attempt in outputs:
1168
- checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
1169
- if not checkpoint_paths:
1170
- continue
1171
- result_path = f"{directory}/result.json"
1172
- retained_paths = [result_path, *checkpoint_paths]
1173
- paths.update(dict.fromkeys(retained_paths))
1174
- reducer_paths.update(retained_paths)
1175
- reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
1176
- continue
1177
- if worker["kind"] != "discovery":
1178
- continue
1179
- output, attempt = outputs[0]
1180
- paths[f"{output}/result.json"] = worker["id"]
1181
- current_results.add(f"{output}/result.json")
1182
- for archived, archived_attempt in outputs[1:]:
1183
- paths[f"{archived}/result.json"] = worker["id"]
1184
- checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
1185
- checkpoints(f"{output}/checkpoints", worker["id"], attempt)
1186
- if worker["result_manifest_path"]:
1187
- try:
1188
- current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
1189
- paths[current_path] = worker["id"]
1190
- current_results.add(current_path)
1191
- except ValueError:
1192
- warnings.append("Skipped a worker result outside the scan directory.")
1193
-
1194
- if frozen_source_digests is None:
1195
- for relative, worker_id in list(paths.items()):
1196
- if Path(relative).name != "checkpoint-head.json":
1197
- continue
1198
- del paths[relative]
1199
- try:
1200
- captured = _capture_saved_source(scan_dir, relative, scan_id)
1201
- paths.update({path: worker_id for path in captured})
1202
- except (ContractError, OSError, ValueError) as exc:
1203
- if (scan_dir / relative).exists():
1204
- warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1205
-
1206
- if frozen_source_digests is not None:
1207
- source_digests.update(
1208
- {
1209
- path: digest
1210
- for path, digest in frozen_source_digests.items()
1211
- if _is_source_order_snapshot(path)
1212
- }
1213
- )
1214
- paths = {
1215
- relative: worker_id
1216
- for relative, worker_id in paths.items()
1217
- if relative in frozen_source_digests
1218
- }
1219
- current_results.intersection_update(frozen_source_digests)
1220
- if latest_reducer not in frozen_source_digests:
1221
- latest_reducer = None
1222
-
1223
- for relative, worker_id in paths.items():
1224
- try:
1225
- draft, digest, observed = _read_saved_result(
1226
- scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
1227
- )
1228
- if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
1229
- raise ContractError("checkpoint changed after the scan stopped")
1230
- source_digests[relative] = digest
1231
- source_times.setdefault(relative, observed)
1232
- source_order[relative] = (0, source_times[relative])
1233
- if _checkpoint_head_directory(relative) is not None:
1234
- saved_heads[relative] = draft["checkpoint"]
1235
- continue
1236
- # Recovery expects coverage, but reducer results only contain findings
1237
- # and context. Add an empty value after hashing the original result.
1238
- sources.append((relative, {"coverage": {}, **draft}, worker_id))
1239
- except (ContractError, OSError, ValueError) as exc:
1240
- if (scan_dir / relative).exists():
1241
- warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1242
- if frozen_source_digests is not None:
1243
- if frozen_source_digests.keys() - source_digests.keys():
1244
- raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
1245
-
1246
- # Frozen observations retain checkpoint selection even if a worker moves its head.
1247
- headed_workers = {
1248
- worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
1249
- for head in saved_heads
1250
- if _checkpoint_head_directory(head) != Path(".")
1251
- }
1252
- for relative, _, worker_id in sources:
1253
- if worker_id not in headed_workers:
1254
- continue
1255
- directory = Path(relative).parent
1256
- if directory.name == "checkpoints":
1257
- directory = directory.parent
1258
- _, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
1259
- source_order[relative] = (attempt, source_order[relative][1])
1260
- selected_observations: dict[str, tuple[int, int]] = {}
1261
- parent_heads: list[tuple[int, str]] = []
1262
- for head, checkpoint in saved_heads.items():
1263
- directory = _checkpoint_head_directory(head)
1264
- selected = (directory / "checkpoints" / checkpoint).as_posix()
1265
- if selected not in source_order:
1266
- raise ContractError("Checkpoint head is outside the saved source set.")
1267
- observed = source_order[head][1]
1268
- if directory == Path("."):
1269
- order = (0, max(source_order[selected][1], observed))
1270
- parent_heads.append((observed, selected))
1271
- else:
1272
- _, attempt = worker_attempts[directory.as_posix()]
1273
- order = (attempt, observed)
1274
- selected_observations[selected] = max(selected_observations.get(selected, order), order)
1275
- source_order.update(selected_observations)
1276
-
1277
- drafts_by_path = {relative: draft for relative, draft, _ in sources}
1278
- latest_reducer_key = (
1279
- (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
1280
- if reducer is not None and latest_reducer in drafts_by_path
1281
- else None
1282
- )
1283
- if latest_reducer_key is None:
1284
- latest_reducer = None
1285
- for worker, result_path, checkpoint_paths, attempt in reducer_outputs:
1286
- result = drafts_by_path.get(result_path)
1287
- if result is None or not any(
1288
- drafts_by_path.get(checkpoint_path) == result for checkpoint_path in checkpoint_paths
1289
- ):
1290
- continue
1291
- current_results.add(result_path)
1292
- candidate_key = (worker["completed_at"] or "", worker["id"], attempt)
1293
- if latest_reducer_key is None or candidate_key > latest_reducer_key:
1294
- latest_reducer_key = candidate_key
1295
- latest_reducer = result_path
1296
-
1297
- if parent_heads:
1298
- latest_observation = max(observed for observed, _ in parent_heads)
1299
- for observed, parent_path in parent_heads:
1300
- if observed != latest_observation:
1301
- continue
1302
- draft = drafts_by_path[parent_path]
1303
- modified = source_order[parent_path][1]
1304
- if parent is None or modified > parent_modified:
1305
- parent = draft
1306
- parent_modified = modified
1307
- parent_is_canonical = False
1308
- elif modified == parent_modified and draft != parent:
1309
- parent = _merge_tied_parent_observations(parent, draft)
1310
- parent_is_canonical = False
1311
- if parent is None and latest_reducer is not None:
1312
- parent = drafts_by_path[latest_reducer]
1313
-
1314
- project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
1315
- sources, parent, source_order, enabled=project_worker_coverage
1316
- )
1317
- # Keep the actual retained row even when recovery adds previous findings.
1318
- linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
1319
- linked_surface_outputs: dict[int, dict[str, Any]] = {}
1320
- worker_row_owners: dict[int, str] = {}
1321
- worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
1322
-
1323
- def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
1324
- # Parent aggregates can carry worker metadata without originating in
1325
- # that worker's retained source. Track the actual source of output rows.
1326
- if project_worker_coverage and owner is not None and isinstance(retained, dict):
1327
- worker_row_owners[id(retained)] = owner
1328
- if field == "surfaces" and isinstance(item.get("id"), str):
1329
- worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
1330
-
1331
- def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
1332
- retained = _project_worker_coverage_row(
1333
- item, owner if project_worker_coverage else None, field
1334
- )
1335
- if retained not in output:
1336
- output.append(retained)
1337
- retained = output[output.index(retained)]
1338
- record_coverage_owner(retained, item, owner, field)
1339
- return retained
1340
-
1341
- def linked_tasks(
1342
- task: dict[str, Any],
1343
- published_id: str,
1344
- source_task: dict[str, Any] | None,
1345
- candidates: list[tuple[str | None, Any]],
1346
- ) -> list[Any]:
1347
- expected = (
1348
- _project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
1349
- if source_task is not None
1350
- else None
1351
- )
1352
- return [
1353
- retained
1354
- for owner, retained in candidates
1355
- if retained == task
1356
- or (
1357
- expected is not None
1358
- and owner == expected["workerId"]
1359
- and isinstance(retained, dict)
1360
- and retained.get("id") == expected.get("id")
1361
- and isinstance(retained.get("surfaceIds"), list)
1362
- and published_id in retained["surfaceIds"]
1363
- )
1364
- ]
1365
-
1366
- def linked_surface(surface, outputs, named_outputs):
1367
- retained = outputs.get(id(surface))
1368
- if (
1369
- retained is None
1370
- and isinstance(owner := surface.get("workerId"), str)
1371
- and isinstance(identity := surface.get("id"), str)
1372
- ):
1373
- matches = named_outputs.get((owner, identity), {})
1374
- if len(matches) == 1:
1375
- retained = next(iter(matches.values()))
1376
- return retained
1377
-
1378
- all_sources = ([("parent", parent, None)] if parent else []) + sources
1379
- # Older checkpoints can omit IDs already assigned in their published output.
1380
- for field in ("deferred", "surfaces"):
1381
- named_rows: dict[str | None, list[dict[str, Any]]] = {}
1382
- for _, draft, owner in all_sources:
1383
- rows = draft["coverage"].get(field, [])
1384
- named_rows.setdefault(owner, []).extend(
1385
- row
1386
- for row in (rows if isinstance(rows, list) else [])
1387
- if isinstance(row, dict) and isinstance(row.get("id"), str)
1388
- )
1389
- for _, draft, owner in all_sources:
1390
- rows = draft["coverage"].get(field, [])
1391
- if not isinstance(rows, list):
1392
- continue
1393
- reserved = {
1394
- row["id"]
1395
- for row in rows
1396
- if isinstance(row, dict) and isinstance(row.get("id"), str)
1397
- }
1398
- for row in rows:
1399
- if not isinstance(row, dict) or "id" in row:
1400
- continue
1401
- if field == "deferred" and any(
1402
- key in row for key in ("candidateId", "candidate", "finding")
1403
- ):
1404
- continue
1405
- content = {"receiptRefs": [], **row} if field == "surfaces" else row
1406
- identity = next(
1407
- (
1408
- named["id"]
1409
- for named in named_rows[owner]
1410
- if named["id"] not in reserved
1411
- and {
1412
- **({"receiptRefs": []} if field == "surfaces" else {}),
1413
- **{key: value for key, value in named.items() if key != "id"},
1414
- }
1415
- == content
1416
- ),
1417
- None,
1418
- )
1419
- if identity is not None:
1420
- row["id"] = identity
1421
- reserved.add(identity)
1422
-
1423
- if parent is None and not sources:
1424
- return None
1425
- if (
1426
- parent_manifest
1427
- and parent_manifest["scan"].get("sealedAt")
1428
- and parent_manifest["scan"].get("status") == binding["status"]
1429
- and parent_manifest["scan"].get("preservedSources") == source_digests
1430
- and all(warning in initial_warnings for warning in warnings)
1431
- ):
1432
- return None
1433
-
1434
- if (
1435
- frozen_source_digests is None
1436
- or any(_is_source_order_snapshot(path) for path in source_digests)
1437
- or allow_frozen_legacy_parent
1438
- ):
1439
- _freeze_source_times(scan_dir, scan_id, source_digests, source_times)
1440
-
1441
- target_kind = binding["allowedTargetKinds"][0]
1442
- if (
1443
- target_kind == "git_worktree"
1444
- and "snapshotDigest" not in binding["target"]
1445
- and "git_revision" in binding["allowedTargetKinds"]
1446
- ):
1447
- target_kind = "git_revision"
1448
- target = {"kind": target_kind, **binding["target"]}
1449
- if target["kind"] == "git_diff" and "snapshotDigest" not in target:
1450
- diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
1451
- target["snapshotDigest"] = committed_diff_snapshot_digest(
1452
- diff_kind, target["baseRevision"], target["headRevision"]
1453
- )
1454
- manifest = (
1455
- copy.deepcopy(parent_manifest)
1456
- if parent_manifest
1457
- else {"scan": {"target": target, "scope": binding["scope"]}}
1458
- )
1459
- for key in ("sealedAt", "artifacts"):
1460
- manifest["scan"].pop(key, None)
1461
- manifest["scan"]["preservedSources"] = source_digests
1462
- # Completion follows the selected parent, including legacy terminal drafts
1463
- # that omit the optional marker, rather than an older canonical manifest.
1464
- if parent is not None:
1465
- manifest["scan"].pop("complete", None)
1466
- if "complete" in parent:
1467
- manifest["scan"]["complete"] = parent["complete"]
1468
- for key in ("scope", "threatModel"):
1469
- if isinstance(parent.get(key), dict):
1470
- manifest["scan"][key] = copy.deepcopy(parent[key])
1471
- if isinstance(manifest["scan"].get("scope"), dict):
1472
- manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
1473
- if frozen_model_source is not None:
1474
- # Publication retries retain the choice made with the frozen source set.
1475
- model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
1476
- if not isinstance(model, dict):
1477
- raise ContractError("Frozen stopped-scan model source is unavailable.")
1478
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
1479
- if paths[frozen_model_source] is not None:
1480
- manifest["scan"]["threatModel"]["origin"] = "recovered"
1481
- if selected_model_source is not None:
1482
- selected_model_source[:] = [frozen_model_source]
1483
- coverage = (
1484
- copy.deepcopy(parent["coverage"])
1485
- if parent and parent["coverage"]
1486
- else {
1487
- "mode": binding["coverageMode"],
1488
- "inventoryStrategy": "diff"
1489
- if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
1490
- else "scoped_path"
1491
- if binding["coverageMode"] == "scoped_path"
1492
- else "repository",
1493
- **binding["scope"],
1494
- "surfaces": [],
1495
- "explicitExclusions": [],
1496
- "deferred": [],
1497
- }
1498
- )
1499
- if isinstance(coverage.get("openQuestions"), list):
1500
- coverage["openQuestions"] = [
1501
- {"question": item.strip()} if isinstance(item, str) else item
1502
- for item in coverage["openQuestions"]
1503
- ]
1504
- canonical_rows = (
1505
- {
1506
- id(item)
1507
- for field in ("surfaces", "explicitExclusions", "deferred")
1508
- for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
1509
- }
1510
- if parent_is_canonical
1511
- else set()
1512
- )
1513
- findings: list[dict[str, Any]] = []
1514
- finding_positions: dict[str, int] = {}
1515
- represented: dict[str, str | None] = {}
1516
- represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}
1517
- represented_history: dict[str, set[str]] = {}
1518
- represented_candidate_history: dict[tuple[str, str, Any, Any, Any], set[str]] = {}
1519
- rejected_history: dict[tuple[str, str], list[dict[str, Any]]] = {}
1520
- stopped_parent_seal = bool(
1521
- stopped and parent_manifest and parent_manifest["scan"].get("sealedAt")
1522
- )
1523
-
1524
- def valid_finding(value: Any) -> bool:
1525
- # Use the finalizer's own per-record recovery before a draft can suppress
1526
- # an earlier checkpoint. Invalid latest records must not hide valid history.
1527
- document = {"scanId": scan_id, "findings": [copy.deepcopy(value)]}
1528
- if isinstance(document["findings"][0], dict):
1529
- _ensure_finding_identity(document["findings"][0], candidate_only=True)
1530
- _recover_unsealed_findings(
1531
- {"scan": {"id": scan_id, "target": binding["target"]}},
1532
- document,
1533
- Path(__file__).resolve().parent.parent / "schemas",
1534
- scan_dir,
1535
- [],
1536
- )
1537
- return bool(document["findings"])
1538
-
1539
- source_order["parent"] = (0, parent_modified)
1540
- deferred_rows = {
1541
- relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
1542
- }
1543
- # A legacy source can contain independent tasks with the same explicit ID.
1544
- # Later rewrites cannot make an ID-only closure identify one of those tasks.
1545
- ambiguous_deferred: set[tuple[str | None, str]] = set()
1546
- for relative, _, owner in all_sources:
1547
- by_id: dict[str, dict[str, Any]] = {}
1548
- candidate_aliases = {
1549
- identity
1550
- for row in deferred_rows[relative]
1551
- if isinstance(row, dict)
1552
- and any(key in row for key in ("candidateId", "candidate", "finding"))
1553
- for identity in (row.get("id"), row.get("candidateId"))
1554
- if isinstance(identity, str)
1555
- }
1556
- for row in deferred_rows[relative]:
1557
- if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
1558
- continue
1559
- if any(key in row for key in ("candidateId", "candidate", "finding")):
1560
- continue
1561
- if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
1562
- ambiguous_deferred.add((owner, identity))
1563
- by_id[identity] = row
1564
- current_drafts = ([("parent", parent, None)] if parent else []) + [
1565
- source for source in sources if source[0] in current_results | selected_observations.keys()
1566
- ]
1567
- # Generic closures belong to one logical scan or worker, just like candidates.
1568
- # Keep them when recovering a terminal checkpoint without its canonical write.
1569
- closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1570
-
1571
- candidate_ids: set[tuple[str | None, str]] = set()
1572
- accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
1573
- active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1574
- coverage_schema = _read_json(
1575
- Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
1576
- )["properties"]
1577
- for relative, draft, owner in all_sources:
1578
- order = source_order[relative]
1579
- for item in deferred_rows[relative]:
1580
- if not isinstance(item, dict):
1581
- continue
1582
- if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
1583
- candidate_ids.update(
1584
- (owner, identity)
1585
- for identity in (item.get("id"), item.get("candidateId"))
1586
- if isinstance(identity, str)
1587
- )
1588
- identity = item.get("id") or item.get("candidateId")
1589
- if isinstance(identity, str):
1590
- key = (owner, identity)
1591
- previous = active_deferred.get(key)
1592
- if previous is None or order > previous[0]:
1593
- active_deferred[key] = (order, item, relative)
1594
- if (
1595
- relative == "parent"
1596
- or relative in current_results
1597
- or relative in selected_observations
1598
- ):
1599
- try:
1600
- # A malformed update cannot discard valid saved evidence.
1601
- _validate_schema_node(
1602
- item, coverage_schema["deferred"]["items"], "coverage.deferred"
1603
- )
1604
- except ContractError:
1605
- continue
1606
- accepted_deferred_orders[key] = max(
1607
- accepted_deferred_orders.get(key, order), order
1608
- )
1609
- for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
1610
- key = (owner, closure["id"])
1611
- previous = closed_deferred.get(key)
1612
- if previous is None or order > previous[0]:
1613
- # The parent comes first, preserving its reason on equal timestamps.
1614
- closed_deferred[key] = (order, closure, relative)
1615
- reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
1616
- reopened_generic: set[tuple[str | None, str]] = set()
1617
- for key, (order, _, _) in list(closed_deferred.items()):
1618
- # Equal timestamps cannot distinguish closure from reopened work.
1619
- reopened = [
1620
- active
1621
- for (owner, identity), active in active_deferred.items()
1622
- if owner == key[0]
1623
- and (identity == key[1] or active[1].get("candidateId") == key[1])
1624
- and active[0] >= order
1625
- ]
1626
- if key in candidate_ids or key in ambiguous_deferred or reopened:
1627
- del closed_deferred[key]
1628
- for _, item, _ in reopened:
1629
- reopened_rows.append((key[0], item))
1630
- if key not in candidate_ids:
1631
- reopened_generic.add(key)
1632
- # Retain every distinct ambiguous task even when a terminal result supersedes
1633
- # its source. Existing output ID normalization keeps each row distinct.
1634
- for relative, _, owner in all_sources:
1635
- for row in deferred_rows[relative]:
1636
- if (
1637
- isinstance(row, dict)
1638
- and isinstance(identity := row.get("id"), str)
1639
- and (owner, identity) in ambiguous_deferred
1640
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1641
- and (owner, row) not in reopened_rows
1642
- ):
1643
- reopened_rows.append((owner, row))
1644
- parent_closures = [
1645
- closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
1646
- ]
1647
- coverage.pop("resolvedDeferred", None)
1648
- if parent_closures:
1649
- coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
1650
- if parent:
1651
- coverage["deferred"] = [
1652
- row
1653
- for row in deferred_rows["parent"]
1654
- if not isinstance(row, dict)
1655
- or not isinstance(row.get("id"), str)
1656
- or (None, row["id"]) not in closed_deferred
1657
- ]
1658
- resolved: dict[tuple[str | None, str], str] = {}
1659
-
1660
- ordered_candidates = {
1661
- (owner, identity)
1662
- for owner, row in reopened_rows
1663
- if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
1664
- and (owner, identity) in candidate_ids
1665
- }
1666
- ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
1667
-
1668
- outcomes: list[tuple[str, str | None, str, str]] = []
1669
- for relative, draft, owner in current_drafts:
1670
- for finding in draft["findings"]:
1671
- if (
1672
- isinstance(finding, dict)
1673
- and valid_finding(finding)
1674
- and (candidate_id := finding_candidate_id(finding))
1675
- ):
1676
- outcomes.append((relative, owner, candidate_id, "reported"))
1677
- for field in ("surfaces", "explicitExclusions"):
1678
- items = draft["coverage"].get(field, [])
1679
- for item in items if isinstance(items, list) else []:
1680
- if (
1681
- isinstance(item, dict)
1682
- and isinstance(item.get("candidateId"), str)
1683
- and item.get("disposition") in {"rejected", "not_applicable"}
1684
- ):
1685
- outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
1686
- ordered_candidates.update(
1687
- (owner, candidate_id)
1688
- for relative, owner, candidate_id, _ in outcomes
1689
- if owner is not None and relative in selected_observations
1690
- )
1691
- # Reopened work and selected checkpoint outcomes follow the saved source order.
1692
- for relative, owner, candidate_id, disposition in outcomes:
1693
- key = (owner, candidate_id)
1694
- if key not in ordered_candidates:
1695
- if relative == "parent" or relative in current_results:
1696
- resolved.setdefault(key, disposition)
1697
- continue
1698
- order = source_order[relative]
1699
- if any(
1700
- saved_owner == owner
1701
- and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
1702
- and modified >= order
1703
- for (saved_owner, _), (modified, row, _) in active_deferred.items()
1704
- ):
1705
- continue
1706
- if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
1707
- resolved[key] = disposition
1708
- ordered_outcomes[key] = (order, relative)
1709
- # Only the current parent may claim that another worker finding was absorbed.
1710
- # A superseded checkpoint must not suppress a newer independent result.
1711
- if parent:
1712
- for finding in parent["findings"]:
1713
- if valid_finding(finding):
1714
- canonical_key = _finding_key(finding)
1715
- for retained in _retained_findings(finding):
1716
- retained_key = _finding_key(retained)
1717
- if retained is not finding:
1718
- represented_history.setdefault(retained_key, set()).add(
1719
- _digest(_finding_content(retained))
1720
- )
1721
- previous_key = represented.get(retained_key)
1722
- if retained_key not in represented:
1723
- represented[retained_key] = canonical_key
1724
- elif previous_key != canonical_key:
1725
- # Ambiguous history cannot suppress an independent source.
1726
- represented[retained_key] = None
1727
- originals = finding["provenance"].get("sourceFindings", [])
1728
- for original in originals if isinstance(originals, list) else []:
1729
- if isinstance(original, dict) and isinstance(original.get("finding"), dict):
1730
- source_id = original.get("id")
1731
- candidate_id = finding_candidate_id(original["finding"])
1732
- if isinstance(source_id, str) and ":" in source_id and candidate_id:
1733
- candidate_key = _worker_candidate_key(
1734
- source_id.rsplit(":", 1)[0],
1735
- candidate_id,
1736
- original["finding"],
1737
- )
1738
- previous_key = represented_candidates.get(candidate_key)
1739
- if candidate_key not in represented_candidates:
1740
- represented_candidates[candidate_key] = canonical_key
1741
- elif previous_key != canonical_key:
1742
- # Candidate ids are only authoritative within one
1743
- # logical worker. Multiple canonical owners make
1744
- # that worker-local identity ambiguous.
1745
- represented_candidates[candidate_key] = None
1746
- represented_candidate_history.setdefault(candidate_key, set()).add(
1747
- _digest(_finding_content(original["finding"]))
1748
- )
1749
- selected_terminal_orders: dict[str, tuple[int, int]] = {}
1750
- for relative, draft, worker_id in sources:
1751
- if (
1752
- worker_id is not None
1753
- and relative in selected_observations
1754
- and draft.get("complete") is not False
1755
- ):
1756
- order = source_order[relative]
1757
- selected_terminal_orders[worker_id] = max(
1758
- selected_terminal_orders.get(worker_id, order), order
1759
- )
1760
- terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
1761
- for relative, draft, worker_id in sources:
1762
- if relative in current_results and draft.get("complete") is not False:
1763
- order = source_order[relative]
1764
- terminal_worker_orders[worker_id] = max(
1765
- terminal_worker_orders.get(worker_id, order), order
1766
- )
1767
-
1768
- def source_coverage_selection(
1769
- relative: str,
1770
- worker_id: str | None,
1771
- ) -> tuple[bool, set[str], bool, bool]:
1772
- worker_result_order = terminal_worker_orders.get(worker_id)
1773
- selected_coverage_superseded = worker_id in selected_terminal_orders and (
1774
- source_order[relative] < selected_terminal_orders[worker_id]
1775
- or (
1776
- relative in current_results
1777
- and source_order[relative] <= selected_terminal_orders[worker_id]
1778
- )
1779
- )
1780
- selected_candidates = {
1781
- candidate_id
1782
- for (owner, candidate_id), (_, source) in ordered_outcomes.items()
1783
- if owner == worker_id and source == relative
1784
- }
1785
- superseded = (
1786
- worker_id is None
1787
- and parent is not None
1788
- and parent.get("complete") is not False
1789
- and relative != "parent"
1790
- and source_order[relative] <= (0, parent_modified)
1791
- and (not stopped_parent_seal or relative in parent_preserved_sources)
1792
- ) or (
1793
- relative not in current_results
1794
- and worker_result_order is not None
1795
- and (
1796
- relative not in selected_observations
1797
- or source_order[relative] < worker_result_order
1798
- )
1799
- )
1800
- # A failed result write can leave pending work outside the accepted result.
1801
- accepted_order = (
1802
- (0, parent_modified)
1803
- if worker_id is None and parent is not None
1804
- else worker_result_order
1805
- )
1806
- retain_pending = (
1807
- superseded
1808
- and (worker_id in headed_workers or (worker_id is None and parent_heads))
1809
- and accepted_order is not None
1810
- and source_order[relative] >= accepted_order
1811
- and any(isinstance(row, dict) for row in deferred_rows[relative])
1812
- )
1813
- return selected_coverage_superseded, selected_candidates, superseded, retain_pending
1814
-
1815
- def coverage_row_is_retained(
1816
- field: str,
1817
- item: Any,
1818
- worker_id: str | None,
1819
- selection: tuple[bool, set[str], bool, bool],
1820
- ) -> bool:
1821
- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
1822
- # A selected terminal checkpoint replaces ordinary coverage, while
1823
- # candidate evidence and pending tasks retain their own reconciliation.
1824
- if (
1825
- selected_coverage_superseded
1826
- and field != "deferred"
1827
- and not (
1828
- isinstance(item, dict)
1829
- and any(key in item for key in ("candidateId", "candidate", "finding"))
1830
- )
1831
- ):
1832
- return False
1833
- # A selected outcome must retain its evidence even if its result write failed.
1834
- if superseded and not (
1835
- isinstance(item, dict)
1836
- and (
1837
- (field == "deferred" and retain_pending)
1838
- or (
1839
- isinstance(item.get("candidateId"), str)
1840
- and item["candidateId"] in selected_candidates
1841
- and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
1842
- )
1843
- )
1844
- ):
1845
- return False
1846
- if (
1847
- isinstance(item, dict)
1848
- and isinstance(
1849
- identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
1850
- if field == "deferred"
1851
- else item.get("candidateId"),
1852
- str,
1853
- )
1854
- and (worker_id, identity) in resolved
1855
- and (field == "deferred" or item.get("disposition") == "needs_follow_up")
1856
- ):
1857
- return False
1858
- return True
1859
-
1860
- ambiguous_surface_ids = {
1861
- (owner, identity)
1862
- for owner, row in reopened_rows
1863
- if isinstance(row.get("id"), str)
1864
- and (owner, row["id"]) in ambiguous_deferred
1865
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1866
- for identity in [
1867
- row["id"],
1868
- *(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
1869
- ]
1870
- if isinstance(identity, str)
1871
- }
1872
- for _, draft, owner in all_sources:
1873
- surfaces = draft["coverage"].get("surfaces", [])
1874
- by_id: dict[str, dict[str, Any]] = {}
1875
- for surface in surfaces if isinstance(surfaces, list) else []:
1876
- if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
1877
- continue
1878
- if identity in by_id and surface != by_id[identity]:
1879
- ambiguous_surface_ids.add((owner, identity))
1880
- by_id[identity] = surface
1881
-
1882
- replaced_deferred = {
1883
- id(row)
1884
- for relative, _, owner in all_sources
1885
- for row in deferred_rows[relative]
1886
- if isinstance(row, dict)
1887
- and isinstance(identity := row.get("id"), str)
1888
- and (key := (owner, identity)) not in candidate_ids
1889
- and key not in ambiguous_deferred
1890
- and (updated := accepted_deferred_orders.get(key)) is not None
1891
- and updated > source_order[relative]
1892
- }
1893
- fallback_surfaces = []
1894
- if surface_links and isinstance(coverage.get("surfaces"), list):
1895
- # Plan the surviving associations before historical receipts can claim
1896
- # a spelling needed by a surface that must be restored for its task.
1897
- prospective_tasks = []
1898
- prospective_surfaces = {}
1899
- prospective_named = {}
1900
- for relative, draft, owner in all_sources:
1901
- selection = source_coverage_selection(relative, owner)
1902
- for row in deferred_rows[relative]:
1903
- if (
1904
- coverage_row_is_retained("deferred", row, owner, selection)
1905
- and id(row) not in replaced_deferred
1906
- and not (
1907
- isinstance(row, dict)
1908
- and isinstance(row.get("id"), str)
1909
- and (owner, row["id"]) in closed_deferred
1910
- )
1911
- ):
1912
- prospective_tasks.append(
1913
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
1914
- )
1915
- rows = draft["coverage"].get("surfaces", [])
1916
- for row in rows if isinstance(rows, list) else []:
1917
- if owner is None or not isinstance(row, dict):
1918
- continue
1919
- emitted = coverage_row_is_retained("surfaces", row, owner, selection)
1920
- ambiguous = (
1921
- isinstance(row.get("id"), str)
1922
- and (owner, row["id"]) in ambiguous_surface_ids
1923
- and row.get("disposition") == "needs_follow_up"
1924
- and not any(key in row for key in ("candidateId", "candidate", "finding"))
1925
- )
1926
- if not emitted and not ambiguous:
1927
- continue
1928
- projected = _project_worker_coverage_row(row, owner, "surfaces")
1929
- if isinstance(row.get("id"), str):
1930
- named = prospective_named.setdefault((owner, row["id"]), {})
1931
- projected = next(
1932
- (existing for existing in named.values() if existing == projected),
1933
- projected,
1934
- )
1935
- named[id(projected)] = projected
1936
- if emitted:
1937
- prospective_surfaces[id(row)] = projected
1938
- prospective_tasks.extend(
1939
- (owner, _project_worker_coverage_row(row, owner, "deferred"))
1940
- for owner, row in reopened_rows
1941
- if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
1942
- )
1943
- bound = {
1944
- (id(retained), published_id)
1945
- for task, published_id, surface, _, task_source in surface_links
1946
- if linked_surface(surface, prospective_surfaces, prospective_named) is not None
1947
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
1948
- }
1949
- for task, published_id, _, parent_surface, task_source in surface_links:
1950
- for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
1951
- key = (id(retained), published_id)
1952
- if key not in bound:
1953
- fallback_surfaces.append(parent_surface)
1954
- bound.add(key)
441
+ return _saved_results_changed(db, connection, scan)
1955
442
 
1956
- replaced_surfaces, surface_updates = _generic_surface_updates(
1957
- all_sources,
1958
- source_order,
1959
- closed_deferred,
1960
- active_deferred,
1961
- resolved,
1962
- reopened_generic,
1963
- coverage_schema["surfaces"]["items"],
1964
- deferred_rows,
1965
- ambiguous_deferred,
1966
- receipt_root=scan_dir if project_worker_coverage else None,
1967
- receipt_warnings=warnings,
1968
- fallback_surfaces=fallback_surfaces,
1969
- surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
1970
- "surfaces", row, owner, source_coverage_selection(relative, owner)
1971
- ),
1972
- eligible_receipt_sources={
1973
- relative
1974
- for relative, draft, _ in all_sources
1975
- if draft.get("complete") is not False or relative in selected_observations
1976
- },
1977
- )
1978
- replaced_rows = {
1979
- "surfaces": replaced_surfaces,
1980
- "deferred": replaced_deferred,
1981
- }
1982
- for field, replaced in replaced_rows.items():
1983
- if parent and isinstance(coverage.get(field), list):
1984
- previous = parent["coverage"].get(field, [])
1985
- if isinstance(previous, list):
1986
- removed_parent = [row for row in previous if id(row) in replaced]
1987
- coverage[field] = [row for row in coverage[field] if row not in removed_parent]
1988
- if isinstance(coverage.get("surfaces"), list):
1989
- for owner, surface, replaced_sources in surface_updates:
1990
- retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
1991
- for source_id in replaced_sources & linked_surface_sources:
1992
- linked_surface_outputs[source_id] = retained
1993
- for surface in coverage["surfaces"]:
1994
- if isinstance(surface, dict):
1995
- surface.setdefault("receiptRefs", [])
1996
443
 
1997
- # Reopened work survives a superseded checkpoint, but current candidate
1998
- # outcomes still apply. Parent closures cannot remove another worker's row.
1999
- for owner, item in reopened_rows:
2000
- if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
2001
- owner,
2002
- identity,
2003
- ) in resolved:
2004
- continue
2005
- pending = coverage.setdefault("deferred", [])
2006
- if isinstance(pending, list):
2007
- append_coverage_row(pending, item, owner, "deferred")
2008
- for _, draft, owner in all_sources:
2009
- surfaces = draft["coverage"].get("surfaces", [])
2010
- for surface in surfaces if isinstance(surfaces, list) else []:
2011
- if (
2012
- isinstance(surface, dict)
2013
- and isinstance(surface.get("id"), str)
2014
- and (owner, surface["id"]) in ambiguous_surface_ids
2015
- and surface.get("disposition") == "needs_follow_up"
2016
- and not any(key in surface for key in ("candidateId", "candidate", "finding"))
2017
- and isinstance(coverage.get("surfaces"), list)
2018
- ):
2019
- append_coverage_row(
2020
- coverage["surfaces"],
2021
- {**surface, "receiptRefs": surface.get("receiptRefs", [])},
2022
- owner,
2023
- "surfaces",
2024
- )
2025
- for relative, draft, worker_id in all_sources:
2026
- selection = source_coverage_selection(relative, worker_id)
2027
- selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
2028
- skip_superseded_findings = (
2029
- superseded
2030
- and not stopped
2031
- and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
444
+ @contextmanager
445
+ def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
446
+ """Keep rejected completion attempts from becoming accepted parent observations."""
447
+ head_path = scan_dir / "checkpoint-head.json"
448
+ previous = None
449
+ try:
450
+ head_path.lstat()
451
+ except FileNotFoundError:
452
+ pass
453
+ else:
454
+ descriptor = open_scan_local_file_descriptor(
455
+ scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
2032
456
  )
2033
- if skip_superseded_findings and not selected_candidates and not retain_pending:
2034
- continue
2035
- if (
2036
- not skip_superseded_findings
2037
- and "threatModel" not in manifest["scan"]
2038
- and isinstance(draft.get("threatModel"), dict)
2039
- ):
2040
- model = draft["threatModel"]
2041
- model_path = relative
2042
- checkpoint_dir = Path(relative).parent
2043
- prefer_worker_head = worker_id is not None and (
2044
- checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
2045
- )
2046
- if worker_id is not None:
2047
- if checkpoint_dir.name != "checkpoints":
2048
- checkpoint_dir /= "checkpoints"
2049
- selected_models = [
2050
- path
2051
- for path in selected_observations
2052
- if Path(path).parent == checkpoint_dir
2053
- and isinstance(drafts_by_path[path].get("threatModel"), dict)
2054
- ]
2055
- if selected_models:
2056
- head_path = max(selected_models, key=source_order.__getitem__)
2057
- current = drafts_by_path[head_path]
2058
- # A terminal checkpoint is committed before result.json is replaced.
2059
- # Use the admitted observation, including its frozen ordering on retries.
2060
- if not prefer_worker_head and current.get("complete") is not False:
2061
- prefer_worker_head = source_order[head_path] >= source_order[relative]
2062
- if prefer_worker_head:
2063
- model = current["threatModel"]
2064
- model_path = head_path
2065
- manifest["scan"]["threatModel"] = copy.deepcopy(model)
2066
- if worker_id is not None:
2067
- manifest["scan"]["threatModel"]["origin"] = "recovered"
2068
- if selected_model_source is not None and worker_id is not None:
2069
- selected_model_source[:] = [model_path]
2070
- for value in draft["findings"]:
2071
- if skip_superseded_findings and not (
2072
- isinstance(value, dict)
2073
- and (candidate_id := finding_candidate_id(value)) in selected_candidates
2074
- and resolved.get((worker_id, candidate_id)) == "reported"
2075
- ):
2076
- continue
2077
- if relative == "parent" and parent_is_canonical:
2078
- finding = copy.deepcopy(value)
2079
- _ensure_finding_identity(finding, candidate_only=True)
2080
- provenance = finding.get("provenance") if isinstance(finding, dict) else None
2081
- owner = provenance.get("workerId") if isinstance(provenance, dict) else None
2082
- candidate_id = finding_candidate_id(finding) if isinstance(finding, dict) else None
2083
- if (
2084
- stopped_parent_seal
2085
- and isinstance(owner, str)
2086
- and candidate_id
2087
- and resolved.get((owner, candidate_id)) in {"rejected", "not_applicable"}
2088
- ):
2089
- rejected_history.setdefault((owner, candidate_id), []).append(finding)
2090
- continue
2091
- if valid_finding(finding):
2092
- finding_positions.setdefault(_finding_key(finding), len(findings))
2093
- findings.append(finding)
2094
- continue
2095
- if relative != "parent" and parent and value in parent["findings"]:
2096
- continue
2097
- if not isinstance(value, dict):
2098
- warnings.append(f"Retained malformed finding evidence in {relative}.")
2099
- continue
2100
- finding = copy.deepcopy(value)
2101
- candidate_id = finding_candidate_id(finding)
2102
- if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
2103
- "rejected",
2104
- "not_applicable",
2105
- }:
2106
- surfaces = coverage.get("surfaces")
2107
- for item in surfaces if isinstance(surfaces, list) else []:
2108
- if isinstance(item, dict) and item.get("candidateId") == candidate_id:
2109
- if not isinstance(item.get("previousFindings"), list):
2110
- item["previousFindings"] = []
2111
- history = item["previousFindings"]
2112
- if not any(
2113
- isinstance(previous, dict)
2114
- and _finding_key(previous) == _finding_key(finding)
2115
- and _finding_content(previous) == _finding_content(finding)
2116
- for previous in history
2117
- ):
2118
- history.append(finding)
2119
- continue
2120
- for key in ("findingId", "occurrenceId", "fingerprints"):
2121
- finding.pop(key, None)
2122
- locations = finding.get("locations", [])
2123
- if not isinstance(locations, list) or not any(
2124
- isinstance(location, dict)
2125
- and isinstance(location.get("path"), str)
2126
- and any(
2127
- path_within_scope(location["path"], path)
2128
- for path in binding["scope"]["includePaths"]
2129
- )
2130
- for location in locations
2131
- ):
2132
- warnings.append(f"Skipped out-of-scope finding from {relative}.")
2133
- continue
2134
- provenance = finding.setdefault("provenance", {"source": "local_plugin"})
2135
- if not isinstance(provenance, dict):
2136
- findings.append(finding)
2137
- continue
2138
- if worker_id:
2139
- provenance.setdefault("workerId", worker_id)
2140
- _ensure_finding_identity(finding)
2141
- if not valid_finding(finding):
2142
- findings.append(finding)
2143
- continue
2144
- key = _finding_key(finding)
2145
- represented_by_parent = False
2146
- if relative != "parent":
2147
- if key in represented:
2148
- mapped_key = represented[key]
2149
- historical_contents = represented_history.get(key, set())
2150
- elif worker_id and candidate_id:
2151
- candidate_key = _worker_candidate_key(worker_id, candidate_id, finding)
2152
- if candidate_key not in represented_candidates:
2153
- represented_candidates[candidate_key] = key
2154
- mapped_key = represented_candidates[candidate_key]
2155
- historical_contents = represented_candidate_history.get(candidate_key, set())
2156
- else:
2157
- mapped_key = None
2158
- historical_contents = set()
2159
- if mapped_key is not None:
2160
- key = mapped_key
2161
- represented_by_parent = _digest(_finding_content(value)) in historical_contents
2162
- if key in finding_positions:
2163
- retained = findings[finding_positions[key]]
2164
- if finding != retained:
2165
- if not represented_by_parent and _finding_strength(finding) > _finding_strength(
2166
- retained
2167
- ):
2168
- previous = copy.deepcopy(retained)
2169
- previous_history = previous["provenance"].pop("previousFindings", [])
2170
- retained = finding
2171
- findings[finding_positions[key]] = retained
2172
- else:
2173
- previous = copy.deepcopy(value)
2174
- previous_history = previous.get("provenance", {}).pop(
2175
- "previousFindings", []
2176
- )
2177
- retained_provenance = retained["provenance"]
2178
- retained_history = retained_provenance.get("previousFindings")
2179
- history = (
2180
- [item for item in retained_history if isinstance(item, dict)]
2181
- if isinstance(retained_history, list)
2182
- else []
2183
- )
2184
- retained_provenance["previousFindings"] = history
2185
- for original in [
2186
- *(previous_history if isinstance(previous_history, list) else []),
2187
- previous,
2188
- ]:
2189
- if not isinstance(original, dict):
2190
- continue
2191
- source_key = _finding_key(original)
2192
- source_content = _finding_content(original)
2193
- already_retained = any(
2194
- source_key == _finding_key(historical)
2195
- and source_content == _finding_content(historical)
2196
- for historical in _retained_findings(retained)
2197
- )
2198
- if (
2199
- not already_retained
2200
- and original not in history
2201
- and original != retained
2202
- ):
2203
- history.append(original)
2204
- continue
2205
- finding_positions[key] = len(findings)
2206
- findings.append(finding)
2207
- if superseded and not selected_candidates and not retain_pending:
2208
- continue
2209
- for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
2210
- if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
2211
- continue
2212
- items = draft["coverage"].get(field, [])
2213
- if field == "deferred":
2214
- items = deferred_rows[relative]
2215
- if not isinstance(items, list):
2216
- continue
2217
- output = coverage.setdefault(field, [])
2218
- if not isinstance(output, list):
2219
- # Keep malformed canonical collections for the existing finalizer's
2220
- # recovery and warnings rather than silently changing its contract.
2221
- continue
2222
- for item in items:
2223
- source_item = item
2224
- if not coverage_row_is_retained(field, item, worker_id, selection):
2225
- continue
2226
- if id(item) in replaced_rows.get(field, ()):
2227
- continue
2228
- if (
2229
- field == "deferred"
2230
- and isinstance(item, dict)
2231
- and isinstance(item.get("id"), str)
2232
- and (worker_id, item["id"]) in closed_deferred
2233
- ):
2234
- continue
2235
- if field == "openQuestions" and isinstance(item, str):
2236
- item = {"question": item.strip()}
2237
- if (
2238
- field == "surfaces"
2239
- and isinstance(item, dict)
2240
- and item.get("disposition") in {"rejected", "not_applicable"}
2241
- and isinstance(item.get("candidateId"), str)
2242
- and (history_findings := rejected_history.get((worker_id, item["candidateId"])))
2243
- ):
2244
- item = copy.deepcopy(item)
2245
- if not isinstance(item.get("previousFindings"), list):
2246
- item["previousFindings"] = []
2247
- history = item["previousFindings"]
2248
- for finding in history_findings:
2249
- if not any(
2250
- isinstance(previous, dict)
2251
- and _finding_key(previous) == _finding_key(finding)
2252
- and _finding_content(previous) == _finding_content(finding)
2253
- for previous in history
2254
- ):
2255
- history.append(copy.deepcopy(finding))
2256
- if field == "surfaces" and isinstance(item, dict):
2257
- item = {**item, "receiptRefs": item.get("receiptRefs", [])}
2258
- if isinstance(item, dict) and "id" not in item:
2259
- semantic_item = dict(item)
2260
- existing = next(
2261
- (
2262
- row
2263
- for row in output
2264
- if isinstance(row, dict)
2265
- and {key: value for key, value in row.items() if key != "id"}
2266
- == semantic_item
2267
- ),
2268
- None,
2269
- )
2270
- if existing is not None:
2271
- if project_worker_coverage and worker_id is not None:
2272
- worker_row_owners[id(existing)] = worker_id
2273
- if field == "surfaces" and id(source_item) in linked_surface_sources:
2274
- linked_surface_outputs[id(source_item)] = existing
2275
- continue
2276
- retained = append_coverage_row(output, item, worker_id, field)
2277
- if field == "surfaces" and id(source_item) in linked_surface_sources:
2278
- linked_surface_outputs[id(source_item)] = retained
2279
-
2280
- identities: dict[str, str] = {}
2281
- for finding in findings:
2282
- if not valid_finding(finding):
2283
- continue
2284
- identity = finding.get("identity")
2285
- if not isinstance(identity, dict):
2286
- continue
2287
- key = _encoded([finding.get("ruleId"), identity]).decode()
2288
- variant = _finding_key(finding)
2289
- if key in identities and identities[key] != variant:
2290
- finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)
2291
- identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"
2292
- identities[key] = variant
2293
- # Bind links to retained output objects before their IDs become unique.
2294
- # Candidate metadata can give an anonymous row the same preliminary ID as
2295
- # another surface, so only the final surface ID can be used for its links.
2296
- retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
2297
- output_tasks = coverage.get("deferred", [])
2298
- task_candidates = [
2299
- (worker_row_owners.get(id(row)), row)
2300
- for row in (output_tasks if isinstance(output_tasks, list) else [])
2301
- ]
2302
- for task, published_id, surface, _, task_source in surface_links:
2303
- retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
2304
- if retained_surface is None or not isinstance(retained_surface.get("id"), str):
2305
- continue
2306
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
2307
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
2308
- links.setdefault(published_id, retained_surface)
2309
- # A newer worker checkpoint may omit a surface still needed by a retained
2310
- # task. Restore its parent copy only when no worker replacement survived.
2311
- for task, published_id, surface, parent_surface, task_source in surface_links:
2312
- for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
2313
- _, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
2314
- if published_id not in links and isinstance(coverage.get("surfaces"), list):
2315
- retained_surface = append_coverage_row(
2316
- coverage["surfaces"], parent_surface, None, "surfaces"
2317
- )
2318
- record_coverage_owner(
2319
- retained_surface, surface, surface.get("workerId"), "surfaces"
2320
- )
2321
- links[published_id] = retained_surface
2322
- if project_worker_coverage:
2323
- worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
2324
- named_surface_outputs = {
2325
- identity for outputs in worker_surface_outputs.values() for identity in outputs
2326
- }
2327
- surfaces = coverage.get("surfaces", [])
2328
- for surface in surfaces if isinstance(surfaces, list) else []:
2329
- if not isinstance(surface, dict):
2330
- continue
2331
- owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
2332
- if isinstance(owner, str) and isinstance(identity, str):
2333
- key = (owner, identity)
2334
- existing = worker_surfaces.get(key)
2335
- # An anonymous candidate ID can collide with an explicit surface ID.
2336
- # Bind a named worker link to the explicitly named source.
2337
- if existing is None or (
2338
- id(surface) in named_surface_outputs
2339
- and id(existing) not in named_surface_outputs
2340
- ):
2341
- worker_surfaces[key] = surface
2342
- for task in output_tasks if isinstance(output_tasks, list) else []:
2343
- if (
2344
- not isinstance(task, dict)
2345
- or (owner := worker_row_owners.get(id(task))) is None
2346
- or not isinstance(task.get("surfaceIds"), list)
2347
- ):
2348
- continue
2349
- for identity in task["surfaceIds"]:
2350
- if (
2351
- isinstance(identity, str)
2352
- and (surface := worker_surfaces.get((owner, identity))) is not None
2353
- ):
2354
- _, links = retained_surface_links.setdefault(id(task), (task, {}))
2355
- links.setdefault(identity, surface)
2356
- for field in ("surfaces", "explicitExclusions", "deferred"):
2357
- used: set[str] = set()
2358
- items = coverage.setdefault(field, [])
2359
- reserved = {
2360
- row["id"]
2361
- for row in (items if isinstance(items, list) else [])
2362
- if isinstance(row, dict) and isinstance(row.get("id"), str)
457
+ with os.fdopen(descriptor, "rb") as handle:
458
+ metadata = os.fstat(handle.fileno())
459
+ previous = (handle.read(), metadata)
460
+ directories = ("checkpoints", "checkpoint-heads")
461
+ previous_files = {
462
+ relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
463
+ }
464
+ try:
465
+ yield
466
+ except BaseException:
467
+ if previous is None:
468
+ _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
469
+ else:
470
+ payload, metadata = previous
471
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
472
+ os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
473
+ current_files = {
474
+ relative
475
+ for directory in directories
476
+ for relative in _checkpoint_paths(scan_dir, directory)
2363
477
  }
2364
- for item in items if isinstance(items, list) else []:
2365
- if not isinstance(item, dict):
2366
- continue
2367
- if id(item) in canonical_rows:
2368
- if isinstance(item.get("id"), str):
2369
- used.add(item["id"])
2370
- continue
2371
- item.setdefault("id", _saved_coverage_id(item))
2372
- if not isinstance(item["id"], str):
2373
- # Preserve malformed rows for per-record recovery, including frozen replay.
2374
- continue
2375
- if item["id"] in used:
2376
- if project_worker_coverage:
2377
- base = item["id"]
2378
- suffix = 2
2379
- while item["id"] in used or item["id"] in reserved:
2380
- item["id"] = f"{base}-{suffix}"
2381
- suffix += 1
2382
- else:
2383
- item["id"] = f"{item['id']}-{_digest(item)[:16]}"
2384
- used.add(item["id"])
2385
- if field == "surfaces":
2386
- item.setdefault("receiptRefs", [])
2387
- for task, links in retained_surface_links.values():
2388
- task["surfaceIds"] = [
2389
- links[identity]["id"] if isinstance(identity, str) and identity in links else identity
2390
- for identity in task["surfaceIds"]
2391
- ]
2392
- coverage.pop("completeness", None)
2393
- if stopped:
2394
- if not isinstance(coverage.get("deferred"), list):
2395
- coverage["deferred"] = []
2396
- item = {"id": "scan-stopped", "reason": reason}
2397
- if item not in coverage["deferred"]:
2398
- coverage["deferred"].append(item)
2399
- return manifest, {"findings": findings}, coverage
478
+ for relative in current_files - previous_files:
479
+ _remove_scan_local_file_if_exists(scan_dir, relative)
480
+ raise
2400
481
 
2401
482
 
2402
483
  def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
@@ -2413,107 +494,141 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
2413
494
  return prepared.coverage
2414
495
 
2415
496
 
2416
- def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]:
2417
- snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {}
2418
- for relative in _PUBLISHED_OUTPUTS:
2419
- descriptor = -1
2420
- try:
2421
- descriptor = open_scan_local_file_descriptor(
2422
- scan_dir, relative, "Published scan output"
2423
- )
2424
- with os.fdopen(descriptor, "rb") as handle:
2425
- descriptor = -1
2426
- metadata = os.fstat(handle.fileno())
2427
- snapshots[relative] = (handle.read(), metadata)
2428
- except ContractError:
2429
- path = scan_dir / relative
2430
- if path.exists() or path.is_symlink():
2431
- if relative == "threatmodel.md":
2432
- # This optional projection will not replace an unsafe destination.
2433
- continue
2434
- raise
2435
- snapshots[relative] = None
2436
- finally:
2437
- if descriptor >= 0:
2438
- os.close(descriptor)
2439
- return snapshots
2440
-
2441
-
2442
- def _restore_published_outputs(
2443
- scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
2444
- ) -> None:
2445
- for relative, snapshot in snapshots.items():
2446
- if snapshot is None:
2447
- path = scan_dir / relative
2448
- if path.exists() or path.is_symlink():
2449
- _remove_scan_local_file_if_exists(scan_dir, relative)
2450
- else:
2451
- contents, metadata = snapshot
2452
- write_scan_local_bytes(scan_dir, relative, contents)
2453
- # Saved-result selection uses mtime; rollback must preserve its ordering.
2454
- os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
2455
-
2456
-
2457
- @contextmanager
2458
- def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
2459
- """Recover a settled, zero-accepted Deep deadline without sealing the parent."""
497
+ def preserve_deep_deadline_draft(
498
+ db: Any,
499
+ connection: Any,
500
+ scan: Any,
501
+ *,
502
+ preserve_reduction: bool = False,
503
+ allow_empty: bool = False,
504
+ ) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:
505
+ """Recover unfinished reviews at a settled Deep deadline without sealing the parent."""
2460
506
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2461
- snapshots = _snapshot_published_outputs(scan_dir)
2462
507
  warnings = json.loads(scan["completion_warnings_json"])
2463
508
  binding = db.workbench_completion_binding(scan, db.now())
2464
- with preserve_parent_head_on_error(scan_dir):
2465
- try:
2466
- documents = merge_saved_results(
2467
- scan_dir,
2468
- scan["id"],
2469
- binding,
2470
- connection.execute(
2471
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
2472
- (scan["id"],),
2473
- ).fetchall(),
2474
- warnings,
2475
- stopped=True,
2476
- reason=(
2477
- "The configured discovery time limit elapsed before any source review completed. "
2478
- "Saved findings and pending review were preserved without a completed reduction."
2479
- ),
2480
- project_worker_coverage=True,
2481
- )
2482
- if documents is None:
509
+ workers = connection.execute(
510
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
511
+ (scan["id"],),
512
+ ).fetchall()
513
+ if preserve_reduction:
514
+ # Completed reviews already went through reduction. Only add evidence
515
+ # from unfinished reviews; raw accepted drafts must not undo that result.
516
+ workers = [
517
+ worker
518
+ for worker in workers
519
+ if worker["kind"] == "discovery" and worker["status"] == "canceled"
520
+ ]
521
+ if not workers:
522
+ return None
523
+ _prepare_scan_finalization(
524
+ scan_dir,
525
+ expected_coverage_mode=db.expected_coverage_mode(scan),
526
+ completion_binding=binding,
527
+ allow_missing_writeups=True,
528
+ )
529
+ snapshots = _snapshot_published_outputs(
530
+ scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json")
531
+ )
532
+ created_paths: set[str] = set()
533
+ try:
534
+ documents = merge_saved_results(
535
+ scan_dir,
536
+ scan["id"],
537
+ binding,
538
+ workers,
539
+ warnings,
540
+ stopped=not preserve_reduction,
541
+ preserve_canonical_parent=preserve_reduction,
542
+ project_worker_coverage=True,
543
+ # Without reduction, the coordinator's empty draft cannot replace worker evidence.
544
+ scan_mode="deep" if preserve_reduction else None,
545
+ finding_scope=db.requested_finding_scope(scan),
546
+ created_paths=created_paths,
547
+ current_contract=True,
548
+ reason=(
549
+ "The configured discovery time limit elapsed before any source review completed. "
550
+ "Saved findings and pending review were preserved without a completed reduction."
551
+ ),
552
+ )
553
+ if documents is None:
554
+ if not allow_empty:
2483
555
  raise ContractError("Deep Scan deadline has no saved parent draft.")
2484
- documents[0]["scan"].pop("complete", None)
2485
- # Validate recovered evidence against the current finding contract,
2486
- # scope, source lineage, and receipts before admitting this draft.
2487
- prepared = _prepare_scan_finalization(
2488
- scan_dir,
2489
- expected_coverage_mode=db.expected_coverage_mode(scan),
2490
- completion_binding=binding,
2491
- recover_unsealed=True,
2492
- draft_documents=documents,
2493
- )
2494
- warnings.extend(prepared.warnings)
2495
- manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
2496
- manifest["scan"].pop("sealedAt", None)
2497
- manifest["scan"].pop("artifacts", None)
2498
- checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
2499
- contents = _encoded(checkpoint)
2500
- name = f"{hashlib.sha256(contents).hexdigest()}.json"
2501
- write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
2502
- write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
2503
- for filename, document in (
2504
- ("findings.json", findings),
2505
- ("coverage.json", coverage),
2506
- ("scan-manifest.json", manifest),
2507
- ):
2508
- write_scan_local_bytes(scan_dir, filename, _encoded(document))
556
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
2509
557
  connection.execute(
2510
558
  "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
2511
559
  (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
2512
560
  )
2513
- yield
2514
- except BaseException:
2515
- _restore_published_outputs(scan_dir, snapshots)
2516
- raise
561
+ return None
562
+ if preserve_reduction:
563
+ marker = {
564
+ "id": "scan-deadline",
565
+ "reason": (
566
+ "The configured discovery time limit elapsed. Saved findings and "
567
+ "pending work from unfinished source reviews were preserved."
568
+ ),
569
+ }
570
+ if not any(
571
+ marker["id"] in (row.get("id"), row.get("candidateId"))
572
+ for key in ("deferred", "resolvedDeferred")
573
+ for row in documents[2].get(key, [])
574
+ ):
575
+ documents[2]["deferred"].append(marker)
576
+ documents[0]["scan"].pop("complete", None)
577
+ expected_coverage_mode = db.expected_coverage_mode(scan)
578
+ if not preserve_reduction:
579
+ # A capped deadline can retain interrupted records even though its
580
+ # completion binding requires strict validation of completed findings.
581
+ _populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)
582
+ _populate_unsealed_artifact_envelope(*documents, binding)
583
+ if expected_coverage_mode == "deep_repository":
584
+ documents[2]["inventoryStrategy"] = "repository"
585
+ elif expected_coverage_mode == "scoped_path":
586
+ documents[2]["inventoryStrategy"] = "scoped_path"
587
+ schema_dir = Path(__file__).resolve().parent.parent / "schemas"
588
+ discarded_findings = _recover_unsealed_findings(
589
+ documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True
590
+ )
591
+ _recover_unsealed_coverage(
592
+ documents[2], schema_dir, scan_dir, warnings, discarded_findings
593
+ )
594
+ # Reuse stopped-result recovery's per-record validation, scope checks,
595
+ # source lineage, and receipt validation before admitting this draft.
596
+ prepared = _prepare_scan_finalization(
597
+ scan_dir,
598
+ expected_coverage_mode=expected_coverage_mode,
599
+ completion_binding=binding,
600
+ recover_unsealed=not preserve_reduction,
601
+ allow_missing_writeups=preserve_reduction,
602
+ draft_documents=documents,
603
+ finding_scope=db.requested_finding_scope(scan),
604
+ )
605
+ warnings.extend(prepared.warnings)
606
+ manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
607
+ manifest["scan"].pop("sealedAt", None)
608
+ manifest["scan"].pop("artifacts", None)
609
+ checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
610
+ contents = _encoded(checkpoint)
611
+ name = f"{hashlib.sha256(contents).hexdigest()}.json"
612
+ checkpoint_path = f"checkpoints/{name}"
613
+ # This draft is recoverable parent evidence too. Restore its prior state
614
+ # on local publication failure or a later caller transaction rollback.
615
+ snapshots.update(_snapshot_published_outputs(scan_dir, (checkpoint_path,)))
616
+ write_scan_local_bytes(scan_dir, checkpoint_path, contents)
617
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
618
+ for filename, document in (
619
+ ("findings.json", findings),
620
+ ("coverage.json", coverage),
621
+ ("scan-manifest.json", manifest),
622
+ ):
623
+ write_scan_local_bytes(scan_dir, filename, _encoded(document))
624
+ connection.execute(
625
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
626
+ (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
627
+ )
628
+ except BaseException:
629
+ _restore_published_outputs(scan_dir, snapshots, created_paths)
630
+ raise
631
+ return scan_dir, snapshots, created_paths
2517
632
 
2518
633
 
2519
634
  def preserve_scan_results_locked(
@@ -2522,25 +637,44 @@ def preserve_scan_results_locked(
2522
637
  scan_id: str,
2523
638
  *,
2524
639
  recovery_source_digests: dict[str, str] | None = None,
640
+ recovery_checkpoint_heads: dict[str, str | None] | None = None,
2525
641
  include_parent_with_recovery: bool = False,
642
+ recovery_snapshot_error: _CheckpointSnapshotError | None = None,
2526
643
  ) -> bool:
2527
644
  """Publish or verify retained terminal results through the workbench host."""
2528
645
  scan = db.require_scan(connection, scan_id)
2529
646
  if scan["status"] != "failed":
2530
647
  return False
2531
648
  frozen_source_digests: dict[str, str] | None = None
649
+ checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
2532
650
  model_source: list[str] = []
2533
651
  saved_model_source: str | None = None
2534
652
  raw_frozen_sources = scan["retained_source_digests_json"]
653
+ frozen_source_times = None
2535
654
  if raw_frozen_sources is not None:
2536
- frozen_source_digests, saved_model_source = _retained_source_state(
2537
- json.loads(raw_frozen_sources)
2538
- )
655
+ retained = json.loads(raw_frozen_sources)
656
+ frozen_source_digests, saved_model_source = _retained_source_state(retained)
657
+ if isinstance(retained.get("sources"), dict):
658
+ frozen_source_times = retained.get("sourceTimes")
2539
659
  if saved_model_source is not None:
2540
660
  model_source.append(saved_model_source)
661
+ canonical_sources = frozen_source_digests
2541
662
  if recovery_source_digests is not None:
2542
663
  frozen_source_digests = recovery_source_digests
664
+ checkpoint_heads = recovery_checkpoint_heads
2543
665
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
666
+ if frozen_source_digests is not None:
667
+ _restore_frozen_model_capture(
668
+ scan_dir, scan_id, frozen_source_digests, frozen_source_times, saved_model_source
669
+ )
670
+ if frozen_source_times is not None and recovery_source_digests is None:
671
+ _freeze_source_times(
672
+ scan_dir,
673
+ scan_id,
674
+ frozen_source_digests,
675
+ frozen_source_times,
676
+ retained.get("parentContext"),
677
+ )
2544
678
  deep_run = connection.execute(
2545
679
  "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
2546
680
  ).fetchone()
@@ -2591,11 +725,15 @@ def preserve_scan_results_locked(
2591
725
  db.index_findings(connection, scan_id, findings, scan["completed_at"])
2592
726
  connection.execute(
2593
727
  "UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
728
+ "retained_checkpoint_heads_json = ?, "
2594
729
  "completion_warnings_json = ?, "
2595
730
  "updated_at = ? WHERE id = ? AND status = 'failed'",
2596
731
  (
2597
732
  digest,
2598
733
  _encode_retained_sources(retained_sources, model_source),
734
+ json.dumps(
735
+ manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True
736
+ ),
2599
737
  json.dumps(list(dict.fromkeys(warnings))),
2600
738
  timestamp,
2601
739
  scan_id,
@@ -2611,10 +749,7 @@ def preserve_scan_results_locked(
2611
749
  existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
2612
750
  existing = None
2613
751
  if scan["seal_manifest_digest"] is not None or (
2614
- isinstance(existing_scan, dict)
2615
- and (
2616
- existing_scan.get("sealedAt") is not None or existing_scan.get("artifacts") is not None
2617
- )
752
+ isinstance(existing_scan, dict) and manifest_is_sealed(existing_scan)
2618
753
  ):
2619
754
  db.require_recorded_manifest_digest(scan, scan_dir)
2620
755
  existing, existing_findings, _ = finalize_scan(
@@ -2625,11 +760,23 @@ def preserve_scan_results_locked(
2625
760
  db.verify_manifest_binding(scan, existing)
2626
761
  if existing_scan.get("status") == outcome:
2627
762
  existing_sources = existing_scan.get("preservedSources")
763
+ existing_heads = existing_scan.get("preservedCheckpointHeads")
2628
764
  if frozen_source_digests is None:
2629
765
  frozen_source_digests = _source_digests(
2630
766
  existing_sources, "Stopped scan source digests could not be frozen."
2631
767
  )
2632
- if existing_sources == frozen_source_digests:
768
+ checkpoint_heads = existing_heads
769
+ unchanged = existing_sources == frozen_source_digests and (
770
+ existing_heads == checkpoint_heads
771
+ or (
772
+ "preservedCheckpointHeads" not in existing_scan
773
+ and recovery_source_digests is None
774
+ and not pending_recovery
775
+ )
776
+ )
777
+ if unchanged and not any(
778
+ Path(path).name == "checkpoint-head.json" for path in frozen_source_digests
779
+ ):
2633
780
  if (
2634
781
  raw_frozen_sources is not None
2635
782
  and scan["seal_manifest_digest"] is not None
@@ -2639,43 +786,116 @@ def preserve_scan_results_locked(
2639
786
  return True
2640
787
  record_publication(existing, existing_findings)
2641
788
  return True
2642
- if recovery_source_digests is None:
789
+ if not unchanged and recovery_source_digests is None and not pending_recovery:
2643
790
  raise ContractError("Stopped scan sources changed after terminal publication.")
2644
791
  binding = {
2645
792
  **db.workbench_completion_binding(scan, scan["completed_at"], existing),
2646
793
  "status": outcome,
2647
794
  }
2648
- if recovery_source_digests is not None:
795
+ canonical_model_source = saved_model_source
796
+ canonical_model = existing_scan.get("threatModel") if isinstance(existing_scan, dict) else None
797
+ if existing is None and frozen_source_digests is not None:
798
+ # An unsealed live draft is a fallback only when its complete identity
799
+ # belongs to the frozen set. Otherwise recover from the captured sources.
800
+ canonical_model = None
801
+ try:
802
+ _, canonical_draft = _read_saved_parent_result(scan_dir, scan_id)
803
+ digest = _digest(canonical_draft)
804
+ path = f"checkpoints/{digest}.json"
805
+ if frozen_source_digests.get(path) == digest:
806
+ canonical_model_source = path
807
+ canonical_model = canonical_draft.get("threatModel")
808
+ except (ContractError, OSError, ValueError):
809
+ # This live fallback is optional.
810
+ pass
811
+ if recovery_source_digests is not None and not pending_recovery:
2649
812
  model_source.clear()
2650
- documents = merge_saved_results(
2651
- scan_dir,
2652
- scan_id,
2653
- binding,
2654
- connection.execute(
2655
- "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
2656
- (scan_id,),
2657
- ).fetchall(),
2658
- warnings,
2659
- stopped=True,
2660
- reason=(
2661
- f"Scan {outcome}; saved findings and pending review were preserved. "
2662
- f"{scan['failure_message'] or ''}"
2663
- ).strip(),
2664
- frozen_source_digests=frozen_source_digests,
2665
- frozen_model_source=model_source[0] if model_source else None,
2666
- selected_model_source=model_source,
2667
- project_worker_coverage=scan["mode"] == "deep",
2668
- allow_frozen_legacy_parent=(
2669
- include_parent_with_recovery
2670
- or (
2671
- recovery_source_digests is None
2672
- and frozen_source_digests == {}
2673
- and isinstance(existing_scan, dict)
2674
- and existing_scan.get("sealedAt") is not None
2675
- and "preservedSources" not in existing_scan
2676
- )
2677
- ),
813
+ # Older pending recoveries retained the parent union without recording the mode.
814
+ legacy_parent = (
815
+ include_parent_with_recovery
816
+ or (pending_recovery is not None and pending_recovery.get("legacyParent", True) is True)
817
+ or (
818
+ recovery_source_digests is None
819
+ and frozen_source_digests == {}
820
+ and isinstance(existing_scan, dict)
821
+ and existing_scan.get("sealedAt") is not None
822
+ and "preservedSources" not in existing_scan
823
+ )
2678
824
  )
825
+
826
+ def pending_selection(heads: dict[str, str | None]) -> dict[str, Any]:
827
+ return {
828
+ "heads": heads,
829
+ "recoveryBaseDigest": scan["seal_manifest_digest"],
830
+ "legacyParent": legacy_parent,
831
+ }
832
+
833
+ try:
834
+ documents = merge_saved_results(
835
+ scan_dir,
836
+ scan_id,
837
+ binding,
838
+ connection.execute(
839
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
840
+ (scan_id,),
841
+ ).fetchall(),
842
+ warnings,
843
+ stopped=True,
844
+ scan_mode=scan["mode"],
845
+ finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
846
+ reason=(
847
+ f"Scan {outcome}; saved findings and pending review were preserved. "
848
+ f"{scan['failure_message'] or ''}"
849
+ ).strip(),
850
+ frozen_source_digests=frozen_source_digests,
851
+ checkpoint_heads=checkpoint_heads,
852
+ frozen_model_source=model_source[0] if model_source else None,
853
+ selected_model_source=model_source,
854
+ project_worker_coverage=scan["mode"] == "deep",
855
+ canonical_model=canonical_model,
856
+ canonical_model_source=canonical_model_source,
857
+ canonical_sources=canonical_sources,
858
+ recovery_snapshot_error=recovery_snapshot_error,
859
+ allow_frozen_legacy_parent=legacy_parent,
860
+ parent_already_captured=pending_recovery is not None,
861
+ )
862
+ except _CheckpointSnapshotError as exc:
863
+ if frozen_source_digests is None or recovery_snapshot_error is not None:
864
+ # The stop has already committed. Keep the source set and times in
865
+ # the database even when auxiliary snapshot files cannot be written.
866
+ # Changed raw heads remain rejected on retry.
867
+ captured = exc.captured
868
+ retained_sources = {path: value[0] for path, value in captured.items()}
869
+ source_times = {path: value[1] for path, value in captured.items()}
870
+ retained_heads = exc.heads
871
+ if recovery_snapshot_error is not None:
872
+ _prepare_scan_finalization(
873
+ scan_dir,
874
+ expected_coverage_mode=db.expected_coverage_mode(scan),
875
+ completion_binding=binding,
876
+ recover_unsealed=True,
877
+ draft_documents=exc.documents,
878
+ finding_scope=db.requested_finding_scope(scan)
879
+ if scan["mode"] == "deep"
880
+ else None,
881
+ )
882
+ retained_heads = pending_selection(retained_heads)
883
+ with connection:
884
+ connection.execute(
885
+ "UPDATE scans SET retained_source_digests_json = ?, "
886
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
887
+ "WHERE id = ? AND retained_source_digests_json IS ?",
888
+ (
889
+ _encode_retained_sources(
890
+ retained_sources, exc.model_source, source_times, exc.parent_context
891
+ ),
892
+ json.dumps(retained_heads, sort_keys=True),
893
+ json.dumps(list(dict.fromkeys(warnings))),
894
+ scan_id,
895
+ raw_frozen_sources,
896
+ ),
897
+ )
898
+ raise
2679
899
  if documents is None:
2680
900
  unpublished_warnings = list(dict.fromkeys([*warnings, *publication_follow_up_warnings]))
2681
901
  if unpublished_warnings != stored_warnings:
@@ -2686,23 +906,46 @@ def preserve_scan_results_locked(
2686
906
  (json.dumps(unpublished_warnings), db.now(), scan_id),
2687
907
  )
2688
908
  return False
2689
- if frozen_source_digests is None or (
2690
- recovery_source_digests is None and saved_model_source is None and model_source
909
+ retained_state = None
910
+ if (
911
+ frozen_source_digests is None
912
+ or scan["retained_checkpoint_heads_json"] is None
913
+ or recovery_source_digests is not None
914
+ or documents[0]["scan"]["preservedSources"] != frozen_source_digests
915
+ or model_source != ([saved_model_source] if saved_model_source is not None else [])
2691
916
  ):
2692
917
  retained_sources = _source_digests(
2693
918
  documents[0].get("scan", {}).get("preservedSources"),
2694
919
  "Stopped scan source digests could not be frozen.",
2695
920
  )
2696
- with connection:
2697
- connection.execute(
2698
- "UPDATE scans SET retained_source_digests_json = ? "
2699
- "WHERE id = ? AND retained_source_digests_json IS ?",
2700
- (
2701
- _encode_retained_sources(retained_sources, model_source),
2702
- scan_id,
2703
- raw_frozen_sources,
2704
- ),
2705
- )
921
+ retained_heads = documents[0]["scan"]["preservedCheckpointHeads"]
922
+ if (
923
+ recovery_source_digests is not None
924
+ or pending_recovery
925
+ or (existing is not None and existing_scan.get("status") == outcome)
926
+ ):
927
+ retained_heads = pending_selection(retained_heads)
928
+ retained_state = (
929
+ _encode_retained_sources(retained_sources, model_source),
930
+ json.dumps(retained_heads, sort_keys=True),
931
+ json.dumps(list(dict.fromkeys(warnings))),
932
+ scan_id,
933
+ raw_frozen_sources,
934
+ )
935
+
936
+ def retain_sources() -> None:
937
+ if retained_state is not None:
938
+ with connection:
939
+ connection.execute(
940
+ "UPDATE scans SET retained_source_digests_json = ?, "
941
+ "retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
942
+ "WHERE id = ? AND retained_source_digests_json IS ?",
943
+ retained_state,
944
+ )
945
+
946
+ first_preservation = recovery_source_digests is None and not pending_recovery
947
+ if first_preservation:
948
+ retain_sources()
2706
949
  prepared = _prepare_scan_finalization(
2707
950
  scan_dir,
2708
951
  expected_coverage_mode=db.expected_coverage_mode(scan),
@@ -2711,6 +954,8 @@ def preserve_scan_results_locked(
2711
954
  draft_documents=documents,
2712
955
  finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
2713
956
  )
957
+ if not first_preservation:
958
+ retain_sources()
2714
959
  warnings.extend(prepared.warnings)
2715
960
  snapshots = _snapshot_published_outputs(scan_dir)
2716
961
  try:
@@ -2733,13 +978,24 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2733
978
  raise SystemExit("Only a stopped scan can recover terminal results.")
2734
979
  if scan["canceled_at"] is not None:
2735
980
  raise SystemExit("Canceled scans cannot recover terminal results.")
2736
- recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan)
981
+ recovery_snapshot_error = None
982
+ try:
983
+ recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests(
984
+ db, connection, scan
985
+ )
986
+ except _CheckpointSnapshotError as exc:
987
+ recovery_snapshot_error = exc
988
+ recovery_source_digests = {path: value[0] for path, value in exc.captured.items()}
989
+ include_parent = exc.include_parent
990
+ checkpoint_heads = exc.heads
2737
991
  if not preserve_scan_results_locked(
2738
992
  db,
2739
993
  connection,
2740
994
  scan_id,
2741
995
  recovery_source_digests=recovery_source_digests,
996
+ recovery_checkpoint_heads=checkpoint_heads,
2742
997
  include_parent_with_recovery=include_parent,
998
+ recovery_snapshot_error=recovery_snapshot_error,
2743
999
  ):
2744
1000
  raise SystemExit("No saved stopped-scan results were available to recover.")
2745
1001
  db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
@@ -2807,7 +1063,7 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2807
1063
  manifest_path = db.artifact_path(scan_dir, "scan-manifest.json", required=False)
2808
1064
  if manifest_path is not None:
2809
1065
  manifest = db.read_json_object(manifest_path).get("scan", {})
2810
- if manifest.get("sealedAt") is not None or manifest.get("artifacts") is not None:
1066
+ if manifest_is_sealed(manifest):
2811
1067
  raise SystemExit("The scan is sealed; its artifacts cannot be modified.")
2812
1068
  output = args.artifact_path
2813
1069
  key = output.lower()
@@ -2823,66 +1079,6 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2823
1079
  return {"scanId": scan_id, "path": str(scan_dir / output)}
2824
1080
 
2825
1081
 
2826
- def _require_current_deep_publication(
2827
- db: Any, connection: Any, scan: Any, draft: dict[str, Any]
2828
- ) -> None:
2829
- scan_id = scan["id"]
2830
- run = connection.execute(
2831
- "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
2832
- ).fetchone()
2833
- publication = draft.get("deepScanPublication")
2834
- if (
2835
- publication is None
2836
- and draft["manifest"]["scan"].get("complete") is False
2837
- and (run is None or run["status"] == "running")
2838
- ):
2839
- return
2840
- if run is None:
2841
- run = db.deep_scan.require_deep_scan_run(connection, scan_id)
2842
- # Finished discovery-only runs hand remaining publication back to the parent.
2843
- if (
2844
- publication is None
2845
- and run["status"] != "running"
2846
- and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
2847
- ):
2848
- return
2849
- db.deep_scan.require_current_coordinator(
2850
- run,
2851
- argparse.Namespace(
2852
- coordinator_generation=publication.get("coordinatorGeneration") if publication else None
2853
- ),
2854
- )
2855
- # Once finish selects the canonical parent, only completion may replay it.
2856
- if publication is None:
2857
- if run["status"] != "running" and run["manifest_path"] == str(
2858
- Path(scan["scan_dir"]) / "scan-manifest.json"
2859
- ):
2860
- raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
2861
- return
2862
-
2863
- # Match the durable reducer sequence used by coordinator recovery.
2864
- def reducer_order(worker: Any) -> tuple[int, str]:
2865
- match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
2866
- return (int(match[1]) if match else 0, worker["id"])
2867
-
2868
- reducer = max(
2869
- (
2870
- worker
2871
- for worker in connection.execute(
2872
- "SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
2873
- )
2874
- if worker["kind"] == "dedup"
2875
- and worker["status"] == "succeeded"
2876
- and worker["result_manifest_path"]
2877
- ),
2878
- key=reducer_order,
2879
- default=None,
2880
- )
2881
- selected_result = reducer["result_manifest_path"] if reducer is not None else None
2882
- if publication["resultPath"] != selected_result:
2883
- raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
2884
-
2885
-
2886
1082
  def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2887
1083
  scan_id = db.require_uuid(args.scan_id, "scan-id")
2888
1084
  with db.scan_completion_lock(scan_id):
@@ -2895,13 +1091,6 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2895
1091
  "The scan stopped; its saved checkpoint was retained without replacing sealed results."
2896
1092
  )
2897
1093
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2898
- if (
2899
- args.expected_draft_digest is not None
2900
- and args.expected_draft_digest != _scan_draft_digest(scan_dir)
2901
- ):
2902
- raise SystemExit(
2903
- "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
2904
- )
2905
1094
  try:
2906
1095
  relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
2907
1096
  except ValueError as exc:
@@ -2911,12 +1100,16 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2911
1100
  if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
2912
1101
  raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
2913
1102
  draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
1103
+ accepted_input = copy.deepcopy(draft)
1104
+ accepted_checkpoint = None
2914
1105
  manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
2915
1106
  binding = db.workbench_completion_binding(scan, db.now())
2916
1107
  # Save scan IDs without sealing the draft.
2917
1108
  _populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
2918
1109
  _populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
2919
1110
  _validate_completion_binding(manifest, findings, coverage, binding)
1111
+ projected = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
1112
+ admitted_input = None
2920
1113
  checkpoint = manifest["scan"]
2921
1114
  checkpoint_contents = None
2922
1115
  if args.checkpoint_path is not None:
@@ -2933,8 +1126,13 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2933
1126
  checkpoint, checkpoint_contents = _read_scan_local_json_bytes(
2934
1127
  scan_dir, checkpoint_relative, "Staged scan checkpoint"
2935
1128
  )
1129
+ accepted_checkpoint = checkpoint
2936
1130
  if checkpoint.get("scanId") != scan_id:
2937
1131
  raise SystemExit("Staged scan checkpoint belongs to another scan.")
1132
+ if scan["mode"] == "deep":
1133
+ checkpoint_contents = _admit_parent_checkpoint(
1134
+ scan_dir, scan_id, checkpoint, projected
1135
+ )
2938
1136
  checkpoint_only = (
2939
1137
  scan["mode"] == "deep"
2940
1138
  and draft.get("deepScanPublication") is None
@@ -2948,8 +1146,40 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2948
1146
  )
2949
1147
  if scan["mode"] == "deep" and not checkpoint_only:
2950
1148
  _require_current_deep_publication(db, connection, scan, draft)
1149
+ acceptance_input = {**accepted_input, "checkpoint": accepted_checkpoint}
1150
+ acceptance_relative = Path(relative).with_suffix(".accepted.json").as_posix()
1151
+ acknowledge = (
1152
+ scan["mode"] == "deep"
1153
+ and draft.get("deepScanPublication") is not None
1154
+ and db.deep_scan.require_deep_scan_run(connection, scan_id)["workflow_version"]
1155
+ in {"deep-scan-mcp/v1", "deep-security-scan/v1"}
1156
+ )
1157
+ if acknowledge and (scan_dir / acceptance_relative).exists():
1158
+ accepted = _read_scan_local_json(scan_dir, acceptance_relative, "Accepted scan draft")
1159
+ if (
1160
+ accepted.get("status") == "draft_written"
1161
+ and accepted.get("input") == acceptance_input
1162
+ ):
1163
+ if accepted.get("draftDigest") != _scan_draft_digest(scan_dir):
1164
+ raise SystemExit(
1165
+ "scan_draft_conflict: canonical scan results changed after publication."
1166
+ )
1167
+ return {
1168
+ "scanId": scan_id,
1169
+ "status": "draft_written",
1170
+ **({"warnings": accepted["warnings"]} if accepted.get("warnings") else {}),
1171
+ }
1172
+ if (
1173
+ args.expected_draft_digest is not None
1174
+ and args.expected_draft_digest != _scan_draft_digest(scan_dir)
1175
+ ):
1176
+ raise SystemExit(
1177
+ "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
1178
+ )
2951
1179
  if checkpoint_contents is not None:
2952
1180
  checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
1181
+ if scan["mode"] == "deep":
1182
+ admitted_input = f"{checkpoint_digest}.json"
2953
1183
  write_scan_local_bytes(
2954
1184
  scan_dir,
2955
1185
  f"checkpoints/{checkpoint_digest}.json",
@@ -2957,27 +1187,27 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2957
1187
  )
2958
1188
  if checkpoint_only:
2959
1189
  return {"scanId": scan_id, "status": "draft_written"}
2960
- checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
2961
- checkpoint_contents = _encoded(checkpoint)
2962
- checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
1190
+ projected_contents = _encoded(projected)
1191
+ checkpoint_name = f"{hashlib.sha256(projected_contents).hexdigest()}.json"
2963
1192
  checkpoint_relative = f"checkpoints/{checkpoint_name}"
2964
1193
  if not (scan_dir / checkpoint_relative).exists():
2965
- write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
1194
+ write_scan_local_bytes(scan_dir, checkpoint_relative, projected_contents)
2966
1195
  if checkpoint_only:
2967
1196
  return {"scanId": scan_id, "status": "draft_written"}
2968
- write_scan_local_bytes(
2969
- scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
2970
- )
2971
- for filename, document in (
2972
- ("findings.json", findings),
2973
- ("coverage.json", coverage),
2974
- ("scan-manifest.json", manifest),
2975
- ):
2976
- write_scan_local_bytes(
2977
- scan_dir,
2978
- filename,
2979
- (json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
1197
+ head = {"checkpoint": checkpoint_name}
1198
+ if admitted_input is not None:
1199
+ head["inputCheckpoint"] = admitted_input
1200
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded(head))
1201
+ documents = {
1202
+ filename: (json.dumps(document, allow_nan=False, indent=2) + "\n").encode()
1203
+ for filename, document in (
1204
+ ("findings.json", findings),
1205
+ ("coverage.json", coverage),
1206
+ ("scan-manifest.json", manifest),
2980
1207
  )
1208
+ }
1209
+ for filename, contents in documents.items():
1210
+ write_scan_local_bytes(scan_dir, filename, contents)
2981
1211
  if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
2982
1212
  # Select the accepted final publication before releasing the write lock.
2983
1213
  with connection:
@@ -2987,6 +1217,19 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
2987
1217
  (str(scan_dir / "scan-manifest.json"), scan_id),
2988
1218
  )
2989
1219
  model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
1220
+ if acknowledge:
1221
+ # Optional recovery evidence is written only after the publication succeeds.
1222
+ try:
1223
+ acceptance = {
1224
+ "status": "draft_written",
1225
+ "input": acceptance_input,
1226
+ "draftDigest": _scan_draft_contents_digest(documents),
1227
+ **({"warnings": [model_warning]} if model_warning else {}),
1228
+ }
1229
+ write_scan_local_bytes(scan_dir, acceptance_relative, _encoded(acceptance))
1230
+ except (OSError, ScanLocalIOError):
1231
+ # A missing receipt prevents recovery but does not undo accepted results.
1232
+ pass
2990
1233
  # Accepted Standard drafts are evidence of review or report assembly,
2991
1234
  # even when the parent omitted its explicit progress call.
2992
1235
  model_only_checkpoint = (
@@ -3025,20 +1268,31 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
3025
1268
 
3026
1269
 
3027
1270
  def _scan_draft_digest(scan_dir: Path) -> str:
3028
- digest = hashlib.sha256()
1271
+ documents: dict[str, bytes | None] = {}
3029
1272
  for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
3030
- digest.update(filename.encode())
3031
- digest.update(b"\0")
3032
1273
  try:
3033
1274
  (scan_dir / filename).lstat()
3034
1275
  except FileNotFoundError:
3035
- digest.update(b"missing\0")
1276
+ documents[filename] = None
3036
1277
  continue
3037
- digest.update(b"present\0")
3038
1278
  descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
3039
1279
  with os.fdopen(descriptor, "rb") as handle:
3040
- digest.update(handle.read())
1280
+ documents[filename] = handle.read()
1281
+ return _scan_draft_contents_digest(documents)
1282
+
1283
+
1284
+ def _scan_draft_contents_digest(documents: dict[str, bytes | None]) -> str:
1285
+ digest = hashlib.sha256()
1286
+ for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
1287
+ digest.update(filename.encode())
3041
1288
  digest.update(b"\0")
1289
+ contents = documents[filename]
1290
+ if contents is None:
1291
+ digest.update(b"missing\0")
1292
+ else:
1293
+ digest.update(b"present\0")
1294
+ digest.update(contents)
1295
+ digest.update(b"\0")
3042
1296
  return digest.hexdigest()
3043
1297
 
3044
1298
 
@@ -3064,7 +1318,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
3064
1318
  args.claim_token,
3065
1319
  error_message="Scan failure is owned by another continuation.",
3066
1320
  )
3067
- message = db.optional_text(args.message, maximum=2400)
1321
+ message = db.optional_text(args.message)
3068
1322
  updated = connection.execute(
3069
1323
  """
3070
1324
  UPDATE scans