@openai/codex-security 0.1.31 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (314) hide show
  1. package/README.md +295 -32
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +4 -4
  5. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  6. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  7. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  8. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  9. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  10. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  11. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  12. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  13. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  14. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  15. package/_bundled_plugin/references/artifact-storage.md +4 -4
  16. package/_bundled_plugin/references/config-preflight.md +1 -1
  17. package/_bundled_plugin/references/core-scan.md +14 -2
  18. package/_bundled_plugin/references/final-report.md +4 -2
  19. package/_bundled_plugin/references/scan-artifacts.md +2 -2
  20. package/_bundled_plugin/references/scan-contract.md +2 -2
  21. package/_bundled_plugin/references/threat-model.md +2 -2
  22. package/_bundled_plugin/schemas/coverage.schema.json +22 -0
  23. package/_bundled_plugin/schemas/scan-manifest.schema.json +100 -85
  24. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +116 -102
  25. package/_bundled_plugin/scripts/config_preflight.py +4 -8
  26. package/_bundled_plugin/scripts/deep_scan_workbench.py +83 -217
  27. package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
  28. package/_bundled_plugin/scripts/finalize_scan_contract.py +441 -166
  29. package/_bundled_plugin/scripts/finding_preview.py +42 -82
  30. package/_bundled_plugin/scripts/generate_in_scope_files.py +30 -34
  31. package/_bundled_plugin/scripts/generate_rank_input.py +58 -99
  32. package/_bundled_plugin/scripts/launch_codex_security_mcp +5 -1
  33. package/_bundled_plugin/scripts/rank_preview.py +85 -68
  34. package/_bundled_plugin/scripts/report_projection.py +37 -58
  35. package/_bundled_plugin/scripts/threat_model_projection.py +164 -0
  36. package/_bundled_plugin/scripts/windows_scan_local_files.py +11 -6
  37. package/_bundled_plugin/scripts/workbench/storage.py +11 -0
  38. package/_bundled_plugin/scripts/workbench_cli.py +11 -17
  39. package/_bundled_plugin/scripts/workbench_constants.py +6 -14
  40. package/_bundled_plugin/scripts/workbench_db.py +116 -228
  41. package/_bundled_plugin/scripts/workbench_finding_index.py +1 -6
  42. package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -4
  43. package/_bundled_plugin/scripts/workbench_progress.py +12 -34
  44. package/_bundled_plugin/scripts/workbench_publication.py +47 -62
  45. package/_bundled_plugin/scripts/workbench_remediation.py +2 -20
  46. package/_bundled_plugin/scripts/workbench_saved_results.py +1246 -230
  47. package/_bundled_plugin/scripts/workbench_scan_start.py +4 -11
  48. package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -15
  49. package/_bundled_plugin/scripts/workbench_schema.py +86 -271
  50. package/_bundled_plugin/scripts/workbench_severity.py +3 -0
  51. package/_bundled_plugin/scripts/workbench_source_excerpt.py +5 -6
  52. package/_bundled_plugin/scripts/workbench_target.py +98 -10
  53. package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
  54. package/_bundled_plugin/scripts/workbench_validation.py +6 -20
  55. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
  56. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +3 -3
  57. package/_bundled_plugin/skills/security-scan/SKILL.md +2 -2
  58. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  59. package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
  60. package/dist/api.d.ts +3 -5
  61. package/dist/api.d.ts.map +1 -1
  62. package/dist/api.js +201 -207
  63. package/dist/api.js.map +1 -1
  64. package/dist/artifact-export.d.ts +70 -0
  65. package/dist/artifact-export.d.ts.map +1 -0
  66. package/dist/artifact-export.js +277 -0
  67. package/dist/artifact-export.js.map +1 -0
  68. package/dist/auth.d.ts.map +1 -1
  69. package/dist/auth.js +8 -3
  70. package/dist/auth.js.map +1 -1
  71. package/dist/bulk-scan-discovery.d.ts +1 -0
  72. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  73. package/dist/bulk-scan-discovery.js +8 -9
  74. package/dist/bulk-scan-discovery.js.map +1 -1
  75. package/dist/classify-scan-severity.d.ts.map +1 -1
  76. package/dist/classify-scan-severity.js +2 -5
  77. package/dist/classify-scan-severity.js.map +1 -1
  78. package/dist/classify-severity.d.ts +2 -8
  79. package/dist/classify-severity.d.ts.map +1 -1
  80. package/dist/classify-severity.js +3 -5
  81. package/dist/classify-severity.js.map +1 -1
  82. package/dist/cli-help.d.ts +3 -0
  83. package/dist/cli-help.d.ts.map +1 -0
  84. package/dist/cli-help.js +281 -0
  85. package/dist/cli-help.js.map +1 -0
  86. package/dist/cli.d.ts +19 -22
  87. package/dist/cli.d.ts.map +1 -1
  88. package/dist/cli.js +631 -714
  89. package/dist/cli.js.map +1 -1
  90. package/dist/cloud-publish.d.ts.map +1 -1
  91. package/dist/cloud-publish.js +1 -8
  92. package/dist/cloud-publish.js.map +1 -1
  93. package/dist/codex-prompt.d.ts.map +1 -1
  94. package/dist/codex-prompt.js +1 -2
  95. package/dist/codex-prompt.js.map +1 -1
  96. package/dist/codex-sdk-environment.d.ts +2 -0
  97. package/dist/codex-sdk-environment.d.ts.map +1 -0
  98. package/dist/codex-sdk-environment.js +30 -0
  99. package/dist/codex-sdk-environment.js.map +1 -0
  100. package/dist/component-plan.d.ts.map +1 -1
  101. package/dist/component-plan.js +2 -6
  102. package/dist/component-plan.js.map +1 -1
  103. package/dist/component-scan.d.ts +1 -0
  104. package/dist/component-scan.d.ts.map +1 -1
  105. package/dist/component-scan.js +31 -9
  106. package/dist/component-scan.js.map +1 -1
  107. package/dist/config.d.ts.map +1 -1
  108. package/dist/config.js +32 -33
  109. package/dist/config.js.map +1 -1
  110. package/dist/contract.d.ts +2 -1
  111. package/dist/contract.d.ts.map +1 -1
  112. package/dist/contract.js +63 -122
  113. package/dist/contract.js.map +1 -1
  114. package/dist/cost-model.d.ts.map +1 -1
  115. package/dist/cost-model.js +65 -34
  116. package/dist/cost-model.js.map +1 -1
  117. package/dist/cost.d.ts.map +1 -1
  118. package/dist/cost.js +15 -31
  119. package/dist/cost.js.map +1 -1
  120. package/dist/custom-validation-prompt.js +4 -4
  121. package/dist/custom-validation-prompt.js.map +1 -1
  122. package/dist/custom-validation.d.ts.map +1 -1
  123. package/dist/custom-validation.js +17 -16
  124. package/dist/custom-validation.js.map +1 -1
  125. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  126. package/dist/deduplication/checkpointed-review.js +10 -11
  127. package/dist/deduplication/checkpointed-review.js.map +1 -1
  128. package/dist/deduplication/codex-review.d.ts +2 -0
  129. package/dist/deduplication/codex-review.d.ts.map +1 -1
  130. package/dist/deduplication/codex-review.js +3 -3
  131. package/dist/deduplication/codex-review.js.map +1 -1
  132. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  133. package/dist/deduplication/deduplication-reviewer.js +2 -0
  134. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  135. package/dist/deduplication/deduplication.js +40 -62
  136. package/dist/deduplication/deduplication.js.map +1 -1
  137. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  138. package/dist/deduplication/records-protocol.js +1 -4
  139. package/dist/deduplication/records-protocol.js.map +1 -1
  140. package/dist/deduplication/records.d.ts +2 -1
  141. package/dist/deduplication/records.d.ts.map +1 -1
  142. package/dist/deduplication/records.js +24 -6
  143. package/dist/deduplication/records.js.map +1 -1
  144. package/dist/deduplication/review.d.ts +7 -0
  145. package/dist/deduplication/review.d.ts.map +1 -1
  146. package/dist/deduplication/review.js.map +1 -1
  147. package/dist/deduplication/scan.js +3 -7
  148. package/dist/deduplication/scan.js.map +1 -1
  149. package/dist/deep-config.d.ts.map +1 -1
  150. package/dist/deep-config.js +3 -19
  151. package/dist/deep-config.js.map +1 -1
  152. package/dist/deep-progress.d.ts +0 -1
  153. package/dist/deep-progress.d.ts.map +1 -1
  154. package/dist/deep-progress.js +22 -40
  155. package/dist/deep-progress.js.map +1 -1
  156. package/dist/errors.d.ts +3 -2
  157. package/dist/errors.d.ts.map +1 -1
  158. package/dist/errors.js +5 -7
  159. package/dist/errors.js.map +1 -1
  160. package/dist/finding-catalogue.d.ts.map +1 -1
  161. package/dist/finding-catalogue.js +22 -22
  162. package/dist/finding-catalogue.js.map +1 -1
  163. package/dist/finding-workflow.d.ts.map +1 -1
  164. package/dist/finding-workflow.js +4 -7
  165. package/dist/finding-workflow.js.map +1 -1
  166. package/dist/findings-import.d.ts.map +1 -1
  167. package/dist/findings-import.js +7 -17
  168. package/dist/findings-import.js.map +1 -1
  169. package/dist/github.d.ts.map +1 -1
  170. package/dist/github.js +2 -3
  171. package/dist/github.js.map +1 -1
  172. package/dist/import-scan.js +2 -2
  173. package/dist/import-scan.js.map +1 -1
  174. package/dist/index.d.ts +4 -2
  175. package/dist/index.d.ts.map +1 -1
  176. package/dist/index.js +2 -1
  177. package/dist/index.js.map +1 -1
  178. package/dist/knowledge-base.js +2 -2
  179. package/dist/knowledge-base.js.map +1 -1
  180. package/dist/linear.d.ts.map +1 -1
  181. package/dist/linear.js +2 -3
  182. package/dist/linear.js.map +1 -1
  183. package/dist/models.d.ts +24 -0
  184. package/dist/models.d.ts.map +1 -1
  185. package/dist/models.js.map +1 -1
  186. package/dist/multiscan.d.ts.map +1 -1
  187. package/dist/multiscan.js +36 -31
  188. package/dist/multiscan.js.map +1 -1
  189. package/dist/owner-evidence.d.ts.map +1 -1
  190. package/dist/owner-evidence.js +2 -6
  191. package/dist/owner-evidence.js.map +1 -1
  192. package/dist/patch-tui.d.ts.map +1 -1
  193. package/dist/patch-tui.js +2 -4
  194. package/dist/patch-tui.js.map +1 -1
  195. package/dist/project-config-schema.d.ts +10 -0
  196. package/dist/project-config-schema.d.ts.map +1 -1
  197. package/dist/project-config-schema.js +1 -0
  198. package/dist/project-config-schema.js.map +1 -1
  199. package/dist/project-config.d.ts +1 -0
  200. package/dist/project-config.d.ts.map +1 -1
  201. package/dist/project-config.js +2 -0
  202. package/dist/project-config.js.map +1 -1
  203. package/dist/prompt-files.d.ts +0 -1
  204. package/dist/prompt-files.d.ts.map +1 -1
  205. package/dist/prompt-files.js +2 -4
  206. package/dist/prompt-files.js.map +1 -1
  207. package/dist/publication-events.d.ts.map +1 -1
  208. package/dist/publication-events.js +15 -29
  209. package/dist/publication-events.js.map +1 -1
  210. package/dist/publication-store.d.ts.map +1 -1
  211. package/dist/publication-store.js +9 -13
  212. package/dist/publication-store.js.map +1 -1
  213. package/dist/publication.d.ts.map +1 -1
  214. package/dist/publication.js +11 -14
  215. package/dist/publication.js.map +1 -1
  216. package/dist/publish.d.ts.map +1 -1
  217. package/dist/publish.js +31 -41
  218. package/dist/publish.js.map +1 -1
  219. package/dist/record.d.ts +2 -0
  220. package/dist/record.d.ts.map +1 -0
  221. package/dist/record.js +4 -0
  222. package/dist/record.js.map +1 -0
  223. package/dist/result.d.ts +4 -1
  224. package/dist/result.d.ts.map +1 -1
  225. package/dist/result.js +7 -0
  226. package/dist/result.js.map +1 -1
  227. package/dist/runtime.d.ts +10 -13
  228. package/dist/runtime.d.ts.map +1 -1
  229. package/dist/runtime.js +203 -131
  230. package/dist/runtime.js.map +1 -1
  231. package/dist/saved-scan.d.ts.map +1 -1
  232. package/dist/saved-scan.js +10 -20
  233. package/dist/saved-scan.js.map +1 -1
  234. package/dist/scan-activity.d.ts.map +1 -1
  235. package/dist/scan-activity.js +8 -22
  236. package/dist/scan-activity.js.map +1 -1
  237. package/dist/scan-comparison.d.ts +3 -3
  238. package/dist/scan-comparison.d.ts.map +1 -1
  239. package/dist/scan-comparison.js +38 -73
  240. package/dist/scan-comparison.js.map +1 -1
  241. package/dist/scan-dashboard.d.ts +0 -1
  242. package/dist/scan-dashboard.d.ts.map +1 -1
  243. package/dist/scan-dashboard.js +10 -19
  244. package/dist/scan-dashboard.js.map +1 -1
  245. package/dist/scan-history-renderer.d.ts.map +1 -1
  246. package/dist/scan-history-renderer.js +1 -2
  247. package/dist/scan-history-renderer.js.map +1 -1
  248. package/dist/scan-logs.d.ts.map +1 -1
  249. package/dist/scan-logs.js +2 -6
  250. package/dist/scan-logs.js.map +1 -1
  251. package/dist/scan-sessions.d.ts.map +1 -1
  252. package/dist/scan-sessions.js +1 -3
  253. package/dist/scan-sessions.js.map +1 -1
  254. package/dist/scan-settings.d.ts +5 -0
  255. package/dist/scan-settings.d.ts.map +1 -1
  256. package/dist/scan-settings.js +4 -0
  257. package/dist/scan-settings.js.map +1 -1
  258. package/dist/security-policy-cli.d.ts.map +1 -1
  259. package/dist/security-policy-cli.js +8 -6
  260. package/dist/security-policy-cli.js.map +1 -1
  261. package/dist/security-policy.d.ts +8 -4
  262. package/dist/security-policy.d.ts.map +1 -1
  263. package/dist/security-policy.js +98 -84
  264. package/dist/security-policy.js.map +1 -1
  265. package/dist/server/dashboard/app.js +5 -5
  266. package/dist/server/embeddings.d.ts.map +1 -1
  267. package/dist/server/embeddings.js +1 -0
  268. package/dist/server/embeddings.js.map +1 -1
  269. package/dist/server/routes.d.ts +3 -2
  270. package/dist/server/routes.d.ts.map +1 -1
  271. package/dist/server/routes.js +19 -7
  272. package/dist/server/routes.js.map +1 -1
  273. package/dist/server/server.d.ts.map +1 -1
  274. package/dist/server/server.js +2 -3
  275. package/dist/server/server.js.map +1 -1
  276. package/dist/server/sqlite-store.d.ts.map +1 -1
  277. package/dist/server/sqlite-store.js +4 -14
  278. package/dist/server/sqlite-store.js.map +1 -1
  279. package/dist/severity-store.js +8 -15
  280. package/dist/severity-store.js.map +1 -1
  281. package/dist/suggest-owners.d.ts +2 -3
  282. package/dist/suggest-owners.d.ts.map +1 -1
  283. package/dist/suggest-owners.js +4 -5
  284. package/dist/suggest-owners.js.map +1 -1
  285. package/dist/targets.d.ts +1 -0
  286. package/dist/targets.d.ts.map +1 -1
  287. package/dist/targets.js +24 -21
  288. package/dist/targets.js.map +1 -1
  289. package/dist/trusted-executable.d.ts +5 -0
  290. package/dist/trusted-executable.d.ts.map +1 -1
  291. package/dist/trusted-executable.js +22 -5
  292. package/dist/trusted-executable.js.map +1 -1
  293. package/dist/value.d.ts +2 -0
  294. package/dist/value.d.ts.map +1 -0
  295. package/dist/value.js +20 -0
  296. package/dist/value.js.map +1 -0
  297. package/dist/version.d.ts.map +1 -1
  298. package/dist/version.js +3 -6
  299. package/dist/version.js.map +1 -1
  300. package/dist/windows-path.d.ts.map +1 -1
  301. package/dist/windows-path.js +1 -2
  302. package/dist/windows-path.js.map +1 -1
  303. package/dist/worker-progress.d.ts +1 -0
  304. package/dist/worker-progress.d.ts.map +1 -1
  305. package/dist/worker-progress.js +13 -34
  306. package/dist/worker-progress.js.map +1 -1
  307. package/docs/dedupe-records.md +24 -0
  308. package/package.json +11 -10
  309. package/schemas/project-config.schema.json +5 -0
  310. package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
  311. package/dist/server/findings-service.d.ts +0 -20
  312. package/dist/server/findings-service.d.ts.map +0 -1
  313. package/dist/server/findings-service.js +0 -31
  314. package/dist/server/findings-service.js.map +0 -1
@@ -12,10 +12,9 @@ import re
12
12
  import sqlite3
13
13
  import stat
14
14
  import sys
15
- from collections.abc import Callable, Iterator
16
- from dataclasses import dataclass
15
+ from collections.abc import Iterator
16
+ from contextlib import contextmanager
17
17
  from pathlib import Path
18
- from types import ModuleType
19
18
  from typing import Any
20
19
 
21
20
  sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -25,18 +24,25 @@ from finalize_scan_contract import (
25
24
  _populate_unsealed_artifact_envelope,
26
25
  _populate_unsealed_manifest_envelope,
27
26
  _prepare_scan_finalization,
27
+ _read_json,
28
+ _read_saved_threat_model,
28
29
  _read_scan_local_json,
29
30
  _read_scan_local_json_bytes,
31
+ _read_scan_local_json_with_metadata,
30
32
  _recover_unsealed_findings,
31
33
  _remove_scan_local_file_if_exists,
32
34
  _validate_completion_binding,
35
+ _validate_resolved_deferred,
36
+ _validate_schema_node,
33
37
  _write_prepared_scan_finalization,
34
38
  finalize_scan,
35
39
  finding_candidate_id,
36
40
  open_scan_local_file_descriptor,
37
41
  write_scan_local_bytes,
42
+ write_threat_model_projection_if_possible,
38
43
  )
39
44
  from workbench_constants import PHASES
45
+ from workbench_target import committed_diff_snapshot_digest
40
46
  from workbench_validation import path_within_scope
41
47
 
42
48
  _PUBLISHED_OUTPUTS = (
@@ -44,6 +50,7 @@ _PUBLISHED_OUTPUTS = (
44
50
  "coverage.json",
45
51
  "scan-manifest.json",
46
52
  "report.md",
53
+ "threatmodel.md",
47
54
  "report.html",
48
55
  "exports/results.sarif",
49
56
  )
@@ -55,29 +62,59 @@ _RESERVED_ARTIFACT_PATHS = json.loads(
55
62
  )
56
63
 
57
64
 
58
- @dataclass(frozen=True)
59
- class WorkbenchDbContext:
60
- ARTIFACTS: dict[str, str]
61
- artifact_path: Callable[..., Path | None]
62
- deep_scan: ModuleType
63
- expected_coverage_mode: Callable[..., str]
64
- handoff: ModuleType
65
- index_findings: Callable[..., None]
66
- now: Callable[[], str]
67
- optional_text: Callable[..., str | None]
68
- parse_scan_cost: Callable[..., dict[str, Any] | None]
69
- published_manifest_digest: Callable[..., str]
70
- read_json_object: Callable[[Path], dict[str, Any]]
71
- require_canonical_scan_directory: Callable[[Path], Path]
72
- require_recorded_manifest_digest: Callable[..., None]
73
- require_scan: Callable[..., Any]
74
- require_uuid: Callable[[str, str], str]
75
- require_workspace: Callable[..., Any]
76
- scan_completion_lock: Callable[..., Any]
77
- scan_context: Callable[..., dict[str, Any]]
78
- verify_manifest_binding: Callable[..., None]
79
- workbench_completion_binding: Callable[..., dict[str, Any]]
80
- workspace_state: Callable[..., dict[str, Any]]
65
+ def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]:
66
+ scan_dir = Path(scan["scan_dir"])
67
+ fields: dict[str, Any] = {"threatModelAvailable": False}
68
+ try:
69
+ db.require_recorded_manifest_digest(scan, scan_dir)
70
+ db.verify_manifest_binding(
71
+ scan, _read_scan_local_json(scan_dir, db.ARTIFACTS["manifest"], "scan manifest")
72
+ )
73
+ saved_model = _read_saved_threat_model(scan_dir)
74
+ if saved_model is not None:
75
+ description = saved_model[0]
76
+ fields.update(
77
+ threatModelAvailable=True,
78
+ threatModelProvenance=description["provenance"],
79
+ )
80
+ if description["path"] is not None:
81
+ fields["threatModelPath"] = description["path"]
82
+ except (ContractError, OSError, SystemExit):
83
+ # Unavailable optional model data must not prevent reading the saved scan.
84
+ pass
85
+ return fields
86
+
87
+
88
+ def refresh_completed_scan(
89
+ db: Any,
90
+ connection: sqlite3.Connection,
91
+ scan: sqlite3.Row,
92
+ cost_json: str | None,
93
+ ) -> dict[str, Any]:
94
+ warnings = json.loads(scan["completion_warnings_json"])
95
+ scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
96
+ db.require_recorded_manifest_digest(scan, scan_dir)
97
+ db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"]))
98
+ try:
99
+ manifest, _, _ = finalize_scan(
100
+ scan_dir,
101
+ expected_coverage_mode=db.expected_coverage_mode(scan),
102
+ projection_warnings=warnings,
103
+ )
104
+ except ContractError as exc:
105
+ raise SystemExit(str(exc)) from exc
106
+ db.verify_manifest_binding(scan, manifest)
107
+ manifest_digest = db.published_manifest_digest(scan_dir, manifest)
108
+ db.pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
109
+ if cost_json is not None and scan["recipe_json"] is not None:
110
+ db.scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json)
111
+ if warnings != json.loads(scan["completion_warnings_json"]):
112
+ with connection:
113
+ connection.execute(
114
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
115
+ (json.dumps(warnings), scan["id"]),
116
+ )
117
+ return db.scan_context(connection, scan["id"])
81
118
 
82
119
 
83
120
  def _encoded(value: Any) -> bytes:
@@ -118,35 +155,51 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
118
155
  )
119
156
 
120
157
 
121
- def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
122
- latest_reducer = _latest_successful_reducer(workers)
158
+ def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
159
+ return [
160
+ f"{directory}/{name}"
161
+ for name in _children(scan_dir, directory)
162
+ if re.fullmatch(r"[0-9a-f]{64}\.json", name)
163
+ ]
164
+
165
+
166
+ def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
167
+ output = Path(worker["artifact_dir"]).relative_to(scan_dir)
168
+ attempts = (output.parent if output.name == "output" else output) / "attempts"
169
+ archived = [
170
+ ((attempts / name).as_posix(), int(name.split("-")[1]))
171
+ for name in _children(scan_dir, attempts.as_posix())
172
+ if re.fullmatch(r"attempt-\d+", name)
173
+ ]
174
+ attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
175
+ if worker["kind"] == "discovery" and not attempt:
176
+ attempt = max((attempt for _, attempt in archived), default=0) + 1
177
+ return [(output.as_posix(), attempt), *archived]
123
178
 
124
- def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
125
- for name in _children(scan_dir, directory):
126
- if re.fullmatch(r"[0-9a-f]{64}\.json", name):
127
- yield f"{directory}/{name}", kind
128
179
 
129
- yield from checkpoints("checkpoints")
180
+ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
181
+ latest_reducer = _latest_successful_reducer(workers)
182
+ yield "checkpoint-head.json", None
183
+ for directory in ("checkpoint-heads", "checkpoints"):
184
+ yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
130
185
  for worker in workers:
131
186
  if worker["kind"] not in {"dedup", "discovery"}:
132
187
  continue
133
188
  try:
134
- output = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix()
189
+ outputs = _worker_outputs(scan_dir, worker)
135
190
  except (TypeError, ValueError):
136
191
  continue
137
- attempts = (Path(output).parent if Path(output).name == "output" else Path(output)) / (
138
- "attempts"
139
- )
140
- directories = [output] + [
141
- (attempts / name).as_posix()
142
- for name in _children(scan_dir, attempts.as_posix())
143
- if re.fullmatch(r"attempt-\d+", name)
144
- ]
145
- for directory in directories:
146
- checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
192
+ for directory, _ in outputs:
193
+ if worker["kind"] == "discovery":
194
+ yield f"{directory}/checkpoint-head.json", worker["kind"]
195
+ yield from (
196
+ (path, worker["kind"])
197
+ for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
198
+ )
199
+ checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
147
200
  if worker["kind"] == "discovery" or checkpoint_paths:
148
201
  yield f"{directory}/result.json", worker["kind"]
149
- yield from checkpoint_paths
202
+ yield from ((path, worker["kind"]) for path in checkpoint_paths)
150
203
  if worker["result_manifest_path"] and (
151
204
  worker["kind"] == "discovery"
152
205
  or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
@@ -160,17 +213,143 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st
160
213
  continue
161
214
 
162
215
 
216
+ def _checkpoint_head_directory(relative: str) -> Path | None:
217
+ path = Path(relative)
218
+ if path.name == "checkpoint-head.json":
219
+ return path.parent
220
+ if path.parent.name == "checkpoint-heads":
221
+ return path.parent.parent
222
+ return None
223
+
224
+
225
+ def _capture_saved_source(
226
+ scan_dir: Path,
227
+ relative: str,
228
+ scan_id: str,
229
+ *,
230
+ kind: str | None = None,
231
+ snapshot_head: bool = True,
232
+ write: bool = True,
233
+ ) -> dict[str, tuple[str, int]]:
234
+ if not snapshot_head or Path(relative).name != "checkpoint-head.json":
235
+ _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
236
+ return {relative: (digest, observed)}
237
+ head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
238
+ observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
239
+ directory = Path(relative).parent
240
+ selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
241
+ _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
242
+ digest = _digest(observation)
243
+ snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
244
+ # Capture the selected file even if the worker created it after directory enumeration.
245
+ if write and not (scan_dir / snapshot).exists():
246
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
247
+ return {
248
+ snapshot: (digest, int(observation["observedAtNs"])),
249
+ selected: (selected_digest, selected_time),
250
+ }
251
+
252
+
253
+ def _is_source_order_snapshot(relative: str) -> bool:
254
+ path = Path(relative)
255
+ return path.parent == Path("source-order") and bool(
256
+ re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
257
+ )
258
+
259
+
163
260
  def _read_saved_result(
164
261
  scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
165
- ) -> tuple[dict[str, Any], str]:
166
- draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint")
262
+ ) -> tuple[dict[str, Any], str, int]:
263
+ draft, _, metadata = _read_scan_local_json_with_metadata(
264
+ scan_dir, relative, "Saved scan checkpoint"
265
+ )
266
+ directory = _checkpoint_head_directory(relative)
267
+ if directory is not None:
268
+ checkpoint = draft.get("checkpoint")
269
+ if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
270
+ raise ContractError("checkpoint head does not name a saved checkpoint")
271
+ _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
272
+ if Path(relative).name == "checkpoint-head.json":
273
+ return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
274
+ observed = draft.get("observedAtNs")
275
+ if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
276
+ raise ContractError("checkpoint head has no observation time")
277
+ return draft, _digest(draft), int(observed)
167
278
  if draft.get("scanId") != scan_id:
168
279
  raise ContractError("checkpoint belongs to a different scan")
169
- if not isinstance(draft.get("findings"), list) or not isinstance(
170
- draft.get("coverage", {} if kind == "dedup" else None), dict
280
+ if not _is_source_order_snapshot(relative) and (
281
+ not isinstance(draft.get("findings"), list)
282
+ or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
171
283
  ):
172
284
  raise ContractError("checkpoint has no semantic findings or coverage")
173
- return draft, _digest(draft)
285
+ return draft, _digest(draft), metadata.st_mtime_ns
286
+
287
+
288
+ def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
289
+ times: dict[str, int] = {}
290
+ snapshots = [path for path in sources if _is_source_order_snapshot(path)]
291
+ for path in snapshots:
292
+ record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
293
+ if digest != sources[path] or not isinstance(record.get("sources"), dict):
294
+ raise ContractError("saved source ordering changed after the scan stopped")
295
+ for relative, observation in record["sources"].items():
296
+ if (
297
+ not isinstance(observation, dict)
298
+ or relative not in sources
299
+ or observation.get("digest") != sources[relative]
300
+ or not isinstance(observation.get("observedAtNs"), str)
301
+ or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
302
+ ):
303
+ raise ContractError("saved source ordering does not match its frozen sources")
304
+ observed = int(observation["observedAtNs"])
305
+ if relative in times and times[relative] != observed:
306
+ raise ContractError("saved source ordering has conflicting observations")
307
+ times[relative] = observed
308
+ if snapshots and sources.keys() - set(snapshots) - times.keys():
309
+ raise ContractError("saved source ordering is incomplete")
310
+ return times
311
+
312
+
313
+ def _freeze_source_times(
314
+ scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
315
+ ) -> None:
316
+ # Identical result rewrites must not change the order of frozen review evidence.
317
+ observations = {
318
+ path: {"digest": digest, "observedAtNs": str(times[path])}
319
+ for path, digest in sources.items()
320
+ if not _is_source_order_snapshot(path)
321
+ }
322
+ if not observations:
323
+ return
324
+ record = {"scanId": scan_id, "sources": observations}
325
+ digest = _digest(record)
326
+ path = f"source-order/{digest}.json"
327
+ if not (scan_dir / path).exists():
328
+ write_scan_local_bytes(scan_dir, path, _encoded(record))
329
+ if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
330
+ raise ContractError("saved source ordering does not match its digest")
331
+ sources[path] = digest
332
+
333
+
334
+ def _parent_scan_draft(
335
+ scan_id: str,
336
+ parent_scan: dict[str, Any],
337
+ findings: dict[str, Any],
338
+ coverage: dict[str, Any],
339
+ ) -> dict[str, Any]:
340
+ parent = {
341
+ "scanId": scan_id,
342
+ "findings": findings.get("findings"),
343
+ "coverage": coverage,
344
+ **{
345
+ key: parent_scan[key]
346
+ for key in ("scope", "threatModel", "complete")
347
+ if key in parent_scan
348
+ },
349
+ }
350
+ if not isinstance(parent["findings"], list):
351
+ raise ContractError("Saved parent draft has no findings array")
352
+ return parent
174
353
 
175
354
 
176
355
  def _read_saved_parent_result(
@@ -188,29 +367,34 @@ def _read_saved_parent_result(
188
367
  or coverage.get("scanId", scan_id) != scan_id
189
368
  ):
190
369
  raise ContractError("Saved parent documents belong to a different scan")
191
- parent = {
192
- "scanId": scan_id,
193
- "findings": findings.get("findings"),
194
- "coverage": coverage,
195
- **{
196
- key: parent_scan[key]
197
- for key in ("scope", "threatModel", "complete")
198
- if key in parent_scan
199
- },
200
- }
201
- if not isinstance(parent["findings"], list):
202
- raise ContractError("Saved parent draft has no findings array")
203
- return manifest, parent
370
+ return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
204
371
 
205
372
 
206
- def _source_digests(value: Any, label: str) -> dict[str, str]:
373
+ def _source_digests(value: Any, error: str) -> dict[str, str]:
207
374
  if not isinstance(value, dict) or not all(
208
375
  isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
209
376
  ):
210
- raise ContractError(f"{label} source digests are malformed.")
377
+ raise ContractError(error)
211
378
  return value
212
379
 
213
380
 
381
+ def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
382
+ if isinstance(value, dict) and isinstance(value.get("sources"), dict):
383
+ sources = _source_digests(
384
+ value["sources"], "Saved stopped-scan source digests are malformed."
385
+ )
386
+ model_source = value.get("threatModelSource")
387
+ if not isinstance(model_source, str) or model_source not in sources:
388
+ raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
389
+ return sources, model_source
390
+ return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
391
+
392
+
393
+ def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
394
+ state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
395
+ return json.dumps(state, sort_keys=True)
396
+
397
+
214
398
  def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
215
399
  try:
216
400
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
@@ -234,7 +418,7 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
234
418
 
235
419
  if manifest_path is None:
236
420
  if frozen_sources is not None:
237
- return bool(_source_digests(json.loads(frozen_sources), "Frozen stopped-scan"))
421
+ return bool(_retained_source_state(json.loads(frozen_sources))[0])
238
422
  return has_saved_source()
239
423
  if scan["seal_manifest_digest"] is None:
240
424
  try:
@@ -252,14 +436,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
252
436
  if not isinstance(manifest_scan, dict):
253
437
  return True
254
438
  published_sources = _source_digests(
255
- manifest_scan.get("preservedSources", {}), "Published scan"
439
+ manifest_scan.get("preservedSources", {}),
440
+ "Published scan source digests are malformed.",
256
441
  )
257
442
  current_sources = dict(published_sources)
443
+ paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
258
444
  for path in paths:
259
445
  try:
260
- _, current_sources[path] = _read_saved_result(
261
- scan_dir, path, scan["id"], kind=paths[path]
446
+ captured = _capture_saved_source(
447
+ scan_dir,
448
+ path,
449
+ scan["id"],
450
+ kind=paths[path],
451
+ snapshot_head=path not in published_sources,
452
+ write=False,
262
453
  )
454
+ current_sources.update({path: value[0] for path, value in captured.items()})
263
455
  except (ContractError, OSError, ValueError):
264
456
  continue
265
457
  return current_sources != published_sources
@@ -273,7 +465,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
273
465
  include_parent = True
274
466
  raw_frozen_sources = scan["retained_source_digests_json"]
275
467
  if raw_frozen_sources is not None:
276
- frozen_sources = _source_digests(json.loads(raw_frozen_sources), "Saved stopped-scan")
468
+ frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
277
469
  include_parent = False
278
470
 
279
471
  manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
@@ -291,7 +483,8 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
291
483
  ):
292
484
  if "preservedSources" in manifest_scan:
293
485
  published_sources = _source_digests(
294
- manifest_scan["preservedSources"], "Published scan"
486
+ manifest_scan["preservedSources"],
487
+ "Published scan source digests are malformed.",
295
488
  )
296
489
  include_parent = not published_sources
297
490
  if published_sources:
@@ -312,21 +505,30 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
312
505
  ).fetchall()
313
506
  paths = dict(_saved_result_paths(scan_dir, workers))
314
507
  recovery_sources = dict(frozen_sources or {})
508
+ source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
315
509
  for relative, expected_digest in recovery_sources.items():
316
510
  try:
317
- _, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative))
511
+ _, digest, observed = _read_saved_result(
512
+ scan_dir, relative, scan["id"], kind=paths.get(relative)
513
+ )
318
514
  except (ContractError, OSError, ValueError) as exc:
319
515
  raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
320
516
  if digest != expected_digest:
321
517
  raise ContractError("checkpoint changed after the scan stopped")
518
+ if not _is_source_order_snapshot(relative):
519
+ source_times.setdefault(relative, observed)
322
520
 
323
521
  for relative in paths.keys() - recovery_sources.keys():
324
522
  try:
325
- _, recovery_sources[relative] = _read_saved_result(
326
- scan_dir, relative, scan["id"], kind=paths[relative]
327
- )
523
+ captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
328
524
  except (ContractError, OSError, ValueError):
329
525
  continue
526
+ for path, (digest, observed) in captured.items():
527
+ if path in recovery_sources and recovery_sources[path] != digest:
528
+ raise ContractError("checkpoint changed after the scan stopped")
529
+ recovery_sources[path] = digest
530
+ source_times.setdefault(path, observed)
531
+ _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
330
532
  return recovery_sources, include_parent
331
533
 
332
534
 
@@ -358,15 +560,10 @@ def _finding_key(finding: dict[str, Any]) -> str:
358
560
  else finding.get("identity")
359
561
  )
360
562
  if not isinstance(identity, dict):
361
- extensions = finding.get("extensions")
362
- source = str(
363
- (extensions.get("candidateId") if isinstance(extensions, dict) else None)
364
- or finding.get("title")
365
- or "finding"
366
- )
367
- identity = {
368
- "anchor": re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
369
- }
563
+ normalized = dict(finding)
564
+ normalized.pop("identity", None)
565
+ _ensure_finding_identity(normalized)
566
+ identity = normalized["identity"]
370
567
  locations = finding.get("locations", [])
371
568
  if not isinstance(locations, list):
372
569
  locations = []
@@ -459,6 +656,257 @@ def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
459
656
  )
460
657
 
461
658
 
659
+ def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
660
+ rows = coverage.get("deferred", [])
661
+ return rows if isinstance(rows, list) else []
662
+
663
+
664
+ def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
665
+ if draft.get("complete") is False:
666
+ return []
667
+ coverage = draft["coverage"]
668
+ rows = coverage.get("resolvedDeferred", [])
669
+ try:
670
+ # Invalid closure metadata cannot discard the evidence it names.
671
+ _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
672
+ _validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
673
+ except ContractError:
674
+ return []
675
+ return rows
676
+
677
+
678
+ def _merge_tied_parent_observations(
679
+ current: dict[str, Any], previous: dict[str, Any]
680
+ ) -> dict[str, Any]:
681
+ merged = copy.deepcopy(current)
682
+ for finding in previous["findings"]:
683
+ if finding not in merged["findings"]:
684
+ merged["findings"].append(copy.deepcopy(finding))
685
+ coverage = merged["coverage"]
686
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
687
+ rows = previous["coverage"].get(field, [])
688
+ output = coverage.setdefault(field, [])
689
+ if isinstance(rows, list) and isinstance(output, list):
690
+ for row in rows:
691
+ if row not in output:
692
+ output.append(copy.deepcopy(row))
693
+ pending_ids = {
694
+ identity
695
+ for row in _deferred_rows(coverage)
696
+ if isinstance(row, dict)
697
+ for identity in (row.get("id"), row.get("candidateId"))
698
+ if isinstance(identity, str)
699
+ }
700
+ closure_schema = _read_json(
701
+ Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
702
+ )["properties"]["resolvedDeferred"]
703
+ closures = {}
704
+ for observed in (current, previous):
705
+ for row in _resolved_deferred_rows(observed, closure_schema):
706
+ if row["id"] not in pending_ids:
707
+ closures.setdefault(row["id"], copy.deepcopy(row))
708
+ coverage.pop("resolvedDeferred", None)
709
+ if closures:
710
+ coverage["resolvedDeferred"] = list(closures.values())
711
+ if current.get("complete") is False or previous.get("complete") is False:
712
+ merged["complete"] = False
713
+ if (
714
+ coverage.get("deferred")
715
+ or merged.get("complete") is False
716
+ or (
717
+ isinstance(coverage.get("surfaces"), list)
718
+ and any(
719
+ isinstance(row, dict) and row.get("disposition") == "needs_follow_up"
720
+ for row in coverage["surfaces"]
721
+ )
722
+ )
723
+ or previous["coverage"].get("completeness") == "partial"
724
+ ):
725
+ coverage["completeness"] = "partial"
726
+ return merged
727
+
728
+
729
+ def _saved_coverage_id(item: dict[str, Any]) -> str:
730
+ return item.get("candidateId") or f"saved-{_digest(item)[:16]}"
731
+
732
+
733
+ def _deferred_candidate_id(
734
+ row: dict[str, Any],
735
+ owner: str | None,
736
+ ambiguous_deferred: set[tuple[str | None, str]],
737
+ ) -> str | None:
738
+ identity = row.get("candidateId") or row.get("id")
739
+ if not isinstance(identity, str):
740
+ return None
741
+ if (owner, identity) in ambiguous_deferred and not any(
742
+ key in row for key in ("candidateId", "candidate", "finding")
743
+ ):
744
+ return None
745
+ return identity
746
+
747
+
748
+ def _generic_surface_updates(
749
+ sources: list[tuple[str, dict[str, Any], str | None]],
750
+ source_order: dict[str, tuple[int, int]],
751
+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
752
+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
753
+ resolved_candidates: dict[tuple[str | None, str], str],
754
+ reopened_generic: set[tuple[str | None, str]],
755
+ surface_schema: dict[str, Any],
756
+ deferred_rows: dict[str, list[Any]],
757
+ ambiguous_deferred: set[tuple[str | None, str]],
758
+ ) -> tuple[set[int], list[dict[str, Any]]]:
759
+ if not closed_deferred and not reopened_generic:
760
+ return set(), []
761
+
762
+ def linked(row: dict[str, Any], identity: str | None) -> bool:
763
+ surface_ids = row.get("surfaceIds", [])
764
+ return identity is not None and (
765
+ row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
766
+ )
767
+
768
+ saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
769
+ for relative, draft, owner in sources:
770
+ rows = draft["coverage"].get("surfaces", [])
771
+ for row in rows if isinstance(rows, list) else []:
772
+ if isinstance(row, dict) and isinstance(row.get("id"), str):
773
+ saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
774
+ replaced: set[int] = set()
775
+ updates: list[dict[str, Any]] = []
776
+ for relative, draft, owner in sources:
777
+ closed_ids = {
778
+ identity
779
+ for (saved_owner, identity), (_, _, source) in closed_deferred.items()
780
+ if saved_owner == owner and source == relative
781
+ }
782
+ reopened_ids = {
783
+ identity
784
+ for (saved_owner, identity), (_, _, source) in active_deferred.items()
785
+ if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
786
+ }
787
+ if not closed_ids and not reopened_ids:
788
+ continue
789
+ current = draft["coverage"].get("surfaces", [])
790
+ for surface in current if isinstance(current, list) else []:
791
+ if not isinstance(surface, dict):
792
+ continue
793
+ reopening = surface.get("disposition") == "needs_follow_up"
794
+ work_ids = reopened_ids if reopening else closed_ids
795
+ if not work_ids:
796
+ continue
797
+ identity = surface.get("id")
798
+ if not isinstance(identity, str):
799
+ continue
800
+ matches = saved_surfaces[(owner, identity)]
801
+ # Older writers could assign one ID to distinct surfaces in a draft.
802
+ # A closure cannot identify which of those observations it replaces.
803
+ by_source = dict(matches)
804
+ if any(row != by_source[saved_path] for saved_path, row in matches):
805
+ continue
806
+ if any(
807
+ "candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
808
+ ):
809
+ continue
810
+ if any(
811
+ row is not surface
812
+ and source_order[saved_path] >= source_order[relative]
813
+ and row.get("disposition") != surface.get("disposition")
814
+ for saved_path, row in matches
815
+ ):
816
+ continue
817
+
818
+ if not reopening and any(
819
+ saved_owner == owner
820
+ and (
821
+ not isinstance(row.get("id") or row.get("candidateId"), str)
822
+ or (
823
+ isinstance(row.get("id"), str)
824
+ and (owner, row["id"]) in ambiguous_deferred
825
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
826
+ )
827
+ )
828
+ and linked(row, identity)
829
+ for saved_path, _, saved_owner in sources
830
+ for row in deferred_rows[saved_path]
831
+ if isinstance(row, dict)
832
+ ):
833
+ continue
834
+ if not reopening and any(
835
+ saved_owner == owner
836
+ and (owner, deferred_id) not in closed_deferred
837
+ and (
838
+ (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
839
+ or (owner, candidate_id) not in resolved_candidates
840
+ )
841
+ and linked(row, identity)
842
+ for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
843
+ ):
844
+ continue
845
+ # An accepted checkpoint can update a saved surface by ID without
846
+ # optional surfaceIds links on its generic task.
847
+ latest_surface = max(matches, key=lambda match: source_order[match[0]])[1]
848
+ update = copy.deepcopy(latest_surface)
849
+ refs = update.setdefault("receiptRefs", [])
850
+ if isinstance(refs, list):
851
+ for _, row in matches:
852
+ previous_refs = row.get("receiptRefs", [])
853
+ for ref in previous_refs if isinstance(previous_refs, list) else []:
854
+ if ref not in refs:
855
+ refs.append(ref)
856
+ try:
857
+ _validate_schema_node(update, surface_schema, "coverage.surfaces")
858
+ except ContractError:
859
+ continue
860
+ replaced.update(
861
+ id(row)
862
+ for _, row in matches
863
+ if reopening
864
+ or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
865
+ )
866
+ if update not in updates:
867
+ updates.append(update)
868
+ return replaced, updates
869
+
870
+
871
+ @contextmanager
872
+ def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
873
+ """Keep rejected completion attempts from becoming accepted parent observations."""
874
+ head_path = scan_dir / "checkpoint-head.json"
875
+ previous = None
876
+ try:
877
+ head_path.lstat()
878
+ except FileNotFoundError:
879
+ pass
880
+ else:
881
+ descriptor = open_scan_local_file_descriptor(
882
+ scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
883
+ )
884
+ with os.fdopen(descriptor, "rb") as handle:
885
+ metadata = os.fstat(handle.fileno())
886
+ previous = (handle.read(), metadata)
887
+ directories = ("checkpoints", "checkpoint-heads")
888
+ previous_files = {
889
+ relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
890
+ }
891
+ try:
892
+ yield
893
+ except ContractError:
894
+ if previous is None:
895
+ _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
896
+ else:
897
+ payload, metadata = previous
898
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
899
+ os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
900
+ current_files = {
901
+ relative
902
+ for directory in directories
903
+ for relative in _checkpoint_paths(scan_dir, directory)
904
+ }
905
+ for relative in current_files - previous_files:
906
+ _remove_scan_local_file_if_exists(scan_dir, relative)
907
+ raise
908
+
909
+
462
910
  def merge_saved_results(
463
911
  scan_dir: Path,
464
912
  scan_id: str,
@@ -470,14 +918,25 @@ def merge_saved_results(
470
918
  reason: str,
471
919
  frozen_source_digests: dict[str, str] | None = None,
472
920
  allow_frozen_legacy_parent: bool = False,
921
+ frozen_model_source: str | None = None,
922
+ selected_model_source: list[str] | None = None,
473
923
  ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
474
924
  """Read only bound parent/worker files; return an unsealed loss-preserving union."""
475
925
  initial_warnings = set(warnings)
926
+ try:
927
+ source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
928
+ except (ContractError, OSError, ValueError) as exc:
929
+ raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
476
930
  parent: dict[str, Any] | None = None
477
931
  parent_manifest: dict[str, Any] | None = None
932
+ parent_is_canonical = False
933
+ parent_modified = 0
478
934
  if frozen_source_digests is None or allow_frozen_legacy_parent:
479
935
  try:
480
936
  parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
937
+ # Without an accepted head, file-authored coverage is a full replacement.
938
+ parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
939
+ parent_is_canonical = True
481
940
  except (ContractError, OSError, ValueError) as exc:
482
941
  if not stopped:
483
942
  raise
@@ -487,25 +946,66 @@ def merge_saved_results(
487
946
  parent = None
488
947
  if parent_manifest is not None and parent is not None:
489
948
  parent_scan = parent_manifest["scan"]
949
+ try:
950
+ previous_head, _, head_modified = _read_saved_result(
951
+ scan_dir, "checkpoint-head.json", scan_id
952
+ )
953
+ except (ContractError, OSError, ValueError):
954
+ head_modified = None
955
+ if head_modified is not None:
956
+ # A partial tool publication must not outrank its accepted head.
957
+ parent_modified = min(
958
+ (scan_dir / name).lstat().st_mtime_ns
959
+ for name in ("findings.json", "coverage.json", "scan-manifest.json")
960
+ )
490
961
  if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
962
+ head_path = scan_dir / "checkpoint-head.json"
963
+ tied_observations = False
964
+ if head_modified == parent_modified:
965
+ previous_parent, _, _ = _read_saved_result(
966
+ scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
967
+ )
968
+ if previous_parent != parent:
969
+ # Tied observations cannot decide which pending work came last.
970
+ parent = _merge_tied_parent_observations(parent, previous_parent)
971
+ parent_is_canonical = False
972
+ tied_observations = True
491
973
  payload = _encoded(parent)
492
974
  parent_digest = hashlib.sha256(payload).hexdigest()
493
975
  parent_checkpoint = f"checkpoints/{parent_digest}.json"
494
- write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
976
+ checkpoint_path = scan_dir / parent_checkpoint
977
+ if not checkpoint_path.exists():
978
+ write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
979
+ # A recovery copy must not appear newer than the review it copies.
980
+ os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
981
+ if head_modified is None or head_modified < parent_modified or tied_observations:
982
+ write_scan_local_bytes(
983
+ scan_dir,
984
+ "checkpoint-head.json",
985
+ _encoded({"checkpoint": checkpoint_path.name}),
986
+ )
987
+ os.utime(head_path, ns=(parent_modified, parent_modified))
495
988
  if frozen_source_digests is not None:
989
+ captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
496
990
  frozen_source_digests = {
497
991
  **frozen_source_digests,
498
992
  parent_checkpoint: parent_digest,
993
+ **{path: value[0] for path, value in captured.items()},
499
994
  }
500
995
 
501
996
  sources: list[tuple[str, dict[str, Any], str | None]] = []
502
997
  parent_preserved_sources: dict[str, str] = {}
503
998
  source_digests: dict[str, str] = {}
999
+ source_order: dict[str, tuple[int, int]] = {}
1000
+ worker_attempts: dict[str, tuple[str, int]] = {}
1001
+ saved_heads: dict[str, str] = {}
504
1002
  if parent_manifest:
505
1003
  recorded = parent_manifest["scan"].get("preservedSources", {})
506
1004
  if isinstance(recorded, dict):
507
1005
  parent_preserved_sources = recorded
508
1006
  source_digests.update(parent_preserved_sources)
1007
+ if frozen_source_digests is None:
1008
+ source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
509
1009
  paths: dict[str, str | None] = {}
510
1010
  reducer_paths: set[str] = set()
511
1011
  current_results: set[str] = set()
@@ -520,57 +1020,43 @@ def merge_saved_results(
520
1020
  except ValueError:
521
1021
  warnings.append("Skipped a reducer result outside the scan directory.")
522
1022
 
523
- def checkpoints(directory: str, worker_id: str | None) -> None:
524
- for name in _children(scan_dir, directory):
525
- if re.fullmatch(r"[0-9a-f]{64}\.json", name):
526
- paths[f"{directory}/{name}"] = worker_id
1023
+ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
1024
+ if worker_id is not None:
1025
+ root = Path(directory).parent.as_posix()
1026
+ worker_attempts[root] = (worker_id, attempt)
1027
+ paths[f"{root}/checkpoint-head.json"] = worker_id
1028
+ head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
1029
+ for saved_directory in (directory, head_snapshots):
1030
+ paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
527
1031
 
1032
+ paths["checkpoint-head.json"] = None
528
1033
  checkpoints("checkpoints", None)
529
1034
  for worker in workers:
530
1035
  try:
531
- output = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix()
1036
+ outputs = _worker_outputs(scan_dir, worker)
532
1037
  except (TypeError, ValueError):
533
1038
  warnings.append("Skipped a worker checkpoint outside the scan directory.")
534
1039
  continue
535
1040
  if worker["kind"] == "dedup":
536
-
537
- def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None:
538
- result_path = f"{directory}/result.json"
539
- checkpoint_paths = [
540
- f"{directory}/checkpoints/{name}"
541
- for name in _children(scan_dir, f"{directory}/checkpoints")
542
- if re.fullmatch(r"[0-9a-f]{64}\.json", name)
543
- ]
1041
+ for directory, attempt in outputs:
1042
+ checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
544
1043
  if not checkpoint_paths:
545
- return
546
- paths[result_path] = None
547
- for checkpoint_path in checkpoint_paths:
548
- paths[checkpoint_path] = None
549
- reducer_paths.update([result_path, *checkpoint_paths])
550
- reducer_outputs.append((reducer_worker, result_path, checkpoint_paths, attempt))
551
-
552
- reducer_output(output, int(worker["attempt"] or 0), worker)
553
- attempts = (
554
- Path(output).parent if Path(output).name == "output" else Path(output)
555
- ) / "attempts"
556
- for name in _children(scan_dir, attempts.as_posix()):
557
- match = re.fullmatch(r"attempt-(\d+)", name)
558
- if match:
559
- reducer_output((attempts / name).as_posix(), int(match.group(1)), worker)
1044
+ continue
1045
+ result_path = f"{directory}/result.json"
1046
+ retained_paths = [result_path, *checkpoint_paths]
1047
+ paths.update(dict.fromkeys(retained_paths))
1048
+ reducer_paths.update(retained_paths)
1049
+ reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
560
1050
  continue
561
1051
  if worker["kind"] != "discovery":
562
1052
  continue
1053
+ output, attempt = outputs[0]
563
1054
  paths[f"{output}/result.json"] = worker["id"]
564
1055
  current_results.add(f"{output}/result.json")
565
- checkpoints(f"{output}/checkpoints", worker["id"])
566
- attempts = (
567
- Path(output).parent if Path(output).name == "output" else Path(output)
568
- ) / "attempts"
569
- for name in _children(scan_dir, attempts.as_posix()):
570
- if re.fullmatch(r"attempt-\d+", name):
571
- archived = (attempts / name).as_posix()
572
- paths[f"{archived}/result.json"] = worker["id"]
573
- checkpoints(f"{archived}/checkpoints", worker["id"])
1056
+ for archived, archived_attempt in outputs[1:]:
1057
+ paths[f"{archived}/result.json"] = worker["id"]
1058
+ checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
1059
+ checkpoints(f"{output}/checkpoints", worker["id"], attempt)
574
1060
  if worker["result_manifest_path"]:
575
1061
  try:
576
1062
  current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
@@ -579,7 +1065,26 @@ def merge_saved_results(
579
1065
  except ValueError:
580
1066
  warnings.append("Skipped a worker result outside the scan directory.")
581
1067
 
1068
+ if frozen_source_digests is None:
1069
+ for relative, worker_id in list(paths.items()):
1070
+ if Path(relative).name != "checkpoint-head.json":
1071
+ continue
1072
+ del paths[relative]
1073
+ try:
1074
+ captured = _capture_saved_source(scan_dir, relative, scan_id)
1075
+ paths.update({path: worker_id for path in captured})
1076
+ except (ContractError, OSError, ValueError) as exc:
1077
+ if (scan_dir / relative).exists():
1078
+ warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1079
+
582
1080
  if frozen_source_digests is not None:
1081
+ source_digests.update(
1082
+ {
1083
+ path: digest
1084
+ for path, digest in frozen_source_digests.items()
1085
+ if _is_source_order_snapshot(path)
1086
+ }
1087
+ )
583
1088
  paths = {
584
1089
  relative: worker_id
585
1090
  for relative, worker_id in paths.items()
@@ -591,12 +1096,17 @@ def merge_saved_results(
591
1096
 
592
1097
  for relative, worker_id in paths.items():
593
1098
  try:
594
- draft, digest = _read_saved_result(
1099
+ draft, digest, observed = _read_saved_result(
595
1100
  scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
596
1101
  )
597
1102
  if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
598
1103
  raise ContractError("checkpoint changed after the scan stopped")
599
1104
  source_digests[relative] = digest
1105
+ source_times.setdefault(relative, observed)
1106
+ source_order[relative] = (0, source_times[relative])
1107
+ if _checkpoint_head_directory(relative) is not None:
1108
+ saved_heads[relative] = draft["checkpoint"]
1109
+ continue
600
1110
  # Recovery expects coverage, but reducer results only contain findings
601
1111
  # and context. Add an empty value after hashing the original result.
602
1112
  sources.append((relative, {"coverage": {}, **draft}, worker_id))
@@ -607,6 +1117,37 @@ def merge_saved_results(
607
1117
  if frozen_source_digests.keys() - source_digests.keys():
608
1118
  raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
609
1119
 
1120
+ # Frozen observations retain checkpoint selection even if a worker moves its head.
1121
+ headed_workers = {
1122
+ worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
1123
+ for head in saved_heads
1124
+ if _checkpoint_head_directory(head) != Path(".")
1125
+ }
1126
+ for relative, _, worker_id in sources:
1127
+ if worker_id not in headed_workers:
1128
+ continue
1129
+ directory = Path(relative).parent
1130
+ if directory.name == "checkpoints":
1131
+ directory = directory.parent
1132
+ _, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
1133
+ source_order[relative] = (attempt, source_order[relative][1])
1134
+ selected_observations: dict[str, tuple[int, int]] = {}
1135
+ parent_heads: list[tuple[int, str]] = []
1136
+ for head, checkpoint in saved_heads.items():
1137
+ directory = _checkpoint_head_directory(head)
1138
+ selected = (directory / "checkpoints" / checkpoint).as_posix()
1139
+ if selected not in source_order:
1140
+ raise ContractError("Checkpoint head is outside the saved source set.")
1141
+ observed = source_order[head][1]
1142
+ if directory == Path("."):
1143
+ order = (0, max(source_order[selected][1], observed))
1144
+ parent_heads.append((observed, selected))
1145
+ else:
1146
+ _, attempt = worker_attempts[directory.as_posix()]
1147
+ order = (attempt, observed)
1148
+ selected_observations[selected] = max(selected_observations.get(selected, order), order)
1149
+ source_order.update(selected_observations)
1150
+
610
1151
  drafts_by_path = {relative: draft for relative, draft, _ in sources}
611
1152
  latest_reducer_key = (
612
1153
  (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
@@ -627,8 +1168,67 @@ def merge_saved_results(
627
1168
  latest_reducer_key = candidate_key
628
1169
  latest_reducer = result_path
629
1170
 
1171
+ if parent_heads:
1172
+ latest_observation = max(observed for observed, _ in parent_heads)
1173
+ for observed, parent_path in parent_heads:
1174
+ if observed != latest_observation:
1175
+ continue
1176
+ draft = drafts_by_path[parent_path]
1177
+ modified = source_order[parent_path][1]
1178
+ if parent is None or modified > parent_modified:
1179
+ parent = draft
1180
+ parent_modified = modified
1181
+ parent_is_canonical = False
1182
+ elif modified == parent_modified and draft != parent:
1183
+ parent = _merge_tied_parent_observations(parent, draft)
1184
+ parent_is_canonical = False
630
1185
  if parent is None and latest_reducer is not None:
631
- parent = next((draft for relative, draft, _ in sources if relative == latest_reducer), None)
1186
+ parent = drafts_by_path[latest_reducer]
1187
+
1188
+ all_sources = ([("parent", parent, None)] if parent else []) + sources
1189
+ # Older checkpoints can omit IDs already assigned in their published output.
1190
+ for field in ("deferred", "surfaces"):
1191
+ named_rows: dict[str | None, list[dict[str, Any]]] = {}
1192
+ for _, draft, owner in all_sources:
1193
+ rows = draft["coverage"].get(field, [])
1194
+ named_rows.setdefault(owner, []).extend(
1195
+ row
1196
+ for row in (rows if isinstance(rows, list) else [])
1197
+ if isinstance(row, dict) and isinstance(row.get("id"), str)
1198
+ )
1199
+ for _, draft, owner in all_sources:
1200
+ rows = draft["coverage"].get(field, [])
1201
+ if not isinstance(rows, list):
1202
+ continue
1203
+ reserved = {
1204
+ row["id"]
1205
+ for row in rows
1206
+ if isinstance(row, dict) and isinstance(row.get("id"), str)
1207
+ }
1208
+ for row in rows:
1209
+ if not isinstance(row, dict) or "id" in row:
1210
+ continue
1211
+ if field == "deferred" and any(
1212
+ key in row for key in ("candidateId", "candidate", "finding")
1213
+ ):
1214
+ continue
1215
+ content = {"receiptRefs": [], **row} if field == "surfaces" else row
1216
+ identity = next(
1217
+ (
1218
+ named["id"]
1219
+ for named in named_rows[owner]
1220
+ if named["id"] not in reserved
1221
+ and {
1222
+ **({"receiptRefs": []} if field == "surfaces" else {}),
1223
+ **{key: value for key, value in named.items() if key != "id"},
1224
+ }
1225
+ == content
1226
+ ),
1227
+ None,
1228
+ )
1229
+ if identity is not None:
1230
+ row["id"] = identity
1231
+ reserved.add(identity)
632
1232
 
633
1233
  if parent is None and not sources:
634
1234
  return None
@@ -641,6 +1241,13 @@ def merge_saved_results(
641
1241
  ):
642
1242
  return None
643
1243
 
1244
+ if (
1245
+ frozen_source_digests is None
1246
+ or any(_is_source_order_snapshot(path) for path in source_digests)
1247
+ or allow_frozen_legacy_parent
1248
+ ):
1249
+ _freeze_source_times(scan_dir, scan_id, source_digests, source_times)
1250
+
644
1251
  target_kind = binding["allowedTargetKinds"][0]
645
1252
  if (
646
1253
  target_kind == "git_worktree"
@@ -651,15 +1258,9 @@ def merge_saved_results(
651
1258
  target = {"kind": target_kind, **binding["target"]}
652
1259
  if target["kind"] == "git_diff" and "snapshotDigest" not in target:
653
1260
  diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
654
- digest = hashlib.sha256(
655
- b"codex-security-diff/v1\0"
656
- + diff_kind.encode()
657
- + b"\0"
658
- + target["baseRevision"].encode()
659
- + b"\0"
660
- + target["headRevision"].encode()
661
- ).hexdigest()
662
- target["snapshotDigest"] = f"codex-security-snapshot/v1:sha256:{digest}"
1261
+ target["snapshotDigest"] = committed_diff_snapshot_digest(
1262
+ diff_kind, target["baseRevision"], target["headRevision"]
1263
+ )
663
1264
  manifest = (
664
1265
  copy.deepcopy(parent_manifest)
665
1266
  if parent_manifest
@@ -668,6 +1269,27 @@ def merge_saved_results(
668
1269
  for key in ("sealedAt", "artifacts"):
669
1270
  manifest["scan"].pop(key, None)
670
1271
  manifest["scan"]["preservedSources"] = source_digests
1272
+ # Completion follows the selected parent, including legacy terminal drafts
1273
+ # that omit the optional marker, rather than an older canonical manifest.
1274
+ if parent is not None:
1275
+ manifest["scan"].pop("complete", None)
1276
+ if "complete" in parent:
1277
+ manifest["scan"]["complete"] = parent["complete"]
1278
+ for key in ("scope", "threatModel"):
1279
+ if isinstance(parent.get(key), dict):
1280
+ manifest["scan"][key] = copy.deepcopy(parent[key])
1281
+ if isinstance(manifest["scan"].get("scope"), dict):
1282
+ manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
1283
+ if frozen_model_source is not None:
1284
+ # Publication retries retain the choice made with the frozen source set.
1285
+ model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
1286
+ if not isinstance(model, dict):
1287
+ raise ContractError("Frozen stopped-scan model source is unavailable.")
1288
+ manifest["scan"]["threatModel"] = copy.deepcopy(model)
1289
+ if paths[frozen_model_source] is not None:
1290
+ manifest["scan"]["threatModel"]["origin"] = "recovered"
1291
+ if selected_model_source is not None:
1292
+ selected_model_source[:] = [frozen_model_source]
671
1293
  coverage = (
672
1294
  copy.deepcopy(parent["coverage"])
673
1295
  if parent and parent["coverage"]
@@ -696,7 +1318,7 @@ def merge_saved_results(
696
1318
  for field in ("surfaces", "explicitExclusions", "deferred")
697
1319
  for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
698
1320
  }
699
- if parent_manifest
1321
+ if parent_is_canonical
700
1322
  else set()
701
1323
  )
702
1324
  findings: list[dict[str, Any]] = []
@@ -725,32 +1347,180 @@ def merge_saved_results(
725
1347
  )
726
1348
  return bool(document["findings"])
727
1349
 
728
- all_sources = ([("parent", parent, None)] if parent else []) + sources
729
- current_drafts = ([(None, parent)] if parent else []) + [
730
- (worker_id, draft) for relative, draft, worker_id in sources if relative in current_results
1350
+ source_order["parent"] = (0, parent_modified)
1351
+ deferred_rows = {
1352
+ relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
1353
+ }
1354
+ # A legacy source can contain independent tasks with the same explicit ID.
1355
+ # Later rewrites cannot make an ID-only closure identify one of those tasks.
1356
+ ambiguous_deferred: set[tuple[str | None, str]] = set()
1357
+ for relative, _, owner in all_sources:
1358
+ by_id: dict[str, dict[str, Any]] = {}
1359
+ candidate_aliases = {
1360
+ identity
1361
+ for row in deferred_rows[relative]
1362
+ if isinstance(row, dict)
1363
+ and any(key in row for key in ("candidateId", "candidate", "finding"))
1364
+ for identity in (row.get("id"), row.get("candidateId"))
1365
+ if isinstance(identity, str)
1366
+ }
1367
+ for row in deferred_rows[relative]:
1368
+ if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
1369
+ continue
1370
+ if any(key in row for key in ("candidateId", "candidate", "finding")):
1371
+ continue
1372
+ if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
1373
+ ambiguous_deferred.add((owner, identity))
1374
+ by_id[identity] = row
1375
+ current_drafts = ([("parent", parent, None)] if parent else []) + [
1376
+ source for source in sources if source[0] in current_results | selected_observations.keys()
1377
+ ]
1378
+ # Generic closures belong to one logical scan or worker, just like candidates.
1379
+ # Keep them when recovering a terminal checkpoint without its canonical write.
1380
+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1381
+
1382
+ candidate_ids: set[tuple[str | None, str]] = set()
1383
+ accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
1384
+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1385
+ coverage_schema = _read_json(
1386
+ Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
1387
+ )["properties"]
1388
+ for relative, draft, owner in all_sources:
1389
+ order = source_order[relative]
1390
+ for item in deferred_rows[relative]:
1391
+ if not isinstance(item, dict):
1392
+ continue
1393
+ if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
1394
+ candidate_ids.update(
1395
+ (owner, identity)
1396
+ for identity in (item.get("id"), item.get("candidateId"))
1397
+ if isinstance(identity, str)
1398
+ )
1399
+ identity = item.get("id") or item.get("candidateId")
1400
+ if isinstance(identity, str):
1401
+ key = (owner, identity)
1402
+ previous = active_deferred.get(key)
1403
+ if previous is None or order > previous[0]:
1404
+ active_deferred[key] = (order, item, relative)
1405
+ if (
1406
+ relative == "parent"
1407
+ or relative in current_results
1408
+ or relative in selected_observations
1409
+ ):
1410
+ try:
1411
+ # A malformed update cannot discard valid saved evidence.
1412
+ _validate_schema_node(
1413
+ item, coverage_schema["deferred"]["items"], "coverage.deferred"
1414
+ )
1415
+ except ContractError:
1416
+ continue
1417
+ accepted_deferred_orders[key] = max(
1418
+ accepted_deferred_orders.get(key, order), order
1419
+ )
1420
+ for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
1421
+ key = (owner, closure["id"])
1422
+ previous = closed_deferred.get(key)
1423
+ if previous is None or order > previous[0]:
1424
+ # The parent comes first, preserving its reason on equal timestamps.
1425
+ closed_deferred[key] = (order, closure, relative)
1426
+ reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
1427
+ reopened_generic: set[tuple[str | None, str]] = set()
1428
+ for key, (order, _, _) in list(closed_deferred.items()):
1429
+ # Equal timestamps cannot distinguish closure from reopened work.
1430
+ reopened = [
1431
+ active
1432
+ for (owner, identity), active in active_deferred.items()
1433
+ if owner == key[0]
1434
+ and (identity == key[1] or active[1].get("candidateId") == key[1])
1435
+ and active[0] >= order
1436
+ ]
1437
+ if key in candidate_ids or key in ambiguous_deferred or reopened:
1438
+ del closed_deferred[key]
1439
+ for _, item, _ in reopened:
1440
+ reopened_rows.append((key[0], item))
1441
+ if key not in candidate_ids:
1442
+ reopened_generic.add(key)
1443
+ # Retain every distinct ambiguous task even when a terminal result supersedes
1444
+ # its source. Existing output ID normalization keeps each row distinct.
1445
+ for relative, _, owner in all_sources:
1446
+ for row in deferred_rows[relative]:
1447
+ if (
1448
+ isinstance(row, dict)
1449
+ and isinstance(identity := row.get("id"), str)
1450
+ and (owner, identity) in ambiguous_deferred
1451
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
1452
+ and (owner, row) not in reopened_rows
1453
+ ):
1454
+ reopened_rows.append((owner, row))
1455
+ parent_closures = [
1456
+ closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
731
1457
  ]
1458
+ coverage.pop("resolvedDeferred", None)
1459
+ if parent_closures:
1460
+ coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
1461
+ if parent:
1462
+ coverage["deferred"] = [
1463
+ row
1464
+ for row in deferred_rows["parent"]
1465
+ if not isinstance(row, dict)
1466
+ or not isinstance(row.get("id"), str)
1467
+ or (None, row["id"]) not in closed_deferred
1468
+ ]
732
1469
  resolved: dict[tuple[str | None, str], str] = {}
733
- for owner, draft in current_drafts:
1470
+
1471
+ ordered_candidates = {
1472
+ (owner, identity)
1473
+ for owner, row in reopened_rows
1474
+ if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
1475
+ and (owner, identity) in candidate_ids
1476
+ }
1477
+ ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
1478
+
1479
+ outcomes: list[tuple[str, str | None, str, str]] = []
1480
+ for relative, draft, owner in current_drafts:
734
1481
  for finding in draft["findings"]:
735
1482
  if (
736
1483
  isinstance(finding, dict)
737
1484
  and valid_finding(finding)
738
1485
  and (candidate_id := finding_candidate_id(finding))
739
1486
  ):
740
- resolved.setdefault((owner, candidate_id), "reported")
1487
+ outcomes.append((relative, owner, candidate_id, "reported"))
741
1488
  for field in ("surfaces", "explicitExclusions"):
742
1489
  items = draft["coverage"].get(field, [])
743
1490
  for item in items if isinstance(items, list) else []:
744
1491
  if (
745
1492
  isinstance(item, dict)
746
1493
  and isinstance(item.get("candidateId"), str)
747
- and item.get("disposition") in {"reported", "rejected", "not_applicable"}
1494
+ and item.get("disposition") in {"rejected", "not_applicable"}
748
1495
  ):
749
- resolved.setdefault((owner, item["candidateId"]), item["disposition"])
1496
+ outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
1497
+ ordered_candidates.update(
1498
+ (owner, candidate_id)
1499
+ for relative, owner, candidate_id, _ in outcomes
1500
+ if owner is not None and relative in selected_observations
1501
+ )
1502
+ # Reopened work and selected checkpoint outcomes follow the saved source order.
1503
+ for relative, owner, candidate_id, disposition in outcomes:
1504
+ key = (owner, candidate_id)
1505
+ if key not in ordered_candidates:
1506
+ if relative == "parent" or relative in current_results:
1507
+ resolved.setdefault(key, disposition)
1508
+ continue
1509
+ order = source_order[relative]
1510
+ if any(
1511
+ saved_owner == owner
1512
+ and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
1513
+ and modified >= order
1514
+ for (saved_owner, _), (modified, row, _) in active_deferred.items()
1515
+ ):
1516
+ continue
1517
+ if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
1518
+ resolved[key] = disposition
1519
+ ordered_outcomes[key] = (order, relative)
750
1520
  # Only the current parent may claim that another worker finding was absorbed.
751
1521
  # A superseded checkpoint must not suppress a newer independent result.
752
- for draft in [parent] if parent else []:
753
- for finding in draft["findings"]:
1522
+ if parent:
1523
+ for finding in parent["findings"]:
754
1524
  if valid_finding(finding):
755
1525
  canonical_key = _finding_key(finding)
756
1526
  for retained in _retained_findings(finding):
@@ -787,26 +1557,155 @@ def merge_saved_results(
787
1557
  represented_candidate_history.setdefault(candidate_key, set()).add(
788
1558
  _digest(_finding_content(original["finding"]))
789
1559
  )
790
- resolved.setdefault(candidate_key, "reported")
1560
+ replaced_surfaces, surface_updates = _generic_surface_updates(
1561
+ all_sources,
1562
+ source_order,
1563
+ closed_deferred,
1564
+ active_deferred,
1565
+ resolved,
1566
+ reopened_generic,
1567
+ coverage_schema["surfaces"]["items"],
1568
+ deferred_rows,
1569
+ ambiguous_deferred,
1570
+ )
1571
+ replaced_rows = {
1572
+ "surfaces": replaced_surfaces,
1573
+ "deferred": {
1574
+ id(row)
1575
+ for relative, _, owner in all_sources
1576
+ for row in deferred_rows[relative]
1577
+ if isinstance(row, dict)
1578
+ and isinstance(identity := row.get("id"), str)
1579
+ and (key := (owner, identity)) not in candidate_ids
1580
+ and key not in ambiguous_deferred
1581
+ and (updated := accepted_deferred_orders.get(key)) is not None
1582
+ and updated > source_order[relative]
1583
+ },
1584
+ }
1585
+ for field, replaced in replaced_rows.items():
1586
+ if parent and isinstance(coverage.get(field), list):
1587
+ previous = parent["coverage"].get(field, [])
1588
+ if isinstance(previous, list):
1589
+ removed_parent = [row for row in previous if id(row) in replaced]
1590
+ coverage[field] = [row for row in coverage[field] if row not in removed_parent]
1591
+ if isinstance(coverage.get("surfaces"), list):
1592
+ for surface in surface_updates:
1593
+ if surface not in coverage["surfaces"]:
1594
+ coverage["surfaces"].append(surface)
1595
+ for surface in coverage["surfaces"]:
1596
+ if isinstance(surface, dict):
1597
+ surface.setdefault("receiptRefs", [])
1598
+
1599
+ # Reopened work survives a superseded checkpoint, but current candidate
1600
+ # outcomes still apply. Parent closures cannot remove another worker's row.
1601
+ for owner, item in reopened_rows:
1602
+ if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
1603
+ owner,
1604
+ identity,
1605
+ ) in resolved:
1606
+ continue
1607
+ pending = coverage.setdefault("deferred", [])
1608
+ if isinstance(pending, list) and item not in pending:
1609
+ pending.append(copy.deepcopy(item))
1610
+ ambiguous_surface_ids = {
1611
+ (owner, identity)
1612
+ for owner, row in reopened_rows
1613
+ if isinstance(row.get("id"), str)
1614
+ and (owner, row["id"]) in ambiguous_deferred
1615
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
1616
+ for identity in [
1617
+ row["id"],
1618
+ *(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
1619
+ ]
1620
+ if isinstance(identity, str)
1621
+ }
1622
+ for _, draft, owner in all_sources:
1623
+ surfaces = draft["coverage"].get("surfaces", [])
1624
+ by_id: dict[str, dict[str, Any]] = {}
1625
+ for surface in surfaces if isinstance(surfaces, list) else []:
1626
+ if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
1627
+ continue
1628
+ if identity in by_id and surface != by_id[identity]:
1629
+ ambiguous_surface_ids.add((owner, identity))
1630
+ by_id[identity] = surface
1631
+ for _, draft, owner in all_sources:
1632
+ surfaces = draft["coverage"].get("surfaces", [])
1633
+ for surface in surfaces if isinstance(surfaces, list) else []:
1634
+ if (
1635
+ isinstance(surface, dict)
1636
+ and isinstance(surface.get("id"), str)
1637
+ and (owner, surface["id"]) in ambiguous_surface_ids
1638
+ and surface.get("disposition") == "needs_follow_up"
1639
+ and not any(key in surface for key in ("candidateId", "candidate", "finding"))
1640
+ and isinstance(coverage.get("surfaces"), list)
1641
+ ):
1642
+ retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
1643
+ if retained_surface not in coverage["surfaces"]:
1644
+ coverage["surfaces"].append(copy.deepcopy(retained_surface))
1645
+ selected_terminal_orders: dict[str, tuple[int, int]] = {}
1646
+ for relative, draft, worker_id in sources:
1647
+ if (
1648
+ worker_id is not None
1649
+ and relative in selected_observations
1650
+ and draft.get("complete") is not False
1651
+ ):
1652
+ order = source_order[relative]
1653
+ selected_terminal_orders[worker_id] = max(
1654
+ selected_terminal_orders.get(worker_id, order), order
1655
+ )
1656
+ terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
1657
+ for relative, draft, worker_id in sources:
1658
+ if relative in current_results and draft.get("complete") is not False:
1659
+ order = source_order[relative]
1660
+ terminal_worker_orders[worker_id] = max(
1661
+ terminal_worker_orders.get(worker_id, order), order
1662
+ )
791
1663
  for relative, draft, worker_id in all_sources:
1664
+ worker_result_order = terminal_worker_orders.get(worker_id)
1665
+ selected_coverage_superseded = worker_id in selected_terminal_orders and (
1666
+ source_order[relative] < selected_terminal_orders[worker_id]
1667
+ or (
1668
+ relative in current_results
1669
+ and source_order[relative] <= selected_terminal_orders[worker_id]
1670
+ )
1671
+ )
1672
+ selected_candidates = {
1673
+ candidate_id
1674
+ for (owner, candidate_id), (_, source) in ordered_outcomes.items()
1675
+ if owner == worker_id and source == relative
1676
+ }
792
1677
  superseded = (
793
1678
  worker_id is None
794
1679
  and parent is not None
795
1680
  and parent.get("complete") is not False
796
1681
  and relative != "parent"
1682
+ and source_order[relative] <= (0, parent_modified)
797
1683
  and (not stopped_parent_seal or relative in parent_preserved_sources)
798
1684
  ) or (
799
1685
  relative not in current_results
800
- and any(
801
- saved_worker == worker_id
802
- and saved_path in current_results
803
- and current.get("complete") is not False
804
- for saved_path, current, saved_worker in sources
1686
+ and worker_result_order is not None
1687
+ and (
1688
+ relative not in selected_observations
1689
+ or source_order[relative] < worker_result_order
805
1690
  )
806
1691
  )
1692
+ # A failed result write can leave pending work outside the accepted result.
1693
+ accepted_order = (
1694
+ (0, parent_modified)
1695
+ if worker_id is None and parent is not None
1696
+ else worker_result_order
1697
+ )
1698
+ retain_pending = (
1699
+ superseded
1700
+ and (worker_id in headed_workers or (worker_id is None and parent_heads))
1701
+ and accepted_order is not None
1702
+ and source_order[relative] >= accepted_order
1703
+ and any(isinstance(row, dict) for row in deferred_rows[relative])
1704
+ )
807
1705
  if (
808
- (relative != "parent" or not parent_manifest)
1706
+ (relative != "parent" or not parent_is_canonical)
809
1707
  and not superseded
1708
+ and not selected_coverage_superseded
810
1709
  and (
811
1710
  draft.get("complete") is False
812
1711
  or draft["coverage"].get("completeness") != "complete"
@@ -814,16 +1713,56 @@ def merge_saved_results(
814
1713
  and coverage.get("completeness") in {"complete", "unknown"}
815
1714
  ):
816
1715
  coverage["completeness"] = "partial"
817
- if (
1716
+ skip_superseded_findings = (
818
1717
  superseded
819
1718
  and not stopped
820
1719
  and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
821
- ):
1720
+ )
1721
+ if skip_superseded_findings and not selected_candidates and not retain_pending:
822
1722
  continue
823
- if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict):
824
- manifest["scan"]["threatModel"] = copy.deepcopy(draft["threatModel"])
1723
+ if (
1724
+ not skip_superseded_findings
1725
+ and "threatModel" not in manifest["scan"]
1726
+ and isinstance(draft.get("threatModel"), dict)
1727
+ ):
1728
+ model = draft["threatModel"]
1729
+ model_path = relative
1730
+ checkpoint_dir = Path(relative).parent
1731
+ prefer_worker_head = worker_id is not None and (
1732
+ checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
1733
+ )
1734
+ if worker_id is not None:
1735
+ if checkpoint_dir.name != "checkpoints":
1736
+ checkpoint_dir /= "checkpoints"
1737
+ selected_models = [
1738
+ path
1739
+ for path in selected_observations
1740
+ if Path(path).parent == checkpoint_dir
1741
+ and isinstance(drafts_by_path[path].get("threatModel"), dict)
1742
+ ]
1743
+ if selected_models:
1744
+ head_path = max(selected_models, key=source_order.__getitem__)
1745
+ current = drafts_by_path[head_path]
1746
+ # A terminal checkpoint is committed before result.json is replaced.
1747
+ # Use the admitted observation, including its frozen ordering on retries.
1748
+ if not prefer_worker_head and current.get("complete") is not False:
1749
+ prefer_worker_head = source_order[head_path] >= source_order[relative]
1750
+ if prefer_worker_head:
1751
+ model = current["threatModel"]
1752
+ model_path = head_path
1753
+ manifest["scan"]["threatModel"] = copy.deepcopy(model)
1754
+ if worker_id is not None:
1755
+ manifest["scan"]["threatModel"]["origin"] = "recovered"
1756
+ if selected_model_source is not None and worker_id is not None:
1757
+ selected_model_source[:] = [model_path]
825
1758
  for value in draft["findings"]:
826
- if relative == "parent" and parent_manifest:
1759
+ if skip_superseded_findings and not (
1760
+ isinstance(value, dict)
1761
+ and (candidate_id := finding_candidate_id(value)) in selected_candidates
1762
+ and resolved.get((worker_id, candidate_id)) == "reported"
1763
+ ):
1764
+ continue
1765
+ if relative == "parent" and parent_is_canonical:
827
1766
  finding = copy.deepcopy(value)
828
1767
  _ensure_finding_identity(finding, candidate_only=True)
829
1768
  provenance = finding.get("provenance") if isinstance(finding, dict) else None
@@ -846,7 +1785,6 @@ def merge_saved_results(
846
1785
  if not isinstance(value, dict):
847
1786
  warnings.append(f"Retained malformed finding evidence in {relative}.")
848
1787
  continue
849
- source_value = copy.deepcopy(value)
850
1788
  finding = copy.deepcopy(value)
851
1789
  candidate_id = finding_candidate_id(finding)
852
1790
  if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
@@ -909,24 +1847,19 @@ def merge_saved_results(
909
1847
  historical_contents = set()
910
1848
  if mapped_key is not None:
911
1849
  key = mapped_key
912
- represented_by_parent = (
913
- _digest(_finding_content(source_value)) in historical_contents
914
- )
1850
+ represented_by_parent = _digest(_finding_content(value)) in historical_contents
915
1851
  if key in finding_positions:
916
1852
  retained = findings[finding_positions[key]]
917
1853
  if finding != retained:
918
- if represented_by_parent:
919
- previous = copy.deepcopy(source_value)
920
- previous_history = previous.get("provenance", {}).pop(
921
- "previousFindings", []
922
- )
923
- elif _finding_strength(finding) > _finding_strength(retained):
1854
+ if not represented_by_parent and _finding_strength(finding) > _finding_strength(
1855
+ retained
1856
+ ):
924
1857
  previous = copy.deepcopy(retained)
925
1858
  previous_history = previous["provenance"].pop("previousFindings", [])
926
1859
  retained = finding
927
1860
  findings[finding_positions[key]] = retained
928
1861
  else:
929
- previous = copy.deepcopy(source_value)
1862
+ previous = copy.deepcopy(value)
930
1863
  previous_history = previous.get("provenance", {}).pop(
931
1864
  "previousFindings", []
932
1865
  )
@@ -960,10 +1893,14 @@ def merge_saved_results(
960
1893
  continue
961
1894
  finding_positions[key] = len(findings)
962
1895
  findings.append(finding)
963
- if superseded:
1896
+ if superseded and not selected_candidates and not retain_pending:
964
1897
  continue
965
1898
  for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
1899
+ if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
1900
+ continue
966
1901
  items = draft["coverage"].get(field, [])
1902
+ if field == "deferred":
1903
+ items = deferred_rows[relative]
967
1904
  if not isinstance(items, list):
968
1905
  continue
969
1906
  output = coverage.setdefault(field, [])
@@ -972,6 +1909,40 @@ def merge_saved_results(
972
1909
  # recovery and warnings rather than silently changing its contract.
973
1910
  continue
974
1911
  for item in items:
1912
+ # A selected terminal checkpoint replaces ordinary coverage, while
1913
+ # candidate evidence and pending tasks retain their own reconciliation.
1914
+ if (
1915
+ selected_coverage_superseded
1916
+ and field != "deferred"
1917
+ and not (
1918
+ isinstance(item, dict)
1919
+ and any(key in item for key in ("candidateId", "candidate", "finding"))
1920
+ )
1921
+ ):
1922
+ continue
1923
+ # A selected outcome must retain its evidence even if its result write failed.
1924
+ if superseded and not (
1925
+ isinstance(item, dict)
1926
+ and (
1927
+ (field == "deferred" and retain_pending)
1928
+ or (
1929
+ isinstance(item.get("candidateId"), str)
1930
+ and item["candidateId"] in selected_candidates
1931
+ and item.get("disposition")
1932
+ == resolved.get((worker_id, item["candidateId"]))
1933
+ )
1934
+ )
1935
+ ):
1936
+ continue
1937
+ if id(item) in replaced_rows.get(field, ()):
1938
+ continue
1939
+ if (
1940
+ field == "deferred"
1941
+ and isinstance(item, dict)
1942
+ and isinstance(item.get("id"), str)
1943
+ and (worker_id, item["id"]) in closed_deferred
1944
+ ):
1945
+ continue
975
1946
  if field == "openQuestions" and isinstance(item, str):
976
1947
  item = {"question": item.strip()}
977
1948
  if (
@@ -995,14 +1966,20 @@ def merge_saved_results(
995
1966
  history.append(copy.deepcopy(finding))
996
1967
  if (
997
1968
  isinstance(item, dict)
998
- and (worker_id, item.get("candidateId")) in resolved
1969
+ and isinstance(
1970
+ identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
1971
+ if field == "deferred"
1972
+ else item.get("candidateId"),
1973
+ str,
1974
+ )
1975
+ and (worker_id, identity) in resolved
999
1976
  and (field == "deferred" or item.get("disposition") == "needs_follow_up")
1000
1977
  ):
1001
1978
  continue
1979
+ if field == "surfaces" and isinstance(item, dict):
1980
+ item = {**item, "receiptRefs": item.get("receiptRefs", [])}
1002
1981
  if isinstance(item, dict) and "id" not in item:
1003
1982
  semantic_item = dict(item)
1004
- if field == "surfaces":
1005
- semantic_item.setdefault("receiptRefs", [])
1006
1983
  if any(
1007
1984
  isinstance(existing, dict)
1008
1985
  and {key: value for key, value in existing.items() if key != "id"}
@@ -1036,7 +2013,10 @@ def merge_saved_results(
1036
2013
  if isinstance(item.get("id"), str):
1037
2014
  used.add(item["id"])
1038
2015
  continue
1039
- item.setdefault("id", item.get("candidateId") or f"saved-{_digest(item)[:16]}")
2016
+ item.setdefault("id", _saved_coverage_id(item))
2017
+ if not isinstance(item["id"], str):
2018
+ # Preserve malformed rows for per-record recovery, including frozen replay.
2019
+ continue
1040
2020
  if item["id"] in used:
1041
2021
  item["id"] = f"{item['id']}-{_digest(item)[:16]}"
1042
2022
  used.add(item["id"])
@@ -1081,6 +2061,9 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
1081
2061
  except ContractError:
1082
2062
  path = scan_dir / relative
1083
2063
  if path.exists() or path.is_symlink():
2064
+ if relative == "threatmodel.md":
2065
+ # This optional projection will not replace an unsafe destination.
2066
+ continue
1084
2067
  raise
1085
2068
  snapshots[relative] = None
1086
2069
  finally:
@@ -1112,13 +2095,17 @@ def preserve_scan_results_locked(
1112
2095
  if scan["status"] != "failed":
1113
2096
  return False
1114
2097
  frozen_source_digests: dict[str, str] | None = None
2098
+ model_source: list[str] = []
2099
+ saved_model_source: str | None = None
1115
2100
  raw_frozen_sources = scan["retained_source_digests_json"]
2101
+ if raw_frozen_sources is not None:
2102
+ frozen_source_digests, saved_model_source = _retained_source_state(
2103
+ json.loads(raw_frozen_sources)
2104
+ )
2105
+ if saved_model_source is not None:
2106
+ model_source.append(saved_model_source)
1116
2107
  if recovery_source_digests is not None:
1117
2108
  frozen_source_digests = recovery_source_digests
1118
- elif raw_frozen_sources is not None:
1119
- frozen_source_digests = _source_digests(
1120
- json.loads(raw_frozen_sources), "Saved stopped-scan"
1121
- )
1122
2109
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
1123
2110
  deep_run = connection.execute(
1124
2111
  "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
@@ -1141,12 +2128,10 @@ def preserve_scan_results_locked(
1141
2128
  ]
1142
2129
 
1143
2130
  def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> None:
1144
- retained_sources = manifest.get("scan", {}).get("preservedSources")
1145
- if not isinstance(retained_sources, dict) or not all(
1146
- isinstance(relative, str) and isinstance(source_digest, str)
1147
- for relative, source_digest in retained_sources.items()
1148
- ):
1149
- raise ContractError("Stopped scan source digests could not be frozen.")
2131
+ retained_sources = _source_digests(
2132
+ manifest.get("scan", {}).get("preservedSources"),
2133
+ "Stopped scan source digests could not be frozen.",
2134
+ )
1150
2135
  digest = db.published_manifest_digest(scan_dir, manifest)
1151
2136
  timestamp = db.now()
1152
2137
  with connection:
@@ -1176,7 +2161,7 @@ def preserve_scan_results_locked(
1176
2161
  "updated_at = ? WHERE id = ? AND status = 'failed'",
1177
2162
  (
1178
2163
  digest,
1179
- json.dumps(retained_sources, sort_keys=True),
2164
+ _encode_retained_sources(retained_sources, model_source),
1180
2165
  json.dumps(list(dict.fromkeys(warnings))),
1181
2166
  timestamp,
1182
2167
  scan_id,
@@ -1190,6 +2175,7 @@ def preserve_scan_results_locked(
1190
2175
 
1191
2176
  existing_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
1192
2177
  existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
2178
+ existing = None
1193
2179
  if scan["seal_manifest_digest"] is not None or (
1194
2180
  isinstance(existing_scan, dict)
1195
2181
  and (
@@ -1198,30 +2184,35 @@ def preserve_scan_results_locked(
1198
2184
  ):
1199
2185
  db.require_recorded_manifest_digest(scan, scan_dir)
1200
2186
  existing, existing_findings, _ = finalize_scan(
1201
- scan_dir, expected_coverage_mode=db.expected_coverage_mode(scan)
2187
+ scan_dir,
2188
+ expected_coverage_mode=db.expected_coverage_mode(scan),
2189
+ projection_warnings=warnings,
1202
2190
  )
1203
2191
  db.verify_manifest_binding(scan, existing)
1204
2192
  if existing_scan.get("status") == outcome:
1205
2193
  existing_sources = existing_scan.get("preservedSources")
1206
2194
  if frozen_source_digests is None:
1207
- if not isinstance(existing_sources, dict) or not all(
1208
- isinstance(relative, str) and isinstance(digest, str)
1209
- for relative, digest in existing_sources.items()
1210
- ):
1211
- raise ContractError("Stopped scan source digests could not be frozen.")
1212
- frozen_source_digests = existing_sources
2195
+ frozen_source_digests = _source_digests(
2196
+ existing_sources, "Stopped scan source digests could not be frozen."
2197
+ )
1213
2198
  if existing_sources == frozen_source_digests:
1214
2199
  if (
1215
2200
  raw_frozen_sources is not None
1216
2201
  and scan["seal_manifest_digest"] is not None
1217
2202
  and not publication_follow_up_warnings
2203
+ and warnings == stored_warnings
1218
2204
  ):
1219
2205
  return True
1220
2206
  record_publication(existing, existing_findings)
1221
2207
  return True
1222
2208
  if recovery_source_digests is None:
1223
2209
  raise ContractError("Stopped scan sources changed after terminal publication.")
1224
- binding = {**db.workbench_completion_binding(scan, scan["completed_at"]), "status": outcome}
2210
+ binding = {
2211
+ **db.workbench_completion_binding(scan, scan["completed_at"], existing),
2212
+ "status": outcome,
2213
+ }
2214
+ if recovery_source_digests is not None:
2215
+ model_source.clear()
1225
2216
  documents = merge_saved_results(
1226
2217
  scan_dir,
1227
2218
  scan_id,
@@ -1237,6 +2228,8 @@ def preserve_scan_results_locked(
1237
2228
  f"{scan['failure_message'] or ''}"
1238
2229
  ).strip(),
1239
2230
  frozen_source_digests=frozen_source_digests,
2231
+ frozen_model_source=model_source[0] if model_source else None,
2232
+ selected_model_source=model_source,
1240
2233
  allow_frozen_legacy_parent=(
1241
2234
  include_parent_with_recovery
1242
2235
  or (
@@ -1258,18 +2251,22 @@ def preserve_scan_results_locked(
1258
2251
  (json.dumps(unpublished_warnings), db.now(), scan_id),
1259
2252
  )
1260
2253
  return False
1261
- if frozen_source_digests is None:
1262
- retained_sources = documents[0].get("scan", {}).get("preservedSources")
1263
- if not isinstance(retained_sources, dict) or not all(
1264
- isinstance(relative, str) and isinstance(digest, str)
1265
- for relative, digest in retained_sources.items()
1266
- ):
1267
- raise ContractError("Stopped scan source digests could not be frozen.")
2254
+ if frozen_source_digests is None or (
2255
+ recovery_source_digests is None and saved_model_source is None and model_source
2256
+ ):
2257
+ retained_sources = _source_digests(
2258
+ documents[0].get("scan", {}).get("preservedSources"),
2259
+ "Stopped scan source digests could not be frozen.",
2260
+ )
1268
2261
  with connection:
1269
2262
  connection.execute(
1270
2263
  "UPDATE scans SET retained_source_digests_json = ? "
1271
- "WHERE id = ? AND retained_source_digests_json IS NULL",
1272
- (json.dumps(retained_sources, sort_keys=True), scan_id),
2264
+ "WHERE id = ? AND retained_source_digests_json IS ?",
2265
+ (
2266
+ _encode_retained_sources(retained_sources, model_source),
2267
+ scan_id,
2268
+ raw_frozen_sources,
2269
+ ),
1273
2270
  )
1274
2271
  prepared = _prepare_scan_finalization(
1275
2272
  scan_dir,
@@ -1280,7 +2277,9 @@ def preserve_scan_results_locked(
1280
2277
  )
1281
2278
  snapshots = _snapshot_published_outputs(scan_dir)
1282
2279
  try:
1283
- manifest, findings, _ = _write_prepared_scan_finalization(prepared)
2280
+ manifest, findings, _ = _write_prepared_scan_finalization(
2281
+ prepared, projection_warnings=warnings
2282
+ )
1284
2283
  db.verify_manifest_binding(scan, manifest)
1285
2284
  record_publication(manifest, findings)
1286
2285
  except BaseException:
@@ -1399,6 +2398,28 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1399
2398
  "The scan stopped; its saved checkpoint was retained without replacing sealed results."
1400
2399
  )
1401
2400
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2401
+ if (
2402
+ args.expected_draft_digest is not None
2403
+ and args.expected_draft_digest != _scan_draft_digest(scan_dir)
2404
+ ):
2405
+ raise SystemExit(
2406
+ "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
2407
+ )
2408
+ try:
2409
+ relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
2410
+ except ValueError as exc:
2411
+ raise SystemExit(
2412
+ "Scan draft must be inside the registered scan drafts directory."
2413
+ ) from exc
2414
+ if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
2415
+ raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
2416
+ draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
2417
+ manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
2418
+ binding = db.workbench_completion_binding(scan, db.now())
2419
+ # Save scan IDs without sealing the draft.
2420
+ _populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
2421
+ _populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
2422
+ _validate_completion_binding(manifest, findings, coverage, binding)
1402
2423
  if args.checkpoint_path is not None:
1403
2424
  try:
1404
2425
  checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix()
@@ -1421,33 +2442,15 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1421
2442
  f"checkpoints/{checkpoint_digest}.json",
1422
2443
  checkpoint_contents,
1423
2444
  )
1424
- if (
1425
- args.expected_draft_digest is not None
1426
- and args.expected_draft_digest != _scan_draft_digest(scan_dir)
1427
- ):
1428
- raise SystemExit(
1429
- "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
1430
- )
1431
- try:
1432
- relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
1433
- except ValueError as exc:
1434
- raise SystemExit(
1435
- "Scan draft must be inside the registered scan drafts directory."
1436
- ) from exc
1437
- if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
1438
- raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
1439
- draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
1440
- manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
1441
- binding = db.workbench_completion_binding(scan, db.now())
1442
- # Validate on copies: saved canonical documents remain ordinary unsealed drafts.
1443
- copied_manifest = copy.deepcopy(manifest)
1444
- copied_findings = copy.deepcopy(findings)
1445
- copied_coverage = copy.deepcopy(coverage)
1446
- _populate_unsealed_manifest_envelope(copied_manifest, copied_manifest["scan"], binding)
1447
- _populate_unsealed_artifact_envelope(
1448
- copied_manifest, copied_findings, copied_coverage, binding
2445
+ checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
2446
+ checkpoint_contents = _encoded(checkpoint)
2447
+ checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
2448
+ checkpoint_relative = f"checkpoints/{checkpoint_name}"
2449
+ if not (scan_dir / checkpoint_relative).exists():
2450
+ write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
2451
+ write_scan_local_bytes(
2452
+ scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
1449
2453
  )
1450
- _validate_completion_binding(copied_manifest, copied_findings, copied_coverage, binding)
1451
2454
  for filename, document in (
1452
2455
  ("findings.json", findings),
1453
2456
  ("coverage.json", coverage),
@@ -1458,9 +2461,17 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1458
2461
  filename,
1459
2462
  (json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
1460
2463
  )
2464
+ model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
1461
2465
  # Accepted Standard drafts are evidence of review or report assembly,
1462
2466
  # even when the parent omitted its explicit progress call.
1463
- if scan["mode"] == "standard":
2467
+ model_only_checkpoint = (
2468
+ manifest["scan"].get("complete") is False
2469
+ and isinstance(manifest["scan"].get("threatModel"), dict)
2470
+ and not findings.get("findings")
2471
+ and not coverage.get("surfaces")
2472
+ and not coverage.get("deferred")
2473
+ )
2474
+ if scan["mode"] == "standard" and not model_only_checkpoint:
1464
2475
  phase = "discovery" if manifest["scan"].get("complete") is False else "reporting"
1465
2476
  earlier = PHASES[: PHASES.index(phase)]
1466
2477
  placeholders = ",".join("?" for _ in earlier)
@@ -1481,7 +2492,11 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1481
2492
  )
1482
2493
  except sqlite3.Error as exc:
1483
2494
  print(f"Could not save scan progress: {exc}", file=sys.stderr)
1484
- return {"scanId": scan_id, "status": "draft_written"}
2495
+ return {
2496
+ "scanId": scan_id,
2497
+ "status": "draft_written",
2498
+ **({"warnings": [model_warning]} if model_warning else {}),
2499
+ }
1485
2500
 
1486
2501
 
1487
2502
  def _scan_draft_digest(scan_dir: Path) -> str:
@@ -1494,9 +2509,10 @@ def _scan_draft_digest(scan_dir: Path) -> str:
1494
2509
  except FileNotFoundError:
1495
2510
  digest.update(b"missing\0")
1496
2511
  continue
1497
- _, contents = _read_scan_local_json_bytes(scan_dir, filename, filename)
1498
2512
  digest.update(b"present\0")
1499
- digest.update(contents)
2513
+ descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
2514
+ with os.fdopen(descriptor, "rb") as handle:
2515
+ digest.update(handle.read())
1500
2516
  digest.update(b"\0")
1501
2517
  return digest.hexdigest()
1502
2518