@openai/codex-security 0.1.31 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (514) hide show
  1. package/README.md +442 -2434
  2. package/_bundled_plugin/.app.json +5 -1
  3. package/_bundled_plugin/.codex-plugin/plugin.json +9 -5
  4. package/_bundled_plugin/.mcp.json +3 -0
  5. package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
  6. package/_bundled_plugin/examples/completed-scan/report.md +28 -26
  7. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
  8. package/_bundled_plugin/mcp/helpers.mjs +1 -0
  9. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  10. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +7 -214
  11. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  12. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  13. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +563 -0
  20. package/_bundled_plugin/mcp/server.mjs +1 -0
  21. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  22. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  23. package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
  24. package/_bundled_plugin/references/artifact-storage.md +5 -5
  25. package/_bundled_plugin/references/config-preflight.md +1 -5
  26. package/_bundled_plugin/references/core-scan.md +34 -10
  27. package/_bundled_plugin/references/final-report.md +13 -13
  28. package/_bundled_plugin/references/finding-detail-fields.md +90 -80
  29. package/_bundled_plugin/references/sarif-adapter.md +1 -1
  30. package/_bundled_plugin/references/scan-artifacts.md +3 -4
  31. package/_bundled_plugin/references/scan-contract.md +11 -6
  32. package/_bundled_plugin/references/scan-prologue.md +12 -0
  33. package/_bundled_plugin/references/security-guidance.md +2 -2
  34. package/_bundled_plugin/references/standalone-validation.md +11 -0
  35. package/_bundled_plugin/references/threat-model.md +2 -2
  36. package/_bundled_plugin/schemas/coverage.schema.json +23 -47
  37. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
  38. package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
  39. package/_bundled_plugin/schemas/findings.schema.json +71 -192
  40. package/_bundled_plugin/schemas/scan-manifest.schema.json +83 -95
  41. package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
  42. package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
  43. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
  44. package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
  45. package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
  46. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +245 -356
  47. package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
  48. package/_bundled_plugin/scripts/codex_session.mjs +53 -0
  49. package/_bundled_plugin/scripts/config_preflight.py +29 -69
  50. package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
  51. package/_bundled_plugin/scripts/deep_scan_workbench.py +208 -363
  52. package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
  53. package/_bundled_plugin/scripts/finalize_scan_contract.py +733 -375
  54. package/_bundled_plugin/scripts/finding_preview.py +81 -91
  55. package/_bundled_plugin/scripts/generate_in_scope_files.py +48 -102
  56. package/_bundled_plugin/scripts/generate_rank_input.py +117 -888
  57. package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -6
  58. package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
  59. package/_bundled_plugin/scripts/rank_preview.py +43 -902
  60. package/_bundled_plugin/scripts/report_projection.py +143 -219
  61. package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
  62. package/_bundled_plugin/scripts/threat_model_projection.py +161 -0
  63. package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
  64. package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
  65. package/_bundled_plugin/scripts/windows_scan_local_files.py +59 -9
  66. package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
  67. package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
  68. package/_bundled_plugin/scripts/workbench/storage.py +21 -6
  69. package/_bundled_plugin/scripts/workbench_cli.py +99 -190
  70. package/_bundled_plugin/scripts/workbench_constants.py +6 -14
  71. package/_bundled_plugin/scripts/workbench_db.py +524 -672
  72. package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
  73. package/_bundled_plugin/scripts/workbench_finding_index.py +20 -7
  74. package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
  75. package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
  76. package/_bundled_plugin/scripts/workbench_native_indexes.py +28 -18
  77. package/_bundled_plugin/scripts/workbench_progress.py +26 -92
  78. package/_bundled_plugin/scripts/workbench_publication.py +77 -116
  79. package/_bundled_plugin/scripts/workbench_remediation.py +14 -20
  80. package/_bundled_plugin/scripts/workbench_saved_results.py +1402 -263
  81. package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
  82. package/_bundled_plugin/scripts/workbench_scan_start.py +109 -52
  83. package/_bundled_plugin/scripts/workbench_scan_usage.py +30 -49
  84. package/_bundled_plugin/scripts/workbench_schema.py +71 -1370
  85. package/_bundled_plugin/scripts/workbench_source_excerpt.py +16 -7
  86. package/_bundled_plugin/scripts/workbench_target.py +526 -150
  87. package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
  88. package/_bundled_plugin/scripts/workbench_validation.py +19 -44
  89. package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
  90. package/_bundled_plugin/shared/workbench-migrations.json +397 -0
  91. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
  92. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
  93. package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
  94. package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
  95. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +6 -6
  96. package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
  97. package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
  98. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
  99. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
  100. package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
  101. package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
  102. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
  103. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  104. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +13 -1
  105. package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
  106. package/_bundled_plugin/skills/track-findings/SKILL.md +46 -178
  107. package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
  108. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
  109. package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
  110. package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
  111. package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
  112. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
  113. package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
  114. package/bin/codex-security.mjs +9 -2
  115. package/dist/api.d.ts +63 -15
  116. package/dist/api.d.ts.map +1 -1
  117. package/dist/api.js +970 -553
  118. package/dist/api.js.map +1 -1
  119. package/dist/artifact-export.d.ts +70 -0
  120. package/dist/artifact-export.d.ts.map +1 -0
  121. package/dist/artifact-export.js +292 -0
  122. package/dist/artifact-export.js.map +1 -0
  123. package/dist/auth.d.ts +2 -6
  124. package/dist/auth.d.ts.map +1 -1
  125. package/dist/auth.js +27 -12
  126. package/dist/auth.js.map +1 -1
  127. package/dist/bulk-scan-discovery.d.ts +1 -0
  128. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  129. package/dist/bulk-scan-discovery.js +9 -9
  130. package/dist/bulk-scan-discovery.js.map +1 -1
  131. package/dist/classify-scan-severity.d.ts.map +1 -1
  132. package/dist/classify-scan-severity.js +2 -5
  133. package/dist/classify-scan-severity.js.map +1 -1
  134. package/dist/classify-severity.d.ts +28 -17
  135. package/dist/classify-severity.d.ts.map +1 -1
  136. package/dist/classify-severity.js +16 -35
  137. package/dist/classify-severity.js.map +1 -1
  138. package/dist/cli-help.d.ts +3 -0
  139. package/dist/cli-help.d.ts.map +1 -0
  140. package/dist/cli-help.js +285 -0
  141. package/dist/cli-help.js.map +1 -0
  142. package/dist/cli-signals.d.ts +8 -0
  143. package/dist/cli-signals.d.ts.map +1 -0
  144. package/dist/cli-signals.js +28 -0
  145. package/dist/cli-signals.js.map +1 -0
  146. package/dist/cli.d.ts +22 -24
  147. package/dist/cli.d.ts.map +1 -1
  148. package/dist/cli.js +1660 -1556
  149. package/dist/cli.js.map +1 -1
  150. package/dist/cloud-publish.d.ts.map +1 -1
  151. package/dist/cloud-publish.js +11 -17
  152. package/dist/cloud-publish.js.map +1 -1
  153. package/dist/codeowners.d.ts +19 -0
  154. package/dist/codeowners.d.ts.map +1 -0
  155. package/dist/codeowners.js +138 -0
  156. package/dist/codeowners.js.map +1 -0
  157. package/dist/codex-prompt.d.ts.map +1 -1
  158. package/dist/codex-prompt.js +1 -2
  159. package/dist/codex-prompt.js.map +1 -1
  160. package/dist/codex-sdk-environment.d.ts +3 -0
  161. package/dist/codex-sdk-environment.d.ts.map +1 -0
  162. package/dist/codex-sdk-environment.js +68 -0
  163. package/dist/codex-sdk-environment.js.map +1 -0
  164. package/dist/component-plan.d.ts +2 -5
  165. package/dist/component-plan.d.ts.map +1 -1
  166. package/dist/component-plan.js +38 -34
  167. package/dist/component-plan.js.map +1 -1
  168. package/dist/component-scan.d.ts +7 -5
  169. package/dist/component-scan.d.ts.map +1 -1
  170. package/dist/component-scan.js +54 -44
  171. package/dist/component-scan.js.map +1 -1
  172. package/dist/config.d.ts.map +1 -1
  173. package/dist/config.js +93 -104
  174. package/dist/config.js.map +1 -1
  175. package/dist/contract.d.ts +15 -2
  176. package/dist/contract.d.ts.map +1 -1
  177. package/dist/contract.js +124 -348
  178. package/dist/contract.js.map +1 -1
  179. package/dist/cost-model.d.ts +13 -6
  180. package/dist/cost-model.d.ts.map +1 -1
  181. package/dist/cost-model.js +75 -34
  182. package/dist/cost-model.js.map +1 -1
  183. package/dist/cost.d.ts +10 -2
  184. package/dist/cost.d.ts.map +1 -1
  185. package/dist/cost.js +116 -134
  186. package/dist/cost.js.map +1 -1
  187. package/dist/custom-validation-prompt.d.ts.map +1 -1
  188. package/dist/custom-validation-prompt.js +7 -6
  189. package/dist/custom-validation-prompt.js.map +1 -1
  190. package/dist/custom-validation.d.ts +1 -2
  191. package/dist/custom-validation.d.ts.map +1 -1
  192. package/dist/custom-validation.js +23 -16
  193. package/dist/custom-validation.js.map +1 -1
  194. package/dist/deduplication/checkpointed-review.d.ts +4 -2
  195. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  196. package/dist/deduplication/checkpointed-review.js +33 -20
  197. package/dist/deduplication/checkpointed-review.js.map +1 -1
  198. package/dist/deduplication/codex-review.d.ts +9 -1
  199. package/dist/deduplication/codex-review.d.ts.map +1 -1
  200. package/dist/deduplication/codex-review.js +129 -20
  201. package/dist/deduplication/codex-review.js.map +1 -1
  202. package/dist/deduplication/deduplication-prompts.d.ts +5 -2
  203. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  204. package/dist/deduplication/deduplication-prompts.js +18 -4
  205. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  206. package/dist/deduplication/deduplication-reviewer.d.ts +26 -4
  207. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  208. package/dist/deduplication/deduplication-reviewer.js +59 -47
  209. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  210. package/dist/deduplication/deduplication.d.ts.map +1 -1
  211. package/dist/deduplication/deduplication.js +59 -113
  212. package/dist/deduplication/deduplication.js.map +1 -1
  213. package/dist/deduplication/diagnostics.d.ts +17 -0
  214. package/dist/deduplication/diagnostics.d.ts.map +1 -0
  215. package/dist/deduplication/diagnostics.js +5 -0
  216. package/dist/deduplication/diagnostics.js.map +1 -0
  217. package/dist/deduplication/finding-schema.d.ts.map +1 -1
  218. package/dist/deduplication/finding-schema.js +7 -1
  219. package/dist/deduplication/finding-schema.js.map +1 -1
  220. package/dist/deduplication/local.d.ts +31 -0
  221. package/dist/deduplication/local.d.ts.map +1 -0
  222. package/dist/deduplication/local.js +166 -0
  223. package/dist/deduplication/local.js.map +1 -0
  224. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  225. package/dist/deduplication/records-protocol.js +12 -28
  226. package/dist/deduplication/records-protocol.js.map +1 -1
  227. package/dist/deduplication/records.d.ts +188 -12
  228. package/dist/deduplication/records.d.ts.map +1 -1
  229. package/dist/deduplication/records.js +34 -24
  230. package/dist/deduplication/records.js.map +1 -1
  231. package/dist/deduplication/review.d.ts +7 -0
  232. package/dist/deduplication/review.d.ts.map +1 -1
  233. package/dist/deduplication/review.js.map +1 -1
  234. package/dist/deduplication/scan.d.ts +9 -3
  235. package/dist/deduplication/scan.d.ts.map +1 -1
  236. package/dist/deduplication/scan.js +159 -44
  237. package/dist/deduplication/scan.js.map +1 -1
  238. package/dist/deep-config.d.ts +2 -1
  239. package/dist/deep-config.d.ts.map +1 -1
  240. package/dist/deep-config.js +7 -23
  241. package/dist/deep-config.js.map +1 -1
  242. package/dist/deep-progress.d.ts +2 -1
  243. package/dist/deep-progress.d.ts.map +1 -1
  244. package/dist/deep-progress.js +32 -47
  245. package/dist/deep-progress.js.map +1 -1
  246. package/dist/deep-scan.d.ts +20 -0
  247. package/dist/deep-scan.d.ts.map +1 -0
  248. package/dist/deep-scan.js +113 -0
  249. package/dist/deep-scan.js.map +1 -0
  250. package/dist/errors.d.ts +5 -2
  251. package/dist/errors.d.ts.map +1 -1
  252. package/dist/errors.js +7 -7
  253. package/dist/errors.js.map +1 -1
  254. package/dist/feedback.d.ts.map +1 -1
  255. package/dist/feedback.js +53 -21
  256. package/dist/feedback.js.map +1 -1
  257. package/dist/finding-catalogue.d.ts.map +1 -1
  258. package/dist/finding-catalogue.js +26 -30
  259. package/dist/finding-catalogue.js.map +1 -1
  260. package/dist/finding-normalization.d.ts +5 -0
  261. package/dist/finding-normalization.d.ts.map +1 -0
  262. package/dist/finding-normalization.js +120 -0
  263. package/dist/finding-normalization.js.map +1 -0
  264. package/dist/finding-retrieval.d.ts +9 -0
  265. package/dist/finding-retrieval.d.ts.map +1 -1
  266. package/dist/finding-retrieval.js.map +1 -1
  267. package/dist/finding-workflow.d.ts +13 -8
  268. package/dist/finding-workflow.d.ts.map +1 -1
  269. package/dist/finding-workflow.js +18 -13
  270. package/dist/finding-workflow.js.map +1 -1
  271. package/dist/findings-client.d.ts +2 -1
  272. package/dist/findings-client.d.ts.map +1 -1
  273. package/dist/findings-client.js +36 -9
  274. package/dist/findings-client.js.map +1 -1
  275. package/dist/findings-errors.d.ts +16 -0
  276. package/dist/findings-errors.d.ts.map +1 -0
  277. package/dist/findings-errors.js +24 -0
  278. package/dist/findings-errors.js.map +1 -0
  279. package/dist/findings-import.d.ts +1 -1
  280. package/dist/findings-import.d.ts.map +1 -1
  281. package/dist/findings-import.js +16 -44
  282. package/dist/findings-import.js.map +1 -1
  283. package/dist/github.d.ts.map +1 -1
  284. package/dist/github.js +9 -4
  285. package/dist/github.js.map +1 -1
  286. package/dist/import-scan.d.ts.map +1 -1
  287. package/dist/import-scan.js +49 -39
  288. package/dist/import-scan.js.map +1 -1
  289. package/dist/index.d.ts +11 -5
  290. package/dist/index.d.ts.map +1 -1
  291. package/dist/index.js +2 -1
  292. package/dist/index.js.map +1 -1
  293. package/dist/knowledge-base.d.ts +7 -1
  294. package/dist/knowledge-base.d.ts.map +1 -1
  295. package/dist/knowledge-base.js +115 -47
  296. package/dist/knowledge-base.js.map +1 -1
  297. package/dist/linear.d.ts.map +1 -1
  298. package/dist/linear.js +4 -10
  299. package/dist/linear.js.map +1 -1
  300. package/dist/mock-scan.d.ts.map +1 -1
  301. package/dist/mock-scan.js +0 -1
  302. package/dist/mock-scan.js.map +1 -1
  303. package/dist/models.d.ts +24 -2
  304. package/dist/models.d.ts.map +1 -1
  305. package/dist/models.js.map +1 -1
  306. package/dist/multiscan.d.ts +9 -3
  307. package/dist/multiscan.d.ts.map +1 -1
  308. package/dist/multiscan.js +225 -153
  309. package/dist/multiscan.js.map +1 -1
  310. package/dist/owner-evidence.d.ts +10 -2
  311. package/dist/owner-evidence.d.ts.map +1 -1
  312. package/dist/owner-evidence.js +69 -14
  313. package/dist/owner-evidence.js.map +1 -1
  314. package/dist/patch-tui.d.ts.map +1 -1
  315. package/dist/patch-tui.js +5 -6
  316. package/dist/patch-tui.js.map +1 -1
  317. package/dist/project-config-schema.d.ts +10 -0
  318. package/dist/project-config-schema.d.ts.map +1 -1
  319. package/dist/project-config-schema.js +1 -0
  320. package/dist/project-config-schema.js.map +1 -1
  321. package/dist/project-config.d.ts +1 -0
  322. package/dist/project-config.d.ts.map +1 -1
  323. package/dist/project-config.js +5 -3
  324. package/dist/project-config.js.map +1 -1
  325. package/dist/prompt-files.d.ts +0 -1
  326. package/dist/prompt-files.d.ts.map +1 -1
  327. package/dist/prompt-files.js +2 -4
  328. package/dist/prompt-files.js.map +1 -1
  329. package/dist/provider-profile.d.ts +21 -0
  330. package/dist/provider-profile.d.ts.map +1 -0
  331. package/dist/provider-profile.js +133 -0
  332. package/dist/provider-profile.js.map +1 -0
  333. package/dist/publication-events.d.ts +6 -8
  334. package/dist/publication-events.d.ts.map +1 -1
  335. package/dist/publication-events.js +40 -79
  336. package/dist/publication-events.js.map +1 -1
  337. package/dist/publication-store.d.ts.map +1 -1
  338. package/dist/publication-store.js +48 -76
  339. package/dist/publication-store.js.map +1 -1
  340. package/dist/publication.d.ts +1 -2
  341. package/dist/publication.d.ts.map +1 -1
  342. package/dist/publication.js +7 -16
  343. package/dist/publication.js.map +1 -1
  344. package/dist/publish.d.ts +1 -5
  345. package/dist/publish.d.ts.map +1 -1
  346. package/dist/publish.js +138 -199
  347. package/dist/publish.js.map +1 -1
  348. package/dist/record.d.ts +2 -0
  349. package/dist/record.d.ts.map +1 -0
  350. package/dist/record.js +4 -0
  351. package/dist/record.js.map +1 -0
  352. package/dist/request-metadata.d.ts +9 -0
  353. package/dist/request-metadata.d.ts.map +1 -0
  354. package/dist/request-metadata.js +13 -0
  355. package/dist/request-metadata.js.map +1 -0
  356. package/dist/result.d.ts +4 -1
  357. package/dist/result.d.ts.map +1 -1
  358. package/dist/result.js +7 -0
  359. package/dist/result.js.map +1 -1
  360. package/dist/runtime.d.ts +25 -17
  361. package/dist/runtime.d.ts.map +1 -1
  362. package/dist/runtime.js +619 -255
  363. package/dist/runtime.js.map +1 -1
  364. package/dist/saved-scan-bootstrap.d.ts +14 -0
  365. package/dist/saved-scan-bootstrap.d.ts.map +1 -0
  366. package/dist/saved-scan-bootstrap.js +261 -0
  367. package/dist/saved-scan-bootstrap.js.map +1 -0
  368. package/dist/saved-scan.d.ts +1 -1
  369. package/dist/saved-scan.d.ts.map +1 -1
  370. package/dist/saved-scan.js +16 -32
  371. package/dist/saved-scan.js.map +1 -1
  372. package/dist/scan-activity.d.ts.map +1 -1
  373. package/dist/scan-activity.js +19 -30
  374. package/dist/scan-activity.js.map +1 -1
  375. package/dist/scan-comparison.d.ts +31 -18
  376. package/dist/scan-comparison.d.ts.map +1 -1
  377. package/dist/scan-comparison.js +307 -347
  378. package/dist/scan-comparison.js.map +1 -1
  379. package/dist/scan-dashboard.d.ts +3 -1
  380. package/dist/scan-dashboard.d.ts.map +1 -1
  381. package/dist/scan-dashboard.js +351 -92
  382. package/dist/scan-dashboard.js.map +1 -1
  383. package/dist/scan-history-renderer.d.ts.map +1 -1
  384. package/dist/scan-history-renderer.js +55 -15
  385. package/dist/scan-history-renderer.js.map +1 -1
  386. package/dist/scan-inputs.d.ts +6 -0
  387. package/dist/scan-inputs.d.ts.map +1 -0
  388. package/dist/scan-inputs.js +71 -0
  389. package/dist/scan-inputs.js.map +1 -0
  390. package/dist/scan-logs.d.ts +2 -3
  391. package/dist/scan-logs.d.ts.map +1 -1
  392. package/dist/scan-logs.js +14 -12
  393. package/dist/scan-logs.js.map +1 -1
  394. package/dist/scan-sessions.d.ts +4 -0
  395. package/dist/scan-sessions.d.ts.map +1 -1
  396. package/dist/scan-sessions.js +18 -3
  397. package/dist/scan-sessions.js.map +1 -1
  398. package/dist/scan-settings.d.ts +5 -0
  399. package/dist/scan-settings.d.ts.map +1 -1
  400. package/dist/scan-settings.js +4 -0
  401. package/dist/scan-settings.js.map +1 -1
  402. package/dist/scan-skill-paths.d.ts +8 -0
  403. package/dist/scan-skill-paths.d.ts.map +1 -0
  404. package/dist/scan-skill-paths.js +47 -0
  405. package/dist/scan-skill-paths.js.map +1 -0
  406. package/dist/security-policy-cli.d.ts.map +1 -1
  407. package/dist/security-policy-cli.js +15 -33
  408. package/dist/security-policy-cli.js.map +1 -1
  409. package/dist/security-policy.d.ts +17 -9
  410. package/dist/security-policy.d.ts.map +1 -1
  411. package/dist/security-policy.js +131 -98
  412. package/dist/security-policy.js.map +1 -1
  413. package/dist/server/api.d.ts +0 -1
  414. package/dist/server/api.d.ts.map +1 -1
  415. package/dist/server/api.js +0 -1
  416. package/dist/server/api.js.map +1 -1
  417. package/dist/server/embeddings.d.ts +3 -1
  418. package/dist/server/embeddings.d.ts.map +1 -1
  419. package/dist/server/embeddings.js +11 -3
  420. package/dist/server/embeddings.js.map +1 -1
  421. package/dist/server/errors.d.ts +5 -2
  422. package/dist/server/errors.d.ts.map +1 -1
  423. package/dist/server/errors.js +7 -0
  424. package/dist/server/errors.js.map +1 -1
  425. package/dist/server/sqlite-store.d.ts +0 -2
  426. package/dist/server/sqlite-store.d.ts.map +1 -1
  427. package/dist/server/sqlite-store.js +55 -55
  428. package/dist/server/sqlite-store.js.map +1 -1
  429. package/dist/server/storage.d.ts +0 -2
  430. package/dist/server/storage.d.ts.map +1 -1
  431. package/dist/server/storage.js.map +1 -1
  432. package/dist/severity-store.js +20 -32
  433. package/dist/severity-store.js.map +1 -1
  434. package/dist/standalone-validation-prompt.d.ts +2 -0
  435. package/dist/standalone-validation-prompt.d.ts.map +1 -0
  436. package/dist/standalone-validation-prompt.js +10 -0
  437. package/dist/standalone-validation-prompt.js.map +1 -0
  438. package/dist/suggest-owners.d.ts +6 -5
  439. package/dist/suggest-owners.d.ts.map +1 -1
  440. package/dist/suggest-owners.js +15 -10
  441. package/dist/suggest-owners.js.map +1 -1
  442. package/dist/targets.d.ts +12 -0
  443. package/dist/targets.d.ts.map +1 -1
  444. package/dist/targets.js +124 -62
  445. package/dist/targets.js.map +1 -1
  446. package/dist/trusted-executable.d.ts +6 -1
  447. package/dist/trusted-executable.d.ts.map +1 -1
  448. package/dist/trusted-executable.js +28 -9
  449. package/dist/trusted-executable.js.map +1 -1
  450. package/dist/value.d.ts +2 -0
  451. package/dist/value.d.ts.map +1 -0
  452. package/dist/value.js +27 -0
  453. package/dist/value.js.map +1 -0
  454. package/dist/version.d.ts.map +1 -1
  455. package/dist/version.js +3 -6
  456. package/dist/version.js.map +1 -1
  457. package/dist/windows-path.d.ts.map +1 -1
  458. package/dist/windows-path.js +1 -2
  459. package/dist/windows-path.js.map +1 -1
  460. package/dist/workbench-arguments.d.ts +3 -0
  461. package/dist/workbench-arguments.d.ts.map +1 -0
  462. package/dist/workbench-arguments.js +15 -0
  463. package/dist/workbench-arguments.js.map +1 -0
  464. package/dist/worker-progress.d.ts +2 -0
  465. package/dist/worker-progress.d.ts.map +1 -1
  466. package/dist/worker-progress.js +40 -46
  467. package/dist/worker-progress.js.map +1 -1
  468. package/docs/cli.md +1778 -0
  469. package/docs/dedupe-records.md +24 -0
  470. package/docs/findings-service.md +402 -0
  471. package/package.json +23 -23
  472. package/schemas/project-config.schema.json +5 -0
  473. package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
  474. package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
  475. package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
  476. package/_bundled_plugin/scripts/workbench_findings.py +0 -221
  477. package/_bundled_plugin/scripts/workbench_severity.py +0 -119
  478. package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
  479. package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
  480. package/dist/server/dashboard/app.css +0 -2
  481. package/dist/server/dashboard/app.js +0 -66
  482. package/dist/server/dashboard/index.html +0 -18
  483. package/dist/server/dashboard-types.d.ts +0 -48
  484. package/dist/server/dashboard-types.d.ts.map +0 -1
  485. package/dist/server/dashboard-types.js +0 -2
  486. package/dist/server/dashboard-types.js.map +0 -1
  487. package/dist/server/dashboard.d.ts +0 -6
  488. package/dist/server/dashboard.d.ts.map +0 -1
  489. package/dist/server/dashboard.js +0 -59
  490. package/dist/server/dashboard.js.map +0 -1
  491. package/dist/server/findings-service.d.ts +0 -20
  492. package/dist/server/findings-service.d.ts.map +0 -1
  493. package/dist/server/findings-service.js +0 -31
  494. package/dist/server/findings-service.js.map +0 -1
  495. package/dist/server/index.d.ts +0 -2
  496. package/dist/server/index.d.ts.map +0 -1
  497. package/dist/server/index.js +0 -6
  498. package/dist/server/index.js.map +0 -1
  499. package/dist/server/routes.d.ts +0 -6
  500. package/dist/server/routes.d.ts.map +0 -1
  501. package/dist/server/routes.js +0 -86
  502. package/dist/server/routes.js.map +0 -1
  503. package/dist/server/serve.d.ts +0 -2
  504. package/dist/server/serve.d.ts.map +0 -1
  505. package/dist/server/serve.js +0 -28
  506. package/dist/server/serve.js.map +0 -1
  507. package/dist/server/server.d.ts +0 -10
  508. package/dist/server/server.d.ts.map +0 -1
  509. package/dist/server/server.js +0 -17
  510. package/dist/server/server.js.map +0 -1
  511. package/dist/server/validation.d.ts +0 -17
  512. package/dist/server/validation.d.ts.map +0 -1
  513. package/dist/server/validation.js +0 -53
  514. package/dist/server/validation.js.map +0 -1
@@ -12,10 +12,9 @@ import re
12
12
  import sqlite3
13
13
  import stat
14
14
  import sys
15
- from collections.abc import Callable, Iterator
16
- from dataclasses import dataclass
15
+ from collections.abc import Iterator
16
+ from contextlib import contextmanager
17
17
  from pathlib import Path
18
- from types import ModuleType
19
18
  from typing import Any
20
19
 
21
20
  sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -25,18 +24,25 @@ from finalize_scan_contract import (
25
24
  _populate_unsealed_artifact_envelope,
26
25
  _populate_unsealed_manifest_envelope,
27
26
  _prepare_scan_finalization,
27
+ _read_json,
28
+ _read_saved_threat_model,
28
29
  _read_scan_local_json,
29
30
  _read_scan_local_json_bytes,
31
+ _read_scan_local_json_with_metadata,
30
32
  _recover_unsealed_findings,
31
33
  _remove_scan_local_file_if_exists,
32
34
  _validate_completion_binding,
35
+ _validate_resolved_deferred,
36
+ _validate_schema_node,
33
37
  _write_prepared_scan_finalization,
34
38
  finalize_scan,
35
39
  finding_candidate_id,
36
40
  open_scan_local_file_descriptor,
37
41
  write_scan_local_bytes,
42
+ write_threat_model_projection_if_possible,
38
43
  )
39
44
  from workbench_constants import PHASES
45
+ from workbench_target import committed_diff_snapshot_digest
40
46
  from workbench_validation import path_within_scope
41
47
 
42
48
  _PUBLISHED_OUTPUTS = (
@@ -44,6 +50,7 @@ _PUBLISHED_OUTPUTS = (
44
50
  "coverage.json",
45
51
  "scan-manifest.json",
46
52
  "report.md",
53
+ "threatmodel.md",
47
54
  "report.html",
48
55
  "exports/results.sarif",
49
56
  )
@@ -55,29 +62,59 @@ _RESERVED_ARTIFACT_PATHS = json.loads(
55
62
  )
56
63
 
57
64
 
58
- @dataclass(frozen=True)
59
- class WorkbenchDbContext:
60
- ARTIFACTS: dict[str, str]
61
- artifact_path: Callable[..., Path | None]
62
- deep_scan: ModuleType
63
- expected_coverage_mode: Callable[..., str]
64
- handoff: ModuleType
65
- index_findings: Callable[..., None]
66
- now: Callable[[], str]
67
- optional_text: Callable[..., str | None]
68
- parse_scan_cost: Callable[..., dict[str, Any] | None]
69
- published_manifest_digest: Callable[..., str]
70
- read_json_object: Callable[[Path], dict[str, Any]]
71
- require_canonical_scan_directory: Callable[[Path], Path]
72
- require_recorded_manifest_digest: Callable[..., None]
73
- require_scan: Callable[..., Any]
74
- require_uuid: Callable[[str, str], str]
75
- require_workspace: Callable[..., Any]
76
- scan_completion_lock: Callable[..., Any]
77
- scan_context: Callable[..., dict[str, Any]]
78
- verify_manifest_binding: Callable[..., None]
79
- workbench_completion_binding: Callable[..., dict[str, Any]]
80
- workspace_state: Callable[..., dict[str, Any]]
65
+ def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]:
66
+ scan_dir = Path(scan["scan_dir"])
67
+ fields: dict[str, Any] = {"threatModelAvailable": False}
68
+ try:
69
+ db.require_recorded_manifest_digest(scan, scan_dir)
70
+ db.verify_manifest_binding(
71
+ scan, _read_scan_local_json(scan_dir, db.ARTIFACTS["manifest"], "scan manifest")
72
+ )
73
+ saved_model = _read_saved_threat_model(scan_dir)
74
+ if saved_model is not None:
75
+ description = saved_model[0]
76
+ fields.update(
77
+ threatModelAvailable=True,
78
+ threatModelProvenance=description["provenance"],
79
+ )
80
+ if description["path"] is not None:
81
+ fields["threatModelPath"] = description["path"]
82
+ except (ContractError, OSError, SystemExit):
83
+ # Unavailable optional model data must not prevent reading the saved scan.
84
+ pass
85
+ return fields
86
+
87
+
88
+ def refresh_completed_scan(
89
+ db: Any,
90
+ connection: sqlite3.Connection,
91
+ scan: sqlite3.Row,
92
+ cost_json: str | None,
93
+ ) -> dict[str, Any]:
94
+ warnings = json.loads(scan["completion_warnings_json"])
95
+ scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
96
+ db.require_recorded_manifest_digest(scan, scan_dir)
97
+ db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"]))
98
+ try:
99
+ manifest, _, _ = finalize_scan(
100
+ scan_dir,
101
+ expected_coverage_mode=db.expected_coverage_mode(scan),
102
+ projection_warnings=warnings,
103
+ )
104
+ except ContractError as exc:
105
+ raise SystemExit(str(exc)) from exc
106
+ db.verify_manifest_binding(scan, manifest)
107
+ manifest_digest = db.published_manifest_digest(scan_dir, manifest)
108
+ db.pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
109
+ if cost_json is not None and scan["recipe_json"] is not None:
110
+ db.scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json)
111
+ if warnings != json.loads(scan["completion_warnings_json"]):
112
+ with connection:
113
+ connection.execute(
114
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
115
+ (json.dumps(warnings), scan["id"]),
116
+ )
117
+ return db.scan_context(connection, scan["id"])
81
118
 
82
119
 
83
120
  def _encoded(value: Any) -> bytes:
@@ -118,35 +155,51 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
118
155
  )
119
156
 
120
157
 
121
- def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
122
- latest_reducer = _latest_successful_reducer(workers)
158
+ def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
159
+ return [
160
+ f"{directory}/{name}"
161
+ for name in _children(scan_dir, directory)
162
+ if re.fullmatch(r"[0-9a-f]{64}\.json", name)
163
+ ]
123
164
 
124
- def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
125
- for name in _children(scan_dir, directory):
126
- if re.fullmatch(r"[0-9a-f]{64}\.json", name):
127
- yield f"{directory}/{name}", kind
128
165
 
129
- yield from checkpoints("checkpoints")
166
+ def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
167
+ output = Path(worker["artifact_dir"]).relative_to(scan_dir)
168
+ attempts = (output.parent if output.name == "output" else output) / "attempts"
169
+ archived = [
170
+ ((attempts / name).as_posix(), int(name.split("-")[1]))
171
+ for name in _children(scan_dir, attempts.as_posix())
172
+ if re.fullmatch(r"attempt-\d+", name)
173
+ ]
174
+ attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
175
+ if worker["kind"] == "discovery" and not attempt:
176
+ attempt = max((attempt for _, attempt in archived), default=0) + 1
177
+ return [(output.as_posix(), attempt), *archived]
178
+
179
+
180
+ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
181
+ latest_reducer = _latest_successful_reducer(workers)
182
+ yield "checkpoint-head.json", None
183
+ for directory in ("checkpoint-heads", "checkpoints"):
184
+ yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
130
185
  for worker in workers:
131
186
  if worker["kind"] not in {"dedup", "discovery"}:
132
187
  continue
133
188
  try:
134
- output = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix()
189
+ outputs = _worker_outputs(scan_dir, worker)
135
190
  except (TypeError, ValueError):
136
191
  continue
137
- attempts = (Path(output).parent if Path(output).name == "output" else Path(output)) / (
138
- "attempts"
139
- )
140
- directories = [output] + [
141
- (attempts / name).as_posix()
142
- for name in _children(scan_dir, attempts.as_posix())
143
- if re.fullmatch(r"attempt-\d+", name)
144
- ]
145
- for directory in directories:
146
- checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
192
+ for directory, _ in outputs:
193
+ if worker["kind"] == "discovery":
194
+ yield f"{directory}/checkpoint-head.json", worker["kind"]
195
+ yield from (
196
+ (path, worker["kind"])
197
+ for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
198
+ )
199
+ checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
147
200
  if worker["kind"] == "discovery" or checkpoint_paths:
148
201
  yield f"{directory}/result.json", worker["kind"]
149
- yield from checkpoint_paths
202
+ yield from ((path, worker["kind"]) for path in checkpoint_paths)
150
203
  if worker["result_manifest_path"] and (
151
204
  worker["kind"] == "discovery"
152
205
  or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
@@ -160,17 +213,143 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st
160
213
  continue
161
214
 
162
215
 
216
+ def _checkpoint_head_directory(relative: str) -> Path | None:
217
+ path = Path(relative)
218
+ if path.name == "checkpoint-head.json":
219
+ return path.parent
220
+ if path.parent.name == "checkpoint-heads":
221
+ return path.parent.parent
222
+ return None
223
+
224
+
225
+ def _capture_saved_source(
226
+ scan_dir: Path,
227
+ relative: str,
228
+ scan_id: str,
229
+ *,
230
+ kind: str | None = None,
231
+ snapshot_head: bool = True,
232
+ write: bool = True,
233
+ ) -> dict[str, tuple[str, int]]:
234
+ if not snapshot_head or Path(relative).name != "checkpoint-head.json":
235
+ _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
236
+ return {relative: (digest, observed)}
237
+ head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
238
+ observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
239
+ directory = Path(relative).parent
240
+ selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
241
+ _, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
242
+ digest = _digest(observation)
243
+ snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
244
+ # Capture the selected file even if the worker created it after directory enumeration.
245
+ if write and not (scan_dir / snapshot).exists():
246
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
247
+ return {
248
+ snapshot: (digest, int(observation["observedAtNs"])),
249
+ selected: (selected_digest, selected_time),
250
+ }
251
+
252
+
253
+ def _is_source_order_snapshot(relative: str) -> bool:
254
+ path = Path(relative)
255
+ return path.parent == Path("source-order") and bool(
256
+ re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
257
+ )
258
+
259
+
163
260
  def _read_saved_result(
164
261
  scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
165
- ) -> tuple[dict[str, Any], str]:
166
- draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint")
262
+ ) -> tuple[dict[str, Any], str, int]:
263
+ draft, _, metadata = _read_scan_local_json_with_metadata(
264
+ scan_dir, relative, "Saved scan checkpoint"
265
+ )
266
+ directory = _checkpoint_head_directory(relative)
267
+ if directory is not None:
268
+ checkpoint = draft.get("checkpoint")
269
+ if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
270
+ raise ContractError("checkpoint head does not name a saved checkpoint")
271
+ _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
272
+ if Path(relative).name == "checkpoint-head.json":
273
+ return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
274
+ observed = draft.get("observedAtNs")
275
+ if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
276
+ raise ContractError("checkpoint head has no observation time")
277
+ return draft, _digest(draft), int(observed)
167
278
  if draft.get("scanId") != scan_id:
168
279
  raise ContractError("checkpoint belongs to a different scan")
169
- if not isinstance(draft.get("findings"), list) or not isinstance(
170
- draft.get("coverage", {} if kind == "dedup" else None), dict
280
+ if not _is_source_order_snapshot(relative) and (
281
+ not isinstance(draft.get("findings"), list)
282
+ or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
171
283
  ):
172
284
  raise ContractError("checkpoint has no semantic findings or coverage")
173
- return draft, _digest(draft)
285
+ return draft, _digest(draft), metadata.st_mtime_ns
286
+
287
+
288
+ def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
289
+ times: dict[str, int] = {}
290
+ snapshots = [path for path in sources if _is_source_order_snapshot(path)]
291
+ for path in snapshots:
292
+ record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
293
+ if digest != sources[path] or not isinstance(record.get("sources"), dict):
294
+ raise ContractError("saved source ordering changed after the scan stopped")
295
+ for relative, observation in record["sources"].items():
296
+ if (
297
+ not isinstance(observation, dict)
298
+ or relative not in sources
299
+ or observation.get("digest") != sources[relative]
300
+ or not isinstance(observation.get("observedAtNs"), str)
301
+ or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
302
+ ):
303
+ raise ContractError("saved source ordering does not match its frozen sources")
304
+ observed = int(observation["observedAtNs"])
305
+ if relative in times and times[relative] != observed:
306
+ raise ContractError("saved source ordering has conflicting observations")
307
+ times[relative] = observed
308
+ if snapshots and sources.keys() - set(snapshots) - times.keys():
309
+ raise ContractError("saved source ordering is incomplete")
310
+ return times
311
+
312
+
313
+ def _freeze_source_times(
314
+ scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
315
+ ) -> None:
316
+ # Identical result rewrites must not change the order of frozen review evidence.
317
+ observations = {
318
+ path: {"digest": digest, "observedAtNs": str(times[path])}
319
+ for path, digest in sources.items()
320
+ if not _is_source_order_snapshot(path)
321
+ }
322
+ if not observations:
323
+ return
324
+ record = {"scanId": scan_id, "sources": observations}
325
+ digest = _digest(record)
326
+ path = f"source-order/{digest}.json"
327
+ if not (scan_dir / path).exists():
328
+ write_scan_local_bytes(scan_dir, path, _encoded(record))
329
+ if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
330
+ raise ContractError("saved source ordering does not match its digest")
331
+ sources[path] = digest
332
+
333
+
334
+ def _parent_scan_draft(
335
+ scan_id: str,
336
+ parent_scan: dict[str, Any],
337
+ findings: dict[str, Any],
338
+ coverage: dict[str, Any],
339
+ ) -> dict[str, Any]:
340
+ parent = {
341
+ "scanId": scan_id,
342
+ "findings": findings.get("findings"),
343
+ "coverage": coverage,
344
+ **{
345
+ key: parent_scan[key]
346
+ for key in ("scope", "threatModel", "complete")
347
+ if key in parent_scan
348
+ },
349
+ }
350
+ if not isinstance(parent["findings"], list):
351
+ raise ContractError("Saved parent draft has no findings array")
352
+ return parent
174
353
 
175
354
 
176
355
  def _read_saved_parent_result(
@@ -188,29 +367,34 @@ def _read_saved_parent_result(
188
367
  or coverage.get("scanId", scan_id) != scan_id
189
368
  ):
190
369
  raise ContractError("Saved parent documents belong to a different scan")
191
- parent = {
192
- "scanId": scan_id,
193
- "findings": findings.get("findings"),
194
- "coverage": coverage,
195
- **{
196
- key: parent_scan[key]
197
- for key in ("scope", "threatModel", "complete")
198
- if key in parent_scan
199
- },
200
- }
201
- if not isinstance(parent["findings"], list):
202
- raise ContractError("Saved parent draft has no findings array")
203
- return manifest, parent
370
+ return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
204
371
 
205
372
 
206
- def _source_digests(value: Any, label: str) -> dict[str, str]:
373
+ def _source_digests(value: Any, error: str) -> dict[str, str]:
207
374
  if not isinstance(value, dict) or not all(
208
375
  isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
209
376
  ):
210
- raise ContractError(f"{label} source digests are malformed.")
377
+ raise ContractError(error)
211
378
  return value
212
379
 
213
380
 
381
+ def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
382
+ if isinstance(value, dict) and isinstance(value.get("sources"), dict):
383
+ sources = _source_digests(
384
+ value["sources"], "Saved stopped-scan source digests are malformed."
385
+ )
386
+ model_source = value.get("threatModelSource")
387
+ if not isinstance(model_source, str) or model_source not in sources:
388
+ raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
389
+ return sources, model_source
390
+ return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
391
+
392
+
393
+ def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
394
+ state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
395
+ return json.dumps(state, sort_keys=True)
396
+
397
+
214
398
  def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
215
399
  try:
216
400
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
@@ -234,7 +418,7 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
234
418
 
235
419
  if manifest_path is None:
236
420
  if frozen_sources is not None:
237
- return bool(_source_digests(json.loads(frozen_sources), "Frozen stopped-scan"))
421
+ return bool(_retained_source_state(json.loads(frozen_sources))[0])
238
422
  return has_saved_source()
239
423
  if scan["seal_manifest_digest"] is None:
240
424
  try:
@@ -252,14 +436,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
252
436
  if not isinstance(manifest_scan, dict):
253
437
  return True
254
438
  published_sources = _source_digests(
255
- manifest_scan.get("preservedSources", {}), "Published scan"
439
+ manifest_scan.get("preservedSources", {}),
440
+ "Published scan source digests are malformed.",
256
441
  )
257
442
  current_sources = dict(published_sources)
443
+ paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
258
444
  for path in paths:
259
445
  try:
260
- _, current_sources[path] = _read_saved_result(
261
- scan_dir, path, scan["id"], kind=paths[path]
446
+ captured = _capture_saved_source(
447
+ scan_dir,
448
+ path,
449
+ scan["id"],
450
+ kind=paths[path],
451
+ snapshot_head=path not in published_sources,
452
+ write=False,
262
453
  )
454
+ current_sources.update({path: value[0] for path, value in captured.items()})
263
455
  except (ContractError, OSError, ValueError):
264
456
  continue
265
457
  return current_sources != published_sources
@@ -273,7 +465,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
273
465
  include_parent = True
274
466
  raw_frozen_sources = scan["retained_source_digests_json"]
275
467
  if raw_frozen_sources is not None:
276
- frozen_sources = _source_digests(json.loads(raw_frozen_sources), "Saved stopped-scan")
468
+ frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
277
469
  include_parent = False
278
470
 
279
471
  manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
@@ -291,7 +483,8 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
291
483
  ):
292
484
  if "preservedSources" in manifest_scan:
293
485
  published_sources = _source_digests(
294
- manifest_scan["preservedSources"], "Published scan"
486
+ manifest_scan["preservedSources"],
487
+ "Published scan source digests are malformed.",
295
488
  )
296
489
  include_parent = not published_sources
297
490
  if published_sources:
@@ -312,21 +505,30 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
312
505
  ).fetchall()
313
506
  paths = dict(_saved_result_paths(scan_dir, workers))
314
507
  recovery_sources = dict(frozen_sources or {})
508
+ source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
315
509
  for relative, expected_digest in recovery_sources.items():
316
510
  try:
317
- _, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative))
511
+ _, digest, observed = _read_saved_result(
512
+ scan_dir, relative, scan["id"], kind=paths.get(relative)
513
+ )
318
514
  except (ContractError, OSError, ValueError) as exc:
319
515
  raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
320
516
  if digest != expected_digest:
321
517
  raise ContractError("checkpoint changed after the scan stopped")
518
+ if not _is_source_order_snapshot(relative):
519
+ source_times.setdefault(relative, observed)
322
520
 
323
521
  for relative in paths.keys() - recovery_sources.keys():
324
522
  try:
325
- _, recovery_sources[relative] = _read_saved_result(
326
- scan_dir, relative, scan["id"], kind=paths[relative]
327
- )
523
+ captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
328
524
  except (ContractError, OSError, ValueError):
329
525
  continue
526
+ for path, (digest, observed) in captured.items():
527
+ if path in recovery_sources and recovery_sources[path] != digest:
528
+ raise ContractError("checkpoint changed after the scan stopped")
529
+ recovery_sources[path] = digest
530
+ source_times.setdefault(path, observed)
531
+ _freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
330
532
  return recovery_sources, include_parent
331
533
 
332
534
 
@@ -358,15 +560,10 @@ def _finding_key(finding: dict[str, Any]) -> str:
358
560
  else finding.get("identity")
359
561
  )
360
562
  if not isinstance(identity, dict):
361
- extensions = finding.get("extensions")
362
- source = str(
363
- (extensions.get("candidateId") if isinstance(extensions, dict) else None)
364
- or finding.get("title")
365
- or "finding"
366
- )
367
- identity = {
368
- "anchor": re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
369
- }
563
+ normalized = dict(finding)
564
+ normalized.pop("identity", None)
565
+ _ensure_finding_identity(normalized)
566
+ identity = normalized["identity"]
370
567
  locations = finding.get("locations", [])
371
568
  if not isinstance(locations, list):
372
569
  locations = []
@@ -459,6 +656,245 @@ def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
459
656
  )
460
657
 
461
658
 
659
+ def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
660
+ rows = coverage.get("deferred", [])
661
+ return rows if isinstance(rows, list) else []
662
+
663
+
664
+ def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
665
+ if draft.get("complete") is False:
666
+ return []
667
+ coverage = draft["coverage"]
668
+ rows = coverage.get("resolvedDeferred", [])
669
+ try:
670
+ # Invalid closure metadata cannot discard the evidence it names.
671
+ _validate_schema_node(rows, schema, "coverage.resolvedDeferred")
672
+ _validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
673
+ except ContractError:
674
+ return []
675
+ return rows
676
+
677
+
678
+ def _merge_tied_parent_observations(
679
+ current: dict[str, Any], previous: dict[str, Any]
680
+ ) -> dict[str, Any]:
681
+ merged = copy.deepcopy(current)
682
+ for finding in previous["findings"]:
683
+ if finding not in merged["findings"]:
684
+ merged["findings"].append(copy.deepcopy(finding))
685
+ coverage = merged["coverage"]
686
+ for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
687
+ rows = previous["coverage"].get(field, [])
688
+ output = coverage.setdefault(field, [])
689
+ if isinstance(rows, list) and isinstance(output, list):
690
+ for row in rows:
691
+ if row not in output:
692
+ output.append(copy.deepcopy(row))
693
+ pending_ids = {
694
+ identity
695
+ for row in _deferred_rows(coverage)
696
+ if isinstance(row, dict)
697
+ for identity in (row.get("id"), row.get("candidateId"))
698
+ if isinstance(identity, str)
699
+ }
700
+ closure_schema = _read_json(
701
+ Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
702
+ )["properties"]["resolvedDeferred"]
703
+ closures = {}
704
+ for observed in (current, previous):
705
+ for row in _resolved_deferred_rows(observed, closure_schema):
706
+ if row["id"] not in pending_ids:
707
+ closures.setdefault(row["id"], copy.deepcopy(row))
708
+ coverage.pop("resolvedDeferred", None)
709
+ if closures:
710
+ coverage["resolvedDeferred"] = list(closures.values())
711
+ if current.get("complete") is False or previous.get("complete") is False:
712
+ merged["complete"] = False
713
+ coverage.pop("completeness", None)
714
+ return merged
715
+
716
+
717
+ def _saved_coverage_id(item: dict[str, Any]) -> str:
718
+ return item.get("candidateId") or f"saved-{_digest(item)[:16]}"
719
+
720
+
721
+ def _deferred_candidate_id(
722
+ row: dict[str, Any],
723
+ owner: str | None,
724
+ ambiguous_deferred: set[tuple[str | None, str]],
725
+ ) -> str | None:
726
+ identity = row.get("candidateId") or row.get("id")
727
+ if not isinstance(identity, str):
728
+ return None
729
+ if (owner, identity) in ambiguous_deferred and not any(
730
+ key in row for key in ("candidateId", "candidate", "finding")
731
+ ):
732
+ return None
733
+ return identity
734
+
735
+
736
+ def _generic_surface_updates(
737
+ sources: list[tuple[str, dict[str, Any], str | None]],
738
+ source_order: dict[str, tuple[int, int]],
739
+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
740
+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
741
+ resolved_candidates: dict[tuple[str | None, str], str],
742
+ reopened_generic: set[tuple[str | None, str]],
743
+ surface_schema: dict[str, Any],
744
+ deferred_rows: dict[str, list[Any]],
745
+ ambiguous_deferred: set[tuple[str | None, str]],
746
+ ) -> tuple[set[int], list[dict[str, Any]]]:
747
+ if not closed_deferred and not reopened_generic:
748
+ return set(), []
749
+
750
+ def linked(row: dict[str, Any], identity: str | None) -> bool:
751
+ surface_ids = row.get("surfaceIds", [])
752
+ return identity is not None and (
753
+ row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
754
+ )
755
+
756
+ saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
757
+ for relative, draft, owner in sources:
758
+ rows = draft["coverage"].get("surfaces", [])
759
+ for row in rows if isinstance(rows, list) else []:
760
+ if isinstance(row, dict) and isinstance(row.get("id"), str):
761
+ saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
762
+ replaced: set[int] = set()
763
+ updates: list[dict[str, Any]] = []
764
+ for relative, draft, owner in sources:
765
+ closed_ids = {
766
+ identity
767
+ for (saved_owner, identity), (_, _, source) in closed_deferred.items()
768
+ if saved_owner == owner and source == relative
769
+ }
770
+ reopened_ids = {
771
+ identity
772
+ for (saved_owner, identity), (_, _, source) in active_deferred.items()
773
+ if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
774
+ }
775
+ if not closed_ids and not reopened_ids:
776
+ continue
777
+ current = draft["coverage"].get("surfaces", [])
778
+ for surface in current if isinstance(current, list) else []:
779
+ if not isinstance(surface, dict):
780
+ continue
781
+ reopening = surface.get("disposition") == "needs_follow_up"
782
+ work_ids = reopened_ids if reopening else closed_ids
783
+ if not work_ids:
784
+ continue
785
+ identity = surface.get("id")
786
+ if not isinstance(identity, str):
787
+ continue
788
+ matches = saved_surfaces[(owner, identity)]
789
+ # Older writers could assign one ID to distinct surfaces in a draft.
790
+ # A closure cannot identify which of those observations it replaces.
791
+ by_source = dict(matches)
792
+ if any(row != by_source[saved_path] for saved_path, row in matches):
793
+ continue
794
+ if any(
795
+ "candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
796
+ ):
797
+ continue
798
+ if any(
799
+ row is not surface
800
+ and source_order[saved_path] >= source_order[relative]
801
+ and row.get("disposition") != surface.get("disposition")
802
+ for saved_path, row in matches
803
+ ):
804
+ continue
805
+
806
+ if not reopening and any(
807
+ saved_owner == owner
808
+ and (
809
+ not isinstance(row.get("id") or row.get("candidateId"), str)
810
+ or (
811
+ isinstance(row.get("id"), str)
812
+ and (owner, row["id"]) in ambiguous_deferred
813
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
814
+ )
815
+ )
816
+ and linked(row, identity)
817
+ for saved_path, _, saved_owner in sources
818
+ for row in deferred_rows[saved_path]
819
+ if isinstance(row, dict)
820
+ ):
821
+ continue
822
+ if not reopening and any(
823
+ saved_owner == owner
824
+ and (owner, deferred_id) not in closed_deferred
825
+ and (
826
+ (candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
827
+ or (owner, candidate_id) not in resolved_candidates
828
+ )
829
+ and linked(row, identity)
830
+ for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
831
+ ):
832
+ continue
833
+ # An accepted checkpoint can update a saved surface by ID without
834
+ # optional surfaceIds links on its generic task.
835
+ latest_surface = max(matches, key=lambda match: source_order[match[0]])[1]
836
+ update = copy.deepcopy(latest_surface)
837
+ refs = update.setdefault("receiptRefs", [])
838
+ if isinstance(refs, list):
839
+ for _, row in matches:
840
+ previous_refs = row.get("receiptRefs", [])
841
+ for ref in previous_refs if isinstance(previous_refs, list) else []:
842
+ if ref not in refs:
843
+ refs.append(ref)
844
+ try:
845
+ _validate_schema_node(update, surface_schema, "coverage.surfaces")
846
+ except ContractError:
847
+ continue
848
+ replaced.update(
849
+ id(row)
850
+ for _, row in matches
851
+ if reopening
852
+ or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
853
+ )
854
+ if update not in updates:
855
+ updates.append(update)
856
+ return replaced, updates
857
+
858
+
859
+ @contextmanager
860
+ def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
861
+ """Keep rejected completion attempts from becoming accepted parent observations."""
862
+ head_path = scan_dir / "checkpoint-head.json"
863
+ previous = None
864
+ try:
865
+ head_path.lstat()
866
+ except FileNotFoundError:
867
+ pass
868
+ else:
869
+ descriptor = open_scan_local_file_descriptor(
870
+ scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
871
+ )
872
+ with os.fdopen(descriptor, "rb") as handle:
873
+ metadata = os.fstat(handle.fileno())
874
+ previous = (handle.read(), metadata)
875
+ directories = ("checkpoints", "checkpoint-heads")
876
+ previous_files = {
877
+ relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
878
+ }
879
+ try:
880
+ yield
881
+ except BaseException:
882
+ if previous is None:
883
+ _remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
884
+ else:
885
+ payload, metadata = previous
886
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
887
+ os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
888
+ current_files = {
889
+ relative
890
+ for directory in directories
891
+ for relative in _checkpoint_paths(scan_dir, directory)
892
+ }
893
+ for relative in current_files - previous_files:
894
+ _remove_scan_local_file_if_exists(scan_dir, relative)
895
+ raise
896
+
897
+
462
898
  def merge_saved_results(
463
899
  scan_dir: Path,
464
900
  scan_id: str,
@@ -470,14 +906,25 @@ def merge_saved_results(
470
906
  reason: str,
471
907
  frozen_source_digests: dict[str, str] | None = None,
472
908
  allow_frozen_legacy_parent: bool = False,
909
+ frozen_model_source: str | None = None,
910
+ selected_model_source: list[str] | None = None,
473
911
  ) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
474
912
  """Read only bound parent/worker files; return an unsealed loss-preserving union."""
475
913
  initial_warnings = set(warnings)
914
+ try:
915
+ source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
916
+ except (ContractError, OSError, ValueError) as exc:
917
+ raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
476
918
  parent: dict[str, Any] | None = None
477
919
  parent_manifest: dict[str, Any] | None = None
920
+ parent_is_canonical = False
921
+ parent_modified = 0
478
922
  if frozen_source_digests is None or allow_frozen_legacy_parent:
479
923
  try:
480
924
  parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
925
+ # Without an accepted head, file-authored coverage is a full replacement.
926
+ parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
927
+ parent_is_canonical = True
481
928
  except (ContractError, OSError, ValueError) as exc:
482
929
  if not stopped:
483
930
  raise
@@ -487,25 +934,66 @@ def merge_saved_results(
487
934
  parent = None
488
935
  if parent_manifest is not None and parent is not None:
489
936
  parent_scan = parent_manifest["scan"]
937
+ try:
938
+ previous_head, _, head_modified = _read_saved_result(
939
+ scan_dir, "checkpoint-head.json", scan_id
940
+ )
941
+ except (ContractError, OSError, ValueError):
942
+ head_modified = None
943
+ if head_modified is not None:
944
+ # A partial tool publication must not outrank its accepted head.
945
+ parent_modified = min(
946
+ (scan_dir / name).lstat().st_mtime_ns
947
+ for name in ("findings.json", "coverage.json", "scan-manifest.json")
948
+ )
490
949
  if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
950
+ head_path = scan_dir / "checkpoint-head.json"
951
+ tied_observations = False
952
+ if head_modified == parent_modified:
953
+ previous_parent, _, _ = _read_saved_result(
954
+ scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
955
+ )
956
+ if previous_parent != parent:
957
+ # Tied observations cannot decide which pending work came last.
958
+ parent = _merge_tied_parent_observations(parent, previous_parent)
959
+ parent_is_canonical = False
960
+ tied_observations = True
491
961
  payload = _encoded(parent)
492
962
  parent_digest = hashlib.sha256(payload).hexdigest()
493
963
  parent_checkpoint = f"checkpoints/{parent_digest}.json"
494
- write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
964
+ checkpoint_path = scan_dir / parent_checkpoint
965
+ if not checkpoint_path.exists():
966
+ write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
967
+ # A recovery copy must not appear newer than the review it copies.
968
+ os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
969
+ if head_modified is None or head_modified < parent_modified or tied_observations:
970
+ write_scan_local_bytes(
971
+ scan_dir,
972
+ "checkpoint-head.json",
973
+ _encoded({"checkpoint": checkpoint_path.name}),
974
+ )
975
+ os.utime(head_path, ns=(parent_modified, parent_modified))
495
976
  if frozen_source_digests is not None:
977
+ captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
496
978
  frozen_source_digests = {
497
979
  **frozen_source_digests,
498
980
  parent_checkpoint: parent_digest,
981
+ **{path: value[0] for path, value in captured.items()},
499
982
  }
500
983
 
501
984
  sources: list[tuple[str, dict[str, Any], str | None]] = []
502
985
  parent_preserved_sources: dict[str, str] = {}
503
986
  source_digests: dict[str, str] = {}
987
+ source_order: dict[str, tuple[int, int]] = {}
988
+ worker_attempts: dict[str, tuple[str, int]] = {}
989
+ saved_heads: dict[str, str] = {}
504
990
  if parent_manifest:
505
991
  recorded = parent_manifest["scan"].get("preservedSources", {})
506
992
  if isinstance(recorded, dict):
507
993
  parent_preserved_sources = recorded
508
994
  source_digests.update(parent_preserved_sources)
995
+ if frozen_source_digests is None:
996
+ source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
509
997
  paths: dict[str, str | None] = {}
510
998
  reducer_paths: set[str] = set()
511
999
  current_results: set[str] = set()
@@ -520,57 +1008,43 @@ def merge_saved_results(
520
1008
  except ValueError:
521
1009
  warnings.append("Skipped a reducer result outside the scan directory.")
522
1010
 
523
- def checkpoints(directory: str, worker_id: str | None) -> None:
524
- for name in _children(scan_dir, directory):
525
- if re.fullmatch(r"[0-9a-f]{64}\.json", name):
526
- paths[f"{directory}/{name}"] = worker_id
1011
+ def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
1012
+ if worker_id is not None:
1013
+ root = Path(directory).parent.as_posix()
1014
+ worker_attempts[root] = (worker_id, attempt)
1015
+ paths[f"{root}/checkpoint-head.json"] = worker_id
1016
+ head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
1017
+ for saved_directory in (directory, head_snapshots):
1018
+ paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
527
1019
 
1020
+ paths["checkpoint-head.json"] = None
528
1021
  checkpoints("checkpoints", None)
529
1022
  for worker in workers:
530
1023
  try:
531
- output = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix()
1024
+ outputs = _worker_outputs(scan_dir, worker)
532
1025
  except (TypeError, ValueError):
533
1026
  warnings.append("Skipped a worker checkpoint outside the scan directory.")
534
1027
  continue
535
1028
  if worker["kind"] == "dedup":
536
-
537
- def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None:
538
- result_path = f"{directory}/result.json"
539
- checkpoint_paths = [
540
- f"{directory}/checkpoints/{name}"
541
- for name in _children(scan_dir, f"{directory}/checkpoints")
542
- if re.fullmatch(r"[0-9a-f]{64}\.json", name)
543
- ]
1029
+ for directory, attempt in outputs:
1030
+ checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
544
1031
  if not checkpoint_paths:
545
- return
546
- paths[result_path] = None
547
- for checkpoint_path in checkpoint_paths:
548
- paths[checkpoint_path] = None
549
- reducer_paths.update([result_path, *checkpoint_paths])
550
- reducer_outputs.append((reducer_worker, result_path, checkpoint_paths, attempt))
551
-
552
- reducer_output(output, int(worker["attempt"] or 0), worker)
553
- attempts = (
554
- Path(output).parent if Path(output).name == "output" else Path(output)
555
- ) / "attempts"
556
- for name in _children(scan_dir, attempts.as_posix()):
557
- match = re.fullmatch(r"attempt-(\d+)", name)
558
- if match:
559
- reducer_output((attempts / name).as_posix(), int(match.group(1)), worker)
1032
+ continue
1033
+ result_path = f"{directory}/result.json"
1034
+ retained_paths = [result_path, *checkpoint_paths]
1035
+ paths.update(dict.fromkeys(retained_paths))
1036
+ reducer_paths.update(retained_paths)
1037
+ reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
560
1038
  continue
561
1039
  if worker["kind"] != "discovery":
562
1040
  continue
1041
+ output, attempt = outputs[0]
563
1042
  paths[f"{output}/result.json"] = worker["id"]
564
1043
  current_results.add(f"{output}/result.json")
565
- checkpoints(f"{output}/checkpoints", worker["id"])
566
- attempts = (
567
- Path(output).parent if Path(output).name == "output" else Path(output)
568
- ) / "attempts"
569
- for name in _children(scan_dir, attempts.as_posix()):
570
- if re.fullmatch(r"attempt-\d+", name):
571
- archived = (attempts / name).as_posix()
572
- paths[f"{archived}/result.json"] = worker["id"]
573
- checkpoints(f"{archived}/checkpoints", worker["id"])
1044
+ for archived, archived_attempt in outputs[1:]:
1045
+ paths[f"{archived}/result.json"] = worker["id"]
1046
+ checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
1047
+ checkpoints(f"{output}/checkpoints", worker["id"], attempt)
574
1048
  if worker["result_manifest_path"]:
575
1049
  try:
576
1050
  current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
@@ -579,7 +1053,26 @@ def merge_saved_results(
579
1053
  except ValueError:
580
1054
  warnings.append("Skipped a worker result outside the scan directory.")
581
1055
 
1056
+ if frozen_source_digests is None:
1057
+ for relative, worker_id in list(paths.items()):
1058
+ if Path(relative).name != "checkpoint-head.json":
1059
+ continue
1060
+ del paths[relative]
1061
+ try:
1062
+ captured = _capture_saved_source(scan_dir, relative, scan_id)
1063
+ paths.update({path: worker_id for path in captured})
1064
+ except (ContractError, OSError, ValueError) as exc:
1065
+ if (scan_dir / relative).exists():
1066
+ warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
1067
+
582
1068
  if frozen_source_digests is not None:
1069
+ source_digests.update(
1070
+ {
1071
+ path: digest
1072
+ for path, digest in frozen_source_digests.items()
1073
+ if _is_source_order_snapshot(path)
1074
+ }
1075
+ )
583
1076
  paths = {
584
1077
  relative: worker_id
585
1078
  for relative, worker_id in paths.items()
@@ -591,12 +1084,17 @@ def merge_saved_results(
591
1084
 
592
1085
  for relative, worker_id in paths.items():
593
1086
  try:
594
- draft, digest = _read_saved_result(
1087
+ draft, digest, observed = _read_saved_result(
595
1088
  scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
596
1089
  )
597
1090
  if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
598
1091
  raise ContractError("checkpoint changed after the scan stopped")
599
1092
  source_digests[relative] = digest
1093
+ source_times.setdefault(relative, observed)
1094
+ source_order[relative] = (0, source_times[relative])
1095
+ if _checkpoint_head_directory(relative) is not None:
1096
+ saved_heads[relative] = draft["checkpoint"]
1097
+ continue
600
1098
  # Recovery expects coverage, but reducer results only contain findings
601
1099
  # and context. Add an empty value after hashing the original result.
602
1100
  sources.append((relative, {"coverage": {}, **draft}, worker_id))
@@ -607,6 +1105,37 @@ def merge_saved_results(
607
1105
  if frozen_source_digests.keys() - source_digests.keys():
608
1106
  raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
609
1107
 
1108
+ # Frozen observations retain checkpoint selection even if a worker moves its head.
1109
+ headed_workers = {
1110
+ worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
1111
+ for head in saved_heads
1112
+ if _checkpoint_head_directory(head) != Path(".")
1113
+ }
1114
+ for relative, _, worker_id in sources:
1115
+ if worker_id not in headed_workers:
1116
+ continue
1117
+ directory = Path(relative).parent
1118
+ if directory.name == "checkpoints":
1119
+ directory = directory.parent
1120
+ _, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
1121
+ source_order[relative] = (attempt, source_order[relative][1])
1122
+ selected_observations: dict[str, tuple[int, int]] = {}
1123
+ parent_heads: list[tuple[int, str]] = []
1124
+ for head, checkpoint in saved_heads.items():
1125
+ directory = _checkpoint_head_directory(head)
1126
+ selected = (directory / "checkpoints" / checkpoint).as_posix()
1127
+ if selected not in source_order:
1128
+ raise ContractError("Checkpoint head is outside the saved source set.")
1129
+ observed = source_order[head][1]
1130
+ if directory == Path("."):
1131
+ order = (0, max(source_order[selected][1], observed))
1132
+ parent_heads.append((observed, selected))
1133
+ else:
1134
+ _, attempt = worker_attempts[directory.as_posix()]
1135
+ order = (attempt, observed)
1136
+ selected_observations[selected] = max(selected_observations.get(selected, order), order)
1137
+ source_order.update(selected_observations)
1138
+
610
1139
  drafts_by_path = {relative: draft for relative, draft, _ in sources}
611
1140
  latest_reducer_key = (
612
1141
  (reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
@@ -627,8 +1156,67 @@ def merge_saved_results(
627
1156
  latest_reducer_key = candidate_key
628
1157
  latest_reducer = result_path
629
1158
 
1159
+ if parent_heads:
1160
+ latest_observation = max(observed for observed, _ in parent_heads)
1161
+ for observed, parent_path in parent_heads:
1162
+ if observed != latest_observation:
1163
+ continue
1164
+ draft = drafts_by_path[parent_path]
1165
+ modified = source_order[parent_path][1]
1166
+ if parent is None or modified > parent_modified:
1167
+ parent = draft
1168
+ parent_modified = modified
1169
+ parent_is_canonical = False
1170
+ elif modified == parent_modified and draft != parent:
1171
+ parent = _merge_tied_parent_observations(parent, draft)
1172
+ parent_is_canonical = False
630
1173
  if parent is None and latest_reducer is not None:
631
- parent = next((draft for relative, draft, _ in sources if relative == latest_reducer), None)
1174
+ parent = drafts_by_path[latest_reducer]
1175
+
1176
+ all_sources = ([("parent", parent, None)] if parent else []) + sources
1177
+ # Older checkpoints can omit IDs already assigned in their published output.
1178
+ for field in ("deferred", "surfaces"):
1179
+ named_rows: dict[str | None, list[dict[str, Any]]] = {}
1180
+ for _, draft, owner in all_sources:
1181
+ rows = draft["coverage"].get(field, [])
1182
+ named_rows.setdefault(owner, []).extend(
1183
+ row
1184
+ for row in (rows if isinstance(rows, list) else [])
1185
+ if isinstance(row, dict) and isinstance(row.get("id"), str)
1186
+ )
1187
+ for _, draft, owner in all_sources:
1188
+ rows = draft["coverage"].get(field, [])
1189
+ if not isinstance(rows, list):
1190
+ continue
1191
+ reserved = {
1192
+ row["id"]
1193
+ for row in rows
1194
+ if isinstance(row, dict) and isinstance(row.get("id"), str)
1195
+ }
1196
+ for row in rows:
1197
+ if not isinstance(row, dict) or "id" in row:
1198
+ continue
1199
+ if field == "deferred" and any(
1200
+ key in row for key in ("candidateId", "candidate", "finding")
1201
+ ):
1202
+ continue
1203
+ content = {"receiptRefs": [], **row} if field == "surfaces" else row
1204
+ identity = next(
1205
+ (
1206
+ named["id"]
1207
+ for named in named_rows[owner]
1208
+ if named["id"] not in reserved
1209
+ and {
1210
+ **({"receiptRefs": []} if field == "surfaces" else {}),
1211
+ **{key: value for key, value in named.items() if key != "id"},
1212
+ }
1213
+ == content
1214
+ ),
1215
+ None,
1216
+ )
1217
+ if identity is not None:
1218
+ row["id"] = identity
1219
+ reserved.add(identity)
632
1220
 
633
1221
  if parent is None and not sources:
634
1222
  return None
@@ -641,6 +1229,13 @@ def merge_saved_results(
641
1229
  ):
642
1230
  return None
643
1231
 
1232
+ if (
1233
+ frozen_source_digests is None
1234
+ or any(_is_source_order_snapshot(path) for path in source_digests)
1235
+ or allow_frozen_legacy_parent
1236
+ ):
1237
+ _freeze_source_times(scan_dir, scan_id, source_digests, source_times)
1238
+
644
1239
  target_kind = binding["allowedTargetKinds"][0]
645
1240
  if (
646
1241
  target_kind == "git_worktree"
@@ -651,15 +1246,9 @@ def merge_saved_results(
651
1246
  target = {"kind": target_kind, **binding["target"]}
652
1247
  if target["kind"] == "git_diff" and "snapshotDigest" not in target:
653
1248
  diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
654
- digest = hashlib.sha256(
655
- b"codex-security-diff/v1\0"
656
- + diff_kind.encode()
657
- + b"\0"
658
- + target["baseRevision"].encode()
659
- + b"\0"
660
- + target["headRevision"].encode()
661
- ).hexdigest()
662
- target["snapshotDigest"] = f"codex-security-snapshot/v1:sha256:{digest}"
1249
+ target["snapshotDigest"] = committed_diff_snapshot_digest(
1250
+ diff_kind, target["baseRevision"], target["headRevision"]
1251
+ )
663
1252
  manifest = (
664
1253
  copy.deepcopy(parent_manifest)
665
1254
  if parent_manifest
@@ -668,11 +1257,31 @@ def merge_saved_results(
668
1257
  for key in ("sealedAt", "artifacts"):
669
1258
  manifest["scan"].pop(key, None)
670
1259
  manifest["scan"]["preservedSources"] = source_digests
1260
+ # Completion follows the selected parent, including legacy terminal drafts
1261
+ # that omit the optional marker, rather than an older canonical manifest.
1262
+ if parent is not None:
1263
+ manifest["scan"].pop("complete", None)
1264
+ if "complete" in parent:
1265
+ manifest["scan"]["complete"] = parent["complete"]
1266
+ for key in ("scope", "threatModel"):
1267
+ if isinstance(parent.get(key), dict):
1268
+ manifest["scan"][key] = copy.deepcopy(parent[key])
1269
+ if isinstance(manifest["scan"].get("scope"), dict):
1270
+ manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
1271
+ if frozen_model_source is not None:
1272
+ # Publication retries retain the choice made with the frozen source set.
1273
+ model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
1274
+ if not isinstance(model, dict):
1275
+ raise ContractError("Frozen stopped-scan model source is unavailable.")
1276
+ manifest["scan"]["threatModel"] = copy.deepcopy(model)
1277
+ if paths[frozen_model_source] is not None:
1278
+ manifest["scan"]["threatModel"]["origin"] = "recovered"
1279
+ if selected_model_source is not None:
1280
+ selected_model_source[:] = [frozen_model_source]
671
1281
  coverage = (
672
1282
  copy.deepcopy(parent["coverage"])
673
1283
  if parent and parent["coverage"]
674
1284
  else {
675
- "completeness": "partial",
676
1285
  "mode": binding["coverageMode"],
677
1286
  "inventoryStrategy": "diff"
678
1287
  if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
@@ -696,7 +1305,7 @@ def merge_saved_results(
696
1305
  for field in ("surfaces", "explicitExclusions", "deferred")
697
1306
  for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
698
1307
  }
699
- if parent_manifest
1308
+ if parent_is_canonical
700
1309
  else set()
701
1310
  )
702
1311
  findings: list[dict[str, Any]] = []
@@ -725,32 +1334,180 @@ def merge_saved_results(
725
1334
  )
726
1335
  return bool(document["findings"])
727
1336
 
728
- all_sources = ([("parent", parent, None)] if parent else []) + sources
729
- current_drafts = ([(None, parent)] if parent else []) + [
730
- (worker_id, draft) for relative, draft, worker_id in sources if relative in current_results
1337
+ source_order["parent"] = (0, parent_modified)
1338
+ deferred_rows = {
1339
+ relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
1340
+ }
1341
+ # A legacy source can contain independent tasks with the same explicit ID.
1342
+ # Later rewrites cannot make an ID-only closure identify one of those tasks.
1343
+ ambiguous_deferred: set[tuple[str | None, str]] = set()
1344
+ for relative, _, owner in all_sources:
1345
+ by_id: dict[str, dict[str, Any]] = {}
1346
+ candidate_aliases = {
1347
+ identity
1348
+ for row in deferred_rows[relative]
1349
+ if isinstance(row, dict)
1350
+ and any(key in row for key in ("candidateId", "candidate", "finding"))
1351
+ for identity in (row.get("id"), row.get("candidateId"))
1352
+ if isinstance(identity, str)
1353
+ }
1354
+ for row in deferred_rows[relative]:
1355
+ if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
1356
+ continue
1357
+ if any(key in row for key in ("candidateId", "candidate", "finding")):
1358
+ continue
1359
+ if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
1360
+ ambiguous_deferred.add((owner, identity))
1361
+ by_id[identity] = row
1362
+ current_drafts = ([("parent", parent, None)] if parent else []) + [
1363
+ source for source in sources if source[0] in current_results | selected_observations.keys()
731
1364
  ]
1365
+ # Generic closures belong to one logical scan or worker, just like candidates.
1366
+ # Keep them when recovering a terminal checkpoint without its canonical write.
1367
+ closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1368
+
1369
+ candidate_ids: set[tuple[str | None, str]] = set()
1370
+ accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
1371
+ active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
1372
+ coverage_schema = _read_json(
1373
+ Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
1374
+ )["properties"]
1375
+ for relative, draft, owner in all_sources:
1376
+ order = source_order[relative]
1377
+ for item in deferred_rows[relative]:
1378
+ if not isinstance(item, dict):
1379
+ continue
1380
+ if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
1381
+ candidate_ids.update(
1382
+ (owner, identity)
1383
+ for identity in (item.get("id"), item.get("candidateId"))
1384
+ if isinstance(identity, str)
1385
+ )
1386
+ identity = item.get("id") or item.get("candidateId")
1387
+ if isinstance(identity, str):
1388
+ key = (owner, identity)
1389
+ previous = active_deferred.get(key)
1390
+ if previous is None or order > previous[0]:
1391
+ active_deferred[key] = (order, item, relative)
1392
+ if (
1393
+ relative == "parent"
1394
+ or relative in current_results
1395
+ or relative in selected_observations
1396
+ ):
1397
+ try:
1398
+ # A malformed update cannot discard valid saved evidence.
1399
+ _validate_schema_node(
1400
+ item, coverage_schema["deferred"]["items"], "coverage.deferred"
1401
+ )
1402
+ except ContractError:
1403
+ continue
1404
+ accepted_deferred_orders[key] = max(
1405
+ accepted_deferred_orders.get(key, order), order
1406
+ )
1407
+ for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
1408
+ key = (owner, closure["id"])
1409
+ previous = closed_deferred.get(key)
1410
+ if previous is None or order > previous[0]:
1411
+ # The parent comes first, preserving its reason on equal timestamps.
1412
+ closed_deferred[key] = (order, closure, relative)
1413
+ reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
1414
+ reopened_generic: set[tuple[str | None, str]] = set()
1415
+ for key, (order, _, _) in list(closed_deferred.items()):
1416
+ # Equal timestamps cannot distinguish closure from reopened work.
1417
+ reopened = [
1418
+ active
1419
+ for (owner, identity), active in active_deferred.items()
1420
+ if owner == key[0]
1421
+ and (identity == key[1] or active[1].get("candidateId") == key[1])
1422
+ and active[0] >= order
1423
+ ]
1424
+ if key in candidate_ids or key in ambiguous_deferred or reopened:
1425
+ del closed_deferred[key]
1426
+ for _, item, _ in reopened:
1427
+ reopened_rows.append((key[0], item))
1428
+ if key not in candidate_ids:
1429
+ reopened_generic.add(key)
1430
+ # Retain every distinct ambiguous task even when a terminal result supersedes
1431
+ # its source. Existing output ID normalization keeps each row distinct.
1432
+ for relative, _, owner in all_sources:
1433
+ for row in deferred_rows[relative]:
1434
+ if (
1435
+ isinstance(row, dict)
1436
+ and isinstance(identity := row.get("id"), str)
1437
+ and (owner, identity) in ambiguous_deferred
1438
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
1439
+ and (owner, row) not in reopened_rows
1440
+ ):
1441
+ reopened_rows.append((owner, row))
1442
+ parent_closures = [
1443
+ closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
1444
+ ]
1445
+ coverage.pop("resolvedDeferred", None)
1446
+ if parent_closures:
1447
+ coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
1448
+ if parent:
1449
+ coverage["deferred"] = [
1450
+ row
1451
+ for row in deferred_rows["parent"]
1452
+ if not isinstance(row, dict)
1453
+ or not isinstance(row.get("id"), str)
1454
+ or (None, row["id"]) not in closed_deferred
1455
+ ]
732
1456
  resolved: dict[tuple[str | None, str], str] = {}
733
- for owner, draft in current_drafts:
1457
+
1458
+ ordered_candidates = {
1459
+ (owner, identity)
1460
+ for owner, row in reopened_rows
1461
+ if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
1462
+ and (owner, identity) in candidate_ids
1463
+ }
1464
+ ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
1465
+
1466
+ outcomes: list[tuple[str, str | None, str, str]] = []
1467
+ for relative, draft, owner in current_drafts:
734
1468
  for finding in draft["findings"]:
735
1469
  if (
736
1470
  isinstance(finding, dict)
737
1471
  and valid_finding(finding)
738
1472
  and (candidate_id := finding_candidate_id(finding))
739
1473
  ):
740
- resolved.setdefault((owner, candidate_id), "reported")
1474
+ outcomes.append((relative, owner, candidate_id, "reported"))
741
1475
  for field in ("surfaces", "explicitExclusions"):
742
1476
  items = draft["coverage"].get(field, [])
743
1477
  for item in items if isinstance(items, list) else []:
744
1478
  if (
745
1479
  isinstance(item, dict)
746
1480
  and isinstance(item.get("candidateId"), str)
747
- and item.get("disposition") in {"reported", "rejected", "not_applicable"}
1481
+ and item.get("disposition") in {"rejected", "not_applicable"}
748
1482
  ):
749
- resolved.setdefault((owner, item["candidateId"]), item["disposition"])
1483
+ outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
1484
+ ordered_candidates.update(
1485
+ (owner, candidate_id)
1486
+ for relative, owner, candidate_id, _ in outcomes
1487
+ if owner is not None and relative in selected_observations
1488
+ )
1489
+ # Reopened work and selected checkpoint outcomes follow the saved source order.
1490
+ for relative, owner, candidate_id, disposition in outcomes:
1491
+ key = (owner, candidate_id)
1492
+ if key not in ordered_candidates:
1493
+ if relative == "parent" or relative in current_results:
1494
+ resolved.setdefault(key, disposition)
1495
+ continue
1496
+ order = source_order[relative]
1497
+ if any(
1498
+ saved_owner == owner
1499
+ and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
1500
+ and modified >= order
1501
+ for (saved_owner, _), (modified, row, _) in active_deferred.items()
1502
+ ):
1503
+ continue
1504
+ if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
1505
+ resolved[key] = disposition
1506
+ ordered_outcomes[key] = (order, relative)
750
1507
  # Only the current parent may claim that another worker finding was absorbed.
751
1508
  # A superseded checkpoint must not suppress a newer independent result.
752
- for draft in [parent] if parent else []:
753
- for finding in draft["findings"]:
1509
+ if parent:
1510
+ for finding in parent["findings"]:
754
1511
  if valid_finding(finding):
755
1512
  canonical_key = _finding_key(finding)
756
1513
  for retained in _retained_findings(finding):
@@ -787,43 +1544,201 @@ def merge_saved_results(
787
1544
  represented_candidate_history.setdefault(candidate_key, set()).add(
788
1545
  _digest(_finding_content(original["finding"]))
789
1546
  )
790
- resolved.setdefault(candidate_key, "reported")
1547
+ replaced_surfaces, surface_updates = _generic_surface_updates(
1548
+ all_sources,
1549
+ source_order,
1550
+ closed_deferred,
1551
+ active_deferred,
1552
+ resolved,
1553
+ reopened_generic,
1554
+ coverage_schema["surfaces"]["items"],
1555
+ deferred_rows,
1556
+ ambiguous_deferred,
1557
+ )
1558
+ replaced_rows = {
1559
+ "surfaces": replaced_surfaces,
1560
+ "deferred": {
1561
+ id(row)
1562
+ for relative, _, owner in all_sources
1563
+ for row in deferred_rows[relative]
1564
+ if isinstance(row, dict)
1565
+ and isinstance(identity := row.get("id"), str)
1566
+ and (key := (owner, identity)) not in candidate_ids
1567
+ and key not in ambiguous_deferred
1568
+ and (updated := accepted_deferred_orders.get(key)) is not None
1569
+ and updated > source_order[relative]
1570
+ },
1571
+ }
1572
+ for field, replaced in replaced_rows.items():
1573
+ if parent and isinstance(coverage.get(field), list):
1574
+ previous = parent["coverage"].get(field, [])
1575
+ if isinstance(previous, list):
1576
+ removed_parent = [row for row in previous if id(row) in replaced]
1577
+ coverage[field] = [row for row in coverage[field] if row not in removed_parent]
1578
+ if isinstance(coverage.get("surfaces"), list):
1579
+ for surface in surface_updates:
1580
+ if surface not in coverage["surfaces"]:
1581
+ coverage["surfaces"].append(surface)
1582
+ for surface in coverage["surfaces"]:
1583
+ if isinstance(surface, dict):
1584
+ surface.setdefault("receiptRefs", [])
1585
+
1586
+ # Reopened work survives a superseded checkpoint, but current candidate
1587
+ # outcomes still apply. Parent closures cannot remove another worker's row.
1588
+ for owner, item in reopened_rows:
1589
+ if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
1590
+ owner,
1591
+ identity,
1592
+ ) in resolved:
1593
+ continue
1594
+ pending = coverage.setdefault("deferred", [])
1595
+ if isinstance(pending, list) and item not in pending:
1596
+ pending.append(copy.deepcopy(item))
1597
+ ambiguous_surface_ids = {
1598
+ (owner, identity)
1599
+ for owner, row in reopened_rows
1600
+ if isinstance(row.get("id"), str)
1601
+ and (owner, row["id"]) in ambiguous_deferred
1602
+ and not any(key in row for key in ("candidateId", "candidate", "finding"))
1603
+ for identity in [
1604
+ row["id"],
1605
+ *(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
1606
+ ]
1607
+ if isinstance(identity, str)
1608
+ }
1609
+ for _, draft, owner in all_sources:
1610
+ surfaces = draft["coverage"].get("surfaces", [])
1611
+ by_id: dict[str, dict[str, Any]] = {}
1612
+ for surface in surfaces if isinstance(surfaces, list) else []:
1613
+ if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
1614
+ continue
1615
+ if identity in by_id and surface != by_id[identity]:
1616
+ ambiguous_surface_ids.add((owner, identity))
1617
+ by_id[identity] = surface
1618
+ for _, draft, owner in all_sources:
1619
+ surfaces = draft["coverage"].get("surfaces", [])
1620
+ for surface in surfaces if isinstance(surfaces, list) else []:
1621
+ if (
1622
+ isinstance(surface, dict)
1623
+ and isinstance(surface.get("id"), str)
1624
+ and (owner, surface["id"]) in ambiguous_surface_ids
1625
+ and surface.get("disposition") == "needs_follow_up"
1626
+ and not any(key in surface for key in ("candidateId", "candidate", "finding"))
1627
+ and isinstance(coverage.get("surfaces"), list)
1628
+ ):
1629
+ retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
1630
+ if retained_surface not in coverage["surfaces"]:
1631
+ coverage["surfaces"].append(copy.deepcopy(retained_surface))
1632
+ selected_terminal_orders: dict[str, tuple[int, int]] = {}
1633
+ for relative, draft, worker_id in sources:
1634
+ if (
1635
+ worker_id is not None
1636
+ and relative in selected_observations
1637
+ and draft.get("complete") is not False
1638
+ ):
1639
+ order = source_order[relative]
1640
+ selected_terminal_orders[worker_id] = max(
1641
+ selected_terminal_orders.get(worker_id, order), order
1642
+ )
1643
+ terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
1644
+ for relative, draft, worker_id in sources:
1645
+ if relative in current_results and draft.get("complete") is not False:
1646
+ order = source_order[relative]
1647
+ terminal_worker_orders[worker_id] = max(
1648
+ terminal_worker_orders.get(worker_id, order), order
1649
+ )
791
1650
  for relative, draft, worker_id in all_sources:
1651
+ worker_result_order = terminal_worker_orders.get(worker_id)
1652
+ selected_coverage_superseded = worker_id in selected_terminal_orders and (
1653
+ source_order[relative] < selected_terminal_orders[worker_id]
1654
+ or (
1655
+ relative in current_results
1656
+ and source_order[relative] <= selected_terminal_orders[worker_id]
1657
+ )
1658
+ )
1659
+ selected_candidates = {
1660
+ candidate_id
1661
+ for (owner, candidate_id), (_, source) in ordered_outcomes.items()
1662
+ if owner == worker_id and source == relative
1663
+ }
792
1664
  superseded = (
793
1665
  worker_id is None
794
1666
  and parent is not None
795
1667
  and parent.get("complete") is not False
796
1668
  and relative != "parent"
1669
+ and source_order[relative] <= (0, parent_modified)
797
1670
  and (not stopped_parent_seal or relative in parent_preserved_sources)
798
1671
  ) or (
799
1672
  relative not in current_results
800
- and any(
801
- saved_worker == worker_id
802
- and saved_path in current_results
803
- and current.get("complete") is not False
804
- for saved_path, current, saved_worker in sources
805
- )
806
- )
807
- if (
808
- (relative != "parent" or not parent_manifest)
809
- and not superseded
1673
+ and worker_result_order is not None
810
1674
  and (
811
- draft.get("complete") is False
812
- or draft["coverage"].get("completeness") != "complete"
1675
+ relative not in selected_observations
1676
+ or source_order[relative] < worker_result_order
813
1677
  )
814
- and coverage.get("completeness") in {"complete", "unknown"}
815
- ):
816
- coverage["completeness"] = "partial"
817
- if (
1678
+ )
1679
+ # A failed result write can leave pending work outside the accepted result.
1680
+ accepted_order = (
1681
+ (0, parent_modified)
1682
+ if worker_id is None and parent is not None
1683
+ else worker_result_order
1684
+ )
1685
+ retain_pending = (
1686
+ superseded
1687
+ and (worker_id in headed_workers or (worker_id is None and parent_heads))
1688
+ and accepted_order is not None
1689
+ and source_order[relative] >= accepted_order
1690
+ and any(isinstance(row, dict) for row in deferred_rows[relative])
1691
+ )
1692
+ skip_superseded_findings = (
818
1693
  superseded
819
1694
  and not stopped
820
1695
  and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
821
- ):
1696
+ )
1697
+ if skip_superseded_findings and not selected_candidates and not retain_pending:
822
1698
  continue
823
- if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict):
824
- manifest["scan"]["threatModel"] = copy.deepcopy(draft["threatModel"])
1699
+ if (
1700
+ not skip_superseded_findings
1701
+ and "threatModel" not in manifest["scan"]
1702
+ and isinstance(draft.get("threatModel"), dict)
1703
+ ):
1704
+ model = draft["threatModel"]
1705
+ model_path = relative
1706
+ checkpoint_dir = Path(relative).parent
1707
+ prefer_worker_head = worker_id is not None and (
1708
+ checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
1709
+ )
1710
+ if worker_id is not None:
1711
+ if checkpoint_dir.name != "checkpoints":
1712
+ checkpoint_dir /= "checkpoints"
1713
+ selected_models = [
1714
+ path
1715
+ for path in selected_observations
1716
+ if Path(path).parent == checkpoint_dir
1717
+ and isinstance(drafts_by_path[path].get("threatModel"), dict)
1718
+ ]
1719
+ if selected_models:
1720
+ head_path = max(selected_models, key=source_order.__getitem__)
1721
+ current = drafts_by_path[head_path]
1722
+ # A terminal checkpoint is committed before result.json is replaced.
1723
+ # Use the admitted observation, including its frozen ordering on retries.
1724
+ if not prefer_worker_head and current.get("complete") is not False:
1725
+ prefer_worker_head = source_order[head_path] >= source_order[relative]
1726
+ if prefer_worker_head:
1727
+ model = current["threatModel"]
1728
+ model_path = head_path
1729
+ manifest["scan"]["threatModel"] = copy.deepcopy(model)
1730
+ if worker_id is not None:
1731
+ manifest["scan"]["threatModel"]["origin"] = "recovered"
1732
+ if selected_model_source is not None and worker_id is not None:
1733
+ selected_model_source[:] = [model_path]
825
1734
  for value in draft["findings"]:
826
- if relative == "parent" and parent_manifest:
1735
+ if skip_superseded_findings and not (
1736
+ isinstance(value, dict)
1737
+ and (candidate_id := finding_candidate_id(value)) in selected_candidates
1738
+ and resolved.get((worker_id, candidate_id)) == "reported"
1739
+ ):
1740
+ continue
1741
+ if relative == "parent" and parent_is_canonical:
827
1742
  finding = copy.deepcopy(value)
828
1743
  _ensure_finding_identity(finding, candidate_only=True)
829
1744
  provenance = finding.get("provenance") if isinstance(finding, dict) else None
@@ -846,7 +1761,6 @@ def merge_saved_results(
846
1761
  if not isinstance(value, dict):
847
1762
  warnings.append(f"Retained malformed finding evidence in {relative}.")
848
1763
  continue
849
- source_value = copy.deepcopy(value)
850
1764
  finding = copy.deepcopy(value)
851
1765
  candidate_id = finding_candidate_id(finding)
852
1766
  if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
@@ -880,7 +1794,6 @@ def merge_saved_results(
880
1794
  for location in locations
881
1795
  ):
882
1796
  warnings.append(f"Skipped out-of-scope finding from {relative}.")
883
- coverage["completeness"] = "partial"
884
1797
  continue
885
1798
  provenance = finding.setdefault("provenance", {"source": "local_plugin"})
886
1799
  if not isinstance(provenance, dict):
@@ -909,24 +1822,19 @@ def merge_saved_results(
909
1822
  historical_contents = set()
910
1823
  if mapped_key is not None:
911
1824
  key = mapped_key
912
- represented_by_parent = (
913
- _digest(_finding_content(source_value)) in historical_contents
914
- )
1825
+ represented_by_parent = _digest(_finding_content(value)) in historical_contents
915
1826
  if key in finding_positions:
916
1827
  retained = findings[finding_positions[key]]
917
1828
  if finding != retained:
918
- if represented_by_parent:
919
- previous = copy.deepcopy(source_value)
920
- previous_history = previous.get("provenance", {}).pop(
921
- "previousFindings", []
922
- )
923
- elif _finding_strength(finding) > _finding_strength(retained):
1829
+ if not represented_by_parent and _finding_strength(finding) > _finding_strength(
1830
+ retained
1831
+ ):
924
1832
  previous = copy.deepcopy(retained)
925
1833
  previous_history = previous["provenance"].pop("previousFindings", [])
926
1834
  retained = finding
927
1835
  findings[finding_positions[key]] = retained
928
1836
  else:
929
- previous = copy.deepcopy(source_value)
1837
+ previous = copy.deepcopy(value)
930
1838
  previous_history = previous.get("provenance", {}).pop(
931
1839
  "previousFindings", []
932
1840
  )
@@ -960,10 +1868,14 @@ def merge_saved_results(
960
1868
  continue
961
1869
  finding_positions[key] = len(findings)
962
1870
  findings.append(finding)
963
- if superseded:
1871
+ if superseded and not selected_candidates and not retain_pending:
964
1872
  continue
965
1873
  for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
1874
+ if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
1875
+ continue
966
1876
  items = draft["coverage"].get(field, [])
1877
+ if field == "deferred":
1878
+ items = deferred_rows[relative]
967
1879
  if not isinstance(items, list):
968
1880
  continue
969
1881
  output = coverage.setdefault(field, [])
@@ -972,6 +1884,40 @@ def merge_saved_results(
972
1884
  # recovery and warnings rather than silently changing its contract.
973
1885
  continue
974
1886
  for item in items:
1887
+ # A selected terminal checkpoint replaces ordinary coverage, while
1888
+ # candidate evidence and pending tasks retain their own reconciliation.
1889
+ if (
1890
+ selected_coverage_superseded
1891
+ and field != "deferred"
1892
+ and not (
1893
+ isinstance(item, dict)
1894
+ and any(key in item for key in ("candidateId", "candidate", "finding"))
1895
+ )
1896
+ ):
1897
+ continue
1898
+ # A selected outcome must retain its evidence even if its result write failed.
1899
+ if superseded and not (
1900
+ isinstance(item, dict)
1901
+ and (
1902
+ (field == "deferred" and retain_pending)
1903
+ or (
1904
+ isinstance(item.get("candidateId"), str)
1905
+ and item["candidateId"] in selected_candidates
1906
+ and item.get("disposition")
1907
+ == resolved.get((worker_id, item["candidateId"]))
1908
+ )
1909
+ )
1910
+ ):
1911
+ continue
1912
+ if id(item) in replaced_rows.get(field, ()):
1913
+ continue
1914
+ if (
1915
+ field == "deferred"
1916
+ and isinstance(item, dict)
1917
+ and isinstance(item.get("id"), str)
1918
+ and (worker_id, item["id"]) in closed_deferred
1919
+ ):
1920
+ continue
975
1921
  if field == "openQuestions" and isinstance(item, str):
976
1922
  item = {"question": item.strip()}
977
1923
  if (
@@ -995,14 +1941,20 @@ def merge_saved_results(
995
1941
  history.append(copy.deepcopy(finding))
996
1942
  if (
997
1943
  isinstance(item, dict)
998
- and (worker_id, item.get("candidateId")) in resolved
1944
+ and isinstance(
1945
+ identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
1946
+ if field == "deferred"
1947
+ else item.get("candidateId"),
1948
+ str,
1949
+ )
1950
+ and (worker_id, identity) in resolved
999
1951
  and (field == "deferred" or item.get("disposition") == "needs_follow_up")
1000
1952
  ):
1001
1953
  continue
1954
+ if field == "surfaces" and isinstance(item, dict):
1955
+ item = {**item, "receiptRefs": item.get("receiptRefs", [])}
1002
1956
  if isinstance(item, dict) and "id" not in item:
1003
1957
  semantic_item = dict(item)
1004
- if field == "surfaces":
1005
- semantic_item.setdefault("receiptRefs", [])
1006
1958
  if any(
1007
1959
  isinstance(existing, dict)
1008
1960
  and {key: value for key, value in existing.items() if key != "id"}
@@ -1036,14 +1988,16 @@ def merge_saved_results(
1036
1988
  if isinstance(item.get("id"), str):
1037
1989
  used.add(item["id"])
1038
1990
  continue
1039
- item.setdefault("id", item.get("candidateId") or f"saved-{_digest(item)[:16]}")
1991
+ item.setdefault("id", _saved_coverage_id(item))
1992
+ if not isinstance(item["id"], str):
1993
+ # Preserve malformed rows for per-record recovery, including frozen replay.
1994
+ continue
1040
1995
  if item["id"] in used:
1041
1996
  item["id"] = f"{item['id']}-{_digest(item)[:16]}"
1042
1997
  used.add(item["id"])
1043
1998
  if field == "surfaces":
1044
1999
  item.setdefault("receiptRefs", [])
1045
- if stopped or any(warning not in initial_warnings for warning in warnings):
1046
- coverage["completeness"] = "partial"
2000
+ coverage.pop("completeness", None)
1047
2001
  if stopped:
1048
2002
  if not isinstance(coverage.get("deferred"), list):
1049
2003
  coverage["deferred"] = []
@@ -1059,16 +2013,16 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
1059
2013
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
1060
2014
  db.require_recorded_manifest_digest(scan, scan_dir)
1061
2015
  try:
1062
- _, _, manifest, _, coverage, was_sealed, _ = _prepare_scan_finalization(scan_dir)
2016
+ prepared = _prepare_scan_finalization(scan_dir)
1063
2017
  except ContractError as exc:
1064
2018
  raise SystemExit(str(exc)) from exc
1065
- if not was_sealed or manifest["scan"]["id"] != scan["id"]:
2019
+ if not prepared.was_sealed or prepared.manifest["scan"]["id"] != scan["id"]:
1066
2020
  raise SystemExit("Only sealed scans can be compared.")
1067
- return coverage
2021
+ return prepared.coverage
1068
2022
 
1069
2023
 
1070
- def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
1071
- snapshots: dict[str, bytes | None] = {}
2024
+ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]:
2025
+ snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {}
1072
2026
  for relative in _PUBLISHED_OUTPUTS:
1073
2027
  descriptor = -1
1074
2028
  try:
@@ -1077,10 +2031,14 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
1077
2031
  )
1078
2032
  with os.fdopen(descriptor, "rb") as handle:
1079
2033
  descriptor = -1
1080
- snapshots[relative] = handle.read()
2034
+ metadata = os.fstat(handle.fileno())
2035
+ snapshots[relative] = (handle.read(), metadata)
1081
2036
  except ContractError:
1082
2037
  path = scan_dir / relative
1083
2038
  if path.exists() or path.is_symlink():
2039
+ if relative == "threatmodel.md":
2040
+ # This optional projection will not replace an unsafe destination.
2041
+ continue
1084
2042
  raise
1085
2043
  snapshots[relative] = None
1086
2044
  finally:
@@ -1089,14 +2047,80 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
1089
2047
  return snapshots
1090
2048
 
1091
2049
 
1092
- def _restore_published_outputs(scan_dir: Path, snapshots: dict[str, bytes | None]) -> None:
1093
- for relative, contents in snapshots.items():
1094
- if contents is None:
2050
+ def _restore_published_outputs(
2051
+ scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
2052
+ ) -> None:
2053
+ for relative, snapshot in snapshots.items():
2054
+ if snapshot is None:
1095
2055
  path = scan_dir / relative
1096
2056
  if path.exists() or path.is_symlink():
1097
2057
  _remove_scan_local_file_if_exists(scan_dir, relative)
1098
2058
  else:
2059
+ contents, metadata = snapshot
1099
2060
  write_scan_local_bytes(scan_dir, relative, contents)
2061
+ # Saved-result selection uses mtime; rollback must preserve its ordering.
2062
+ os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
2063
+
2064
+
2065
+ @contextmanager
2066
+ def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
2067
+ """Recover a settled, zero-accepted Deep deadline without sealing the parent."""
2068
+ scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2069
+ snapshots = _snapshot_published_outputs(scan_dir)
2070
+ warnings = json.loads(scan["completion_warnings_json"])
2071
+ binding = db.workbench_completion_binding(scan, db.now())
2072
+ with preserve_parent_head_on_error(scan_dir):
2073
+ try:
2074
+ documents = merge_saved_results(
2075
+ scan_dir,
2076
+ scan["id"],
2077
+ binding,
2078
+ connection.execute(
2079
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
2080
+ (scan["id"],),
2081
+ ).fetchall(),
2082
+ warnings,
2083
+ stopped=True,
2084
+ reason=(
2085
+ "The configured discovery time limit elapsed before any source review completed. "
2086
+ "Saved findings and pending review were preserved without a completed reduction."
2087
+ ),
2088
+ )
2089
+ if documents is None:
2090
+ raise ContractError("Deep Scan deadline has no saved parent draft.")
2091
+ documents[0]["scan"].pop("complete", None)
2092
+ # Validate recovered evidence against the current finding contract,
2093
+ # scope, source lineage, and receipts before admitting this draft.
2094
+ prepared = _prepare_scan_finalization(
2095
+ scan_dir,
2096
+ expected_coverage_mode=db.expected_coverage_mode(scan),
2097
+ completion_binding=binding,
2098
+ recover_unsealed=True,
2099
+ draft_documents=documents,
2100
+ )
2101
+ warnings.extend(prepared.warnings)
2102
+ manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
2103
+ manifest["scan"].pop("sealedAt", None)
2104
+ manifest["scan"].pop("artifacts", None)
2105
+ checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
2106
+ contents = _encoded(checkpoint)
2107
+ name = f"{hashlib.sha256(contents).hexdigest()}.json"
2108
+ write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
2109
+ write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
2110
+ for filename, document in (
2111
+ ("findings.json", findings),
2112
+ ("coverage.json", coverage),
2113
+ ("scan-manifest.json", manifest),
2114
+ ):
2115
+ write_scan_local_bytes(scan_dir, filename, _encoded(document))
2116
+ connection.execute(
2117
+ "UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
2118
+ (json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
2119
+ )
2120
+ yield
2121
+ except BaseException:
2122
+ _restore_published_outputs(scan_dir, snapshots)
2123
+ raise
1100
2124
 
1101
2125
 
1102
2126
  def preserve_scan_results_locked(
@@ -1112,13 +2136,17 @@ def preserve_scan_results_locked(
1112
2136
  if scan["status"] != "failed":
1113
2137
  return False
1114
2138
  frozen_source_digests: dict[str, str] | None = None
2139
+ model_source: list[str] = []
2140
+ saved_model_source: str | None = None
1115
2141
  raw_frozen_sources = scan["retained_source_digests_json"]
2142
+ if raw_frozen_sources is not None:
2143
+ frozen_source_digests, saved_model_source = _retained_source_state(
2144
+ json.loads(raw_frozen_sources)
2145
+ )
2146
+ if saved_model_source is not None:
2147
+ model_source.append(saved_model_source)
1116
2148
  if recovery_source_digests is not None:
1117
2149
  frozen_source_digests = recovery_source_digests
1118
- elif raw_frozen_sources is not None:
1119
- frozen_source_digests = _source_digests(
1120
- json.loads(raw_frozen_sources), "Saved stopped-scan"
1121
- )
1122
2150
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
1123
2151
  deep_run = connection.execute(
1124
2152
  "SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
@@ -1141,12 +2169,10 @@ def preserve_scan_results_locked(
1141
2169
  ]
1142
2170
 
1143
2171
  def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> None:
1144
- retained_sources = manifest.get("scan", {}).get("preservedSources")
1145
- if not isinstance(retained_sources, dict) or not all(
1146
- isinstance(relative, str) and isinstance(source_digest, str)
1147
- for relative, source_digest in retained_sources.items()
1148
- ):
1149
- raise ContractError("Stopped scan source digests could not be frozen.")
2172
+ retained_sources = _source_digests(
2173
+ manifest.get("scan", {}).get("preservedSources"),
2174
+ "Stopped scan source digests could not be frozen.",
2175
+ )
1150
2176
  digest = db.published_manifest_digest(scan_dir, manifest)
1151
2177
  timestamp = db.now()
1152
2178
  with connection:
@@ -1176,7 +2202,7 @@ def preserve_scan_results_locked(
1176
2202
  "updated_at = ? WHERE id = ? AND status = 'failed'",
1177
2203
  (
1178
2204
  digest,
1179
- json.dumps(retained_sources, sort_keys=True),
2205
+ _encode_retained_sources(retained_sources, model_source),
1180
2206
  json.dumps(list(dict.fromkeys(warnings))),
1181
2207
  timestamp,
1182
2208
  scan_id,
@@ -1190,6 +2216,7 @@ def preserve_scan_results_locked(
1190
2216
 
1191
2217
  existing_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
1192
2218
  existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
2219
+ existing = None
1193
2220
  if scan["seal_manifest_digest"] is not None or (
1194
2221
  isinstance(existing_scan, dict)
1195
2222
  and (
@@ -1198,30 +2225,35 @@ def preserve_scan_results_locked(
1198
2225
  ):
1199
2226
  db.require_recorded_manifest_digest(scan, scan_dir)
1200
2227
  existing, existing_findings, _ = finalize_scan(
1201
- scan_dir, expected_coverage_mode=db.expected_coverage_mode(scan)
2228
+ scan_dir,
2229
+ expected_coverage_mode=db.expected_coverage_mode(scan),
2230
+ projection_warnings=warnings,
1202
2231
  )
1203
2232
  db.verify_manifest_binding(scan, existing)
1204
2233
  if existing_scan.get("status") == outcome:
1205
2234
  existing_sources = existing_scan.get("preservedSources")
1206
2235
  if frozen_source_digests is None:
1207
- if not isinstance(existing_sources, dict) or not all(
1208
- isinstance(relative, str) and isinstance(digest, str)
1209
- for relative, digest in existing_sources.items()
1210
- ):
1211
- raise ContractError("Stopped scan source digests could not be frozen.")
1212
- frozen_source_digests = existing_sources
2236
+ frozen_source_digests = _source_digests(
2237
+ existing_sources, "Stopped scan source digests could not be frozen."
2238
+ )
1213
2239
  if existing_sources == frozen_source_digests:
1214
2240
  if (
1215
2241
  raw_frozen_sources is not None
1216
2242
  and scan["seal_manifest_digest"] is not None
1217
2243
  and not publication_follow_up_warnings
2244
+ and warnings == stored_warnings
1218
2245
  ):
1219
2246
  return True
1220
2247
  record_publication(existing, existing_findings)
1221
2248
  return True
1222
2249
  if recovery_source_digests is None:
1223
2250
  raise ContractError("Stopped scan sources changed after terminal publication.")
1224
- binding = {**db.workbench_completion_binding(scan, scan["completed_at"]), "status": outcome}
2251
+ binding = {
2252
+ **db.workbench_completion_binding(scan, scan["completed_at"], existing),
2253
+ "status": outcome,
2254
+ }
2255
+ if recovery_source_digests is not None:
2256
+ model_source.clear()
1225
2257
  documents = merge_saved_results(
1226
2258
  scan_dir,
1227
2259
  scan_id,
@@ -1237,6 +2269,8 @@ def preserve_scan_results_locked(
1237
2269
  f"{scan['failure_message'] or ''}"
1238
2270
  ).strip(),
1239
2271
  frozen_source_digests=frozen_source_digests,
2272
+ frozen_model_source=model_source[0] if model_source else None,
2273
+ selected_model_source=model_source,
1240
2274
  allow_frozen_legacy_parent=(
1241
2275
  include_parent_with_recovery
1242
2276
  or (
@@ -1258,29 +2292,37 @@ def preserve_scan_results_locked(
1258
2292
  (json.dumps(unpublished_warnings), db.now(), scan_id),
1259
2293
  )
1260
2294
  return False
1261
- if frozen_source_digests is None:
1262
- retained_sources = documents[0].get("scan", {}).get("preservedSources")
1263
- if not isinstance(retained_sources, dict) or not all(
1264
- isinstance(relative, str) and isinstance(digest, str)
1265
- for relative, digest in retained_sources.items()
1266
- ):
1267
- raise ContractError("Stopped scan source digests could not be frozen.")
2295
+ if frozen_source_digests is None or (
2296
+ recovery_source_digests is None and saved_model_source is None and model_source
2297
+ ):
2298
+ retained_sources = _source_digests(
2299
+ documents[0].get("scan", {}).get("preservedSources"),
2300
+ "Stopped scan source digests could not be frozen.",
2301
+ )
1268
2302
  with connection:
1269
2303
  connection.execute(
1270
2304
  "UPDATE scans SET retained_source_digests_json = ? "
1271
- "WHERE id = ? AND retained_source_digests_json IS NULL",
1272
- (json.dumps(retained_sources, sort_keys=True), scan_id),
2305
+ "WHERE id = ? AND retained_source_digests_json IS ?",
2306
+ (
2307
+ _encode_retained_sources(retained_sources, model_source),
2308
+ scan_id,
2309
+ raw_frozen_sources,
2310
+ ),
1273
2311
  )
1274
2312
  prepared = _prepare_scan_finalization(
1275
2313
  scan_dir,
1276
2314
  expected_coverage_mode=db.expected_coverage_mode(scan),
1277
2315
  completion_binding=binding,
1278
- completion_warnings=warnings,
2316
+ recover_unsealed=True,
1279
2317
  draft_documents=documents,
2318
+ finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
1280
2319
  )
2320
+ warnings.extend(prepared.warnings)
1281
2321
  snapshots = _snapshot_published_outputs(scan_dir)
1282
2322
  try:
1283
- manifest, findings, _ = _write_prepared_scan_finalization(prepared)
2323
+ manifest, findings, _ = _write_prepared_scan_finalization(
2324
+ prepared, projection_warnings=warnings
2325
+ )
1284
2326
  db.verify_manifest_binding(scan, manifest)
1285
2327
  record_publication(manifest, findings)
1286
2328
  except BaseException:
@@ -1387,6 +2429,66 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1387
2429
  return {"scanId": scan_id, "path": str(scan_dir / output)}
1388
2430
 
1389
2431
 
2432
+ def _require_current_deep_publication(
2433
+ db: Any, connection: Any, scan: Any, draft: dict[str, Any]
2434
+ ) -> None:
2435
+ scan_id = scan["id"]
2436
+ run = connection.execute(
2437
+ "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
2438
+ ).fetchone()
2439
+ publication = draft.get("deepScanPublication")
2440
+ if (
2441
+ publication is None
2442
+ and draft["manifest"]["scan"].get("complete") is False
2443
+ and (run is None or run["status"] == "running")
2444
+ ):
2445
+ return
2446
+ if run is None:
2447
+ run = db.deep_scan.require_deep_scan_run(connection, scan_id)
2448
+ # Finished discovery-only runs hand remaining publication back to the parent.
2449
+ if (
2450
+ publication is None
2451
+ and run["status"] != "running"
2452
+ and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
2453
+ ):
2454
+ return
2455
+ db.deep_scan.require_current_coordinator(
2456
+ run,
2457
+ argparse.Namespace(
2458
+ coordinator_generation=publication.get("coordinatorGeneration") if publication else None
2459
+ ),
2460
+ )
2461
+ # Once finish selects the canonical parent, only completion may replay it.
2462
+ if publication is None:
2463
+ if run["status"] != "running" and run["manifest_path"] == str(
2464
+ Path(scan["scan_dir"]) / "scan-manifest.json"
2465
+ ):
2466
+ raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
2467
+ return
2468
+
2469
+ # Match the durable reducer sequence used by coordinator recovery.
2470
+ def reducer_order(worker: Any) -> tuple[int, str]:
2471
+ match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
2472
+ return (int(match[1]) if match else 0, worker["id"])
2473
+
2474
+ reducer = max(
2475
+ (
2476
+ worker
2477
+ for worker in connection.execute(
2478
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
2479
+ )
2480
+ if worker["kind"] == "dedup"
2481
+ and worker["status"] == "succeeded"
2482
+ and worker["result_manifest_path"]
2483
+ ),
2484
+ key=reducer_order,
2485
+ default=None,
2486
+ )
2487
+ selected_result = reducer["result_manifest_path"] if reducer is not None else None
2488
+ if publication["resultPath"] != selected_result:
2489
+ raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
2490
+
2491
+
1390
2492
  def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1391
2493
  scan_id = db.require_uuid(args.scan_id, "scan-id")
1392
2494
  with db.scan_completion_lock(scan_id):
@@ -1399,6 +2501,30 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1399
2501
  "The scan stopped; its saved checkpoint was retained without replacing sealed results."
1400
2502
  )
1401
2503
  scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
2504
+ if (
2505
+ args.expected_draft_digest is not None
2506
+ and args.expected_draft_digest != _scan_draft_digest(scan_dir)
2507
+ ):
2508
+ raise SystemExit(
2509
+ "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
2510
+ )
2511
+ try:
2512
+ relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
2513
+ except ValueError as exc:
2514
+ raise SystemExit(
2515
+ "Scan draft must be inside the registered scan drafts directory."
2516
+ ) from exc
2517
+ if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
2518
+ raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
2519
+ draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
2520
+ manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
2521
+ binding = db.workbench_completion_binding(scan, db.now())
2522
+ # Save scan IDs without sealing the draft.
2523
+ _populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
2524
+ _populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
2525
+ _validate_completion_binding(manifest, findings, coverage, binding)
2526
+ checkpoint = manifest["scan"]
2527
+ checkpoint_contents = None
1402
2528
  if args.checkpoint_path is not None:
1403
2529
  try:
1404
2530
  checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix()
@@ -1415,39 +2541,39 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1415
2541
  )
1416
2542
  if checkpoint.get("scanId") != scan_id:
1417
2543
  raise SystemExit("Staged scan checkpoint belongs to another scan.")
2544
+ checkpoint_only = (
2545
+ scan["mode"] == "deep"
2546
+ and draft.get("deepScanPublication") is None
2547
+ and checkpoint.get("complete") is False
2548
+ and connection.execute(
2549
+ "SELECT 1 FROM deep_scan_runs WHERE scan_id = ? "
2550
+ "AND status = 'running' AND manifest_path = ?",
2551
+ (scan_id, str(scan_dir / "scan-manifest.json")),
2552
+ ).fetchone()
2553
+ is not None
2554
+ )
2555
+ if scan["mode"] == "deep" and not checkpoint_only:
2556
+ _require_current_deep_publication(db, connection, scan, draft)
2557
+ if checkpoint_contents is not None:
1418
2558
  checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
1419
2559
  write_scan_local_bytes(
1420
2560
  scan_dir,
1421
2561
  f"checkpoints/{checkpoint_digest}.json",
1422
2562
  checkpoint_contents,
1423
2563
  )
1424
- if (
1425
- args.expected_draft_digest is not None
1426
- and args.expected_draft_digest != _scan_draft_digest(scan_dir)
1427
- ):
1428
- raise SystemExit(
1429
- "scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
1430
- )
1431
- try:
1432
- relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
1433
- except ValueError as exc:
1434
- raise SystemExit(
1435
- "Scan draft must be inside the registered scan drafts directory."
1436
- ) from exc
1437
- if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
1438
- raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
1439
- draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
1440
- manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
1441
- binding = db.workbench_completion_binding(scan, db.now())
1442
- # Validate on copies: saved canonical documents remain ordinary unsealed drafts.
1443
- copied_manifest = copy.deepcopy(manifest)
1444
- copied_findings = copy.deepcopy(findings)
1445
- copied_coverage = copy.deepcopy(coverage)
1446
- _populate_unsealed_manifest_envelope(copied_manifest, copied_manifest["scan"], binding)
1447
- _populate_unsealed_artifact_envelope(
1448
- copied_manifest, copied_findings, copied_coverage, binding
2564
+ if checkpoint_only:
2565
+ return {"scanId": scan_id, "status": "draft_written"}
2566
+ checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
2567
+ checkpoint_contents = _encoded(checkpoint)
2568
+ checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
2569
+ checkpoint_relative = f"checkpoints/{checkpoint_name}"
2570
+ if not (scan_dir / checkpoint_relative).exists():
2571
+ write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
2572
+ if checkpoint_only:
2573
+ return {"scanId": scan_id, "status": "draft_written"}
2574
+ write_scan_local_bytes(
2575
+ scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
1449
2576
  )
1450
- _validate_completion_binding(copied_manifest, copied_findings, copied_coverage, binding)
1451
2577
  for filename, document in (
1452
2578
  ("findings.json", findings),
1453
2579
  ("coverage.json", coverage),
@@ -1458,9 +2584,25 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1458
2584
  filename,
1459
2585
  (json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
1460
2586
  )
2587
+ if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
2588
+ # Select the accepted final publication before releasing the write lock.
2589
+ with connection:
2590
+ connection.execute(
2591
+ "UPDATE deep_scan_runs SET manifest_path = ? "
2592
+ "WHERE scan_id = ? AND status = 'running'",
2593
+ (str(scan_dir / "scan-manifest.json"), scan_id),
2594
+ )
2595
+ model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
1461
2596
  # Accepted Standard drafts are evidence of review or report assembly,
1462
2597
  # even when the parent omitted its explicit progress call.
1463
- if scan["mode"] == "standard":
2598
+ model_only_checkpoint = (
2599
+ manifest["scan"].get("complete") is False
2600
+ and isinstance(manifest["scan"].get("threatModel"), dict)
2601
+ and not findings.get("findings")
2602
+ and not coverage.get("surfaces")
2603
+ and not coverage.get("deferred")
2604
+ )
2605
+ if scan["mode"] == "standard" and not model_only_checkpoint:
1464
2606
  phase = "discovery" if manifest["scan"].get("complete") is False else "reporting"
1465
2607
  earlier = PHASES[: PHASES.index(phase)]
1466
2608
  placeholders = ",".join("?" for _ in earlier)
@@ -1481,7 +2623,11 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1481
2623
  )
1482
2624
  except sqlite3.Error as exc:
1483
2625
  print(f"Could not save scan progress: {exc}", file=sys.stderr)
1484
- return {"scanId": scan_id, "status": "draft_written"}
2626
+ return {
2627
+ "scanId": scan_id,
2628
+ "status": "draft_written",
2629
+ **({"warnings": [model_warning]} if model_warning else {}),
2630
+ }
1485
2631
 
1486
2632
 
1487
2633
  def _scan_draft_digest(scan_dir: Path) -> str:
@@ -1494,9 +2640,10 @@ def _scan_draft_digest(scan_dir: Path) -> str:
1494
2640
  except FileNotFoundError:
1495
2641
  digest.update(b"missing\0")
1496
2642
  continue
1497
- _, contents = _read_scan_local_json_bytes(scan_dir, filename, filename)
1498
2643
  digest.update(b"present\0")
1499
- digest.update(contents)
2644
+ descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
2645
+ with os.fdopen(descriptor, "rb") as handle:
2646
+ digest.update(handle.read())
1500
2647
  digest.update(b"\0")
1501
2648
  return digest.hexdigest()
1502
2649
 
@@ -1510,7 +2657,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1510
2657
  scan_id = db.require_uuid(args.scan_id, "scan-id")
1511
2658
  cost_json = db.parse_scan_cost(args.cost_json)
1512
2659
  connection.execute("BEGIN IMMEDIATE")
1513
- try:
2660
+ with connection:
1514
2661
  timestamp = db.now()
1515
2662
  scan = db.require_scan(connection, scan_id)
1516
2663
  if scan["status"] == "failed":
@@ -1542,10 +2689,6 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1542
2689
  )
1543
2690
  if progress_updated.rowcount != 1:
1544
2691
  raise SystemExit("Codex Security scan progress not found.")
1545
- connection.commit()
1546
- except BaseException:
1547
- connection.rollback()
1548
- raise
1549
2692
  preserve_stopped_results_after_transition(db, connection, scan["id"])
1550
2693
  return db.scan_context(connection, scan["id"])
1551
2694
 
@@ -1559,7 +2702,7 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1559
2702
  scan_id = db.require_uuid(args.scan_id, "scan-id")
1560
2703
  thread_id = db.optional_text(args.thread_id, maximum=512)
1561
2704
  connection.execute("BEGIN IMMEDIATE")
1562
- try:
2705
+ with connection:
1563
2706
  timestamp = db.now()
1564
2707
  scan = db.require_scan(connection, scan_id)
1565
2708
  workspace = db.require_workspace(connection, scan["workspace_id"])
@@ -1588,10 +2731,6 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
1588
2731
  )
1589
2732
  if progress_updated.rowcount != 1:
1590
2733
  raise SystemExit("Codex Security scan progress not found.")
1591
- connection.commit()
1592
- except BaseException:
1593
- connection.rollback()
1594
- raise
1595
2734
  preserve_stopped_results_after_transition(db, connection, scan["id"])
1596
2735
  return db.workspace_state(connection, scan["workspace_id"])
1597
2736