@openai/codex-security 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. package/README.md +76 -39
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/examples/completed-scan/findings.json +34 -34
  4. package/_bundled_plugin/examples/completed-scan/report.md +22 -31
  5. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +27 -29
  6. package/_bundled_plugin/examples/{completed-scan → legacy-completed-scan}/coverage.json +1 -0
  7. package/_bundled_plugin/examples/legacy-completed-scan/findings.json +57 -0
  8. package/_bundled_plugin/examples/legacy-completed-scan/report.md +98 -0
  9. package/_bundled_plugin/examples/legacy-completed-scan/scan-manifest.json +43 -0
  10. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  11. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  12. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
  13. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  20. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  21. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  22. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  23. package/_bundled_plugin/references/access-advisory.md +11 -0
  24. package/_bundled_plugin/references/core-scan.md +30 -77
  25. package/_bundled_plugin/references/final-report.md +50 -184
  26. package/_bundled_plugin/references/scan-artifacts.md +5 -5
  27. package/_bundled_plugin/references/scan-contract.md +12 -12
  28. package/_bundled_plugin/references/scan-prologue.md +1 -1
  29. package/_bundled_plugin/references/scan-runtime.md +1 -5
  30. package/_bundled_plugin/references/threat-model.md +33 -41
  31. package/_bundled_plugin/schemas/coverage.schema.json +12 -1
  32. package/_bundled_plugin/schemas/scan-manifest.schema.json +37 -11
  33. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
  34. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +5 -5
  35. package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
  36. package/_bundled_plugin/scripts/deep_scan_workbench.py +146 -86
  37. package/_bundled_plugin/scripts/finalize_scan_contract.py +249 -76
  38. package/_bundled_plugin/scripts/report_projection.py +49 -17
  39. package/_bundled_plugin/scripts/review_state.py +68 -0
  40. package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
  41. package/_bundled_plugin/scripts/validate_scan_contract.py +1 -2
  42. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
  43. package/_bundled_plugin/scripts/workbench/retained_coverage.py +3230 -0
  44. package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
  45. package/_bundled_plugin/scripts/workbench/saved_coverage.py +651 -8
  46. package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
  47. package/_bundled_plugin/scripts/workbench/saved_sources.py +730 -0
  48. package/_bundled_plugin/scripts/workbench_cli.py +3 -0
  49. package/_bundled_plugin/scripts/workbench_constants.py +0 -1
  50. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
  51. package/_bundled_plugin/scripts/workbench_db.py +55 -171
  52. package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
  53. package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
  54. package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
  55. package/_bundled_plugin/scripts/workbench_saved_results.py +808 -2389
  56. package/_bundled_plugin/scripts/workbench_scan_history.py +77 -7
  57. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  58. package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
  59. package/_bundled_plugin/scripts/workbench_target.py +142 -17
  60. package/_bundled_plugin/scripts/workbench_validation.py +9 -2
  61. package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
  62. package/_bundled_plugin/shared/workbench-migrations.json +28 -0
  63. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
  64. package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
  65. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
  66. package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
  67. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
  68. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
  69. package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
  70. package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
  71. package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
  72. package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
  73. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
  74. package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -24
  75. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
  76. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
  77. package/_bundled_plugin/skills/security-scan/SKILL.md +29 -17
  78. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
  79. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
  80. package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
  81. package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
  82. package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
  83. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
  84. package/_bundled_plugin/skills/validation/SKILL.md +42 -104
  85. package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
  86. package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
  87. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
  88. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
  89. package/dist/api.d.ts +5 -3
  90. package/dist/api.d.ts.map +1 -1
  91. package/dist/api.js +359 -53
  92. package/dist/api.js.map +1 -1
  93. package/dist/auth.d.ts.map +1 -1
  94. package/dist/auth.js +4 -1
  95. package/dist/auth.js.map +1 -1
  96. package/dist/cli.d.ts +2 -2
  97. package/dist/cli.d.ts.map +1 -1
  98. package/dist/cli.js +696 -430
  99. package/dist/cli.js.map +1 -1
  100. package/dist/cloud-publish.d.ts.map +1 -1
  101. package/dist/cloud-publish.js +2 -23
  102. package/dist/cloud-publish.js.map +1 -1
  103. package/dist/component-scan.d.ts +8 -2
  104. package/dist/component-scan.d.ts.map +1 -1
  105. package/dist/component-scan.js +2 -0
  106. package/dist/component-scan.js.map +1 -1
  107. package/dist/contract.d.ts +5 -2
  108. package/dist/contract.d.ts.map +1 -1
  109. package/dist/contract.js +93 -45
  110. package/dist/contract.js.map +1 -1
  111. package/dist/cost.d.ts +18 -0
  112. package/dist/cost.d.ts.map +1 -1
  113. package/dist/cost.js +107 -12
  114. package/dist/cost.js.map +1 -1
  115. package/dist/coverage-presentation.d.ts +6 -0
  116. package/dist/coverage-presentation.d.ts.map +1 -0
  117. package/dist/coverage-presentation.js +34 -0
  118. package/dist/coverage-presentation.js.map +1 -0
  119. package/dist/custom-publish.js +2 -2
  120. package/dist/custom-publish.js.map +1 -1
  121. package/dist/custom-validation-prompt.js +5 -5
  122. package/dist/custom-validation-prompt.js.map +1 -1
  123. package/dist/custom-validation.d.ts.map +1 -1
  124. package/dist/custom-validation.js +47 -19
  125. package/dist/custom-validation.js.map +1 -1
  126. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  127. package/dist/deduplication/checkpointed-review.js +3 -1
  128. package/dist/deduplication/checkpointed-review.js.map +1 -1
  129. package/dist/deduplication/codex-review.d.ts.map +1 -1
  130. package/dist/deduplication/codex-review.js +2 -2
  131. package/dist/deduplication/codex-review.js.map +1 -1
  132. package/dist/deduplication/deduplication-prompts.d.ts +1 -0
  133. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  134. package/dist/deduplication/deduplication-prompts.js +3 -0
  135. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  136. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  137. package/dist/deduplication/records-protocol.js +21 -1
  138. package/dist/deduplication/records-protocol.js.map +1 -1
  139. package/dist/deduplication/scan.d.ts.map +1 -1
  140. package/dist/deduplication/scan.js +9 -7
  141. package/dist/deduplication/scan.js.map +1 -1
  142. package/dist/endor-findings.d.ts +4 -3
  143. package/dist/endor-findings.d.ts.map +1 -1
  144. package/dist/endor-findings.js +16 -11
  145. package/dist/endor-findings.js.map +1 -1
  146. package/dist/external-findings-import.d.ts.map +1 -1
  147. package/dist/external-findings-import.js +13 -5
  148. package/dist/external-findings-import.js.map +1 -1
  149. package/dist/external-findings-local.d.ts +1 -0
  150. package/dist/external-findings-local.d.ts.map +1 -1
  151. package/dist/external-findings-local.js +14 -3
  152. package/dist/external-findings-local.js.map +1 -1
  153. package/dist/external-findings-publish.d.ts +2 -1
  154. package/dist/external-findings-publish.d.ts.map +1 -1
  155. package/dist/external-findings-publish.js +64 -28
  156. package/dist/external-findings-publish.js.map +1 -1
  157. package/dist/external-findings-read.d.ts +12 -1
  158. package/dist/external-findings-read.d.ts.map +1 -1
  159. package/dist/external-findings-read.js +25 -6
  160. package/dist/external-findings-read.js.map +1 -1
  161. package/dist/external-findings.d.ts +2 -0
  162. package/dist/external-findings.d.ts.map +1 -1
  163. package/dist/external-findings.js.map +1 -1
  164. package/dist/external-import-contract.d.ts +7 -1
  165. package/dist/external-import-contract.d.ts.map +1 -1
  166. package/dist/external-import-contract.js +63 -31
  167. package/dist/external-import-contract.js.map +1 -1
  168. package/dist/external-imports-feature.d.ts +6 -0
  169. package/dist/external-imports-feature.d.ts.map +1 -0
  170. package/dist/external-imports-feature.js +51 -0
  171. package/dist/external-imports-feature.js.map +1 -0
  172. package/dist/finding-workflow.d.ts +24 -5
  173. package/dist/finding-workflow.d.ts.map +1 -1
  174. package/dist/finding-workflow.js +24 -9
  175. package/dist/finding-workflow.js.map +1 -1
  176. package/dist/github-advisories.d.ts.map +1 -1
  177. package/dist/github-advisories.js +3 -1
  178. package/dist/github-advisories.js.map +1 -1
  179. package/dist/github-findings.d.ts +4 -3
  180. package/dist/github-findings.d.ts.map +1 -1
  181. package/dist/github-findings.js +16 -14
  182. package/dist/github-findings.js.map +1 -1
  183. package/dist/host-protocol.d.ts +1 -1
  184. package/dist/host-protocol.d.ts.map +1 -1
  185. package/dist/host-protocol.js +2 -2
  186. package/dist/host-protocol.js.map +1 -1
  187. package/dist/hosted-remediation.d.ts.map +1 -1
  188. package/dist/hosted-remediation.js +19 -2
  189. package/dist/hosted-remediation.js.map +1 -1
  190. package/dist/hosted-scan.d.ts +1 -1
  191. package/dist/hosted-scan.d.ts.map +1 -1
  192. package/dist/hosted-scan.js +0 -6
  193. package/dist/hosted-scan.js.map +1 -1
  194. package/dist/import-scan.js +17 -17
  195. package/dist/import-scan.js.map +1 -1
  196. package/dist/index.d.ts +4 -3
  197. package/dist/index.d.ts.map +1 -1
  198. package/dist/index.js +2 -1
  199. package/dist/index.js.map +1 -1
  200. package/dist/knowledge-base.d.ts +1 -0
  201. package/dist/knowledge-base.d.ts.map +1 -1
  202. package/dist/knowledge-base.js +20 -8
  203. package/dist/knowledge-base.js.map +1 -1
  204. package/dist/mock-scan.d.ts +1 -2
  205. package/dist/mock-scan.d.ts.map +1 -1
  206. package/dist/mock-scan.js +12 -17
  207. package/dist/mock-scan.js.map +1 -1
  208. package/dist/models.d.ts +10 -1
  209. package/dist/models.d.ts.map +1 -1
  210. package/dist/models.js.map +1 -1
  211. package/dist/multiscan.d.ts.map +1 -1
  212. package/dist/multiscan.js +1 -6
  213. package/dist/multiscan.js.map +1 -1
  214. package/dist/native-json-findings.d.ts +30 -0
  215. package/dist/native-json-findings.d.ts.map +1 -0
  216. package/dist/native-json-findings.js +114 -0
  217. package/dist/native-json-findings.js.map +1 -0
  218. package/dist/opengrep-findings.d.ts +10 -0
  219. package/dist/opengrep-findings.d.ts.map +1 -0
  220. package/dist/opengrep-findings.js +198 -0
  221. package/dist/opengrep-findings.js.map +1 -0
  222. package/dist/owner-evidence.d.ts +2 -0
  223. package/dist/owner-evidence.d.ts.map +1 -1
  224. package/dist/owner-evidence.js +2 -1
  225. package/dist/owner-evidence.js.map +1 -1
  226. package/dist/patch-tui.d.ts.map +1 -1
  227. package/dist/patch-tui.js +2 -0
  228. package/dist/patch-tui.js.map +1 -1
  229. package/dist/publication.js +1 -2
  230. package/dist/publication.js.map +1 -1
  231. package/dist/publish.d.ts +2 -0
  232. package/dist/publish.d.ts.map +1 -1
  233. package/dist/publish.js +1 -1
  234. package/dist/publish.js.map +1 -1
  235. package/dist/record.d.ts +1 -0
  236. package/dist/record.d.ts.map +1 -1
  237. package/dist/record.js +10 -0
  238. package/dist/record.js.map +1 -1
  239. package/dist/remediation.d.ts +1 -25
  240. package/dist/remediation.d.ts.map +1 -1
  241. package/dist/remediation.js +3 -24
  242. package/dist/remediation.js.map +1 -1
  243. package/dist/result.d.ts +5 -3
  244. package/dist/result.d.ts.map +1 -1
  245. package/dist/result.js +9 -6
  246. package/dist/result.js.map +1 -1
  247. package/dist/runtime.d.ts.map +1 -1
  248. package/dist/runtime.js +7 -7
  249. package/dist/runtime.js.map +1 -1
  250. package/dist/scan-dashboard.d.ts +5 -3
  251. package/dist/scan-dashboard.d.ts.map +1 -1
  252. package/dist/scan-dashboard.js +79 -26
  253. package/dist/scan-dashboard.js.map +1 -1
  254. package/dist/scan-history-renderer.d.ts.map +1 -1
  255. package/dist/scan-history-renderer.js +9 -3
  256. package/dist/scan-history-renderer.js.map +1 -1
  257. package/dist/scan-owner-suggestions.d.ts +1 -1
  258. package/dist/scan-owner-suggestions.d.ts.map +1 -1
  259. package/dist/scan-owner-suggestions.js +4 -4
  260. package/dist/scan-owner-suggestions.js.map +1 -1
  261. package/dist/scan-usage-display.d.ts +8 -0
  262. package/dist/scan-usage-display.d.ts.map +1 -0
  263. package/dist/scan-usage-display.js +46 -0
  264. package/dist/scan-usage-display.js.map +1 -0
  265. package/dist/scanner-import.d.ts +1 -1
  266. package/dist/scanner-import.d.ts.map +1 -1
  267. package/dist/scanner-import.js +7 -3
  268. package/dist/scanner-import.js.map +1 -1
  269. package/dist/security-policy.d.ts.map +1 -1
  270. package/dist/security-policy.js +17 -22
  271. package/dist/security-policy.js.map +1 -1
  272. package/dist/semgrep-findings.d.ts +4 -3
  273. package/dist/semgrep-findings.d.ts.map +1 -1
  274. package/dist/semgrep-findings.js +22 -89
  275. package/dist/semgrep-findings.js.map +1 -1
  276. package/dist/skill-authentication.d.ts +18 -0
  277. package/dist/skill-authentication.d.ts.map +1 -0
  278. package/dist/skill-authentication.js +82 -0
  279. package/dist/skill-authentication.js.map +1 -0
  280. package/dist/snyk-findings.d.ts +5 -4
  281. package/dist/snyk-findings.d.ts.map +1 -1
  282. package/dist/snyk-findings.js +18 -17
  283. package/dist/snyk-findings.js.map +1 -1
  284. package/dist/suggest-owners.d.ts +4 -1
  285. package/dist/suggest-owners.d.ts.map +1 -1
  286. package/dist/suggest-owners.js +53 -21
  287. package/dist/suggest-owners.js.map +1 -1
  288. package/dist/targets.d.ts.map +1 -1
  289. package/dist/targets.js +4 -1
  290. package/dist/targets.js.map +1 -1
  291. package/dist/wiz-csv.d.ts +2 -1
  292. package/dist/wiz-csv.d.ts.map +1 -1
  293. package/dist/wiz-csv.js +3 -3
  294. package/dist/wiz-csv.js.map +1 -1
  295. package/dist/wiz-findings.d.ts +2 -1
  296. package/dist/wiz-findings.d.ts.map +1 -1
  297. package/dist/wiz-findings.js +27 -23
  298. package/dist/wiz-findings.js.map +1 -1
  299. package/docs/classify-severity-records.md +3 -4
  300. package/docs/cli.md +171 -41
  301. package/docs/dedupe-records.md +15 -1
  302. package/docs/findings-service.md +119 -118
  303. package/docs/hosted-remediation.md +19 -17
  304. package/docs/hosted-scans.md +37 -11
  305. package/package.json +9 -9
  306. package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
  307. package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
  308. package/_bundled_plugin/scripts/rank_preview.py +0 -129
@@ -0,0 +1,730 @@
1
+ """Read and freeze saved scan checkpoint observations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import re
10
+ import stat
11
+ from collections.abc import Iterator
12
+ from functools import partial
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from finalize_scan_contract import (
17
+ ContractError,
18
+ _read_scan_local_json,
19
+ _read_scan_local_json_with_metadata,
20
+ manifest_is_sealed,
21
+ write_scan_local_bytes,
22
+ )
23
+ from review_state import review_state, uses_legacy_review
24
+ from workbench_coverage_receipts import capture_receipt_checkpoints
25
+
26
+
27
+ class _CheckpointSnapshotError(ContractError):
28
+ """A captured source observation could not be persisted for publication replay."""
29
+
30
+ def __init__(self, message: str, captured: dict[str, tuple[str, int]]) -> None:
31
+ super().__init__(message)
32
+ self.captured = captured
33
+ self.records: dict[str, tuple[dict[str, Any], str, int]] = {}
34
+ self.heads: dict[str, str | None] = {}
35
+ self.model_source: list[str] = []
36
+ self.parent_context: dict[str, str] = {}
37
+ self.include_parent = False
38
+ self.documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None
39
+
40
+
41
+ def _encoded(value: Any) -> bytes:
42
+ return json.dumps(
43
+ value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
44
+ ).encode()
45
+
46
+
47
+ def _digest(value: Any) -> str:
48
+ return hashlib.sha256(_encoded(value)).hexdigest()
49
+
50
+
51
+ def _source_digests(value: Any, error: str) -> dict[str, str]:
52
+ if not isinstance(value, dict) or not all(
53
+ isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
54
+ ):
55
+ raise ContractError(error)
56
+ return value
57
+
58
+
59
+ def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
60
+ if isinstance(value, dict) and isinstance(value.get("sources"), dict):
61
+ sources = _source_digests(
62
+ value["sources"], "Saved stopped-scan source digests are malformed."
63
+ )
64
+ model_source = value.get("threatModelSource")
65
+ if "threatModelSource" in value and (
66
+ not isinstance(model_source, str) or model_source not in sources
67
+ ):
68
+ raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
69
+ return sources, model_source
70
+ return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
71
+
72
+
73
+ def _retained_checkpoint_state(
74
+ scan: Any,
75
+ ) -> tuple[dict[str, str | None] | None, dict[str, Any] | None]:
76
+ raw = scan["retained_checkpoint_heads_json"]
77
+ if raw is None:
78
+ return None, None
79
+ saved = json.loads(raw)
80
+ if not isinstance(saved.get("heads"), dict):
81
+ return saved, None
82
+ if saved["recoveryBaseDigest"] != scan["seal_manifest_digest"]:
83
+ raise ContractError("Stopped scan recovery publication changed after selection.")
84
+ return saved["heads"], saved
85
+
86
+
87
+ def _encode_retained_sources(
88
+ sources: dict[str, str],
89
+ model_source: list[str],
90
+ source_times: dict[str, int] | None = None,
91
+ parent_context: dict[str, str] | None = None,
92
+ ) -> str:
93
+ state: dict[str, Any] = sources
94
+ if model_source or source_times is not None:
95
+ state = {"sources": sources}
96
+ if model_source:
97
+ state["threatModelSource"] = model_source[0]
98
+ if source_times is not None:
99
+ state["sourceTimes"] = source_times
100
+ if parent_context:
101
+ state["parentContext"] = parent_context
102
+ return json.dumps(state, sort_keys=True)
103
+
104
+
105
+ def _children(scan_dir: Path, relative: str) -> list[str]:
106
+ cursor = scan_dir
107
+ for part in Path(relative).parts:
108
+ if part in {"..", "."}:
109
+ return []
110
+ cursor = cursor / part
111
+ try:
112
+ if not stat.S_ISDIR(cursor.lstat().st_mode):
113
+ return []
114
+ except FileNotFoundError:
115
+ return []
116
+ return sorted(child.name for child in cursor.iterdir())
117
+
118
+
119
+ def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
120
+ return [
121
+ f"{directory}/{name}"
122
+ for name in _children(scan_dir, directory)
123
+ if re.fullmatch(r"[0-9a-f]{64}\.json", name)
124
+ ]
125
+
126
+
127
+ def _checkpoint_head_directory(relative: str) -> Path | None:
128
+ path = Path(relative)
129
+ if path.name == "checkpoint-head.json":
130
+ return path.parent
131
+ if path.parent.name == "checkpoint-heads":
132
+ return path.parent.parent
133
+ return None
134
+
135
+
136
+ def _unselected_root_order(
137
+ saved_heads: dict[str, str],
138
+ checkpoint_heads: dict[str, str | None],
139
+ source_order: dict[str, tuple[int, int]],
140
+ ) -> tuple[int, int]:
141
+ # One payload can have several accepted observations; retain their order.
142
+ return max(
143
+ (
144
+ source_order[head]
145
+ for head, checkpoint in saved_heads.items()
146
+ if _checkpoint_head_directory(head) == Path(".")
147
+ and checkpoint_heads.get(".") not in (None, head, f"checkpoints/{checkpoint}")
148
+ ),
149
+ default=(-1, 0),
150
+ )
151
+
152
+
153
+ def _capture_saved_source(
154
+ scan_dir: Path,
155
+ relative: str,
156
+ scan_id: str,
157
+ *,
158
+ kind: str | None = None,
159
+ snapshot_head: bool = True,
160
+ write: bool = True,
161
+ created_paths: set[str] | None = None,
162
+ checkpoint_heads: dict[str, str | None] | None = None,
163
+ expected_digest: str | None = None,
164
+ ) -> dict[str, tuple[str, int]]:
165
+ if not snapshot_head or Path(relative).name != "checkpoint-head.json":
166
+ _, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
167
+ return capture_receipt_checkpoints(
168
+ relative,
169
+ kind,
170
+ {relative: (digest, observed)},
171
+ partial(_checkpoint_paths, scan_dir),
172
+ partial(_read_saved_result, scan_dir, scan_id=scan_id),
173
+ )
174
+ head, head_digest, observed = _read_saved_result(scan_dir, relative, scan_id)
175
+ if expected_digest is not None and head_digest != expected_digest:
176
+ raise ContractError("checkpoint changed after the scan stopped")
177
+ observation = {
178
+ **{key: head[key] for key in ("checkpoint", "inputCheckpoint") if key in head},
179
+ "observedAtNs": str(observed),
180
+ }
181
+ directory = Path(relative).parent
182
+ selected_records = {}
183
+ for key in ("checkpoint", "inputCheckpoint"):
184
+ if key in head:
185
+ selected = (directory / "checkpoints" / head[key]).as_posix()
186
+ selected_records[selected] = _read_saved_result(scan_dir, selected, scan_id)
187
+ selected_sources = {
188
+ path: (digest, seen) for path, (_, digest, seen) in selected_records.items()
189
+ }
190
+ digest = _digest(observation)
191
+ snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
192
+ # Capture the selected file even if the worker created it after directory enumeration.
193
+ if checkpoint_heads is not None:
194
+ checkpoint_heads[directory.as_posix()] = snapshot
195
+ if write and not (scan_dir / snapshot).exists():
196
+ if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
197
+ created_paths.add(snapshot)
198
+ try:
199
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
200
+ except (ContractError, OSError) as exc:
201
+ if checkpoint_heads is not None:
202
+ checkpoint_heads[directory.as_posix()] = relative
203
+ failure = _CheckpointSnapshotError(
204
+ str(exc), {relative: (head_digest, observed), **selected_sources}
205
+ )
206
+ failure.records = {relative: (head, head_digest, observed), **selected_records}
207
+ raise failure from exc
208
+ return {
209
+ snapshot: (digest, observed),
210
+ **selected_sources,
211
+ }
212
+
213
+
214
+ def _read_saved_result(
215
+ scan_dir: Path,
216
+ relative: str,
217
+ scan_id: str,
218
+ *,
219
+ kind: str | None = None,
220
+ normalize_review: bool = True,
221
+ ) -> tuple[dict[str, Any], str, int]:
222
+ draft, _, metadata = _read_scan_local_json_with_metadata(
223
+ scan_dir, relative, "Saved scan checkpoint"
224
+ )
225
+ directory = _checkpoint_head_directory(relative)
226
+ if directory is not None:
227
+ checkpoint = draft.get("checkpoint")
228
+ if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
229
+ raise ContractError("checkpoint head does not name a saved checkpoint")
230
+ _read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
231
+ if "inputCheckpoint" in draft:
232
+ raw = draft["inputCheckpoint"]
233
+ if not isinstance(raw, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", raw):
234
+ raise ContractError("checkpoint head does not name a saved input checkpoint")
235
+ _read_saved_result(scan_dir, (directory / "checkpoints" / raw).as_posix(), scan_id)
236
+ if Path(relative).name == "checkpoint-head.json":
237
+ return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
238
+ observed = draft.get("observedAtNs")
239
+ if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
240
+ raise ContractError("checkpoint head has no observation time")
241
+ return draft, _digest(draft), int(observed)
242
+ if draft.get("scanId") != scan_id:
243
+ raise ContractError("checkpoint belongs to a different scan")
244
+ if not _is_source_order_snapshot(relative) and (
245
+ not isinstance(draft.get("findings"), list)
246
+ or not isinstance(
247
+ draft.get("reviewState", draft.get("coverage", {} if kind == "dedup" else None)), dict
248
+ )
249
+ ):
250
+ raise ContractError("checkpoint has no semantic findings or coverage")
251
+ digest = _digest(draft)
252
+ if normalize_review and "coverage" in draft and "reviewState" not in draft:
253
+ draft["reviewState"] = draft.pop("coverage")
254
+ return draft, digest, metadata.st_mtime_ns
255
+
256
+
257
+ def _is_source_order_snapshot(relative: str) -> bool:
258
+ path = Path(relative)
259
+ return path.parent == Path("source-order") and bool(
260
+ re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
261
+ )
262
+
263
+
264
+ def _frozen_source_times(
265
+ scan_dir: Path,
266
+ scan_id: str,
267
+ sources: dict[str, str],
268
+ parent_context: dict[str, str] | None = None,
269
+ ) -> dict[str, int]:
270
+ times: dict[str, int] = {}
271
+ snapshots = [path for path in sources if _is_source_order_snapshot(path)]
272
+ for path in snapshots:
273
+ record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
274
+ if digest != sources[path] or not isinstance(record.get("sources"), dict):
275
+ raise ContractError("saved source ordering changed after the scan stopped")
276
+ context = record.get("parentContext")
277
+ if parent_context is not None and context is not None:
278
+ if not isinstance(context, dict) or any(
279
+ (
280
+ "canonical" not in context
281
+ or not isinstance(relative, str)
282
+ or not re.fullmatch(r"-?[0-9]+", relative)
283
+ )
284
+ if role == "canonicalObservedAtNs"
285
+ else (
286
+ role not in ("canonical", "head")
287
+ or not isinstance(relative, str)
288
+ or relative not in sources
289
+ or (role == "canonical" and Path(relative).parent != Path("checkpoints"))
290
+ or (role == "head" and _checkpoint_head_directory(relative) != Path("."))
291
+ )
292
+ for role, relative in context.items()
293
+ ):
294
+ raise ContractError("saved parent context is outside the frozen sources")
295
+ if any(
296
+ role in parent_context and parent_context[role] != relative
297
+ for role, relative in context.items()
298
+ ):
299
+ raise ContractError("saved parent context has conflicting observations")
300
+ parent_context.update(context)
301
+ for relative, observation in record["sources"].items():
302
+ if (
303
+ not isinstance(observation, dict)
304
+ or relative not in sources
305
+ or observation.get("digest") != sources[relative]
306
+ or not isinstance(observation.get("observedAtNs"), str)
307
+ or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
308
+ ):
309
+ raise ContractError("saved source ordering does not match its frozen sources")
310
+ observed = int(observation["observedAtNs"])
311
+ if relative in times and times[relative] != observed:
312
+ raise ContractError("saved source ordering has conflicting observations")
313
+ times[relative] = observed
314
+ if snapshots and sources.keys() - set(snapshots) - times.keys():
315
+ raise ContractError("saved source ordering is incomplete")
316
+ return times
317
+
318
+
319
+ def _frozen_checkpoint_heads(
320
+ scan_dir: Path, scan_id: str, sources: dict[str, str], *, skip_unreadable: bool = False
321
+ ) -> dict[str, str]:
322
+ parent_context: dict[str, str] = {}
323
+ try:
324
+ times = _frozen_source_times(scan_dir, scan_id, sources, parent_context)
325
+ except (ContractError, OSError, ValueError):
326
+ if not skip_unreadable:
327
+ raise
328
+ times = {}
329
+ parent_context.clear()
330
+ # A frozen parent can identify its accepted head even when observation times tie.
331
+ accepted_root = None
332
+ heads: dict[str, tuple[int, str]] = {}
333
+ ambiguous: set[str] = set()
334
+ for relative, expected_digest in sources.items():
335
+ directory = _checkpoint_head_directory(relative)
336
+ if directory is None:
337
+ continue
338
+ try:
339
+ head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
340
+ if digest != expected_digest:
341
+ raise ContractError("checkpoint changed after the scan stopped")
342
+ except (ContractError, OSError, ValueError):
343
+ if skip_unreadable:
344
+ continue
345
+ raise
346
+ selected = (directory / "checkpoints" / head["checkpoint"]).as_posix()
347
+ observation = (times.get(relative, observed), selected)
348
+ key = directory.as_posix()
349
+ if relative == parent_context.get("head"):
350
+ accepted_root = relative
351
+ if key not in heads or observation[0] > heads[key][0]:
352
+ heads[key] = observation
353
+ ambiguous.discard(key)
354
+ elif observation[0] == heads[key][0] and selected != heads[key][1]:
355
+ ambiguous.add(key)
356
+ # A legacy tie retains both observations until recovery selects a head.
357
+ frozen = {
358
+ directory: selected
359
+ for directory, (_, selected) in heads.items()
360
+ if directory not in ambiguous
361
+ }
362
+ if accepted_root is not None:
363
+ frozen["."] = accepted_root
364
+ return frozen
365
+
366
+
367
+ def _legacy_checkpoint_observed_at(
368
+ scan_dir: Path, scan_id: str, sources: dict[str, str], selected: str
369
+ ) -> int | None:
370
+ times = _frozen_source_times(scan_dir, scan_id, sources)
371
+ checkpoint = Path(selected)
372
+ observations = []
373
+ for relative, expected_digest in sources.items():
374
+ if _checkpoint_head_directory(relative) != checkpoint.parent.parent:
375
+ continue
376
+ head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
377
+ if digest != expected_digest:
378
+ raise ContractError("checkpoint changed after the scan stopped")
379
+ if head["checkpoint"] == checkpoint.name:
380
+ observations.append(times.get(relative, observed))
381
+ # Payload-only selections used the newest retained observation for that payload.
382
+ return max(observations, default=None)
383
+
384
+
385
+ def _restore_checkpoint_head(
386
+ heads: dict[str, str | None],
387
+ previous_heads: dict[str, str | None],
388
+ sources: dict[str, str],
389
+ relative: str,
390
+ ) -> None:
391
+ path = Path(relative)
392
+ if path.name != "checkpoint-head.json":
393
+ return
394
+ directory = path.parent.as_posix()
395
+ if directory in previous_heads:
396
+ heads[directory] = previous_heads[directory]
397
+ return
398
+ directories = {path.parent, path.parent / "checkpoints", path.parent / "checkpoint-heads"}
399
+ if any(Path(source).parent in directories for source in sources):
400
+ # Legacy evidence can remain accepted without one selected head.
401
+ heads.pop(directory, None)
402
+
403
+
404
+ def _freeze_source_times(
405
+ scan_dir: Path,
406
+ scan_id: str,
407
+ sources: dict[str, str],
408
+ times: dict[str, int],
409
+ parent_context: dict[str, str] | None = None,
410
+ *,
411
+ created_paths: set[str] | None = None,
412
+ ) -> None:
413
+ # Identical result rewrites must not change the order of frozen review evidence.
414
+ observations = {
415
+ path: {"digest": digest, "observedAtNs": str(times[path])}
416
+ for path, digest in sources.items()
417
+ if not _is_source_order_snapshot(path)
418
+ }
419
+ if not observations:
420
+ return
421
+ record = {"scanId": scan_id, "sources": observations}
422
+ if parent_context:
423
+ record["parentContext"] = parent_context
424
+ digest = _digest(record)
425
+ path = f"source-order/{digest}.json"
426
+ if not (scan_dir / path).exists():
427
+ if created_paths is not None and not os.path.lexists(scan_dir / path):
428
+ created_paths.add(path)
429
+ write_scan_local_bytes(scan_dir, path, _encoded(record))
430
+ if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
431
+ raise ContractError("saved source ordering does not match its digest")
432
+ if parent_context:
433
+ for previous in [relative for relative in sources if _is_source_order_snapshot(relative)]:
434
+ previous_record, _, _ = _read_saved_result(scan_dir, previous, scan_id)
435
+ if previous_record.get("parentContext"):
436
+ del sources[previous]
437
+ sources[path] = digest
438
+
439
+
440
+ def _parent_publication_ancestors(
441
+ input_checkpoint: str | None, drafts: dict[str, dict[str, Any]]
442
+ ) -> set[str]:
443
+ """Find stale canonical projections without relying on publication clocks."""
444
+ ancestors = set()
445
+ inputs = set()
446
+ # Input hashes distinguish repeated publication of the same projected result.
447
+ while isinstance(input_checkpoint, str) and input_checkpoint not in inputs:
448
+ inputs.add(input_checkpoint)
449
+ publication = drafts.get(f"checkpoints/{input_checkpoint}", {}).get("publication")
450
+ if not isinstance(publication, dict):
451
+ break
452
+ previous = publication.get("previousCheckpoint")
453
+ if isinstance(previous, str):
454
+ ancestors.add(f"checkpoints/{previous}")
455
+ input_checkpoint = publication.get("previousInputCheckpoint")
456
+ return ancestors
457
+
458
+
459
+ def _admit_parent_checkpoint(
460
+ scan_dir: Path, scan_id: str, checkpoint: dict[str, Any], projected: dict[str, Any]
461
+ ) -> bytes:
462
+ """Bind a Deep input to the accepted head under the host's publication lock."""
463
+ previous = {}
464
+ try:
465
+ previous, _, _ = _read_saved_result(scan_dir, "checkpoint-head.json", scan_id)
466
+ if (
467
+ previous.get("checkpoint") == f"{_digest(projected)}.json"
468
+ and "inputCheckpoint" in previous
469
+ ):
470
+ accepted, _, _ = _read_saved_result(
471
+ scan_dir,
472
+ f"checkpoints/{previous['inputCheckpoint']}",
473
+ scan_id,
474
+ normalize_review=False,
475
+ )
476
+ if {key: value for key, value in accepted.items() if key != "publication"} == {
477
+ key: value for key, value in checkpoint.items() if key != "publication"
478
+ }:
479
+ for relative in _checkpoint_paths(scan_dir, "checkpoints"):
480
+ pending, _, _ = _read_saved_result(scan_dir, relative, scan_id)
481
+ publication = pending.get("publication")
482
+ if (
483
+ isinstance(publication, dict)
484
+ and publication.get("previousInputCheckpoint")
485
+ == previous["inputCheckpoint"]
486
+ ):
487
+ break
488
+ else:
489
+ return _encoded(accepted)
490
+ except (ContractError, OSError, ValueError):
491
+ pass # An absent or unreadable previous head contributes no predecessor.
492
+ return _encoded(
493
+ {
494
+ **checkpoint,
495
+ "publication": {
496
+ "previousCheckpoint": previous.get("checkpoint"),
497
+ "previousInputCheckpoint": previous.get("inputCheckpoint"),
498
+ },
499
+ }
500
+ )
501
+
502
+
503
+ def _capture_parent_observation(
504
+ scan_dir: Path,
505
+ scan_id: str,
506
+ checkpoint: str,
507
+ observed: int,
508
+ legacy_head: bool = False,
509
+ *,
510
+ created_paths: set[str] | None = None,
511
+ ) -> dict[str, tuple[str, int]]:
512
+ """Freeze canonical input, preserving the legacy sealed-parent migration."""
513
+ if legacy_head:
514
+ write_scan_local_bytes(
515
+ scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint})
516
+ )
517
+ os.utime(scan_dir / "checkpoint-head.json", ns=(observed, observed))
518
+ return _capture_saved_source(
519
+ scan_dir, "checkpoint-head.json", scan_id, created_paths=created_paths
520
+ )
521
+ observation = {"checkpoint": checkpoint, "observedAtNs": str(observed)}
522
+ digest = _digest(observation)
523
+ snapshot = f"checkpoint-heads/{digest}.json"
524
+ if not (scan_dir / snapshot).exists():
525
+ if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
526
+ created_paths.add(snapshot)
527
+ try:
528
+ write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
529
+ except (ContractError, OSError) as exc:
530
+ # The canonical checkpoint carries this observation without a live head.
531
+ raise _CheckpointSnapshotError(str(exc), {}) from exc
532
+ return {snapshot: (digest, observed)}
533
+
534
+
535
+ def _parent_scan_draft(
536
+ scan_id: str,
537
+ parent_scan: dict[str, Any],
538
+ findings: dict[str, Any],
539
+ coverage: dict[str, Any],
540
+ ) -> dict[str, Any]:
541
+ parent = {
542
+ "scanId": scan_id,
543
+ "findings": findings.get("findings"),
544
+ "reviewState": coverage,
545
+ **{
546
+ key: parent_scan[key]
547
+ for key in ("scope", "threatModel", "complete")
548
+ if key in parent_scan
549
+ },
550
+ }
551
+ if not isinstance(parent["findings"], list):
552
+ raise ContractError("Saved parent draft has no findings array")
553
+ return parent
554
+
555
+
556
+ def _model_capture(scan_id: str, model: dict[str, Any]) -> dict[str, Any]:
557
+ return _parent_scan_draft(scan_id, {"threatModel": model}, {"findings": []}, {})
558
+
559
+
560
+ def _restore_frozen_model_capture(
561
+ scan_dir: Path,
562
+ scan_id: str,
563
+ sources: dict[str, str],
564
+ times: dict[str, int] | None,
565
+ model_source: str | None,
566
+ ) -> None:
567
+ if model_source not in (times or {}) or (scan_dir / model_source).exists():
568
+ return
569
+ manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved scan manifest")
570
+ scan = manifest.get("scan")
571
+ if not isinstance(scan, dict) or not isinstance(scan.get("threatModel"), dict):
572
+ return
573
+ draft = _model_capture(scan_id, scan["threatModel"])
574
+ digest = _digest(draft)
575
+ if model_source != f"checkpoints/{digest}.json" or sources.get(model_source) != digest:
576
+ return
577
+ # A failed first capture can retry only the model and time bound by that stop.
578
+ write_scan_local_bytes(scan_dir, model_source, _encoded(draft))
579
+ observed = times[model_source]
580
+ os.utime(scan_dir / model_source, ns=(observed, observed))
581
+
582
+
583
+ def _read_saved_parent_result(
584
+ scan_dir: Path, scan_id: str
585
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
586
+ manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
587
+ findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
588
+ parent_scan = manifest.get("scan")
589
+ if not isinstance(parent_scan, dict):
590
+ raise ContractError("Saved parent manifest has no scan object")
591
+ coverage = (
592
+ _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
593
+ if uses_legacy_review(parent_scan)
594
+ else review_state(parent_scan)
595
+ )
596
+ if manifest_is_sealed(parent_scan) and (
597
+ parent_scan.get("id", scan_id) != scan_id
598
+ or findings.get("scanId", scan_id) != scan_id
599
+ or coverage.get("scanId", scan_id) != scan_id
600
+ ):
601
+ raise ContractError("Saved parent documents belong to a different scan")
602
+ return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
603
+
604
+
605
+ def _require_current_deep_publication(
606
+ db: Any, connection: Any, scan: Any, draft: dict[str, Any]
607
+ ) -> None:
608
+ scan_id = scan["id"]
609
+ run = connection.execute(
610
+ "SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
611
+ ).fetchone()
612
+ publication = draft.get("deepScanPublication")
613
+ if (
614
+ publication is None
615
+ and draft["manifest"]["scan"].get("complete") is False
616
+ and (run is None or run["status"] == "running")
617
+ ):
618
+ return
619
+ if run is None:
620
+ run = db.deep_scan.require_deep_scan_run(connection, scan_id)
621
+ # Finished discovery-only runs hand remaining publication back to the parent.
622
+ if (
623
+ publication is None
624
+ and run["status"] != "running"
625
+ and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
626
+ ):
627
+ return
628
+ db.deep_scan.require_current_coordinator(
629
+ run,
630
+ argparse.Namespace(
631
+ coordinator_generation=publication.get("coordinatorGeneration") if publication else None
632
+ ),
633
+ )
634
+ # Once finish selects the canonical parent, only completion may replay it.
635
+ if publication is None:
636
+ if run["status"] != "running" and run["manifest_path"] == str(
637
+ Path(scan["scan_dir"]) / "scan-manifest.json"
638
+ ):
639
+ raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
640
+ return
641
+
642
+ # Match the durable reducer sequence used by coordinator recovery.
643
+ def reducer_order(worker: Any) -> tuple[int, str]:
644
+ match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
645
+ return (int(match[1]) if match else 0, worker["id"])
646
+
647
+ reducer = max(
648
+ (
649
+ worker
650
+ for worker in connection.execute(
651
+ "SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
652
+ )
653
+ if worker["kind"] == "dedup"
654
+ and worker["status"] == "succeeded"
655
+ and worker["result_manifest_path"]
656
+ ),
657
+ key=reducer_order,
658
+ default=None,
659
+ )
660
+ selected_result = reducer["result_manifest_path"] if reducer is not None else None
661
+ if publication["resultPath"] != selected_result:
662
+ raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
663
+
664
+
665
+ if __name__ == "__main__":
666
+ argparse.ArgumentParser(description=__doc__).parse_args()
667
+
668
+
669
+ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
670
+ return max(
671
+ (
672
+ worker
673
+ for worker in workers
674
+ if worker["kind"] == "dedup"
675
+ and worker["status"] == "succeeded"
676
+ and worker["result_manifest_path"]
677
+ ),
678
+ key=lambda worker: (worker["completed_at"] or "", worker["id"]),
679
+ default=None,
680
+ )
681
+
682
+
683
+ def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
684
+ output = Path(worker["artifact_dir"]).relative_to(scan_dir)
685
+ attempts = (output.parent if output.name == "output" else output) / "attempts"
686
+ archived = [
687
+ ((attempts / name).as_posix(), int(name.split("-")[1]))
688
+ for name in _children(scan_dir, attempts.as_posix())
689
+ if re.fullmatch(r"attempt-\d+", name)
690
+ ]
691
+ attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
692
+ if worker["kind"] == "discovery" and not attempt:
693
+ attempt = max((attempt for _, attempt in archived), default=0) + 1
694
+ return [(output.as_posix(), attempt), *archived]
695
+
696
+
697
+ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
698
+ latest_reducer = _latest_successful_reducer(workers)
699
+ yield "checkpoint-head.json", None
700
+ for directory in ("checkpoint-heads", "checkpoints"):
701
+ yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
702
+ for worker in workers:
703
+ if worker["kind"] not in {"dedup", "discovery"}:
704
+ continue
705
+ try:
706
+ outputs = _worker_outputs(scan_dir, worker)
707
+ except (TypeError, ValueError):
708
+ continue
709
+ for directory, _ in outputs:
710
+ if worker["kind"] == "discovery":
711
+ yield f"{directory}/checkpoint-head.json", worker["kind"]
712
+ yield from (
713
+ (path, worker["kind"])
714
+ for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
715
+ )
716
+ checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
717
+ if worker["kind"] == "discovery" or checkpoint_paths:
718
+ yield f"{directory}/result.json", worker["kind"]
719
+ yield from ((path, worker["kind"]) for path in checkpoint_paths)
720
+ if worker["result_manifest_path"] and (
721
+ worker["kind"] == "discovery"
722
+ or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
723
+ ):
724
+ try:
725
+ yield (
726
+ Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
727
+ worker["kind"],
728
+ )
729
+ except ValueError:
730
+ continue