@openai/codex-security 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -39
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +34 -34
- package/_bundled_plugin/examples/completed-scan/report.md +22 -31
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +27 -29
- package/_bundled_plugin/examples/{completed-scan → legacy-completed-scan}/coverage.json +1 -0
- package/_bundled_plugin/examples/legacy-completed-scan/findings.json +57 -0
- package/_bundled_plugin/examples/legacy-completed-scan/report.md +98 -0
- package/_bundled_plugin/examples/legacy-completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +50 -184
- package/_bundled_plugin/references/scan-artifacts.md +5 -5
- package/_bundled_plugin/references/scan-contract.md +12 -12
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +37 -11
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +5 -5
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +146 -86
- package/_bundled_plugin/scripts/finalize_scan_contract.py +249 -76
- package/_bundled_plugin/scripts/report_projection.py +49 -17
- package/_bundled_plugin/scripts/review_state.py +68 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3230 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +651 -8
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +730 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_constants.py +0 -1
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +55 -171
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +808 -2389
- package/_bundled_plugin/scripts/workbench_scan_history.py +77 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +28 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -24
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +29 -17
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +5 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +359 -53
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +696 -430
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +2 -23
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +5 -2
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +93 -45
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-publish.js +2 -2
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.js +5 -5
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +47 -19
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +2 -2
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -0
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +3 -0
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +21 -1
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +9 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/endor-findings.d.ts +4 -3
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +16 -11
- package/dist/endor-findings.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +13 -5
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +1 -0
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +14 -3
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +2 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +64 -28
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +12 -1
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +25 -6
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +2 -0
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +7 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +63 -31
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +24 -5
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +24 -9
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +3 -1
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-findings.d.ts +4 -3
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +16 -14
- package/dist/github-findings.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -1
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +19 -2
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +0 -6
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +17 -17
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +20 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/mock-scan.d.ts +1 -2
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +12 -17
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +10 -1
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +1 -6
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +30 -0
- package/dist/native-json-findings.d.ts.map +1 -0
- package/dist/native-json-findings.js +114 -0
- package/dist/native-json-findings.js.map +1 -0
- package/dist/opengrep-findings.d.ts +10 -0
- package/dist/opengrep-findings.d.ts.map +1 -0
- package/dist/opengrep-findings.js +198 -0
- package/dist/opengrep-findings.js.map +1 -0
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +2 -0
- package/dist/patch-tui.js.map +1 -1
- package/dist/publication.js +1 -2
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +1 -0
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +10 -0
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +1 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -24
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +5 -3
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +9 -6
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +7 -7
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +9 -3
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +1 -1
- package/dist/scan-owner-suggestions.d.ts.map +1 -1
- package/dist/scan-owner-suggestions.js +4 -4
- package/dist/scan-owner-suggestions.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +1 -1
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +7 -3
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +17 -22
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +4 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +22 -89
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +5 -4
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +18 -17
- package/dist/snyk-findings.js.map +1 -1
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +4 -1
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -1
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +3 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -1
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +27 -23
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +3 -4
- package/docs/cli.md +171 -41
- package/docs/dedupe-records.md +15 -1
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +19 -17
- package/docs/hosted-scans.md +37 -11
- package/package.json +9 -9
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -0,0 +1,730 @@
|
|
|
1
|
+
"""Read and freeze saved scan checkpoint observations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import stat
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from functools import partial
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from finalize_scan_contract import (
|
|
17
|
+
ContractError,
|
|
18
|
+
_read_scan_local_json,
|
|
19
|
+
_read_scan_local_json_with_metadata,
|
|
20
|
+
manifest_is_sealed,
|
|
21
|
+
write_scan_local_bytes,
|
|
22
|
+
)
|
|
23
|
+
from review_state import review_state, uses_legacy_review
|
|
24
|
+
from workbench_coverage_receipts import capture_receipt_checkpoints
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _CheckpointSnapshotError(ContractError):
|
|
28
|
+
"""A captured source observation could not be persisted for publication replay."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, message: str, captured: dict[str, tuple[str, int]]) -> None:
|
|
31
|
+
super().__init__(message)
|
|
32
|
+
self.captured = captured
|
|
33
|
+
self.records: dict[str, tuple[dict[str, Any], str, int]] = {}
|
|
34
|
+
self.heads: dict[str, str | None] = {}
|
|
35
|
+
self.model_source: list[str] = []
|
|
36
|
+
self.parent_context: dict[str, str] = {}
|
|
37
|
+
self.include_parent = False
|
|
38
|
+
self.documents: tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _encoded(value: Any) -> bytes:
|
|
42
|
+
return json.dumps(
|
|
43
|
+
value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
|
|
44
|
+
).encode()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _digest(value: Any) -> str:
|
|
48
|
+
return hashlib.sha256(_encoded(value)).hexdigest()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _source_digests(value: Any, error: str) -> dict[str, str]:
|
|
52
|
+
if not isinstance(value, dict) or not all(
|
|
53
|
+
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
54
|
+
):
|
|
55
|
+
raise ContractError(error)
|
|
56
|
+
return value
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
|
|
60
|
+
if isinstance(value, dict) and isinstance(value.get("sources"), dict):
|
|
61
|
+
sources = _source_digests(
|
|
62
|
+
value["sources"], "Saved stopped-scan source digests are malformed."
|
|
63
|
+
)
|
|
64
|
+
model_source = value.get("threatModelSource")
|
|
65
|
+
if "threatModelSource" in value and (
|
|
66
|
+
not isinstance(model_source, str) or model_source not in sources
|
|
67
|
+
):
|
|
68
|
+
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
|
|
69
|
+
return sources, model_source
|
|
70
|
+
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _retained_checkpoint_state(
|
|
74
|
+
scan: Any,
|
|
75
|
+
) -> tuple[dict[str, str | None] | None, dict[str, Any] | None]:
|
|
76
|
+
raw = scan["retained_checkpoint_heads_json"]
|
|
77
|
+
if raw is None:
|
|
78
|
+
return None, None
|
|
79
|
+
saved = json.loads(raw)
|
|
80
|
+
if not isinstance(saved.get("heads"), dict):
|
|
81
|
+
return saved, None
|
|
82
|
+
if saved["recoveryBaseDigest"] != scan["seal_manifest_digest"]:
|
|
83
|
+
raise ContractError("Stopped scan recovery publication changed after selection.")
|
|
84
|
+
return saved["heads"], saved
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _encode_retained_sources(
|
|
88
|
+
sources: dict[str, str],
|
|
89
|
+
model_source: list[str],
|
|
90
|
+
source_times: dict[str, int] | None = None,
|
|
91
|
+
parent_context: dict[str, str] | None = None,
|
|
92
|
+
) -> str:
|
|
93
|
+
state: dict[str, Any] = sources
|
|
94
|
+
if model_source or source_times is not None:
|
|
95
|
+
state = {"sources": sources}
|
|
96
|
+
if model_source:
|
|
97
|
+
state["threatModelSource"] = model_source[0]
|
|
98
|
+
if source_times is not None:
|
|
99
|
+
state["sourceTimes"] = source_times
|
|
100
|
+
if parent_context:
|
|
101
|
+
state["parentContext"] = parent_context
|
|
102
|
+
return json.dumps(state, sort_keys=True)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _children(scan_dir: Path, relative: str) -> list[str]:
|
|
106
|
+
cursor = scan_dir
|
|
107
|
+
for part in Path(relative).parts:
|
|
108
|
+
if part in {"..", "."}:
|
|
109
|
+
return []
|
|
110
|
+
cursor = cursor / part
|
|
111
|
+
try:
|
|
112
|
+
if not stat.S_ISDIR(cursor.lstat().st_mode):
|
|
113
|
+
return []
|
|
114
|
+
except FileNotFoundError:
|
|
115
|
+
return []
|
|
116
|
+
return sorted(child.name for child in cursor.iterdir())
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
|
|
120
|
+
return [
|
|
121
|
+
f"{directory}/{name}"
|
|
122
|
+
for name in _children(scan_dir, directory)
|
|
123
|
+
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
128
|
+
path = Path(relative)
|
|
129
|
+
if path.name == "checkpoint-head.json":
|
|
130
|
+
return path.parent
|
|
131
|
+
if path.parent.name == "checkpoint-heads":
|
|
132
|
+
return path.parent.parent
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _unselected_root_order(
|
|
137
|
+
saved_heads: dict[str, str],
|
|
138
|
+
checkpoint_heads: dict[str, str | None],
|
|
139
|
+
source_order: dict[str, tuple[int, int]],
|
|
140
|
+
) -> tuple[int, int]:
|
|
141
|
+
# One payload can have several accepted observations; retain their order.
|
|
142
|
+
return max(
|
|
143
|
+
(
|
|
144
|
+
source_order[head]
|
|
145
|
+
for head, checkpoint in saved_heads.items()
|
|
146
|
+
if _checkpoint_head_directory(head) == Path(".")
|
|
147
|
+
and checkpoint_heads.get(".") not in (None, head, f"checkpoints/{checkpoint}")
|
|
148
|
+
),
|
|
149
|
+
default=(-1, 0),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _capture_saved_source(
|
|
154
|
+
scan_dir: Path,
|
|
155
|
+
relative: str,
|
|
156
|
+
scan_id: str,
|
|
157
|
+
*,
|
|
158
|
+
kind: str | None = None,
|
|
159
|
+
snapshot_head: bool = True,
|
|
160
|
+
write: bool = True,
|
|
161
|
+
created_paths: set[str] | None = None,
|
|
162
|
+
checkpoint_heads: dict[str, str | None] | None = None,
|
|
163
|
+
expected_digest: str | None = None,
|
|
164
|
+
) -> dict[str, tuple[str, int]]:
|
|
165
|
+
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
|
|
166
|
+
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
|
|
167
|
+
return capture_receipt_checkpoints(
|
|
168
|
+
relative,
|
|
169
|
+
kind,
|
|
170
|
+
{relative: (digest, observed)},
|
|
171
|
+
partial(_checkpoint_paths, scan_dir),
|
|
172
|
+
partial(_read_saved_result, scan_dir, scan_id=scan_id),
|
|
173
|
+
)
|
|
174
|
+
head, head_digest, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
175
|
+
if expected_digest is not None and head_digest != expected_digest:
|
|
176
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
177
|
+
observation = {
|
|
178
|
+
**{key: head[key] for key in ("checkpoint", "inputCheckpoint") if key in head},
|
|
179
|
+
"observedAtNs": str(observed),
|
|
180
|
+
}
|
|
181
|
+
directory = Path(relative).parent
|
|
182
|
+
selected_records = {}
|
|
183
|
+
for key in ("checkpoint", "inputCheckpoint"):
|
|
184
|
+
if key in head:
|
|
185
|
+
selected = (directory / "checkpoints" / head[key]).as_posix()
|
|
186
|
+
selected_records[selected] = _read_saved_result(scan_dir, selected, scan_id)
|
|
187
|
+
selected_sources = {
|
|
188
|
+
path: (digest, seen) for path, (_, digest, seen) in selected_records.items()
|
|
189
|
+
}
|
|
190
|
+
digest = _digest(observation)
|
|
191
|
+
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
|
|
192
|
+
# Capture the selected file even if the worker created it after directory enumeration.
|
|
193
|
+
if checkpoint_heads is not None:
|
|
194
|
+
checkpoint_heads[directory.as_posix()] = snapshot
|
|
195
|
+
if write and not (scan_dir / snapshot).exists():
|
|
196
|
+
if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
|
|
197
|
+
created_paths.add(snapshot)
|
|
198
|
+
try:
|
|
199
|
+
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
200
|
+
except (ContractError, OSError) as exc:
|
|
201
|
+
if checkpoint_heads is not None:
|
|
202
|
+
checkpoint_heads[directory.as_posix()] = relative
|
|
203
|
+
failure = _CheckpointSnapshotError(
|
|
204
|
+
str(exc), {relative: (head_digest, observed), **selected_sources}
|
|
205
|
+
)
|
|
206
|
+
failure.records = {relative: (head, head_digest, observed), **selected_records}
|
|
207
|
+
raise failure from exc
|
|
208
|
+
return {
|
|
209
|
+
snapshot: (digest, observed),
|
|
210
|
+
**selected_sources,
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _read_saved_result(
|
|
215
|
+
scan_dir: Path,
|
|
216
|
+
relative: str,
|
|
217
|
+
scan_id: str,
|
|
218
|
+
*,
|
|
219
|
+
kind: str | None = None,
|
|
220
|
+
normalize_review: bool = True,
|
|
221
|
+
) -> tuple[dict[str, Any], str, int]:
|
|
222
|
+
draft, _, metadata = _read_scan_local_json_with_metadata(
|
|
223
|
+
scan_dir, relative, "Saved scan checkpoint"
|
|
224
|
+
)
|
|
225
|
+
directory = _checkpoint_head_directory(relative)
|
|
226
|
+
if directory is not None:
|
|
227
|
+
checkpoint = draft.get("checkpoint")
|
|
228
|
+
if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
|
|
229
|
+
raise ContractError("checkpoint head does not name a saved checkpoint")
|
|
230
|
+
_read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
|
|
231
|
+
if "inputCheckpoint" in draft:
|
|
232
|
+
raw = draft["inputCheckpoint"]
|
|
233
|
+
if not isinstance(raw, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", raw):
|
|
234
|
+
raise ContractError("checkpoint head does not name a saved input checkpoint")
|
|
235
|
+
_read_saved_result(scan_dir, (directory / "checkpoints" / raw).as_posix(), scan_id)
|
|
236
|
+
if Path(relative).name == "checkpoint-head.json":
|
|
237
|
+
return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
|
|
238
|
+
observed = draft.get("observedAtNs")
|
|
239
|
+
if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
|
|
240
|
+
raise ContractError("checkpoint head has no observation time")
|
|
241
|
+
return draft, _digest(draft), int(observed)
|
|
242
|
+
if draft.get("scanId") != scan_id:
|
|
243
|
+
raise ContractError("checkpoint belongs to a different scan")
|
|
244
|
+
if not _is_source_order_snapshot(relative) and (
|
|
245
|
+
not isinstance(draft.get("findings"), list)
|
|
246
|
+
or not isinstance(
|
|
247
|
+
draft.get("reviewState", draft.get("coverage", {} if kind == "dedup" else None)), dict
|
|
248
|
+
)
|
|
249
|
+
):
|
|
250
|
+
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
251
|
+
digest = _digest(draft)
|
|
252
|
+
if normalize_review and "coverage" in draft and "reviewState" not in draft:
|
|
253
|
+
draft["reviewState"] = draft.pop("coverage")
|
|
254
|
+
return draft, digest, metadata.st_mtime_ns
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _is_source_order_snapshot(relative: str) -> bool:
|
|
258
|
+
path = Path(relative)
|
|
259
|
+
return path.parent == Path("source-order") and bool(
|
|
260
|
+
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _frozen_source_times(
|
|
265
|
+
scan_dir: Path,
|
|
266
|
+
scan_id: str,
|
|
267
|
+
sources: dict[str, str],
|
|
268
|
+
parent_context: dict[str, str] | None = None,
|
|
269
|
+
) -> dict[str, int]:
|
|
270
|
+
times: dict[str, int] = {}
|
|
271
|
+
snapshots = [path for path in sources if _is_source_order_snapshot(path)]
|
|
272
|
+
for path in snapshots:
|
|
273
|
+
record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
|
|
274
|
+
if digest != sources[path] or not isinstance(record.get("sources"), dict):
|
|
275
|
+
raise ContractError("saved source ordering changed after the scan stopped")
|
|
276
|
+
context = record.get("parentContext")
|
|
277
|
+
if parent_context is not None and context is not None:
|
|
278
|
+
if not isinstance(context, dict) or any(
|
|
279
|
+
(
|
|
280
|
+
"canonical" not in context
|
|
281
|
+
or not isinstance(relative, str)
|
|
282
|
+
or not re.fullmatch(r"-?[0-9]+", relative)
|
|
283
|
+
)
|
|
284
|
+
if role == "canonicalObservedAtNs"
|
|
285
|
+
else (
|
|
286
|
+
role not in ("canonical", "head")
|
|
287
|
+
or not isinstance(relative, str)
|
|
288
|
+
or relative not in sources
|
|
289
|
+
or (role == "canonical" and Path(relative).parent != Path("checkpoints"))
|
|
290
|
+
or (role == "head" and _checkpoint_head_directory(relative) != Path("."))
|
|
291
|
+
)
|
|
292
|
+
for role, relative in context.items()
|
|
293
|
+
):
|
|
294
|
+
raise ContractError("saved parent context is outside the frozen sources")
|
|
295
|
+
if any(
|
|
296
|
+
role in parent_context and parent_context[role] != relative
|
|
297
|
+
for role, relative in context.items()
|
|
298
|
+
):
|
|
299
|
+
raise ContractError("saved parent context has conflicting observations")
|
|
300
|
+
parent_context.update(context)
|
|
301
|
+
for relative, observation in record["sources"].items():
|
|
302
|
+
if (
|
|
303
|
+
not isinstance(observation, dict)
|
|
304
|
+
or relative not in sources
|
|
305
|
+
or observation.get("digest") != sources[relative]
|
|
306
|
+
or not isinstance(observation.get("observedAtNs"), str)
|
|
307
|
+
or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
|
|
308
|
+
):
|
|
309
|
+
raise ContractError("saved source ordering does not match its frozen sources")
|
|
310
|
+
observed = int(observation["observedAtNs"])
|
|
311
|
+
if relative in times and times[relative] != observed:
|
|
312
|
+
raise ContractError("saved source ordering has conflicting observations")
|
|
313
|
+
times[relative] = observed
|
|
314
|
+
if snapshots and sources.keys() - set(snapshots) - times.keys():
|
|
315
|
+
raise ContractError("saved source ordering is incomplete")
|
|
316
|
+
return times
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _frozen_checkpoint_heads(
|
|
320
|
+
scan_dir: Path, scan_id: str, sources: dict[str, str], *, skip_unreadable: bool = False
|
|
321
|
+
) -> dict[str, str]:
|
|
322
|
+
parent_context: dict[str, str] = {}
|
|
323
|
+
try:
|
|
324
|
+
times = _frozen_source_times(scan_dir, scan_id, sources, parent_context)
|
|
325
|
+
except (ContractError, OSError, ValueError):
|
|
326
|
+
if not skip_unreadable:
|
|
327
|
+
raise
|
|
328
|
+
times = {}
|
|
329
|
+
parent_context.clear()
|
|
330
|
+
# A frozen parent can identify its accepted head even when observation times tie.
|
|
331
|
+
accepted_root = None
|
|
332
|
+
heads: dict[str, tuple[int, str]] = {}
|
|
333
|
+
ambiguous: set[str] = set()
|
|
334
|
+
for relative, expected_digest in sources.items():
|
|
335
|
+
directory = _checkpoint_head_directory(relative)
|
|
336
|
+
if directory is None:
|
|
337
|
+
continue
|
|
338
|
+
try:
|
|
339
|
+
head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
340
|
+
if digest != expected_digest:
|
|
341
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
342
|
+
except (ContractError, OSError, ValueError):
|
|
343
|
+
if skip_unreadable:
|
|
344
|
+
continue
|
|
345
|
+
raise
|
|
346
|
+
selected = (directory / "checkpoints" / head["checkpoint"]).as_posix()
|
|
347
|
+
observation = (times.get(relative, observed), selected)
|
|
348
|
+
key = directory.as_posix()
|
|
349
|
+
if relative == parent_context.get("head"):
|
|
350
|
+
accepted_root = relative
|
|
351
|
+
if key not in heads or observation[0] > heads[key][0]:
|
|
352
|
+
heads[key] = observation
|
|
353
|
+
ambiguous.discard(key)
|
|
354
|
+
elif observation[0] == heads[key][0] and selected != heads[key][1]:
|
|
355
|
+
ambiguous.add(key)
|
|
356
|
+
# A legacy tie retains both observations until recovery selects a head.
|
|
357
|
+
frozen = {
|
|
358
|
+
directory: selected
|
|
359
|
+
for directory, (_, selected) in heads.items()
|
|
360
|
+
if directory not in ambiguous
|
|
361
|
+
}
|
|
362
|
+
if accepted_root is not None:
|
|
363
|
+
frozen["."] = accepted_root
|
|
364
|
+
return frozen
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _legacy_checkpoint_observed_at(
|
|
368
|
+
scan_dir: Path, scan_id: str, sources: dict[str, str], selected: str
|
|
369
|
+
) -> int | None:
|
|
370
|
+
times = _frozen_source_times(scan_dir, scan_id, sources)
|
|
371
|
+
checkpoint = Path(selected)
|
|
372
|
+
observations = []
|
|
373
|
+
for relative, expected_digest in sources.items():
|
|
374
|
+
if _checkpoint_head_directory(relative) != checkpoint.parent.parent:
|
|
375
|
+
continue
|
|
376
|
+
head, digest, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
377
|
+
if digest != expected_digest:
|
|
378
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
379
|
+
if head["checkpoint"] == checkpoint.name:
|
|
380
|
+
observations.append(times.get(relative, observed))
|
|
381
|
+
# Payload-only selections used the newest retained observation for that payload.
|
|
382
|
+
return max(observations, default=None)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _restore_checkpoint_head(
|
|
386
|
+
heads: dict[str, str | None],
|
|
387
|
+
previous_heads: dict[str, str | None],
|
|
388
|
+
sources: dict[str, str],
|
|
389
|
+
relative: str,
|
|
390
|
+
) -> None:
|
|
391
|
+
path = Path(relative)
|
|
392
|
+
if path.name != "checkpoint-head.json":
|
|
393
|
+
return
|
|
394
|
+
directory = path.parent.as_posix()
|
|
395
|
+
if directory in previous_heads:
|
|
396
|
+
heads[directory] = previous_heads[directory]
|
|
397
|
+
return
|
|
398
|
+
directories = {path.parent, path.parent / "checkpoints", path.parent / "checkpoint-heads"}
|
|
399
|
+
if any(Path(source).parent in directories for source in sources):
|
|
400
|
+
# Legacy evidence can remain accepted without one selected head.
|
|
401
|
+
heads.pop(directory, None)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _freeze_source_times(
|
|
405
|
+
scan_dir: Path,
|
|
406
|
+
scan_id: str,
|
|
407
|
+
sources: dict[str, str],
|
|
408
|
+
times: dict[str, int],
|
|
409
|
+
parent_context: dict[str, str] | None = None,
|
|
410
|
+
*,
|
|
411
|
+
created_paths: set[str] | None = None,
|
|
412
|
+
) -> None:
|
|
413
|
+
# Identical result rewrites must not change the order of frozen review evidence.
|
|
414
|
+
observations = {
|
|
415
|
+
path: {"digest": digest, "observedAtNs": str(times[path])}
|
|
416
|
+
for path, digest in sources.items()
|
|
417
|
+
if not _is_source_order_snapshot(path)
|
|
418
|
+
}
|
|
419
|
+
if not observations:
|
|
420
|
+
return
|
|
421
|
+
record = {"scanId": scan_id, "sources": observations}
|
|
422
|
+
if parent_context:
|
|
423
|
+
record["parentContext"] = parent_context
|
|
424
|
+
digest = _digest(record)
|
|
425
|
+
path = f"source-order/{digest}.json"
|
|
426
|
+
if not (scan_dir / path).exists():
|
|
427
|
+
if created_paths is not None and not os.path.lexists(scan_dir / path):
|
|
428
|
+
created_paths.add(path)
|
|
429
|
+
write_scan_local_bytes(scan_dir, path, _encoded(record))
|
|
430
|
+
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
|
|
431
|
+
raise ContractError("saved source ordering does not match its digest")
|
|
432
|
+
if parent_context:
|
|
433
|
+
for previous in [relative for relative in sources if _is_source_order_snapshot(relative)]:
|
|
434
|
+
previous_record, _, _ = _read_saved_result(scan_dir, previous, scan_id)
|
|
435
|
+
if previous_record.get("parentContext"):
|
|
436
|
+
del sources[previous]
|
|
437
|
+
sources[path] = digest
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _parent_publication_ancestors(
|
|
441
|
+
input_checkpoint: str | None, drafts: dict[str, dict[str, Any]]
|
|
442
|
+
) -> set[str]:
|
|
443
|
+
"""Find stale canonical projections without relying on publication clocks."""
|
|
444
|
+
ancestors = set()
|
|
445
|
+
inputs = set()
|
|
446
|
+
# Input hashes distinguish repeated publication of the same projected result.
|
|
447
|
+
while isinstance(input_checkpoint, str) and input_checkpoint not in inputs:
|
|
448
|
+
inputs.add(input_checkpoint)
|
|
449
|
+
publication = drafts.get(f"checkpoints/{input_checkpoint}", {}).get("publication")
|
|
450
|
+
if not isinstance(publication, dict):
|
|
451
|
+
break
|
|
452
|
+
previous = publication.get("previousCheckpoint")
|
|
453
|
+
if isinstance(previous, str):
|
|
454
|
+
ancestors.add(f"checkpoints/{previous}")
|
|
455
|
+
input_checkpoint = publication.get("previousInputCheckpoint")
|
|
456
|
+
return ancestors
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _admit_parent_checkpoint(
|
|
460
|
+
scan_dir: Path, scan_id: str, checkpoint: dict[str, Any], projected: dict[str, Any]
|
|
461
|
+
) -> bytes:
|
|
462
|
+
"""Bind a Deep input to the accepted head under the host's publication lock."""
|
|
463
|
+
previous = {}
|
|
464
|
+
try:
|
|
465
|
+
previous, _, _ = _read_saved_result(scan_dir, "checkpoint-head.json", scan_id)
|
|
466
|
+
if (
|
|
467
|
+
previous.get("checkpoint") == f"{_digest(projected)}.json"
|
|
468
|
+
and "inputCheckpoint" in previous
|
|
469
|
+
):
|
|
470
|
+
accepted, _, _ = _read_saved_result(
|
|
471
|
+
scan_dir,
|
|
472
|
+
f"checkpoints/{previous['inputCheckpoint']}",
|
|
473
|
+
scan_id,
|
|
474
|
+
normalize_review=False,
|
|
475
|
+
)
|
|
476
|
+
if {key: value for key, value in accepted.items() if key != "publication"} == {
|
|
477
|
+
key: value for key, value in checkpoint.items() if key != "publication"
|
|
478
|
+
}:
|
|
479
|
+
for relative in _checkpoint_paths(scan_dir, "checkpoints"):
|
|
480
|
+
pending, _, _ = _read_saved_result(scan_dir, relative, scan_id)
|
|
481
|
+
publication = pending.get("publication")
|
|
482
|
+
if (
|
|
483
|
+
isinstance(publication, dict)
|
|
484
|
+
and publication.get("previousInputCheckpoint")
|
|
485
|
+
== previous["inputCheckpoint"]
|
|
486
|
+
):
|
|
487
|
+
break
|
|
488
|
+
else:
|
|
489
|
+
return _encoded(accepted)
|
|
490
|
+
except (ContractError, OSError, ValueError):
|
|
491
|
+
pass # An absent or unreadable previous head contributes no predecessor.
|
|
492
|
+
return _encoded(
|
|
493
|
+
{
|
|
494
|
+
**checkpoint,
|
|
495
|
+
"publication": {
|
|
496
|
+
"previousCheckpoint": previous.get("checkpoint"),
|
|
497
|
+
"previousInputCheckpoint": previous.get("inputCheckpoint"),
|
|
498
|
+
},
|
|
499
|
+
}
|
|
500
|
+
)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _capture_parent_observation(
|
|
504
|
+
scan_dir: Path,
|
|
505
|
+
scan_id: str,
|
|
506
|
+
checkpoint: str,
|
|
507
|
+
observed: int,
|
|
508
|
+
legacy_head: bool = False,
|
|
509
|
+
*,
|
|
510
|
+
created_paths: set[str] | None = None,
|
|
511
|
+
) -> dict[str, tuple[str, int]]:
|
|
512
|
+
"""Freeze canonical input, preserving the legacy sealed-parent migration."""
|
|
513
|
+
if legacy_head:
|
|
514
|
+
write_scan_local_bytes(
|
|
515
|
+
scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint})
|
|
516
|
+
)
|
|
517
|
+
os.utime(scan_dir / "checkpoint-head.json", ns=(observed, observed))
|
|
518
|
+
return _capture_saved_source(
|
|
519
|
+
scan_dir, "checkpoint-head.json", scan_id, created_paths=created_paths
|
|
520
|
+
)
|
|
521
|
+
observation = {"checkpoint": checkpoint, "observedAtNs": str(observed)}
|
|
522
|
+
digest = _digest(observation)
|
|
523
|
+
snapshot = f"checkpoint-heads/{digest}.json"
|
|
524
|
+
if not (scan_dir / snapshot).exists():
|
|
525
|
+
if created_paths is not None and not os.path.lexists(scan_dir / snapshot):
|
|
526
|
+
created_paths.add(snapshot)
|
|
527
|
+
try:
|
|
528
|
+
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
529
|
+
except (ContractError, OSError) as exc:
|
|
530
|
+
# The canonical checkpoint carries this observation without a live head.
|
|
531
|
+
raise _CheckpointSnapshotError(str(exc), {}) from exc
|
|
532
|
+
return {snapshot: (digest, observed)}
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _parent_scan_draft(
|
|
536
|
+
scan_id: str,
|
|
537
|
+
parent_scan: dict[str, Any],
|
|
538
|
+
findings: dict[str, Any],
|
|
539
|
+
coverage: dict[str, Any],
|
|
540
|
+
) -> dict[str, Any]:
|
|
541
|
+
parent = {
|
|
542
|
+
"scanId": scan_id,
|
|
543
|
+
"findings": findings.get("findings"),
|
|
544
|
+
"reviewState": coverage,
|
|
545
|
+
**{
|
|
546
|
+
key: parent_scan[key]
|
|
547
|
+
for key in ("scope", "threatModel", "complete")
|
|
548
|
+
if key in parent_scan
|
|
549
|
+
},
|
|
550
|
+
}
|
|
551
|
+
if not isinstance(parent["findings"], list):
|
|
552
|
+
raise ContractError("Saved parent draft has no findings array")
|
|
553
|
+
return parent
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _model_capture(scan_id: str, model: dict[str, Any]) -> dict[str, Any]:
|
|
557
|
+
return _parent_scan_draft(scan_id, {"threatModel": model}, {"findings": []}, {})
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _restore_frozen_model_capture(
|
|
561
|
+
scan_dir: Path,
|
|
562
|
+
scan_id: str,
|
|
563
|
+
sources: dict[str, str],
|
|
564
|
+
times: dict[str, int] | None,
|
|
565
|
+
model_source: str | None,
|
|
566
|
+
) -> None:
|
|
567
|
+
if model_source not in (times or {}) or (scan_dir / model_source).exists():
|
|
568
|
+
return
|
|
569
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved scan manifest")
|
|
570
|
+
scan = manifest.get("scan")
|
|
571
|
+
if not isinstance(scan, dict) or not isinstance(scan.get("threatModel"), dict):
|
|
572
|
+
return
|
|
573
|
+
draft = _model_capture(scan_id, scan["threatModel"])
|
|
574
|
+
digest = _digest(draft)
|
|
575
|
+
if model_source != f"checkpoints/{digest}.json" or sources.get(model_source) != digest:
|
|
576
|
+
return
|
|
577
|
+
# A failed first capture can retry only the model and time bound by that stop.
|
|
578
|
+
write_scan_local_bytes(scan_dir, model_source, _encoded(draft))
|
|
579
|
+
observed = times[model_source]
|
|
580
|
+
os.utime(scan_dir / model_source, ns=(observed, observed))
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _read_saved_parent_result(
|
|
584
|
+
scan_dir: Path, scan_id: str
|
|
585
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
586
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
|
|
587
|
+
findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
|
|
588
|
+
parent_scan = manifest.get("scan")
|
|
589
|
+
if not isinstance(parent_scan, dict):
|
|
590
|
+
raise ContractError("Saved parent manifest has no scan object")
|
|
591
|
+
coverage = (
|
|
592
|
+
_read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
|
|
593
|
+
if uses_legacy_review(parent_scan)
|
|
594
|
+
else review_state(parent_scan)
|
|
595
|
+
)
|
|
596
|
+
if manifest_is_sealed(parent_scan) and (
|
|
597
|
+
parent_scan.get("id", scan_id) != scan_id
|
|
598
|
+
or findings.get("scanId", scan_id) != scan_id
|
|
599
|
+
or coverage.get("scanId", scan_id) != scan_id
|
|
600
|
+
):
|
|
601
|
+
raise ContractError("Saved parent documents belong to a different scan")
|
|
602
|
+
return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _require_current_deep_publication(
|
|
606
|
+
db: Any, connection: Any, scan: Any, draft: dict[str, Any]
|
|
607
|
+
) -> None:
|
|
608
|
+
scan_id = scan["id"]
|
|
609
|
+
run = connection.execute(
|
|
610
|
+
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
611
|
+
).fetchone()
|
|
612
|
+
publication = draft.get("deepScanPublication")
|
|
613
|
+
if (
|
|
614
|
+
publication is None
|
|
615
|
+
and draft["manifest"]["scan"].get("complete") is False
|
|
616
|
+
and (run is None or run["status"] == "running")
|
|
617
|
+
):
|
|
618
|
+
return
|
|
619
|
+
if run is None:
|
|
620
|
+
run = db.deep_scan.require_deep_scan_run(connection, scan_id)
|
|
621
|
+
# Finished discovery-only runs hand remaining publication back to the parent.
|
|
622
|
+
if (
|
|
623
|
+
publication is None
|
|
624
|
+
and run["status"] != "running"
|
|
625
|
+
and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
626
|
+
):
|
|
627
|
+
return
|
|
628
|
+
db.deep_scan.require_current_coordinator(
|
|
629
|
+
run,
|
|
630
|
+
argparse.Namespace(
|
|
631
|
+
coordinator_generation=publication.get("coordinatorGeneration") if publication else None
|
|
632
|
+
),
|
|
633
|
+
)
|
|
634
|
+
# Once finish selects the canonical parent, only completion may replay it.
|
|
635
|
+
if publication is None:
|
|
636
|
+
if run["status"] != "running" and run["manifest_path"] == str(
|
|
637
|
+
Path(scan["scan_dir"]) / "scan-manifest.json"
|
|
638
|
+
):
|
|
639
|
+
raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
|
|
640
|
+
return
|
|
641
|
+
|
|
642
|
+
# Match the durable reducer sequence used by coordinator recovery.
|
|
643
|
+
def reducer_order(worker: Any) -> tuple[int, str]:
|
|
644
|
+
match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
|
|
645
|
+
return (int(match[1]) if match else 0, worker["id"])
|
|
646
|
+
|
|
647
|
+
reducer = max(
|
|
648
|
+
(
|
|
649
|
+
worker
|
|
650
|
+
for worker in connection.execute(
|
|
651
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
|
|
652
|
+
)
|
|
653
|
+
if worker["kind"] == "dedup"
|
|
654
|
+
and worker["status"] == "succeeded"
|
|
655
|
+
and worker["result_manifest_path"]
|
|
656
|
+
),
|
|
657
|
+
key=reducer_order,
|
|
658
|
+
default=None,
|
|
659
|
+
)
|
|
660
|
+
selected_result = reducer["result_manifest_path"] if reducer is not None else None
|
|
661
|
+
if publication["resultPath"] != selected_result:
|
|
662
|
+
raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
if __name__ == "__main__":
|
|
666
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
670
|
+
return max(
|
|
671
|
+
(
|
|
672
|
+
worker
|
|
673
|
+
for worker in workers
|
|
674
|
+
if worker["kind"] == "dedup"
|
|
675
|
+
and worker["status"] == "succeeded"
|
|
676
|
+
and worker["result_manifest_path"]
|
|
677
|
+
),
|
|
678
|
+
key=lambda worker: (worker["completed_at"] or "", worker["id"]),
|
|
679
|
+
default=None,
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
|
|
684
|
+
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
685
|
+
attempts = (output.parent if output.name == "output" else output) / "attempts"
|
|
686
|
+
archived = [
|
|
687
|
+
((attempts / name).as_posix(), int(name.split("-")[1]))
|
|
688
|
+
for name in _children(scan_dir, attempts.as_posix())
|
|
689
|
+
if re.fullmatch(r"attempt-\d+", name)
|
|
690
|
+
]
|
|
691
|
+
attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
|
|
692
|
+
if worker["kind"] == "discovery" and not attempt:
|
|
693
|
+
attempt = max((attempt for _, attempt in archived), default=0) + 1
|
|
694
|
+
return [(output.as_posix(), attempt), *archived]
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
698
|
+
latest_reducer = _latest_successful_reducer(workers)
|
|
699
|
+
yield "checkpoint-head.json", None
|
|
700
|
+
for directory in ("checkpoint-heads", "checkpoints"):
|
|
701
|
+
yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
|
|
702
|
+
for worker in workers:
|
|
703
|
+
if worker["kind"] not in {"dedup", "discovery"}:
|
|
704
|
+
continue
|
|
705
|
+
try:
|
|
706
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
707
|
+
except (TypeError, ValueError):
|
|
708
|
+
continue
|
|
709
|
+
for directory, _ in outputs:
|
|
710
|
+
if worker["kind"] == "discovery":
|
|
711
|
+
yield f"{directory}/checkpoint-head.json", worker["kind"]
|
|
712
|
+
yield from (
|
|
713
|
+
(path, worker["kind"])
|
|
714
|
+
for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
|
|
715
|
+
)
|
|
716
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
717
|
+
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
718
|
+
yield f"{directory}/result.json", worker["kind"]
|
|
719
|
+
yield from ((path, worker["kind"]) for path in checkpoint_paths)
|
|
720
|
+
if worker["result_manifest_path"] and (
|
|
721
|
+
worker["kind"] == "discovery"
|
|
722
|
+
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
723
|
+
):
|
|
724
|
+
try:
|
|
725
|
+
yield (
|
|
726
|
+
Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
|
|
727
|
+
worker["kind"],
|
|
728
|
+
)
|
|
729
|
+
except ValueError:
|
|
730
|
+
continue
|