@openai/codex-security 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -39
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +34 -34
- package/_bundled_plugin/examples/completed-scan/report.md +22 -31
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +27 -29
- package/_bundled_plugin/examples/{completed-scan → legacy-completed-scan}/coverage.json +1 -0
- package/_bundled_plugin/examples/legacy-completed-scan/findings.json +57 -0
- package/_bundled_plugin/examples/legacy-completed-scan/report.md +98 -0
- package/_bundled_plugin/examples/legacy-completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +50 -184
- package/_bundled_plugin/references/scan-artifacts.md +5 -5
- package/_bundled_plugin/references/scan-contract.md +12 -12
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +37 -11
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +5 -5
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +146 -86
- package/_bundled_plugin/scripts/finalize_scan_contract.py +249 -76
- package/_bundled_plugin/scripts/report_projection.py +49 -17
- package/_bundled_plugin/scripts/review_state.py +68 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3230 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +651 -8
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +730 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_constants.py +0 -1
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +55 -171
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +808 -2389
- package/_bundled_plugin/scripts/workbench_scan_history.py +77 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +28 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -24
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +29 -17
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +5 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +359 -53
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +696 -430
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +2 -23
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +5 -2
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +93 -45
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-publish.js +2 -2
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.js +5 -5
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +47 -19
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +2 -2
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -0
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +3 -0
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +21 -1
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +9 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/endor-findings.d.ts +4 -3
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +16 -11
- package/dist/endor-findings.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +13 -5
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +1 -0
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +14 -3
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +2 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +64 -28
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +12 -1
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +25 -6
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +2 -0
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +7 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +63 -31
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +24 -5
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +24 -9
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +3 -1
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-findings.d.ts +4 -3
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +16 -14
- package/dist/github-findings.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -1
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +19 -2
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +0 -6
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +17 -17
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +20 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/mock-scan.d.ts +1 -2
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +12 -17
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +10 -1
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +1 -6
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +30 -0
- package/dist/native-json-findings.d.ts.map +1 -0
- package/dist/native-json-findings.js +114 -0
- package/dist/native-json-findings.js.map +1 -0
- package/dist/opengrep-findings.d.ts +10 -0
- package/dist/opengrep-findings.d.ts.map +1 -0
- package/dist/opengrep-findings.js +198 -0
- package/dist/opengrep-findings.js.map +1 -0
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +2 -0
- package/dist/patch-tui.js.map +1 -1
- package/dist/publication.js +1 -2
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +1 -0
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +10 -0
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +1 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -24
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +5 -3
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +9 -6
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +7 -7
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +9 -3
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +1 -1
- package/dist/scan-owner-suggestions.d.ts.map +1 -1
- package/dist/scan-owner-suggestions.js +4 -4
- package/dist/scan-owner-suggestions.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +1 -1
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +7 -3
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +17 -22
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +4 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +22 -89
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +5 -4
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +18 -17
- package/dist/snyk-findings.js.map +1 -1
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +4 -1
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -1
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +3 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -1
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +27 -23
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +3 -4
- package/docs/cli.md +171 -41
- package/docs/dedupe-records.md +15 -1
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +19 -17
- package/docs/hosted-scans.md +37 -11
- package/package.json +9 -9
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -10,9 +10,8 @@ import json
|
|
|
10
10
|
import os
|
|
11
11
|
import re
|
|
12
12
|
import sqlite3
|
|
13
|
-
import stat
|
|
14
13
|
import sys
|
|
15
|
-
from collections.abc import
|
|
14
|
+
from collections.abc import Iterator
|
|
16
15
|
from contextlib import contextmanager
|
|
17
16
|
from pathlib import Path
|
|
18
17
|
from typing import Any
|
|
@@ -20,47 +19,128 @@ from typing import Any
|
|
|
20
19
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
21
20
|
from finalize_scan_contract import (
|
|
22
21
|
ContractError,
|
|
23
|
-
|
|
22
|
+
ScanLocalIOError,
|
|
24
23
|
_populate_unsealed_artifact_envelope,
|
|
25
24
|
_populate_unsealed_manifest_envelope,
|
|
26
25
|
_prepare_scan_finalization,
|
|
27
|
-
_read_json,
|
|
28
26
|
_read_saved_threat_model,
|
|
29
27
|
_read_scan_local_json,
|
|
30
28
|
_read_scan_local_json_bytes,
|
|
31
|
-
|
|
29
|
+
_recover_unsealed_coverage,
|
|
32
30
|
_recover_unsealed_findings,
|
|
33
31
|
_remove_scan_local_file_if_exists,
|
|
34
|
-
_require_portable_relative_path,
|
|
35
|
-
_require_str,
|
|
36
32
|
_validate_completion_binding,
|
|
37
|
-
_validate_resolved_deferred,
|
|
38
|
-
_validate_schema_node,
|
|
39
33
|
_write_prepared_scan_finalization,
|
|
40
34
|
finalize_scan,
|
|
41
|
-
|
|
35
|
+
manifest_is_sealed,
|
|
42
36
|
open_scan_local_file_descriptor,
|
|
43
37
|
write_scan_local_bytes,
|
|
44
38
|
write_threat_model_projection_if_possible,
|
|
45
39
|
)
|
|
40
|
+
from finalize_scan_contract import finding_candidate_id as finding_candidate_id
|
|
41
|
+
from review_state import checkpoint_review_state, review_state, review_summary, uses_legacy_review
|
|
42
|
+
from workbench.retained_coverage import (
|
|
43
|
+
_append_previous_finding as _append_previous_finding,
|
|
44
|
+
)
|
|
45
|
+
from workbench.retained_coverage import (
|
|
46
|
+
_coverage_rows as _coverage_rows,
|
|
47
|
+
)
|
|
48
|
+
from workbench.retained_coverage import (
|
|
49
|
+
_ensure_finding_identities as _ensure_finding_identities,
|
|
50
|
+
)
|
|
51
|
+
from workbench.retained_coverage import (
|
|
52
|
+
_ensure_finding_identity as _ensure_finding_identity,
|
|
53
|
+
)
|
|
54
|
+
from workbench.retained_coverage import (
|
|
55
|
+
_finding_content as _finding_content,
|
|
56
|
+
)
|
|
57
|
+
from workbench.retained_coverage import (
|
|
58
|
+
_finding_identity as _finding_identity,
|
|
59
|
+
)
|
|
60
|
+
from workbench.retained_coverage import (
|
|
61
|
+
_finding_key as _finding_key,
|
|
62
|
+
)
|
|
63
|
+
from workbench.retained_coverage import (
|
|
64
|
+
_has_candidate_metadata as _has_candidate_metadata,
|
|
65
|
+
)
|
|
66
|
+
from workbench.retained_coverage import (
|
|
67
|
+
_identity_candidate as _identity_candidate,
|
|
68
|
+
)
|
|
69
|
+
from workbench.retained_coverage import (
|
|
70
|
+
_inherit_coverage_sources as _inherit_coverage_sources,
|
|
71
|
+
)
|
|
72
|
+
from workbench.retained_coverage import (
|
|
73
|
+
_retained_findings as _retained_findings,
|
|
74
|
+
)
|
|
75
|
+
from workbench.retained_coverage import (
|
|
76
|
+
_semantic_identity_key as _semantic_identity_key,
|
|
77
|
+
)
|
|
78
|
+
from workbench.retained_coverage import (
|
|
79
|
+
_worker_candidate_key as _worker_candidate_key,
|
|
80
|
+
)
|
|
81
|
+
from workbench.retained_coverage import merge_saved_results as merge_saved_results
|
|
82
|
+
from workbench.saved_coverage import (
|
|
83
|
+
_deferred_rows as _deferred_rows,
|
|
84
|
+
)
|
|
85
|
+
from workbench.saved_coverage import (
|
|
86
|
+
_merge_tied_parent_observations as _merge_tied_parent_observations,
|
|
87
|
+
)
|
|
46
88
|
from workbench.saved_coverage import (
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
89
|
+
_resolved_deferred_rows as _resolved_deferred_rows,
|
|
90
|
+
)
|
|
91
|
+
from workbench.saved_coverage import (
|
|
92
|
+
_saved_coverage_id as _saved_coverage_id,
|
|
93
|
+
)
|
|
94
|
+
from workbench.saved_outputs import _restore_published_outputs, _snapshot_published_outputs
|
|
95
|
+
from workbench.saved_sources import (
|
|
96
|
+
_admit_parent_checkpoint,
|
|
97
|
+
_capture_saved_source,
|
|
98
|
+
_checkpoint_paths,
|
|
99
|
+
_CheckpointSnapshotError,
|
|
100
|
+
_encode_retained_sources,
|
|
101
|
+
_encoded,
|
|
102
|
+
_freeze_source_times,
|
|
103
|
+
_frozen_checkpoint_heads,
|
|
104
|
+
_frozen_source_times,
|
|
105
|
+
_is_source_order_snapshot,
|
|
106
|
+
_legacy_checkpoint_observed_at,
|
|
107
|
+
_parent_scan_draft,
|
|
108
|
+
_read_saved_parent_result,
|
|
109
|
+
_read_saved_result,
|
|
110
|
+
_require_current_deep_publication,
|
|
111
|
+
_restore_checkpoint_head,
|
|
112
|
+
_restore_frozen_model_capture,
|
|
113
|
+
_retained_checkpoint_state,
|
|
114
|
+
_retained_source_state,
|
|
115
|
+
_saved_result_paths,
|
|
116
|
+
_source_digests,
|
|
117
|
+
)
|
|
118
|
+
from workbench.saved_sources import (
|
|
119
|
+
_capture_parent_observation as _capture_parent_observation,
|
|
120
|
+
)
|
|
121
|
+
from workbench.saved_sources import (
|
|
122
|
+
_checkpoint_head_directory as _checkpoint_head_directory,
|
|
123
|
+
)
|
|
124
|
+
from workbench.saved_sources import (
|
|
125
|
+
_digest as _digest,
|
|
126
|
+
)
|
|
127
|
+
from workbench.saved_sources import (
|
|
128
|
+
_latest_successful_reducer as _latest_successful_reducer,
|
|
129
|
+
)
|
|
130
|
+
from workbench.saved_sources import (
|
|
131
|
+
_parent_publication_ancestors as _parent_publication_ancestors,
|
|
132
|
+
)
|
|
133
|
+
from workbench.saved_sources import (
|
|
134
|
+
_worker_outputs as _worker_outputs,
|
|
50
135
|
)
|
|
51
136
|
from workbench_constants import PHASES
|
|
52
|
-
from
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
"coverage.json",
|
|
58
|
-
"scan-manifest.json",
|
|
59
|
-
"report.md",
|
|
60
|
-
"threatmodel.md",
|
|
61
|
-
"report.html",
|
|
62
|
-
"exports/results.sarif",
|
|
137
|
+
from workbench_coverage_receipts import (
|
|
138
|
+
_deferred_candidate_id as _deferred_candidate_id,
|
|
139
|
+
)
|
|
140
|
+
from workbench_coverage_receipts import (
|
|
141
|
+
_generic_surface_updates as _generic_surface_updates,
|
|
63
142
|
)
|
|
143
|
+
|
|
64
144
|
_PUBLICATION_FOLLOW_UP_WARNING = (
|
|
65
145
|
"Saved scan evidence remains on disk; result publication needs follow-up:"
|
|
66
146
|
)
|
|
@@ -124,284 +204,6 @@ def refresh_completed_scan(
|
|
|
124
204
|
return db.scan_context(connection, scan["id"])
|
|
125
205
|
|
|
126
206
|
|
|
127
|
-
def _encoded(value: Any) -> bytes:
|
|
128
|
-
return json.dumps(
|
|
129
|
-
value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
|
|
130
|
-
).encode()
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _digest(value: Any) -> str:
|
|
134
|
-
return hashlib.sha256(_encoded(value)).hexdigest()
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def _children(scan_dir: Path, relative: str) -> list[str]:
|
|
138
|
-
cursor = scan_dir
|
|
139
|
-
for part in Path(relative).parts:
|
|
140
|
-
if part in {"..", "."}:
|
|
141
|
-
return []
|
|
142
|
-
cursor = cursor / part
|
|
143
|
-
try:
|
|
144
|
-
if not stat.S_ISDIR(cursor.lstat().st_mode):
|
|
145
|
-
return []
|
|
146
|
-
except FileNotFoundError:
|
|
147
|
-
return []
|
|
148
|
-
return sorted(child.name for child in cursor.iterdir())
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
152
|
-
return max(
|
|
153
|
-
(
|
|
154
|
-
worker
|
|
155
|
-
for worker in workers
|
|
156
|
-
if worker["kind"] == "dedup"
|
|
157
|
-
and worker["status"] == "succeeded"
|
|
158
|
-
and worker["result_manifest_path"]
|
|
159
|
-
),
|
|
160
|
-
key=lambda worker: (worker["completed_at"] or "", worker["id"]),
|
|
161
|
-
default=None,
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
|
|
166
|
-
return [
|
|
167
|
-
f"{directory}/{name}"
|
|
168
|
-
for name in _children(scan_dir, directory)
|
|
169
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
170
|
-
]
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
|
|
174
|
-
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
175
|
-
attempts = (output.parent if output.name == "output" else output) / "attempts"
|
|
176
|
-
archived = [
|
|
177
|
-
((attempts / name).as_posix(), int(name.split("-")[1]))
|
|
178
|
-
for name in _children(scan_dir, attempts.as_posix())
|
|
179
|
-
if re.fullmatch(r"attempt-\d+", name)
|
|
180
|
-
]
|
|
181
|
-
attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
|
|
182
|
-
if worker["kind"] == "discovery" and not attempt:
|
|
183
|
-
attempt = max((attempt for _, attempt in archived), default=0) + 1
|
|
184
|
-
return [(output.as_posix(), attempt), *archived]
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
188
|
-
latest_reducer = _latest_successful_reducer(workers)
|
|
189
|
-
yield "checkpoint-head.json", None
|
|
190
|
-
for directory in ("checkpoint-heads", "checkpoints"):
|
|
191
|
-
yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
|
|
192
|
-
for worker in workers:
|
|
193
|
-
if worker["kind"] not in {"dedup", "discovery"}:
|
|
194
|
-
continue
|
|
195
|
-
try:
|
|
196
|
-
outputs = _worker_outputs(scan_dir, worker)
|
|
197
|
-
except (TypeError, ValueError):
|
|
198
|
-
continue
|
|
199
|
-
for directory, _ in outputs:
|
|
200
|
-
if worker["kind"] == "discovery":
|
|
201
|
-
yield f"{directory}/checkpoint-head.json", worker["kind"]
|
|
202
|
-
yield from (
|
|
203
|
-
(path, worker["kind"])
|
|
204
|
-
for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
|
|
205
|
-
)
|
|
206
|
-
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
207
|
-
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
208
|
-
yield f"{directory}/result.json", worker["kind"]
|
|
209
|
-
yield from ((path, worker["kind"]) for path in checkpoint_paths)
|
|
210
|
-
if worker["result_manifest_path"] and (
|
|
211
|
-
worker["kind"] == "discovery"
|
|
212
|
-
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
213
|
-
):
|
|
214
|
-
try:
|
|
215
|
-
yield (
|
|
216
|
-
Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
|
|
217
|
-
worker["kind"],
|
|
218
|
-
)
|
|
219
|
-
except ValueError:
|
|
220
|
-
continue
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
224
|
-
path = Path(relative)
|
|
225
|
-
if path.name == "checkpoint-head.json":
|
|
226
|
-
return path.parent
|
|
227
|
-
if path.parent.name == "checkpoint-heads":
|
|
228
|
-
return path.parent.parent
|
|
229
|
-
return None
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
def _capture_saved_source(
|
|
233
|
-
scan_dir: Path,
|
|
234
|
-
relative: str,
|
|
235
|
-
scan_id: str,
|
|
236
|
-
*,
|
|
237
|
-
kind: str | None = None,
|
|
238
|
-
snapshot_head: bool = True,
|
|
239
|
-
write: bool = True,
|
|
240
|
-
) -> dict[str, tuple[str, int]]:
|
|
241
|
-
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
|
|
242
|
-
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
|
|
243
|
-
return {relative: (digest, observed)}
|
|
244
|
-
head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
245
|
-
observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
|
|
246
|
-
directory = Path(relative).parent
|
|
247
|
-
selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
|
|
248
|
-
_, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
|
|
249
|
-
digest = _digest(observation)
|
|
250
|
-
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
|
|
251
|
-
# Capture the selected file even if the worker created it after directory enumeration.
|
|
252
|
-
if write and not (scan_dir / snapshot).exists():
|
|
253
|
-
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
254
|
-
return {
|
|
255
|
-
snapshot: (digest, int(observation["observedAtNs"])),
|
|
256
|
-
selected: (selected_digest, selected_time),
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def _is_source_order_snapshot(relative: str) -> bool:
|
|
261
|
-
path = Path(relative)
|
|
262
|
-
return path.parent == Path("source-order") and bool(
|
|
263
|
-
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
264
|
-
)
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _read_saved_result(
|
|
268
|
-
scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
|
|
269
|
-
) -> tuple[dict[str, Any], str, int]:
|
|
270
|
-
draft, _, metadata = _read_scan_local_json_with_metadata(
|
|
271
|
-
scan_dir, relative, "Saved scan checkpoint"
|
|
272
|
-
)
|
|
273
|
-
directory = _checkpoint_head_directory(relative)
|
|
274
|
-
if directory is not None:
|
|
275
|
-
checkpoint = draft.get("checkpoint")
|
|
276
|
-
if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
|
|
277
|
-
raise ContractError("checkpoint head does not name a saved checkpoint")
|
|
278
|
-
_read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
|
|
279
|
-
if Path(relative).name == "checkpoint-head.json":
|
|
280
|
-
return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
|
|
281
|
-
observed = draft.get("observedAtNs")
|
|
282
|
-
if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
|
|
283
|
-
raise ContractError("checkpoint head has no observation time")
|
|
284
|
-
return draft, _digest(draft), int(observed)
|
|
285
|
-
if draft.get("scanId") != scan_id:
|
|
286
|
-
raise ContractError("checkpoint belongs to a different scan")
|
|
287
|
-
if not _is_source_order_snapshot(relative) and (
|
|
288
|
-
not isinstance(draft.get("findings"), list)
|
|
289
|
-
or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
|
|
290
|
-
):
|
|
291
|
-
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
292
|
-
return draft, _digest(draft), metadata.st_mtime_ns
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
|
|
296
|
-
times: dict[str, int] = {}
|
|
297
|
-
snapshots = [path for path in sources if _is_source_order_snapshot(path)]
|
|
298
|
-
for path in snapshots:
|
|
299
|
-
record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
|
|
300
|
-
if digest != sources[path] or not isinstance(record.get("sources"), dict):
|
|
301
|
-
raise ContractError("saved source ordering changed after the scan stopped")
|
|
302
|
-
for relative, observation in record["sources"].items():
|
|
303
|
-
if (
|
|
304
|
-
not isinstance(observation, dict)
|
|
305
|
-
or relative not in sources
|
|
306
|
-
or observation.get("digest") != sources[relative]
|
|
307
|
-
or not isinstance(observation.get("observedAtNs"), str)
|
|
308
|
-
or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
|
|
309
|
-
):
|
|
310
|
-
raise ContractError("saved source ordering does not match its frozen sources")
|
|
311
|
-
observed = int(observation["observedAtNs"])
|
|
312
|
-
if relative in times and times[relative] != observed:
|
|
313
|
-
raise ContractError("saved source ordering has conflicting observations")
|
|
314
|
-
times[relative] = observed
|
|
315
|
-
if snapshots and sources.keys() - set(snapshots) - times.keys():
|
|
316
|
-
raise ContractError("saved source ordering is incomplete")
|
|
317
|
-
return times
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
def _freeze_source_times(
|
|
321
|
-
scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
|
|
322
|
-
) -> None:
|
|
323
|
-
# Identical result rewrites must not change the order of frozen review evidence.
|
|
324
|
-
observations = {
|
|
325
|
-
path: {"digest": digest, "observedAtNs": str(times[path])}
|
|
326
|
-
for path, digest in sources.items()
|
|
327
|
-
if not _is_source_order_snapshot(path)
|
|
328
|
-
}
|
|
329
|
-
if not observations:
|
|
330
|
-
return
|
|
331
|
-
record = {"scanId": scan_id, "sources": observations}
|
|
332
|
-
digest = _digest(record)
|
|
333
|
-
path = f"source-order/{digest}.json"
|
|
334
|
-
if not (scan_dir / path).exists():
|
|
335
|
-
write_scan_local_bytes(scan_dir, path, _encoded(record))
|
|
336
|
-
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
|
|
337
|
-
raise ContractError("saved source ordering does not match its digest")
|
|
338
|
-
sources[path] = digest
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
def _parent_scan_draft(
|
|
342
|
-
scan_id: str,
|
|
343
|
-
parent_scan: dict[str, Any],
|
|
344
|
-
findings: dict[str, Any],
|
|
345
|
-
coverage: dict[str, Any],
|
|
346
|
-
) -> dict[str, Any]:
|
|
347
|
-
parent = {
|
|
348
|
-
"scanId": scan_id,
|
|
349
|
-
"findings": findings.get("findings"),
|
|
350
|
-
"coverage": coverage,
|
|
351
|
-
**{
|
|
352
|
-
key: parent_scan[key]
|
|
353
|
-
for key in ("scope", "threatModel", "complete")
|
|
354
|
-
if key in parent_scan
|
|
355
|
-
},
|
|
356
|
-
}
|
|
357
|
-
if not isinstance(parent["findings"], list):
|
|
358
|
-
raise ContractError("Saved parent draft has no findings array")
|
|
359
|
-
return parent
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
def _read_saved_parent_result(
|
|
363
|
-
scan_dir: Path, scan_id: str
|
|
364
|
-
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
365
|
-
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
|
|
366
|
-
findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
|
|
367
|
-
coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
|
|
368
|
-
parent_scan = manifest.get("scan")
|
|
369
|
-
if not isinstance(parent_scan, dict):
|
|
370
|
-
raise ContractError("Saved parent manifest has no scan object")
|
|
371
|
-
if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
|
|
372
|
-
parent_scan.get("id", scan_id) != scan_id
|
|
373
|
-
or findings.get("scanId", scan_id) != scan_id
|
|
374
|
-
or coverage.get("scanId", scan_id) != scan_id
|
|
375
|
-
):
|
|
376
|
-
raise ContractError("Saved parent documents belong to a different scan")
|
|
377
|
-
return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
def _source_digests(value: Any, error: str) -> dict[str, str]:
|
|
381
|
-
if not isinstance(value, dict) or not all(
|
|
382
|
-
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
383
|
-
):
|
|
384
|
-
raise ContractError(error)
|
|
385
|
-
return value
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
|
|
389
|
-
if isinstance(value, dict) and isinstance(value.get("sources"), dict):
|
|
390
|
-
sources = _source_digests(
|
|
391
|
-
value["sources"], "Saved stopped-scan source digests are malformed."
|
|
392
|
-
)
|
|
393
|
-
model_source = value.get("threatModelSource")
|
|
394
|
-
if not isinstance(model_source, str) or model_source not in sources:
|
|
395
|
-
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
|
|
396
|
-
return sources, model_source
|
|
397
|
-
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
|
|
401
|
-
state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
|
|
402
|
-
return json.dumps(state, sort_keys=True)
|
|
403
|
-
|
|
404
|
-
|
|
405
207
|
def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
406
208
|
try:
|
|
407
209
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
@@ -446,6 +248,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
446
248
|
manifest_scan.get("preservedSources", {}),
|
|
447
249
|
"Published scan source digests are malformed.",
|
|
448
250
|
)
|
|
251
|
+
_, pending_recovery = _retained_checkpoint_state(scan)
|
|
252
|
+
if pending_recovery:
|
|
253
|
+
return True
|
|
254
|
+
published_heads = manifest_scan.get("preservedCheckpointHeads")
|
|
255
|
+
if published_heads is None:
|
|
256
|
+
published_heads = _frozen_checkpoint_heads(
|
|
257
|
+
scan_dir, scan["id"], published_sources, skip_unreadable=True
|
|
258
|
+
)
|
|
259
|
+
elif "." not in published_heads:
|
|
260
|
+
# Older publications recorded root observations without selecting a root head.
|
|
261
|
+
legacy_heads = _frozen_checkpoint_heads(
|
|
262
|
+
scan_dir, scan["id"], published_sources, skip_unreadable=True
|
|
263
|
+
)
|
|
264
|
+
if "." in legacy_heads:
|
|
265
|
+
published_heads = {**published_heads, ".": legacy_heads["."]}
|
|
266
|
+
checkpoint_heads = dict(published_heads)
|
|
449
267
|
current_sources = dict(published_sources)
|
|
450
268
|
paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
|
|
451
269
|
for path in paths:
|
|
@@ -457,23 +275,53 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
457
275
|
kind=paths[path],
|
|
458
276
|
snapshot_head=path not in published_sources,
|
|
459
277
|
write=False,
|
|
278
|
+
checkpoint_heads=checkpoint_heads,
|
|
460
279
|
)
|
|
461
280
|
current_sources.update({path: value[0] for path, value in captured.items()})
|
|
281
|
+
directory = Path(path).parent.as_posix()
|
|
282
|
+
previous_head = published_heads.get(directory)
|
|
283
|
+
if (
|
|
284
|
+
Path(path).name == "checkpoint-head.json"
|
|
285
|
+
and previous_head in captured
|
|
286
|
+
and Path(previous_head).parent.name == "checkpoints"
|
|
287
|
+
):
|
|
288
|
+
observed = captured[checkpoint_heads[directory]][1]
|
|
289
|
+
if observed == _legacy_checkpoint_observed_at(
|
|
290
|
+
scan_dir, scan["id"], published_sources, previous_head
|
|
291
|
+
):
|
|
292
|
+
checkpoint_heads[directory] = previous_head
|
|
462
293
|
except (ContractError, OSError, ValueError):
|
|
463
|
-
|
|
464
|
-
return current_sources != published_sources
|
|
294
|
+
_restore_checkpoint_head(checkpoint_heads, published_heads, published_sources, path)
|
|
295
|
+
return current_sources != published_sources or checkpoint_heads != published_heads
|
|
465
296
|
except (ContractError, OSError, SystemExit, ValueError):
|
|
466
297
|
return False
|
|
467
298
|
|
|
468
299
|
|
|
469
|
-
def _recovery_source_digests(
|
|
300
|
+
def _recovery_source_digests(
|
|
301
|
+
db: Any, connection: Any, scan: Any
|
|
302
|
+
) -> tuple[dict[str, str], bool, dict[str, str | None]]:
|
|
470
303
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
471
304
|
frozen_sources: dict[str, str] | None = None
|
|
472
305
|
include_parent = True
|
|
473
306
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
474
307
|
if raw_frozen_sources is not None:
|
|
475
|
-
|
|
308
|
+
retained = json.loads(raw_frozen_sources)
|
|
309
|
+
frozen_sources, model_source = _retained_source_state(retained)
|
|
310
|
+
if isinstance(retained.get("sources"), dict) and retained.get("sourceTimes") is not None:
|
|
311
|
+
_restore_frozen_model_capture(
|
|
312
|
+
scan_dir, scan["id"], frozen_sources, retained["sourceTimes"], model_source
|
|
313
|
+
)
|
|
314
|
+
_freeze_source_times(
|
|
315
|
+
scan_dir,
|
|
316
|
+
scan["id"],
|
|
317
|
+
frozen_sources,
|
|
318
|
+
retained["sourceTimes"],
|
|
319
|
+
retained.get("parentContext"),
|
|
320
|
+
)
|
|
476
321
|
include_parent = False
|
|
322
|
+
checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
|
|
323
|
+
if pending_recovery:
|
|
324
|
+
return frozen_sources, False, checkpoint_heads
|
|
477
325
|
|
|
478
326
|
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
479
327
|
if manifest_path is not None:
|
|
@@ -485,9 +333,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
485
333
|
manifest_scan = manifest.get("scan")
|
|
486
334
|
if not isinstance(manifest_scan, dict):
|
|
487
335
|
raise ContractError("Saved scan manifest has no scan object")
|
|
488
|
-
if scan["seal_manifest_digest"] is not None or (
|
|
489
|
-
manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
|
|
490
|
-
):
|
|
336
|
+
if scan["seal_manifest_digest"] is not None or manifest_is_sealed(manifest_scan):
|
|
491
337
|
if "preservedSources" in manifest_scan:
|
|
492
338
|
published_sources = _source_digests(
|
|
493
339
|
manifest_scan["preservedSources"],
|
|
@@ -510,9 +356,14 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
510
356
|
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
511
357
|
(scan["id"],),
|
|
512
358
|
).fetchall()
|
|
359
|
+
previous_heads = checkpoint_heads
|
|
360
|
+
if previous_heads is None:
|
|
361
|
+
previous_heads = _frozen_checkpoint_heads(scan_dir, scan["id"], frozen_sources or {})
|
|
362
|
+
checkpoint_heads = dict(previous_heads)
|
|
513
363
|
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
514
364
|
recovery_sources = dict(frozen_sources or {})
|
|
515
|
-
|
|
365
|
+
parent_context: dict[str, str] = {}
|
|
366
|
+
source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources, parent_context)
|
|
516
367
|
for relative, expected_digest in recovery_sources.items():
|
|
517
368
|
try:
|
|
518
369
|
_, digest, observed = _read_saved_result(
|
|
@@ -525,18 +376,52 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
525
376
|
if not _is_source_order_snapshot(relative):
|
|
526
377
|
source_times.setdefault(relative, observed)
|
|
527
378
|
|
|
379
|
+
snapshot_error = None
|
|
380
|
+
captured_records = {}
|
|
528
381
|
for relative in paths.keys() - recovery_sources.keys():
|
|
529
382
|
try:
|
|
530
|
-
captured = _capture_saved_source(
|
|
383
|
+
captured = _capture_saved_source(
|
|
384
|
+
scan_dir,
|
|
385
|
+
relative,
|
|
386
|
+
scan["id"],
|
|
387
|
+
kind=paths[relative],
|
|
388
|
+
checkpoint_heads=checkpoint_heads,
|
|
389
|
+
)
|
|
390
|
+
except _CheckpointSnapshotError as exc:
|
|
391
|
+
snapshot_error = snapshot_error or exc
|
|
392
|
+
captured = exc.captured
|
|
393
|
+
captured_records.update(exc.records)
|
|
531
394
|
except (ContractError, OSError, ValueError):
|
|
395
|
+
_restore_checkpoint_head(
|
|
396
|
+
checkpoint_heads, previous_heads, frozen_sources or {}, relative
|
|
397
|
+
)
|
|
532
398
|
continue
|
|
399
|
+
if relative == "checkpoint-head.json":
|
|
400
|
+
parent_context["head"] = next(iter(captured))
|
|
533
401
|
for path, (digest, observed) in captured.items():
|
|
534
402
|
if path in recovery_sources and recovery_sources[path] != digest:
|
|
535
403
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
536
404
|
recovery_sources[path] = digest
|
|
537
405
|
source_times.setdefault(path, observed)
|
|
538
|
-
|
|
539
|
-
|
|
406
|
+
try:
|
|
407
|
+
if snapshot_error is not None:
|
|
408
|
+
raise snapshot_error
|
|
409
|
+
_freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
|
|
410
|
+
except (ContractError, OSError) as exc:
|
|
411
|
+
failure = _CheckpointSnapshotError(
|
|
412
|
+
str(exc),
|
|
413
|
+
{
|
|
414
|
+
path: (digest, source_times[path])
|
|
415
|
+
for path, digest in recovery_sources.items()
|
|
416
|
+
if not _is_source_order_snapshot(path)
|
|
417
|
+
},
|
|
418
|
+
)
|
|
419
|
+
failure.records = captured_records
|
|
420
|
+
failure.heads = checkpoint_heads
|
|
421
|
+
failure.include_parent = include_parent
|
|
422
|
+
failure.parent_context = parent_context
|
|
423
|
+
raise failure from exc
|
|
424
|
+
return recovery_sources, include_parent, checkpoint_heads
|
|
540
425
|
|
|
541
426
|
|
|
542
427
|
def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
|
|
@@ -556,1847 +441,44 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
556
441
|
return True
|
|
557
442
|
return _saved_results_changed(db, connection, scan)
|
|
558
443
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
)
|
|
593
|
-
]
|
|
594
|
-
)
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
def _worker_candidate_key(
|
|
598
|
-
worker_id: str, candidate_id: str, finding: dict[str, Any]
|
|
599
|
-
) -> tuple[str, str, Any, Any, Any]:
|
|
600
|
-
"""Identify one worker-local candidate without merging unrelated locations."""
|
|
601
|
-
provenance = finding.get("provenance")
|
|
602
|
-
identity = (
|
|
603
|
-
provenance.get("preservedIdentity", finding.get("identity"))
|
|
604
|
-
if isinstance(provenance, dict)
|
|
605
|
-
else finding.get("identity")
|
|
606
|
-
)
|
|
607
|
-
if not isinstance(identity, dict):
|
|
608
|
-
normalized = dict(finding)
|
|
609
|
-
_ensure_finding_identity(normalized)
|
|
610
|
-
identity = normalized.get("identity")
|
|
611
|
-
anchor = identity.get("anchor") if isinstance(identity, dict) else None
|
|
612
|
-
instance = identity.get("instance") if isinstance(identity, dict) else None
|
|
613
|
-
return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
|
|
617
|
-
"""Return substantive finding content without generated identity or provenance."""
|
|
618
|
-
return {
|
|
619
|
-
key: value
|
|
620
|
-
for key, value in finding.items()
|
|
621
|
-
if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:
|
|
626
|
-
if not isinstance(finding, dict) or "identity" in finding:
|
|
627
|
-
return
|
|
628
|
-
if candidate_only and not finding_candidate_id(finding):
|
|
629
|
-
return
|
|
630
|
-
extensions = finding.get("extensions")
|
|
631
|
-
source = str(
|
|
632
|
-
(extensions.get("candidateId") if isinstance(extensions, dict) else None)
|
|
633
|
-
or finding.get("title")
|
|
634
|
-
or "finding"
|
|
635
|
-
)
|
|
636
|
-
anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
|
|
637
|
-
finding["identity"] = {"anchor": anchor}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
|
|
641
|
-
"""Yield canonical and historical findings without trusting candidate IDs."""
|
|
642
|
-
pending = [finding]
|
|
643
|
-
seen: set[int] = set()
|
|
644
|
-
while pending:
|
|
645
|
-
current = pending.pop()
|
|
646
|
-
marker = id(current)
|
|
647
|
-
if marker in seen:
|
|
648
|
-
continue
|
|
649
|
-
seen.add(marker)
|
|
650
|
-
yield current
|
|
651
|
-
provenance = current.get("provenance")
|
|
652
|
-
if not isinstance(provenance, dict):
|
|
653
|
-
continue
|
|
654
|
-
previous = provenance.get("previousFindings")
|
|
655
|
-
if isinstance(previous, list):
|
|
656
|
-
pending.extend(item for item in reversed(previous) if isinstance(item, dict))
|
|
657
|
-
sources = provenance.get("sourceFindings")
|
|
658
|
-
if isinstance(sources, list):
|
|
659
|
-
pending.extend(
|
|
660
|
-
source["finding"]
|
|
661
|
-
for source in reversed(sources)
|
|
662
|
-
if isinstance(source, dict) and isinstance(source.get("finding"), dict)
|
|
663
|
-
)
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
|
|
667
|
-
rows = coverage.get("deferred", [])
|
|
668
|
-
return rows if isinstance(rows, list) else []
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
672
|
-
if draft.get("complete") is False:
|
|
673
|
-
return []
|
|
674
|
-
coverage = draft["coverage"]
|
|
675
|
-
rows = coverage.get("resolvedDeferred", [])
|
|
676
|
-
try:
|
|
677
|
-
# Invalid closure metadata cannot discard the evidence it names.
|
|
678
|
-
_validate_schema_node(rows, schema, "coverage.resolvedDeferred")
|
|
679
|
-
_validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
|
|
680
|
-
except ContractError:
|
|
681
|
-
return []
|
|
682
|
-
return rows
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
def _merge_tied_parent_observations(
|
|
686
|
-
current: dict[str, Any], previous: dict[str, Any]
|
|
687
|
-
) -> dict[str, Any]:
|
|
688
|
-
merged = copy.deepcopy(current)
|
|
689
|
-
for finding in previous["findings"]:
|
|
690
|
-
if finding not in merged["findings"]:
|
|
691
|
-
merged["findings"].append(copy.deepcopy(finding))
|
|
692
|
-
coverage = merged["coverage"]
|
|
693
|
-
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
694
|
-
rows = previous["coverage"].get(field, [])
|
|
695
|
-
output = coverage.setdefault(field, [])
|
|
696
|
-
if isinstance(rows, list) and isinstance(output, list):
|
|
697
|
-
for row in rows:
|
|
698
|
-
if row not in output:
|
|
699
|
-
output.append(copy.deepcopy(row))
|
|
700
|
-
pending_ids = {
|
|
701
|
-
identity
|
|
702
|
-
for row in _deferred_rows(coverage)
|
|
703
|
-
if isinstance(row, dict)
|
|
704
|
-
for identity in (row.get("id"), row.get("candidateId"))
|
|
705
|
-
if isinstance(identity, str)
|
|
706
|
-
}
|
|
707
|
-
closure_schema = _read_json(
|
|
708
|
-
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
709
|
-
)["properties"]["resolvedDeferred"]
|
|
710
|
-
closures = {}
|
|
711
|
-
for observed in (current, previous):
|
|
712
|
-
for row in _resolved_deferred_rows(observed, closure_schema):
|
|
713
|
-
if row["id"] not in pending_ids:
|
|
714
|
-
closures.setdefault(row["id"], copy.deepcopy(row))
|
|
715
|
-
coverage.pop("resolvedDeferred", None)
|
|
716
|
-
if closures:
|
|
717
|
-
coverage["resolvedDeferred"] = list(closures.values())
|
|
718
|
-
if current.get("complete") is False or previous.get("complete") is False:
|
|
719
|
-
merged["complete"] = False
|
|
720
|
-
coverage.pop("completeness", None)
|
|
721
|
-
return merged
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
def _deferred_candidate_id(
|
|
725
|
-
row: dict[str, Any],
|
|
726
|
-
owner: str | None,
|
|
727
|
-
ambiguous_deferred: set[tuple[str | None, str]],
|
|
728
|
-
) -> str | None:
|
|
729
|
-
identity = row.get("candidateId") or row.get("id")
|
|
730
|
-
if not isinstance(identity, str):
|
|
731
|
-
return None
|
|
732
|
-
if (owner, identity) in ambiguous_deferred and not any(
|
|
733
|
-
key in row for key in ("candidateId", "candidate", "finding")
|
|
734
|
-
):
|
|
735
|
-
return None
|
|
736
|
-
return identity
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
def _generic_surface_updates(
|
|
740
|
-
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
741
|
-
source_order: dict[str, tuple[int, int]],
|
|
742
|
-
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
743
|
-
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
744
|
-
resolved_candidates: dict[tuple[str | None, str], str],
|
|
745
|
-
reopened_generic: set[tuple[str | None, str]],
|
|
746
|
-
surface_schema: dict[str, Any],
|
|
747
|
-
deferred_rows: dict[str, list[Any]],
|
|
748
|
-
ambiguous_deferred: set[tuple[str | None, str]],
|
|
749
|
-
*,
|
|
750
|
-
surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
|
|
751
|
-
eligible_receipt_sources: set[str],
|
|
752
|
-
receipt_root: Path | None = None,
|
|
753
|
-
receipt_warnings: list[str] | None = None,
|
|
754
|
-
fallback_surfaces: list[dict[str, Any]] | None = None,
|
|
755
|
-
) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
|
|
756
|
-
if not closed_deferred and not reopened_generic and receipt_root is None:
|
|
757
|
-
return set(), []
|
|
758
|
-
|
|
759
|
-
def receipt_error(ref: Any) -> ContractError | None:
|
|
760
|
-
if receipt_root is None or not isinstance(ref, str):
|
|
761
|
-
return ContractError("Saved coverage receipt is unavailable.")
|
|
762
|
-
try:
|
|
763
|
-
descriptor = open_scan_local_file_descriptor(
|
|
764
|
-
receipt_root, ref, "Saved coverage receipt"
|
|
765
|
-
)
|
|
766
|
-
os.close(descriptor)
|
|
767
|
-
except ContractError as exc:
|
|
768
|
-
return exc
|
|
769
|
-
return None
|
|
770
|
-
|
|
771
|
-
def linked(row: dict[str, Any], identity: str | None) -> bool:
|
|
772
|
-
surface_ids = row.get("surfaceIds", [])
|
|
773
|
-
return identity is not None and (
|
|
774
|
-
row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
|
|
775
|
-
)
|
|
776
|
-
|
|
777
|
-
saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
|
|
778
|
-
for relative, draft, owner in sources:
|
|
779
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
780
|
-
for row in rows if isinstance(rows, list) else []:
|
|
781
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
782
|
-
saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
|
|
783
|
-
replaced: set[int] = set()
|
|
784
|
-
updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
|
|
785
|
-
# Current selections across all surfaces take precedence over historical refs.
|
|
786
|
-
receipt_paths: dict[str, str] = {}
|
|
787
|
-
retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
|
|
788
|
-
if receipt_root is not None:
|
|
789
|
-
receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
|
|
790
|
-
for key, matches in saved_surfaces.items():
|
|
791
|
-
retained = [
|
|
792
|
-
match for match in matches if surface_is_retained(match[0], key[0], match[1])
|
|
793
|
-
]
|
|
794
|
-
if not retained:
|
|
795
|
-
continue
|
|
796
|
-
latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
|
|
797
|
-
retained_surfaces[key] = latest_surface
|
|
798
|
-
receipt_surfaces.extend(
|
|
799
|
-
(key[0], row)
|
|
800
|
-
for relative, row in retained
|
|
801
|
-
if source_order[relative] == source_order[latest_source]
|
|
802
|
-
)
|
|
803
|
-
for relative, draft, owner in sources:
|
|
804
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
805
|
-
for row in rows if isinstance(rows, list) else []:
|
|
806
|
-
if (
|
|
807
|
-
isinstance(row, dict)
|
|
808
|
-
and "id" not in row
|
|
809
|
-
and surface_is_retained(relative, owner, row)
|
|
810
|
-
):
|
|
811
|
-
receipt_surfaces.append((owner, row))
|
|
812
|
-
receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
|
|
813
|
-
for owner, latest_surface in receipt_surfaces:
|
|
814
|
-
# Recovery repairs disposition and receipts before validating row
|
|
815
|
-
# metadata. A row discarded there cannot reserve a receipt spelling.
|
|
816
|
-
metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
|
|
817
|
-
metadata.setdefault("id", _saved_coverage_id(metadata))
|
|
818
|
-
metadata.update(disposition="needs_follow_up", receiptRefs=[])
|
|
819
|
-
try:
|
|
820
|
-
_require_str(metadata, "id", "coverage.surfaces")
|
|
821
|
-
_validate_schema_node(metadata, surface_schema, "coverage.surfaces")
|
|
822
|
-
except ContractError:
|
|
823
|
-
continue
|
|
824
|
-
refs = latest_surface.get("receiptRefs", [])
|
|
825
|
-
for ref in refs if isinstance(refs, list) else []:
|
|
826
|
-
if receipt_error(ref) is None:
|
|
827
|
-
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
828
|
-
receipt_paths.setdefault(normalized.lower(), normalized)
|
|
829
|
-
for relative, draft, owner in sources:
|
|
830
|
-
closed_ids = {
|
|
831
|
-
identity
|
|
832
|
-
for (saved_owner, identity), (_, _, source) in closed_deferred.items()
|
|
833
|
-
if saved_owner == owner and source == relative
|
|
834
|
-
}
|
|
835
|
-
reopened_ids = {
|
|
836
|
-
identity
|
|
837
|
-
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
838
|
-
if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
|
|
839
|
-
}
|
|
840
|
-
if not closed_ids and not reopened_ids and receipt_root is None:
|
|
841
|
-
continue
|
|
842
|
-
current = draft["coverage"].get("surfaces", [])
|
|
843
|
-
for surface in current if isinstance(current, list) else []:
|
|
844
|
-
if not isinstance(surface, dict):
|
|
845
|
-
continue
|
|
846
|
-
reopening = surface.get("disposition") == "needs_follow_up"
|
|
847
|
-
work_ids = reopened_ids if reopening else closed_ids
|
|
848
|
-
repair_only = not work_ids
|
|
849
|
-
if repair_only and receipt_root is None:
|
|
850
|
-
continue
|
|
851
|
-
identity = surface.get("id")
|
|
852
|
-
if not isinstance(identity, str):
|
|
853
|
-
continue
|
|
854
|
-
matches = saved_surfaces[(owner, identity)]
|
|
855
|
-
# Receipt repair alone cannot close or reopen a saved surface.
|
|
856
|
-
if repair_only and any(
|
|
857
|
-
row.get("disposition") != surface.get("disposition") for _, row in matches
|
|
858
|
-
):
|
|
859
|
-
continue
|
|
860
|
-
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
861
|
-
# A closure cannot identify which of those observations it replaces.
|
|
862
|
-
by_source = dict(matches)
|
|
863
|
-
if any(row != by_source[saved_path] for saved_path, row in matches):
|
|
864
|
-
continue
|
|
865
|
-
if any(
|
|
866
|
-
"candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
|
|
867
|
-
):
|
|
868
|
-
continue
|
|
869
|
-
if any(
|
|
870
|
-
row is not surface
|
|
871
|
-
and source_order[saved_path] >= source_order[relative]
|
|
872
|
-
and row.get("disposition") != surface.get("disposition")
|
|
873
|
-
for saved_path, row in matches
|
|
874
|
-
):
|
|
875
|
-
continue
|
|
876
|
-
|
|
877
|
-
if not reopening and any(
|
|
878
|
-
saved_owner == owner
|
|
879
|
-
and (
|
|
880
|
-
not isinstance(row.get("id") or row.get("candidateId"), str)
|
|
881
|
-
or (
|
|
882
|
-
isinstance(row.get("id"), str)
|
|
883
|
-
and (owner, row["id"]) in ambiguous_deferred
|
|
884
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
885
|
-
)
|
|
886
|
-
)
|
|
887
|
-
and linked(row, identity)
|
|
888
|
-
for saved_path, _, saved_owner in sources
|
|
889
|
-
for row in deferred_rows[saved_path]
|
|
890
|
-
if isinstance(row, dict)
|
|
891
|
-
):
|
|
892
|
-
continue
|
|
893
|
-
if not reopening and any(
|
|
894
|
-
saved_owner == owner
|
|
895
|
-
and (owner, deferred_id) not in closed_deferred
|
|
896
|
-
and (
|
|
897
|
-
(candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
|
|
898
|
-
or (owner, candidate_id) not in resolved_candidates
|
|
899
|
-
)
|
|
900
|
-
and linked(row, identity)
|
|
901
|
-
for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
|
|
902
|
-
):
|
|
903
|
-
continue
|
|
904
|
-
# An accepted checkpoint can update a saved surface by ID without
|
|
905
|
-
# optional surfaceIds links on its generic task.
|
|
906
|
-
receipt_matches = [
|
|
907
|
-
(saved_path, row)
|
|
908
|
-
for saved_path, row in matches
|
|
909
|
-
if receipt_root is None
|
|
910
|
-
or saved_path in eligible_receipt_sources
|
|
911
|
-
or surface_is_retained(saved_path, owner, row)
|
|
912
|
-
]
|
|
913
|
-
if not receipt_matches:
|
|
914
|
-
continue
|
|
915
|
-
latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
|
|
916
|
-
target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
|
|
917
|
-
if target is None:
|
|
918
|
-
continue
|
|
919
|
-
update = copy.deepcopy(target)
|
|
920
|
-
# Historical evidence can repair a retained row without promoting
|
|
921
|
-
# an unselected observation's label, notes, or other metadata.
|
|
922
|
-
refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
|
|
923
|
-
update["receiptRefs"] = refs
|
|
924
|
-
replaced_receipt = False
|
|
925
|
-
replacement_paths = dict(receipt_paths)
|
|
926
|
-
if isinstance(refs, list):
|
|
927
|
-
repaired = (
|
|
928
|
-
not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
|
|
929
|
-
)
|
|
930
|
-
if repaired:
|
|
931
|
-
for ref in refs:
|
|
932
|
-
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
933
|
-
key = normalized.lower()
|
|
934
|
-
if key in replacement_paths and replacement_paths[key] != normalized:
|
|
935
|
-
repaired = False
|
|
936
|
-
break
|
|
937
|
-
replacement_paths[key] = normalized
|
|
938
|
-
for _, row in receipt_matches:
|
|
939
|
-
previous_refs = row.get("receiptRefs", [])
|
|
940
|
-
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
941
|
-
if ref not in refs:
|
|
942
|
-
failure = receipt_error(ref) if repaired else None
|
|
943
|
-
replaceable = failure is not None and isinstance(
|
|
944
|
-
failure.__cause__, FileNotFoundError
|
|
945
|
-
)
|
|
946
|
-
if repaired and isinstance(ref, str):
|
|
947
|
-
try:
|
|
948
|
-
normalized = _require_portable_relative_path(
|
|
949
|
-
ref, "Saved coverage receipt"
|
|
950
|
-
)
|
|
951
|
-
except ContractError as error:
|
|
952
|
-
failure = error
|
|
953
|
-
replaceable = True
|
|
954
|
-
else:
|
|
955
|
-
key = normalized.lower()
|
|
956
|
-
previous = replacement_paths.get(key)
|
|
957
|
-
if previous is not None and previous != normalized:
|
|
958
|
-
failure = ContractError(
|
|
959
|
-
f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
|
|
960
|
-
)
|
|
961
|
-
replaceable = True
|
|
962
|
-
elif not replaceable:
|
|
963
|
-
replacement_paths[key] = normalized
|
|
964
|
-
if replaceable:
|
|
965
|
-
replaced_receipt = True
|
|
966
|
-
if receipt_warnings is not None:
|
|
967
|
-
receipt_warnings.append(
|
|
968
|
-
f"Replaced invalid saved coverage receipt {ref}: {failure}"
|
|
969
|
-
)
|
|
970
|
-
continue
|
|
971
|
-
refs.append(ref)
|
|
972
|
-
if repair_only and not replaced_receipt:
|
|
973
|
-
continue
|
|
974
|
-
try:
|
|
975
|
-
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
976
|
-
except ContractError:
|
|
977
|
-
continue
|
|
978
|
-
replaced_sources = {
|
|
979
|
-
id(row)
|
|
980
|
-
for _, row in matches
|
|
981
|
-
if reopening
|
|
982
|
-
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
983
|
-
}
|
|
984
|
-
replaced.update(replaced_sources)
|
|
985
|
-
existing = next(
|
|
986
|
-
(entry for entry in updates if entry[0] == owner and entry[1] == update), None
|
|
987
|
-
)
|
|
988
|
-
if existing is None:
|
|
989
|
-
receipt_paths.update(replacement_paths)
|
|
990
|
-
updates.append((owner, update, replaced_sources))
|
|
991
|
-
else:
|
|
992
|
-
existing[2].update(replaced_sources)
|
|
993
|
-
return replaced, updates
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
@contextmanager
|
|
997
|
-
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
998
|
-
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
999
|
-
head_path = scan_dir / "checkpoint-head.json"
|
|
1000
|
-
previous = None
|
|
1001
|
-
try:
|
|
1002
|
-
head_path.lstat()
|
|
1003
|
-
except FileNotFoundError:
|
|
1004
|
-
pass
|
|
1005
|
-
else:
|
|
1006
|
-
descriptor = open_scan_local_file_descriptor(
|
|
1007
|
-
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
1008
|
-
)
|
|
1009
|
-
with os.fdopen(descriptor, "rb") as handle:
|
|
1010
|
-
metadata = os.fstat(handle.fileno())
|
|
1011
|
-
previous = (handle.read(), metadata)
|
|
1012
|
-
directories = ("checkpoints", "checkpoint-heads")
|
|
1013
|
-
previous_files = {
|
|
1014
|
-
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
1015
|
-
}
|
|
1016
|
-
try:
|
|
1017
|
-
yield
|
|
1018
|
-
except BaseException:
|
|
1019
|
-
if previous is None:
|
|
1020
|
-
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
1021
|
-
else:
|
|
1022
|
-
payload, metadata = previous
|
|
1023
|
-
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
1024
|
-
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
1025
|
-
current_files = {
|
|
1026
|
-
relative
|
|
1027
|
-
for directory in directories
|
|
1028
|
-
for relative in _checkpoint_paths(scan_dir, directory)
|
|
1029
|
-
}
|
|
1030
|
-
for relative in current_files - previous_files:
|
|
1031
|
-
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
1032
|
-
raise
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
def merge_saved_results(
|
|
1036
|
-
scan_dir: Path,
|
|
1037
|
-
scan_id: str,
|
|
1038
|
-
binding: dict[str, Any],
|
|
1039
|
-
workers: list[Any],
|
|
1040
|
-
warnings: list[str],
|
|
1041
|
-
*,
|
|
1042
|
-
stopped: bool,
|
|
1043
|
-
reason: str,
|
|
1044
|
-
frozen_source_digests: dict[str, str] | None = None,
|
|
1045
|
-
allow_frozen_legacy_parent: bool = False,
|
|
1046
|
-
frozen_model_source: str | None = None,
|
|
1047
|
-
selected_model_source: list[str] | None = None,
|
|
1048
|
-
project_worker_coverage: bool = False,
|
|
1049
|
-
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
1050
|
-
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
1051
|
-
initial_warnings = set(warnings)
|
|
1052
|
-
try:
|
|
1053
|
-
source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
|
|
1054
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1055
|
-
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
1056
|
-
parent: dict[str, Any] | None = None
|
|
1057
|
-
parent_manifest: dict[str, Any] | None = None
|
|
1058
|
-
parent_is_canonical = False
|
|
1059
|
-
parent_modified = 0
|
|
1060
|
-
if frozen_source_digests is None or allow_frozen_legacy_parent:
|
|
1061
|
-
try:
|
|
1062
|
-
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
1063
|
-
# Without an accepted head, file-authored coverage is a full replacement.
|
|
1064
|
-
parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
|
|
1065
|
-
parent_is_canonical = True
|
|
1066
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1067
|
-
if not stopped:
|
|
1068
|
-
raise
|
|
1069
|
-
if (scan_dir / "scan-manifest.json").exists():
|
|
1070
|
-
warnings.append(f"Could not read the saved parent draft: {exc}")
|
|
1071
|
-
parent_manifest = None
|
|
1072
|
-
parent = None
|
|
1073
|
-
if parent_manifest is not None and parent is not None:
|
|
1074
|
-
parent_scan = parent_manifest["scan"]
|
|
1075
|
-
try:
|
|
1076
|
-
previous_head, _, head_modified = _read_saved_result(
|
|
1077
|
-
scan_dir, "checkpoint-head.json", scan_id
|
|
1078
|
-
)
|
|
1079
|
-
except (ContractError, OSError, ValueError):
|
|
1080
|
-
head_modified = None
|
|
1081
|
-
if head_modified is not None:
|
|
1082
|
-
# A partial tool publication must not outrank its accepted head.
|
|
1083
|
-
parent_modified = min(
|
|
1084
|
-
(scan_dir / name).lstat().st_mtime_ns
|
|
1085
|
-
for name in ("findings.json", "coverage.json", "scan-manifest.json")
|
|
1086
|
-
)
|
|
1087
|
-
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
1088
|
-
head_path = scan_dir / "checkpoint-head.json"
|
|
1089
|
-
tied_observations = False
|
|
1090
|
-
if head_modified == parent_modified:
|
|
1091
|
-
previous_parent, _, _ = _read_saved_result(
|
|
1092
|
-
scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
|
|
1093
|
-
)
|
|
1094
|
-
if previous_parent != parent:
|
|
1095
|
-
# Tied observations cannot decide which pending work came last.
|
|
1096
|
-
parent = _merge_tied_parent_observations(parent, previous_parent)
|
|
1097
|
-
parent_is_canonical = False
|
|
1098
|
-
tied_observations = True
|
|
1099
|
-
payload = _encoded(parent)
|
|
1100
|
-
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
1101
|
-
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
1102
|
-
checkpoint_path = scan_dir / parent_checkpoint
|
|
1103
|
-
if not checkpoint_path.exists():
|
|
1104
|
-
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
1105
|
-
# A recovery copy must not appear newer than the review it copies.
|
|
1106
|
-
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
|
|
1107
|
-
if head_modified is None or head_modified < parent_modified or tied_observations:
|
|
1108
|
-
write_scan_local_bytes(
|
|
1109
|
-
scan_dir,
|
|
1110
|
-
"checkpoint-head.json",
|
|
1111
|
-
_encoded({"checkpoint": checkpoint_path.name}),
|
|
1112
|
-
)
|
|
1113
|
-
os.utime(head_path, ns=(parent_modified, parent_modified))
|
|
1114
|
-
if frozen_source_digests is not None:
|
|
1115
|
-
captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
|
|
1116
|
-
frozen_source_digests = {
|
|
1117
|
-
**frozen_source_digests,
|
|
1118
|
-
parent_checkpoint: parent_digest,
|
|
1119
|
-
**{path: value[0] for path, value in captured.items()},
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
1123
|
-
parent_preserved_sources: dict[str, str] = {}
|
|
1124
|
-
source_digests: dict[str, str] = {}
|
|
1125
|
-
source_order: dict[str, tuple[int, int]] = {}
|
|
1126
|
-
worker_attempts: dict[str, tuple[str, int]] = {}
|
|
1127
|
-
saved_heads: dict[str, str] = {}
|
|
1128
|
-
if parent_manifest:
|
|
1129
|
-
recorded = parent_manifest["scan"].get("preservedSources", {})
|
|
1130
|
-
if isinstance(recorded, dict):
|
|
1131
|
-
parent_preserved_sources = recorded
|
|
1132
|
-
source_digests.update(parent_preserved_sources)
|
|
1133
|
-
if frozen_source_digests is None:
|
|
1134
|
-
source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
|
|
1135
|
-
paths: dict[str, str | None] = {}
|
|
1136
|
-
reducer_paths: set[str] = set()
|
|
1137
|
-
current_results: set[str] = set()
|
|
1138
|
-
reducer_outputs: list[tuple[Any, str, list[str], int]] = []
|
|
1139
|
-
reducer = _latest_successful_reducer(workers)
|
|
1140
|
-
latest_reducer: str | None = None
|
|
1141
|
-
if reducer is not None:
|
|
1142
|
-
try:
|
|
1143
|
-
latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
1144
|
-
paths[latest_reducer] = None
|
|
1145
|
-
reducer_paths.add(latest_reducer)
|
|
1146
|
-
except ValueError:
|
|
1147
|
-
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
1148
|
-
|
|
1149
|
-
def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
|
|
1150
|
-
if worker_id is not None:
|
|
1151
|
-
root = Path(directory).parent.as_posix()
|
|
1152
|
-
worker_attempts[root] = (worker_id, attempt)
|
|
1153
|
-
paths[f"{root}/checkpoint-head.json"] = worker_id
|
|
1154
|
-
head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
|
|
1155
|
-
for saved_directory in (directory, head_snapshots):
|
|
1156
|
-
paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
|
|
1157
|
-
|
|
1158
|
-
paths["checkpoint-head.json"] = None
|
|
1159
|
-
checkpoints("checkpoints", None)
|
|
1160
|
-
for worker in workers:
|
|
1161
|
-
try:
|
|
1162
|
-
outputs = _worker_outputs(scan_dir, worker)
|
|
1163
|
-
except (TypeError, ValueError):
|
|
1164
|
-
warnings.append("Skipped a worker checkpoint outside the scan directory.")
|
|
1165
|
-
continue
|
|
1166
|
-
if worker["kind"] == "dedup":
|
|
1167
|
-
for directory, attempt in outputs:
|
|
1168
|
-
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
1169
|
-
if not checkpoint_paths:
|
|
1170
|
-
continue
|
|
1171
|
-
result_path = f"{directory}/result.json"
|
|
1172
|
-
retained_paths = [result_path, *checkpoint_paths]
|
|
1173
|
-
paths.update(dict.fromkeys(retained_paths))
|
|
1174
|
-
reducer_paths.update(retained_paths)
|
|
1175
|
-
reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
|
|
1176
|
-
continue
|
|
1177
|
-
if worker["kind"] != "discovery":
|
|
1178
|
-
continue
|
|
1179
|
-
output, attempt = outputs[0]
|
|
1180
|
-
paths[f"{output}/result.json"] = worker["id"]
|
|
1181
|
-
current_results.add(f"{output}/result.json")
|
|
1182
|
-
for archived, archived_attempt in outputs[1:]:
|
|
1183
|
-
paths[f"{archived}/result.json"] = worker["id"]
|
|
1184
|
-
checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
|
|
1185
|
-
checkpoints(f"{output}/checkpoints", worker["id"], attempt)
|
|
1186
|
-
if worker["result_manifest_path"]:
|
|
1187
|
-
try:
|
|
1188
|
-
current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
1189
|
-
paths[current_path] = worker["id"]
|
|
1190
|
-
current_results.add(current_path)
|
|
1191
|
-
except ValueError:
|
|
1192
|
-
warnings.append("Skipped a worker result outside the scan directory.")
|
|
1193
|
-
|
|
1194
|
-
if frozen_source_digests is None:
|
|
1195
|
-
for relative, worker_id in list(paths.items()):
|
|
1196
|
-
if Path(relative).name != "checkpoint-head.json":
|
|
1197
|
-
continue
|
|
1198
|
-
del paths[relative]
|
|
1199
|
-
try:
|
|
1200
|
-
captured = _capture_saved_source(scan_dir, relative, scan_id)
|
|
1201
|
-
paths.update({path: worker_id for path in captured})
|
|
1202
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1203
|
-
if (scan_dir / relative).exists():
|
|
1204
|
-
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1205
|
-
|
|
1206
|
-
if frozen_source_digests is not None:
|
|
1207
|
-
source_digests.update(
|
|
1208
|
-
{
|
|
1209
|
-
path: digest
|
|
1210
|
-
for path, digest in frozen_source_digests.items()
|
|
1211
|
-
if _is_source_order_snapshot(path)
|
|
1212
|
-
}
|
|
1213
|
-
)
|
|
1214
|
-
paths = {
|
|
1215
|
-
relative: worker_id
|
|
1216
|
-
for relative, worker_id in paths.items()
|
|
1217
|
-
if relative in frozen_source_digests
|
|
1218
|
-
}
|
|
1219
|
-
current_results.intersection_update(frozen_source_digests)
|
|
1220
|
-
if latest_reducer not in frozen_source_digests:
|
|
1221
|
-
latest_reducer = None
|
|
1222
|
-
|
|
1223
|
-
for relative, worker_id in paths.items():
|
|
1224
|
-
try:
|
|
1225
|
-
draft, digest, observed = _read_saved_result(
|
|
1226
|
-
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
1227
|
-
)
|
|
1228
|
-
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
1229
|
-
raise ContractError("checkpoint changed after the scan stopped")
|
|
1230
|
-
source_digests[relative] = digest
|
|
1231
|
-
source_times.setdefault(relative, observed)
|
|
1232
|
-
source_order[relative] = (0, source_times[relative])
|
|
1233
|
-
if _checkpoint_head_directory(relative) is not None:
|
|
1234
|
-
saved_heads[relative] = draft["checkpoint"]
|
|
1235
|
-
continue
|
|
1236
|
-
# Recovery expects coverage, but reducer results only contain findings
|
|
1237
|
-
# and context. Add an empty value after hashing the original result.
|
|
1238
|
-
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
1239
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1240
|
-
if (scan_dir / relative).exists():
|
|
1241
|
-
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1242
|
-
if frozen_source_digests is not None:
|
|
1243
|
-
if frozen_source_digests.keys() - source_digests.keys():
|
|
1244
|
-
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
|
|
1245
|
-
|
|
1246
|
-
# Frozen observations retain checkpoint selection even if a worker moves its head.
|
|
1247
|
-
headed_workers = {
|
|
1248
|
-
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
|
|
1249
|
-
for head in saved_heads
|
|
1250
|
-
if _checkpoint_head_directory(head) != Path(".")
|
|
1251
|
-
}
|
|
1252
|
-
for relative, _, worker_id in sources:
|
|
1253
|
-
if worker_id not in headed_workers:
|
|
1254
|
-
continue
|
|
1255
|
-
directory = Path(relative).parent
|
|
1256
|
-
if directory.name == "checkpoints":
|
|
1257
|
-
directory = directory.parent
|
|
1258
|
-
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
|
|
1259
|
-
source_order[relative] = (attempt, source_order[relative][1])
|
|
1260
|
-
selected_observations: dict[str, tuple[int, int]] = {}
|
|
1261
|
-
parent_heads: list[tuple[int, str]] = []
|
|
1262
|
-
for head, checkpoint in saved_heads.items():
|
|
1263
|
-
directory = _checkpoint_head_directory(head)
|
|
1264
|
-
selected = (directory / "checkpoints" / checkpoint).as_posix()
|
|
1265
|
-
if selected not in source_order:
|
|
1266
|
-
raise ContractError("Checkpoint head is outside the saved source set.")
|
|
1267
|
-
observed = source_order[head][1]
|
|
1268
|
-
if directory == Path("."):
|
|
1269
|
-
order = (0, max(source_order[selected][1], observed))
|
|
1270
|
-
parent_heads.append((observed, selected))
|
|
1271
|
-
else:
|
|
1272
|
-
_, attempt = worker_attempts[directory.as_posix()]
|
|
1273
|
-
order = (attempt, observed)
|
|
1274
|
-
selected_observations[selected] = max(selected_observations.get(selected, order), order)
|
|
1275
|
-
source_order.update(selected_observations)
|
|
1276
|
-
|
|
1277
|
-
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
1278
|
-
latest_reducer_key = (
|
|
1279
|
-
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
1280
|
-
if reducer is not None and latest_reducer in drafts_by_path
|
|
1281
|
-
else None
|
|
1282
|
-
)
|
|
1283
|
-
if latest_reducer_key is None:
|
|
1284
|
-
latest_reducer = None
|
|
1285
|
-
for worker, result_path, checkpoint_paths, attempt in reducer_outputs:
|
|
1286
|
-
result = drafts_by_path.get(result_path)
|
|
1287
|
-
if result is None or not any(
|
|
1288
|
-
drafts_by_path.get(checkpoint_path) == result for checkpoint_path in checkpoint_paths
|
|
1289
|
-
):
|
|
1290
|
-
continue
|
|
1291
|
-
current_results.add(result_path)
|
|
1292
|
-
candidate_key = (worker["completed_at"] or "", worker["id"], attempt)
|
|
1293
|
-
if latest_reducer_key is None or candidate_key > latest_reducer_key:
|
|
1294
|
-
latest_reducer_key = candidate_key
|
|
1295
|
-
latest_reducer = result_path
|
|
1296
|
-
|
|
1297
|
-
if parent_heads:
|
|
1298
|
-
latest_observation = max(observed for observed, _ in parent_heads)
|
|
1299
|
-
for observed, parent_path in parent_heads:
|
|
1300
|
-
if observed != latest_observation:
|
|
1301
|
-
continue
|
|
1302
|
-
draft = drafts_by_path[parent_path]
|
|
1303
|
-
modified = source_order[parent_path][1]
|
|
1304
|
-
if parent is None or modified > parent_modified:
|
|
1305
|
-
parent = draft
|
|
1306
|
-
parent_modified = modified
|
|
1307
|
-
parent_is_canonical = False
|
|
1308
|
-
elif modified == parent_modified and draft != parent:
|
|
1309
|
-
parent = _merge_tied_parent_observations(parent, draft)
|
|
1310
|
-
parent_is_canonical = False
|
|
1311
|
-
if parent is None and latest_reducer is not None:
|
|
1312
|
-
parent = drafts_by_path[latest_reducer]
|
|
1313
|
-
|
|
1314
|
-
project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
|
|
1315
|
-
sources, parent, source_order, enabled=project_worker_coverage
|
|
1316
|
-
)
|
|
1317
|
-
# Keep the actual retained row even when recovery adds previous findings.
|
|
1318
|
-
linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
|
|
1319
|
-
linked_surface_outputs: dict[int, dict[str, Any]] = {}
|
|
1320
|
-
worker_row_owners: dict[int, str] = {}
|
|
1321
|
-
worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
|
|
1322
|
-
|
|
1323
|
-
def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
|
|
1324
|
-
# Parent aggregates can carry worker metadata without originating in
|
|
1325
|
-
# that worker's retained source. Track the actual source of output rows.
|
|
1326
|
-
if project_worker_coverage and owner is not None and isinstance(retained, dict):
|
|
1327
|
-
worker_row_owners[id(retained)] = owner
|
|
1328
|
-
if field == "surfaces" and isinstance(item.get("id"), str):
|
|
1329
|
-
worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
|
|
1330
|
-
|
|
1331
|
-
def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
|
|
1332
|
-
retained = _project_worker_coverage_row(
|
|
1333
|
-
item, owner if project_worker_coverage else None, field
|
|
1334
|
-
)
|
|
1335
|
-
if retained not in output:
|
|
1336
|
-
output.append(retained)
|
|
1337
|
-
retained = output[output.index(retained)]
|
|
1338
|
-
record_coverage_owner(retained, item, owner, field)
|
|
1339
|
-
return retained
|
|
1340
|
-
|
|
1341
|
-
def linked_tasks(
|
|
1342
|
-
task: dict[str, Any],
|
|
1343
|
-
published_id: str,
|
|
1344
|
-
source_task: dict[str, Any] | None,
|
|
1345
|
-
candidates: list[tuple[str | None, Any]],
|
|
1346
|
-
) -> list[Any]:
|
|
1347
|
-
expected = (
|
|
1348
|
-
_project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
|
|
1349
|
-
if source_task is not None
|
|
1350
|
-
else None
|
|
1351
|
-
)
|
|
1352
|
-
return [
|
|
1353
|
-
retained
|
|
1354
|
-
for owner, retained in candidates
|
|
1355
|
-
if retained == task
|
|
1356
|
-
or (
|
|
1357
|
-
expected is not None
|
|
1358
|
-
and owner == expected["workerId"]
|
|
1359
|
-
and isinstance(retained, dict)
|
|
1360
|
-
and retained.get("id") == expected.get("id")
|
|
1361
|
-
and isinstance(retained.get("surfaceIds"), list)
|
|
1362
|
-
and published_id in retained["surfaceIds"]
|
|
1363
|
-
)
|
|
1364
|
-
]
|
|
1365
|
-
|
|
1366
|
-
def linked_surface(surface, outputs, named_outputs):
|
|
1367
|
-
retained = outputs.get(id(surface))
|
|
1368
|
-
if (
|
|
1369
|
-
retained is None
|
|
1370
|
-
and isinstance(owner := surface.get("workerId"), str)
|
|
1371
|
-
and isinstance(identity := surface.get("id"), str)
|
|
1372
|
-
):
|
|
1373
|
-
matches = named_outputs.get((owner, identity), {})
|
|
1374
|
-
if len(matches) == 1:
|
|
1375
|
-
retained = next(iter(matches.values()))
|
|
1376
|
-
return retained
|
|
1377
|
-
|
|
1378
|
-
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
1379
|
-
# Older checkpoints can omit IDs already assigned in their published output.
|
|
1380
|
-
for field in ("deferred", "surfaces"):
|
|
1381
|
-
named_rows: dict[str | None, list[dict[str, Any]]] = {}
|
|
1382
|
-
for _, draft, owner in all_sources:
|
|
1383
|
-
rows = draft["coverage"].get(field, [])
|
|
1384
|
-
named_rows.setdefault(owner, []).extend(
|
|
1385
|
-
row
|
|
1386
|
-
for row in (rows if isinstance(rows, list) else [])
|
|
1387
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1388
|
-
)
|
|
1389
|
-
for _, draft, owner in all_sources:
|
|
1390
|
-
rows = draft["coverage"].get(field, [])
|
|
1391
|
-
if not isinstance(rows, list):
|
|
1392
|
-
continue
|
|
1393
|
-
reserved = {
|
|
1394
|
-
row["id"]
|
|
1395
|
-
for row in rows
|
|
1396
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1397
|
-
}
|
|
1398
|
-
for row in rows:
|
|
1399
|
-
if not isinstance(row, dict) or "id" in row:
|
|
1400
|
-
continue
|
|
1401
|
-
if field == "deferred" and any(
|
|
1402
|
-
key in row for key in ("candidateId", "candidate", "finding")
|
|
1403
|
-
):
|
|
1404
|
-
continue
|
|
1405
|
-
content = {"receiptRefs": [], **row} if field == "surfaces" else row
|
|
1406
|
-
identity = next(
|
|
1407
|
-
(
|
|
1408
|
-
named["id"]
|
|
1409
|
-
for named in named_rows[owner]
|
|
1410
|
-
if named["id"] not in reserved
|
|
1411
|
-
and {
|
|
1412
|
-
**({"receiptRefs": []} if field == "surfaces" else {}),
|
|
1413
|
-
**{key: value for key, value in named.items() if key != "id"},
|
|
1414
|
-
}
|
|
1415
|
-
== content
|
|
1416
|
-
),
|
|
1417
|
-
None,
|
|
1418
|
-
)
|
|
1419
|
-
if identity is not None:
|
|
1420
|
-
row["id"] = identity
|
|
1421
|
-
reserved.add(identity)
|
|
1422
|
-
|
|
1423
|
-
if parent is None and not sources:
|
|
1424
|
-
return None
|
|
1425
|
-
if (
|
|
1426
|
-
parent_manifest
|
|
1427
|
-
and parent_manifest["scan"].get("sealedAt")
|
|
1428
|
-
and parent_manifest["scan"].get("status") == binding["status"]
|
|
1429
|
-
and parent_manifest["scan"].get("preservedSources") == source_digests
|
|
1430
|
-
and all(warning in initial_warnings for warning in warnings)
|
|
1431
|
-
):
|
|
1432
|
-
return None
|
|
1433
|
-
|
|
1434
|
-
if (
|
|
1435
|
-
frozen_source_digests is None
|
|
1436
|
-
or any(_is_source_order_snapshot(path) for path in source_digests)
|
|
1437
|
-
or allow_frozen_legacy_parent
|
|
1438
|
-
):
|
|
1439
|
-
_freeze_source_times(scan_dir, scan_id, source_digests, source_times)
|
|
1440
|
-
|
|
1441
|
-
target_kind = binding["allowedTargetKinds"][0]
|
|
1442
|
-
if (
|
|
1443
|
-
target_kind == "git_worktree"
|
|
1444
|
-
and "snapshotDigest" not in binding["target"]
|
|
1445
|
-
and "git_revision" in binding["allowedTargetKinds"]
|
|
1446
|
-
):
|
|
1447
|
-
target_kind = "git_revision"
|
|
1448
|
-
target = {"kind": target_kind, **binding["target"]}
|
|
1449
|
-
if target["kind"] == "git_diff" and "snapshotDigest" not in target:
|
|
1450
|
-
diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
|
|
1451
|
-
target["snapshotDigest"] = committed_diff_snapshot_digest(
|
|
1452
|
-
diff_kind, target["baseRevision"], target["headRevision"]
|
|
1453
|
-
)
|
|
1454
|
-
manifest = (
|
|
1455
|
-
copy.deepcopy(parent_manifest)
|
|
1456
|
-
if parent_manifest
|
|
1457
|
-
else {"scan": {"target": target, "scope": binding["scope"]}}
|
|
1458
|
-
)
|
|
1459
|
-
for key in ("sealedAt", "artifacts"):
|
|
1460
|
-
manifest["scan"].pop(key, None)
|
|
1461
|
-
manifest["scan"]["preservedSources"] = source_digests
|
|
1462
|
-
# Completion follows the selected parent, including legacy terminal drafts
|
|
1463
|
-
# that omit the optional marker, rather than an older canonical manifest.
|
|
1464
|
-
if parent is not None:
|
|
1465
|
-
manifest["scan"].pop("complete", None)
|
|
1466
|
-
if "complete" in parent:
|
|
1467
|
-
manifest["scan"]["complete"] = parent["complete"]
|
|
1468
|
-
for key in ("scope", "threatModel"):
|
|
1469
|
-
if isinstance(parent.get(key), dict):
|
|
1470
|
-
manifest["scan"][key] = copy.deepcopy(parent[key])
|
|
1471
|
-
if isinstance(manifest["scan"].get("scope"), dict):
|
|
1472
|
-
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
|
|
1473
|
-
if frozen_model_source is not None:
|
|
1474
|
-
# Publication retries retain the choice made with the frozen source set.
|
|
1475
|
-
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
|
|
1476
|
-
if not isinstance(model, dict):
|
|
1477
|
-
raise ContractError("Frozen stopped-scan model source is unavailable.")
|
|
1478
|
-
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1479
|
-
if paths[frozen_model_source] is not None:
|
|
1480
|
-
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1481
|
-
if selected_model_source is not None:
|
|
1482
|
-
selected_model_source[:] = [frozen_model_source]
|
|
1483
|
-
coverage = (
|
|
1484
|
-
copy.deepcopy(parent["coverage"])
|
|
1485
|
-
if parent and parent["coverage"]
|
|
1486
|
-
else {
|
|
1487
|
-
"mode": binding["coverageMode"],
|
|
1488
|
-
"inventoryStrategy": "diff"
|
|
1489
|
-
if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
|
|
1490
|
-
else "scoped_path"
|
|
1491
|
-
if binding["coverageMode"] == "scoped_path"
|
|
1492
|
-
else "repository",
|
|
1493
|
-
**binding["scope"],
|
|
1494
|
-
"surfaces": [],
|
|
1495
|
-
"explicitExclusions": [],
|
|
1496
|
-
"deferred": [],
|
|
1497
|
-
}
|
|
1498
|
-
)
|
|
1499
|
-
if isinstance(coverage.get("openQuestions"), list):
|
|
1500
|
-
coverage["openQuestions"] = [
|
|
1501
|
-
{"question": item.strip()} if isinstance(item, str) else item
|
|
1502
|
-
for item in coverage["openQuestions"]
|
|
1503
|
-
]
|
|
1504
|
-
canonical_rows = (
|
|
1505
|
-
{
|
|
1506
|
-
id(item)
|
|
1507
|
-
for field in ("surfaces", "explicitExclusions", "deferred")
|
|
1508
|
-
for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
|
|
1509
|
-
}
|
|
1510
|
-
if parent_is_canonical
|
|
1511
|
-
else set()
|
|
1512
|
-
)
|
|
1513
|
-
findings: list[dict[str, Any]] = []
|
|
1514
|
-
finding_positions: dict[str, int] = {}
|
|
1515
|
-
represented: dict[str, str | None] = {}
|
|
1516
|
-
represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}
|
|
1517
|
-
represented_history: dict[str, set[str]] = {}
|
|
1518
|
-
represented_candidate_history: dict[tuple[str, str, Any, Any, Any], set[str]] = {}
|
|
1519
|
-
rejected_history: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
1520
|
-
stopped_parent_seal = bool(
|
|
1521
|
-
stopped and parent_manifest and parent_manifest["scan"].get("sealedAt")
|
|
1522
|
-
)
|
|
1523
|
-
|
|
1524
|
-
def valid_finding(value: Any) -> bool:
|
|
1525
|
-
# Use the finalizer's own per-record recovery before a draft can suppress
|
|
1526
|
-
# an earlier checkpoint. Invalid latest records must not hide valid history.
|
|
1527
|
-
document = {"scanId": scan_id, "findings": [copy.deepcopy(value)]}
|
|
1528
|
-
if isinstance(document["findings"][0], dict):
|
|
1529
|
-
_ensure_finding_identity(document["findings"][0], candidate_only=True)
|
|
1530
|
-
_recover_unsealed_findings(
|
|
1531
|
-
{"scan": {"id": scan_id, "target": binding["target"]}},
|
|
1532
|
-
document,
|
|
1533
|
-
Path(__file__).resolve().parent.parent / "schemas",
|
|
1534
|
-
scan_dir,
|
|
1535
|
-
[],
|
|
1536
|
-
)
|
|
1537
|
-
return bool(document["findings"])
|
|
1538
|
-
|
|
1539
|
-
source_order["parent"] = (0, parent_modified)
|
|
1540
|
-
deferred_rows = {
|
|
1541
|
-
relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
|
|
1542
|
-
}
|
|
1543
|
-
# A legacy source can contain independent tasks with the same explicit ID.
|
|
1544
|
-
# Later rewrites cannot make an ID-only closure identify one of those tasks.
|
|
1545
|
-
ambiguous_deferred: set[tuple[str | None, str]] = set()
|
|
1546
|
-
for relative, _, owner in all_sources:
|
|
1547
|
-
by_id: dict[str, dict[str, Any]] = {}
|
|
1548
|
-
candidate_aliases = {
|
|
1549
|
-
identity
|
|
1550
|
-
for row in deferred_rows[relative]
|
|
1551
|
-
if isinstance(row, dict)
|
|
1552
|
-
and any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1553
|
-
for identity in (row.get("id"), row.get("candidateId"))
|
|
1554
|
-
if isinstance(identity, str)
|
|
1555
|
-
}
|
|
1556
|
-
for row in deferred_rows[relative]:
|
|
1557
|
-
if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
|
|
1558
|
-
continue
|
|
1559
|
-
if any(key in row for key in ("candidateId", "candidate", "finding")):
|
|
1560
|
-
continue
|
|
1561
|
-
if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
|
|
1562
|
-
ambiguous_deferred.add((owner, identity))
|
|
1563
|
-
by_id[identity] = row
|
|
1564
|
-
current_drafts = ([("parent", parent, None)] if parent else []) + [
|
|
1565
|
-
source for source in sources if source[0] in current_results | selected_observations.keys()
|
|
1566
|
-
]
|
|
1567
|
-
# Generic closures belong to one logical scan or worker, just like candidates.
|
|
1568
|
-
# Keep them when recovering a terminal checkpoint without its canonical write.
|
|
1569
|
-
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1570
|
-
|
|
1571
|
-
candidate_ids: set[tuple[str | None, str]] = set()
|
|
1572
|
-
accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1573
|
-
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1574
|
-
coverage_schema = _read_json(
|
|
1575
|
-
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
1576
|
-
)["properties"]
|
|
1577
|
-
for relative, draft, owner in all_sources:
|
|
1578
|
-
order = source_order[relative]
|
|
1579
|
-
for item in deferred_rows[relative]:
|
|
1580
|
-
if not isinstance(item, dict):
|
|
1581
|
-
continue
|
|
1582
|
-
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
|
|
1583
|
-
candidate_ids.update(
|
|
1584
|
-
(owner, identity)
|
|
1585
|
-
for identity in (item.get("id"), item.get("candidateId"))
|
|
1586
|
-
if isinstance(identity, str)
|
|
1587
|
-
)
|
|
1588
|
-
identity = item.get("id") or item.get("candidateId")
|
|
1589
|
-
if isinstance(identity, str):
|
|
1590
|
-
key = (owner, identity)
|
|
1591
|
-
previous = active_deferred.get(key)
|
|
1592
|
-
if previous is None or order > previous[0]:
|
|
1593
|
-
active_deferred[key] = (order, item, relative)
|
|
1594
|
-
if (
|
|
1595
|
-
relative == "parent"
|
|
1596
|
-
or relative in current_results
|
|
1597
|
-
or relative in selected_observations
|
|
1598
|
-
):
|
|
1599
|
-
try:
|
|
1600
|
-
# A malformed update cannot discard valid saved evidence.
|
|
1601
|
-
_validate_schema_node(
|
|
1602
|
-
item, coverage_schema["deferred"]["items"], "coverage.deferred"
|
|
1603
|
-
)
|
|
1604
|
-
except ContractError:
|
|
1605
|
-
continue
|
|
1606
|
-
accepted_deferred_orders[key] = max(
|
|
1607
|
-
accepted_deferred_orders.get(key, order), order
|
|
1608
|
-
)
|
|
1609
|
-
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
|
|
1610
|
-
key = (owner, closure["id"])
|
|
1611
|
-
previous = closed_deferred.get(key)
|
|
1612
|
-
if previous is None or order > previous[0]:
|
|
1613
|
-
# The parent comes first, preserving its reason on equal timestamps.
|
|
1614
|
-
closed_deferred[key] = (order, closure, relative)
|
|
1615
|
-
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
|
|
1616
|
-
reopened_generic: set[tuple[str | None, str]] = set()
|
|
1617
|
-
for key, (order, _, _) in list(closed_deferred.items()):
|
|
1618
|
-
# Equal timestamps cannot distinguish closure from reopened work.
|
|
1619
|
-
reopened = [
|
|
1620
|
-
active
|
|
1621
|
-
for (owner, identity), active in active_deferred.items()
|
|
1622
|
-
if owner == key[0]
|
|
1623
|
-
and (identity == key[1] or active[1].get("candidateId") == key[1])
|
|
1624
|
-
and active[0] >= order
|
|
1625
|
-
]
|
|
1626
|
-
if key in candidate_ids or key in ambiguous_deferred or reopened:
|
|
1627
|
-
del closed_deferred[key]
|
|
1628
|
-
for _, item, _ in reopened:
|
|
1629
|
-
reopened_rows.append((key[0], item))
|
|
1630
|
-
if key not in candidate_ids:
|
|
1631
|
-
reopened_generic.add(key)
|
|
1632
|
-
# Retain every distinct ambiguous task even when a terminal result supersedes
|
|
1633
|
-
# its source. Existing output ID normalization keeps each row distinct.
|
|
1634
|
-
for relative, _, owner in all_sources:
|
|
1635
|
-
for row in deferred_rows[relative]:
|
|
1636
|
-
if (
|
|
1637
|
-
isinstance(row, dict)
|
|
1638
|
-
and isinstance(identity := row.get("id"), str)
|
|
1639
|
-
and (owner, identity) in ambiguous_deferred
|
|
1640
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1641
|
-
and (owner, row) not in reopened_rows
|
|
1642
|
-
):
|
|
1643
|
-
reopened_rows.append((owner, row))
|
|
1644
|
-
parent_closures = [
|
|
1645
|
-
closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
|
|
1646
|
-
]
|
|
1647
|
-
coverage.pop("resolvedDeferred", None)
|
|
1648
|
-
if parent_closures:
|
|
1649
|
-
coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
|
|
1650
|
-
if parent:
|
|
1651
|
-
coverage["deferred"] = [
|
|
1652
|
-
row
|
|
1653
|
-
for row in deferred_rows["parent"]
|
|
1654
|
-
if not isinstance(row, dict)
|
|
1655
|
-
or not isinstance(row.get("id"), str)
|
|
1656
|
-
or (None, row["id"]) not in closed_deferred
|
|
1657
|
-
]
|
|
1658
|
-
resolved: dict[tuple[str | None, str], str] = {}
|
|
1659
|
-
|
|
1660
|
-
ordered_candidates = {
|
|
1661
|
-
(owner, identity)
|
|
1662
|
-
for owner, row in reopened_rows
|
|
1663
|
-
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
|
|
1664
|
-
and (owner, identity) in candidate_ids
|
|
1665
|
-
}
|
|
1666
|
-
ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
|
|
1667
|
-
|
|
1668
|
-
outcomes: list[tuple[str, str | None, str, str]] = []
|
|
1669
|
-
for relative, draft, owner in current_drafts:
|
|
1670
|
-
for finding in draft["findings"]:
|
|
1671
|
-
if (
|
|
1672
|
-
isinstance(finding, dict)
|
|
1673
|
-
and valid_finding(finding)
|
|
1674
|
-
and (candidate_id := finding_candidate_id(finding))
|
|
1675
|
-
):
|
|
1676
|
-
outcomes.append((relative, owner, candidate_id, "reported"))
|
|
1677
|
-
for field in ("surfaces", "explicitExclusions"):
|
|
1678
|
-
items = draft["coverage"].get(field, [])
|
|
1679
|
-
for item in items if isinstance(items, list) else []:
|
|
1680
|
-
if (
|
|
1681
|
-
isinstance(item, dict)
|
|
1682
|
-
and isinstance(item.get("candidateId"), str)
|
|
1683
|
-
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
1684
|
-
):
|
|
1685
|
-
outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
|
|
1686
|
-
ordered_candidates.update(
|
|
1687
|
-
(owner, candidate_id)
|
|
1688
|
-
for relative, owner, candidate_id, _ in outcomes
|
|
1689
|
-
if owner is not None and relative in selected_observations
|
|
1690
|
-
)
|
|
1691
|
-
# Reopened work and selected checkpoint outcomes follow the saved source order.
|
|
1692
|
-
for relative, owner, candidate_id, disposition in outcomes:
|
|
1693
|
-
key = (owner, candidate_id)
|
|
1694
|
-
if key not in ordered_candidates:
|
|
1695
|
-
if relative == "parent" or relative in current_results:
|
|
1696
|
-
resolved.setdefault(key, disposition)
|
|
1697
|
-
continue
|
|
1698
|
-
order = source_order[relative]
|
|
1699
|
-
if any(
|
|
1700
|
-
saved_owner == owner
|
|
1701
|
-
and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
|
|
1702
|
-
and modified >= order
|
|
1703
|
-
for (saved_owner, _), (modified, row, _) in active_deferred.items()
|
|
1704
|
-
):
|
|
1705
|
-
continue
|
|
1706
|
-
if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
|
|
1707
|
-
resolved[key] = disposition
|
|
1708
|
-
ordered_outcomes[key] = (order, relative)
|
|
1709
|
-
# Only the current parent may claim that another worker finding was absorbed.
|
|
1710
|
-
# A superseded checkpoint must not suppress a newer independent result.
|
|
1711
|
-
if parent:
|
|
1712
|
-
for finding in parent["findings"]:
|
|
1713
|
-
if valid_finding(finding):
|
|
1714
|
-
canonical_key = _finding_key(finding)
|
|
1715
|
-
for retained in _retained_findings(finding):
|
|
1716
|
-
retained_key = _finding_key(retained)
|
|
1717
|
-
if retained is not finding:
|
|
1718
|
-
represented_history.setdefault(retained_key, set()).add(
|
|
1719
|
-
_digest(_finding_content(retained))
|
|
1720
|
-
)
|
|
1721
|
-
previous_key = represented.get(retained_key)
|
|
1722
|
-
if retained_key not in represented:
|
|
1723
|
-
represented[retained_key] = canonical_key
|
|
1724
|
-
elif previous_key != canonical_key:
|
|
1725
|
-
# Ambiguous history cannot suppress an independent source.
|
|
1726
|
-
represented[retained_key] = None
|
|
1727
|
-
originals = finding["provenance"].get("sourceFindings", [])
|
|
1728
|
-
for original in originals if isinstance(originals, list) else []:
|
|
1729
|
-
if isinstance(original, dict) and isinstance(original.get("finding"), dict):
|
|
1730
|
-
source_id = original.get("id")
|
|
1731
|
-
candidate_id = finding_candidate_id(original["finding"])
|
|
1732
|
-
if isinstance(source_id, str) and ":" in source_id and candidate_id:
|
|
1733
|
-
candidate_key = _worker_candidate_key(
|
|
1734
|
-
source_id.rsplit(":", 1)[0],
|
|
1735
|
-
candidate_id,
|
|
1736
|
-
original["finding"],
|
|
1737
|
-
)
|
|
1738
|
-
previous_key = represented_candidates.get(candidate_key)
|
|
1739
|
-
if candidate_key not in represented_candidates:
|
|
1740
|
-
represented_candidates[candidate_key] = canonical_key
|
|
1741
|
-
elif previous_key != canonical_key:
|
|
1742
|
-
# Candidate ids are only authoritative within one
|
|
1743
|
-
# logical worker. Multiple canonical owners make
|
|
1744
|
-
# that worker-local identity ambiguous.
|
|
1745
|
-
represented_candidates[candidate_key] = None
|
|
1746
|
-
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
1747
|
-
_digest(_finding_content(original["finding"]))
|
|
1748
|
-
)
|
|
1749
|
-
selected_terminal_orders: dict[str, tuple[int, int]] = {}
|
|
1750
|
-
for relative, draft, worker_id in sources:
|
|
1751
|
-
if (
|
|
1752
|
-
worker_id is not None
|
|
1753
|
-
and relative in selected_observations
|
|
1754
|
-
and draft.get("complete") is not False
|
|
1755
|
-
):
|
|
1756
|
-
order = source_order[relative]
|
|
1757
|
-
selected_terminal_orders[worker_id] = max(
|
|
1758
|
-
selected_terminal_orders.get(worker_id, order), order
|
|
1759
|
-
)
|
|
1760
|
-
terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
|
|
1761
|
-
for relative, draft, worker_id in sources:
|
|
1762
|
-
if relative in current_results and draft.get("complete") is not False:
|
|
1763
|
-
order = source_order[relative]
|
|
1764
|
-
terminal_worker_orders[worker_id] = max(
|
|
1765
|
-
terminal_worker_orders.get(worker_id, order), order
|
|
1766
|
-
)
|
|
1767
|
-
|
|
1768
|
-
def source_coverage_selection(
|
|
1769
|
-
relative: str,
|
|
1770
|
-
worker_id: str | None,
|
|
1771
|
-
) -> tuple[bool, set[str], bool, bool]:
|
|
1772
|
-
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
1773
|
-
selected_coverage_superseded = worker_id in selected_terminal_orders and (
|
|
1774
|
-
source_order[relative] < selected_terminal_orders[worker_id]
|
|
1775
|
-
or (
|
|
1776
|
-
relative in current_results
|
|
1777
|
-
and source_order[relative] <= selected_terminal_orders[worker_id]
|
|
1778
|
-
)
|
|
1779
|
-
)
|
|
1780
|
-
selected_candidates = {
|
|
1781
|
-
candidate_id
|
|
1782
|
-
for (owner, candidate_id), (_, source) in ordered_outcomes.items()
|
|
1783
|
-
if owner == worker_id and source == relative
|
|
1784
|
-
}
|
|
1785
|
-
superseded = (
|
|
1786
|
-
worker_id is None
|
|
1787
|
-
and parent is not None
|
|
1788
|
-
and parent.get("complete") is not False
|
|
1789
|
-
and relative != "parent"
|
|
1790
|
-
and source_order[relative] <= (0, parent_modified)
|
|
1791
|
-
and (not stopped_parent_seal or relative in parent_preserved_sources)
|
|
1792
|
-
) or (
|
|
1793
|
-
relative not in current_results
|
|
1794
|
-
and worker_result_order is not None
|
|
1795
|
-
and (
|
|
1796
|
-
relative not in selected_observations
|
|
1797
|
-
or source_order[relative] < worker_result_order
|
|
1798
|
-
)
|
|
1799
|
-
)
|
|
1800
|
-
# A failed result write can leave pending work outside the accepted result.
|
|
1801
|
-
accepted_order = (
|
|
1802
|
-
(0, parent_modified)
|
|
1803
|
-
if worker_id is None and parent is not None
|
|
1804
|
-
else worker_result_order
|
|
1805
|
-
)
|
|
1806
|
-
retain_pending = (
|
|
1807
|
-
superseded
|
|
1808
|
-
and (worker_id in headed_workers or (worker_id is None and parent_heads))
|
|
1809
|
-
and accepted_order is not None
|
|
1810
|
-
and source_order[relative] >= accepted_order
|
|
1811
|
-
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
1812
|
-
)
|
|
1813
|
-
return selected_coverage_superseded, selected_candidates, superseded, retain_pending
|
|
1814
|
-
|
|
1815
|
-
def coverage_row_is_retained(
|
|
1816
|
-
field: str,
|
|
1817
|
-
item: Any,
|
|
1818
|
-
worker_id: str | None,
|
|
1819
|
-
selection: tuple[bool, set[str], bool, bool],
|
|
1820
|
-
) -> bool:
|
|
1821
|
-
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
1822
|
-
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
1823
|
-
# candidate evidence and pending tasks retain their own reconciliation.
|
|
1824
|
-
if (
|
|
1825
|
-
selected_coverage_superseded
|
|
1826
|
-
and field != "deferred"
|
|
1827
|
-
and not (
|
|
1828
|
-
isinstance(item, dict)
|
|
1829
|
-
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1830
|
-
)
|
|
1831
|
-
):
|
|
1832
|
-
return False
|
|
1833
|
-
# A selected outcome must retain its evidence even if its result write failed.
|
|
1834
|
-
if superseded and not (
|
|
1835
|
-
isinstance(item, dict)
|
|
1836
|
-
and (
|
|
1837
|
-
(field == "deferred" and retain_pending)
|
|
1838
|
-
or (
|
|
1839
|
-
isinstance(item.get("candidateId"), str)
|
|
1840
|
-
and item["candidateId"] in selected_candidates
|
|
1841
|
-
and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
|
|
1842
|
-
)
|
|
1843
|
-
)
|
|
1844
|
-
):
|
|
1845
|
-
return False
|
|
1846
|
-
if (
|
|
1847
|
-
isinstance(item, dict)
|
|
1848
|
-
and isinstance(
|
|
1849
|
-
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1850
|
-
if field == "deferred"
|
|
1851
|
-
else item.get("candidateId"),
|
|
1852
|
-
str,
|
|
1853
|
-
)
|
|
1854
|
-
and (worker_id, identity) in resolved
|
|
1855
|
-
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1856
|
-
):
|
|
1857
|
-
return False
|
|
1858
|
-
return True
|
|
1859
|
-
|
|
1860
|
-
ambiguous_surface_ids = {
|
|
1861
|
-
(owner, identity)
|
|
1862
|
-
for owner, row in reopened_rows
|
|
1863
|
-
if isinstance(row.get("id"), str)
|
|
1864
|
-
and (owner, row["id"]) in ambiguous_deferred
|
|
1865
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1866
|
-
for identity in [
|
|
1867
|
-
row["id"],
|
|
1868
|
-
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1869
|
-
]
|
|
1870
|
-
if isinstance(identity, str)
|
|
1871
|
-
}
|
|
1872
|
-
for _, draft, owner in all_sources:
|
|
1873
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
1874
|
-
by_id: dict[str, dict[str, Any]] = {}
|
|
1875
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1876
|
-
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1877
|
-
continue
|
|
1878
|
-
if identity in by_id and surface != by_id[identity]:
|
|
1879
|
-
ambiguous_surface_ids.add((owner, identity))
|
|
1880
|
-
by_id[identity] = surface
|
|
1881
|
-
|
|
1882
|
-
replaced_deferred = {
|
|
1883
|
-
id(row)
|
|
1884
|
-
for relative, _, owner in all_sources
|
|
1885
|
-
for row in deferred_rows[relative]
|
|
1886
|
-
if isinstance(row, dict)
|
|
1887
|
-
and isinstance(identity := row.get("id"), str)
|
|
1888
|
-
and (key := (owner, identity)) not in candidate_ids
|
|
1889
|
-
and key not in ambiguous_deferred
|
|
1890
|
-
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1891
|
-
and updated > source_order[relative]
|
|
1892
|
-
}
|
|
1893
|
-
fallback_surfaces = []
|
|
1894
|
-
if surface_links and isinstance(coverage.get("surfaces"), list):
|
|
1895
|
-
# Plan the surviving associations before historical receipts can claim
|
|
1896
|
-
# a spelling needed by a surface that must be restored for its task.
|
|
1897
|
-
prospective_tasks = []
|
|
1898
|
-
prospective_surfaces = {}
|
|
1899
|
-
prospective_named = {}
|
|
1900
|
-
for relative, draft, owner in all_sources:
|
|
1901
|
-
selection = source_coverage_selection(relative, owner)
|
|
1902
|
-
for row in deferred_rows[relative]:
|
|
1903
|
-
if (
|
|
1904
|
-
coverage_row_is_retained("deferred", row, owner, selection)
|
|
1905
|
-
and id(row) not in replaced_deferred
|
|
1906
|
-
and not (
|
|
1907
|
-
isinstance(row, dict)
|
|
1908
|
-
and isinstance(row.get("id"), str)
|
|
1909
|
-
and (owner, row["id"]) in closed_deferred
|
|
1910
|
-
)
|
|
1911
|
-
):
|
|
1912
|
-
prospective_tasks.append(
|
|
1913
|
-
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1914
|
-
)
|
|
1915
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
1916
|
-
for row in rows if isinstance(rows, list) else []:
|
|
1917
|
-
if owner is None or not isinstance(row, dict):
|
|
1918
|
-
continue
|
|
1919
|
-
emitted = coverage_row_is_retained("surfaces", row, owner, selection)
|
|
1920
|
-
ambiguous = (
|
|
1921
|
-
isinstance(row.get("id"), str)
|
|
1922
|
-
and (owner, row["id"]) in ambiguous_surface_ids
|
|
1923
|
-
and row.get("disposition") == "needs_follow_up"
|
|
1924
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1925
|
-
)
|
|
1926
|
-
if not emitted and not ambiguous:
|
|
1927
|
-
continue
|
|
1928
|
-
projected = _project_worker_coverage_row(row, owner, "surfaces")
|
|
1929
|
-
if isinstance(row.get("id"), str):
|
|
1930
|
-
named = prospective_named.setdefault((owner, row["id"]), {})
|
|
1931
|
-
projected = next(
|
|
1932
|
-
(existing for existing in named.values() if existing == projected),
|
|
1933
|
-
projected,
|
|
1934
|
-
)
|
|
1935
|
-
named[id(projected)] = projected
|
|
1936
|
-
if emitted:
|
|
1937
|
-
prospective_surfaces[id(row)] = projected
|
|
1938
|
-
prospective_tasks.extend(
|
|
1939
|
-
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1940
|
-
for owner, row in reopened_rows
|
|
1941
|
-
if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
|
|
1942
|
-
)
|
|
1943
|
-
bound = {
|
|
1944
|
-
(id(retained), published_id)
|
|
1945
|
-
for task, published_id, surface, _, task_source in surface_links
|
|
1946
|
-
if linked_surface(surface, prospective_surfaces, prospective_named) is not None
|
|
1947
|
-
for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
|
|
1948
|
-
}
|
|
1949
|
-
for task, published_id, _, parent_surface, task_source in surface_links:
|
|
1950
|
-
for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
|
|
1951
|
-
key = (id(retained), published_id)
|
|
1952
|
-
if key not in bound:
|
|
1953
|
-
fallback_surfaces.append(parent_surface)
|
|
1954
|
-
bound.add(key)
|
|
1955
|
-
|
|
1956
|
-
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1957
|
-
all_sources,
|
|
1958
|
-
source_order,
|
|
1959
|
-
closed_deferred,
|
|
1960
|
-
active_deferred,
|
|
1961
|
-
resolved,
|
|
1962
|
-
reopened_generic,
|
|
1963
|
-
coverage_schema["surfaces"]["items"],
|
|
1964
|
-
deferred_rows,
|
|
1965
|
-
ambiguous_deferred,
|
|
1966
|
-
receipt_root=scan_dir if project_worker_coverage else None,
|
|
1967
|
-
receipt_warnings=warnings,
|
|
1968
|
-
fallback_surfaces=fallback_surfaces,
|
|
1969
|
-
surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
|
|
1970
|
-
"surfaces", row, owner, source_coverage_selection(relative, owner)
|
|
1971
|
-
),
|
|
1972
|
-
eligible_receipt_sources={
|
|
1973
|
-
relative
|
|
1974
|
-
for relative, draft, _ in all_sources
|
|
1975
|
-
if draft.get("complete") is not False or relative in selected_observations
|
|
1976
|
-
},
|
|
1977
|
-
)
|
|
1978
|
-
replaced_rows = {
|
|
1979
|
-
"surfaces": replaced_surfaces,
|
|
1980
|
-
"deferred": replaced_deferred,
|
|
1981
|
-
}
|
|
1982
|
-
for field, replaced in replaced_rows.items():
|
|
1983
|
-
if parent and isinstance(coverage.get(field), list):
|
|
1984
|
-
previous = parent["coverage"].get(field, [])
|
|
1985
|
-
if isinstance(previous, list):
|
|
1986
|
-
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1987
|
-
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1988
|
-
if isinstance(coverage.get("surfaces"), list):
|
|
1989
|
-
for owner, surface, replaced_sources in surface_updates:
|
|
1990
|
-
retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
|
|
1991
|
-
for source_id in replaced_sources & linked_surface_sources:
|
|
1992
|
-
linked_surface_outputs[source_id] = retained
|
|
1993
|
-
for surface in coverage["surfaces"]:
|
|
1994
|
-
if isinstance(surface, dict):
|
|
1995
|
-
surface.setdefault("receiptRefs", [])
|
|
1996
|
-
|
|
1997
|
-
# Reopened work survives a superseded checkpoint, but current candidate
|
|
1998
|
-
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
1999
|
-
for owner, item in reopened_rows:
|
|
2000
|
-
if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
|
|
2001
|
-
owner,
|
|
2002
|
-
identity,
|
|
2003
|
-
) in resolved:
|
|
2004
|
-
continue
|
|
2005
|
-
pending = coverage.setdefault("deferred", [])
|
|
2006
|
-
if isinstance(pending, list):
|
|
2007
|
-
append_coverage_row(pending, item, owner, "deferred")
|
|
2008
|
-
for _, draft, owner in all_sources:
|
|
2009
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
2010
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2011
|
-
if (
|
|
2012
|
-
isinstance(surface, dict)
|
|
2013
|
-
and isinstance(surface.get("id"), str)
|
|
2014
|
-
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
2015
|
-
and surface.get("disposition") == "needs_follow_up"
|
|
2016
|
-
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
2017
|
-
and isinstance(coverage.get("surfaces"), list)
|
|
2018
|
-
):
|
|
2019
|
-
append_coverage_row(
|
|
2020
|
-
coverage["surfaces"],
|
|
2021
|
-
{**surface, "receiptRefs": surface.get("receiptRefs", [])},
|
|
2022
|
-
owner,
|
|
2023
|
-
"surfaces",
|
|
2024
|
-
)
|
|
2025
|
-
for relative, draft, worker_id in all_sources:
|
|
2026
|
-
selection = source_coverage_selection(relative, worker_id)
|
|
2027
|
-
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
2028
|
-
skip_superseded_findings = (
|
|
2029
|
-
superseded
|
|
2030
|
-
and not stopped
|
|
2031
|
-
and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
|
|
2032
|
-
)
|
|
2033
|
-
if skip_superseded_findings and not selected_candidates and not retain_pending:
|
|
2034
|
-
continue
|
|
2035
|
-
if (
|
|
2036
|
-
not skip_superseded_findings
|
|
2037
|
-
and "threatModel" not in manifest["scan"]
|
|
2038
|
-
and isinstance(draft.get("threatModel"), dict)
|
|
2039
|
-
):
|
|
2040
|
-
model = draft["threatModel"]
|
|
2041
|
-
model_path = relative
|
|
2042
|
-
checkpoint_dir = Path(relative).parent
|
|
2043
|
-
prefer_worker_head = worker_id is not None and (
|
|
2044
|
-
checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
|
|
2045
|
-
)
|
|
2046
|
-
if worker_id is not None:
|
|
2047
|
-
if checkpoint_dir.name != "checkpoints":
|
|
2048
|
-
checkpoint_dir /= "checkpoints"
|
|
2049
|
-
selected_models = [
|
|
2050
|
-
path
|
|
2051
|
-
for path in selected_observations
|
|
2052
|
-
if Path(path).parent == checkpoint_dir
|
|
2053
|
-
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
2054
|
-
]
|
|
2055
|
-
if selected_models:
|
|
2056
|
-
head_path = max(selected_models, key=source_order.__getitem__)
|
|
2057
|
-
current = drafts_by_path[head_path]
|
|
2058
|
-
# A terminal checkpoint is committed before result.json is replaced.
|
|
2059
|
-
# Use the admitted observation, including its frozen ordering on retries.
|
|
2060
|
-
if not prefer_worker_head and current.get("complete") is not False:
|
|
2061
|
-
prefer_worker_head = source_order[head_path] >= source_order[relative]
|
|
2062
|
-
if prefer_worker_head:
|
|
2063
|
-
model = current["threatModel"]
|
|
2064
|
-
model_path = head_path
|
|
2065
|
-
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
2066
|
-
if worker_id is not None:
|
|
2067
|
-
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
2068
|
-
if selected_model_source is not None and worker_id is not None:
|
|
2069
|
-
selected_model_source[:] = [model_path]
|
|
2070
|
-
for value in draft["findings"]:
|
|
2071
|
-
if skip_superseded_findings and not (
|
|
2072
|
-
isinstance(value, dict)
|
|
2073
|
-
and (candidate_id := finding_candidate_id(value)) in selected_candidates
|
|
2074
|
-
and resolved.get((worker_id, candidate_id)) == "reported"
|
|
2075
|
-
):
|
|
2076
|
-
continue
|
|
2077
|
-
if relative == "parent" and parent_is_canonical:
|
|
2078
|
-
finding = copy.deepcopy(value)
|
|
2079
|
-
_ensure_finding_identity(finding, candidate_only=True)
|
|
2080
|
-
provenance = finding.get("provenance") if isinstance(finding, dict) else None
|
|
2081
|
-
owner = provenance.get("workerId") if isinstance(provenance, dict) else None
|
|
2082
|
-
candidate_id = finding_candidate_id(finding) if isinstance(finding, dict) else None
|
|
2083
|
-
if (
|
|
2084
|
-
stopped_parent_seal
|
|
2085
|
-
and isinstance(owner, str)
|
|
2086
|
-
and candidate_id
|
|
2087
|
-
and resolved.get((owner, candidate_id)) in {"rejected", "not_applicable"}
|
|
2088
|
-
):
|
|
2089
|
-
rejected_history.setdefault((owner, candidate_id), []).append(finding)
|
|
2090
|
-
continue
|
|
2091
|
-
if valid_finding(finding):
|
|
2092
|
-
finding_positions.setdefault(_finding_key(finding), len(findings))
|
|
2093
|
-
findings.append(finding)
|
|
2094
|
-
continue
|
|
2095
|
-
if relative != "parent" and parent and value in parent["findings"]:
|
|
2096
|
-
continue
|
|
2097
|
-
if not isinstance(value, dict):
|
|
2098
|
-
warnings.append(f"Retained malformed finding evidence in {relative}.")
|
|
2099
|
-
continue
|
|
2100
|
-
finding = copy.deepcopy(value)
|
|
2101
|
-
candidate_id = finding_candidate_id(finding)
|
|
2102
|
-
if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
|
|
2103
|
-
"rejected",
|
|
2104
|
-
"not_applicable",
|
|
2105
|
-
}:
|
|
2106
|
-
surfaces = coverage.get("surfaces")
|
|
2107
|
-
for item in surfaces if isinstance(surfaces, list) else []:
|
|
2108
|
-
if isinstance(item, dict) and item.get("candidateId") == candidate_id:
|
|
2109
|
-
if not isinstance(item.get("previousFindings"), list):
|
|
2110
|
-
item["previousFindings"] = []
|
|
2111
|
-
history = item["previousFindings"]
|
|
2112
|
-
if not any(
|
|
2113
|
-
isinstance(previous, dict)
|
|
2114
|
-
and _finding_key(previous) == _finding_key(finding)
|
|
2115
|
-
and _finding_content(previous) == _finding_content(finding)
|
|
2116
|
-
for previous in history
|
|
2117
|
-
):
|
|
2118
|
-
history.append(finding)
|
|
2119
|
-
continue
|
|
2120
|
-
for key in ("findingId", "occurrenceId", "fingerprints"):
|
|
2121
|
-
finding.pop(key, None)
|
|
2122
|
-
locations = finding.get("locations", [])
|
|
2123
|
-
if not isinstance(locations, list) or not any(
|
|
2124
|
-
isinstance(location, dict)
|
|
2125
|
-
and isinstance(location.get("path"), str)
|
|
2126
|
-
and any(
|
|
2127
|
-
path_within_scope(location["path"], path)
|
|
2128
|
-
for path in binding["scope"]["includePaths"]
|
|
2129
|
-
)
|
|
2130
|
-
for location in locations
|
|
2131
|
-
):
|
|
2132
|
-
warnings.append(f"Skipped out-of-scope finding from {relative}.")
|
|
2133
|
-
continue
|
|
2134
|
-
provenance = finding.setdefault("provenance", {"source": "local_plugin"})
|
|
2135
|
-
if not isinstance(provenance, dict):
|
|
2136
|
-
findings.append(finding)
|
|
2137
|
-
continue
|
|
2138
|
-
if worker_id:
|
|
2139
|
-
provenance.setdefault("workerId", worker_id)
|
|
2140
|
-
_ensure_finding_identity(finding)
|
|
2141
|
-
if not valid_finding(finding):
|
|
2142
|
-
findings.append(finding)
|
|
2143
|
-
continue
|
|
2144
|
-
key = _finding_key(finding)
|
|
2145
|
-
represented_by_parent = False
|
|
2146
|
-
if relative != "parent":
|
|
2147
|
-
if key in represented:
|
|
2148
|
-
mapped_key = represented[key]
|
|
2149
|
-
historical_contents = represented_history.get(key, set())
|
|
2150
|
-
elif worker_id and candidate_id:
|
|
2151
|
-
candidate_key = _worker_candidate_key(worker_id, candidate_id, finding)
|
|
2152
|
-
if candidate_key not in represented_candidates:
|
|
2153
|
-
represented_candidates[candidate_key] = key
|
|
2154
|
-
mapped_key = represented_candidates[candidate_key]
|
|
2155
|
-
historical_contents = represented_candidate_history.get(candidate_key, set())
|
|
2156
|
-
else:
|
|
2157
|
-
mapped_key = None
|
|
2158
|
-
historical_contents = set()
|
|
2159
|
-
if mapped_key is not None:
|
|
2160
|
-
key = mapped_key
|
|
2161
|
-
represented_by_parent = _digest(_finding_content(value)) in historical_contents
|
|
2162
|
-
if key in finding_positions:
|
|
2163
|
-
retained = findings[finding_positions[key]]
|
|
2164
|
-
if finding != retained:
|
|
2165
|
-
if not represented_by_parent and _finding_strength(finding) > _finding_strength(
|
|
2166
|
-
retained
|
|
2167
|
-
):
|
|
2168
|
-
previous = copy.deepcopy(retained)
|
|
2169
|
-
previous_history = previous["provenance"].pop("previousFindings", [])
|
|
2170
|
-
retained = finding
|
|
2171
|
-
findings[finding_positions[key]] = retained
|
|
2172
|
-
else:
|
|
2173
|
-
previous = copy.deepcopy(value)
|
|
2174
|
-
previous_history = previous.get("provenance", {}).pop(
|
|
2175
|
-
"previousFindings", []
|
|
2176
|
-
)
|
|
2177
|
-
retained_provenance = retained["provenance"]
|
|
2178
|
-
retained_history = retained_provenance.get("previousFindings")
|
|
2179
|
-
history = (
|
|
2180
|
-
[item for item in retained_history if isinstance(item, dict)]
|
|
2181
|
-
if isinstance(retained_history, list)
|
|
2182
|
-
else []
|
|
2183
|
-
)
|
|
2184
|
-
retained_provenance["previousFindings"] = history
|
|
2185
|
-
for original in [
|
|
2186
|
-
*(previous_history if isinstance(previous_history, list) else []),
|
|
2187
|
-
previous,
|
|
2188
|
-
]:
|
|
2189
|
-
if not isinstance(original, dict):
|
|
2190
|
-
continue
|
|
2191
|
-
source_key = _finding_key(original)
|
|
2192
|
-
source_content = _finding_content(original)
|
|
2193
|
-
already_retained = any(
|
|
2194
|
-
source_key == _finding_key(historical)
|
|
2195
|
-
and source_content == _finding_content(historical)
|
|
2196
|
-
for historical in _retained_findings(retained)
|
|
2197
|
-
)
|
|
2198
|
-
if (
|
|
2199
|
-
not already_retained
|
|
2200
|
-
and original not in history
|
|
2201
|
-
and original != retained
|
|
2202
|
-
):
|
|
2203
|
-
history.append(original)
|
|
2204
|
-
continue
|
|
2205
|
-
finding_positions[key] = len(findings)
|
|
2206
|
-
findings.append(finding)
|
|
2207
|
-
if superseded and not selected_candidates and not retain_pending:
|
|
2208
|
-
continue
|
|
2209
|
-
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
2210
|
-
if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
|
|
2211
|
-
continue
|
|
2212
|
-
items = draft["coverage"].get(field, [])
|
|
2213
|
-
if field == "deferred":
|
|
2214
|
-
items = deferred_rows[relative]
|
|
2215
|
-
if not isinstance(items, list):
|
|
2216
|
-
continue
|
|
2217
|
-
output = coverage.setdefault(field, [])
|
|
2218
|
-
if not isinstance(output, list):
|
|
2219
|
-
# Keep malformed canonical collections for the existing finalizer's
|
|
2220
|
-
# recovery and warnings rather than silently changing its contract.
|
|
2221
|
-
continue
|
|
2222
|
-
for item in items:
|
|
2223
|
-
source_item = item
|
|
2224
|
-
if not coverage_row_is_retained(field, item, worker_id, selection):
|
|
2225
|
-
continue
|
|
2226
|
-
if id(item) in replaced_rows.get(field, ()):
|
|
2227
|
-
continue
|
|
2228
|
-
if (
|
|
2229
|
-
field == "deferred"
|
|
2230
|
-
and isinstance(item, dict)
|
|
2231
|
-
and isinstance(item.get("id"), str)
|
|
2232
|
-
and (worker_id, item["id"]) in closed_deferred
|
|
2233
|
-
):
|
|
2234
|
-
continue
|
|
2235
|
-
if field == "openQuestions" and isinstance(item, str):
|
|
2236
|
-
item = {"question": item.strip()}
|
|
2237
|
-
if (
|
|
2238
|
-
field == "surfaces"
|
|
2239
|
-
and isinstance(item, dict)
|
|
2240
|
-
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
2241
|
-
and isinstance(item.get("candidateId"), str)
|
|
2242
|
-
and (history_findings := rejected_history.get((worker_id, item["candidateId"])))
|
|
2243
|
-
):
|
|
2244
|
-
item = copy.deepcopy(item)
|
|
2245
|
-
if not isinstance(item.get("previousFindings"), list):
|
|
2246
|
-
item["previousFindings"] = []
|
|
2247
|
-
history = item["previousFindings"]
|
|
2248
|
-
for finding in history_findings:
|
|
2249
|
-
if not any(
|
|
2250
|
-
isinstance(previous, dict)
|
|
2251
|
-
and _finding_key(previous) == _finding_key(finding)
|
|
2252
|
-
and _finding_content(previous) == _finding_content(finding)
|
|
2253
|
-
for previous in history
|
|
2254
|
-
):
|
|
2255
|
-
history.append(copy.deepcopy(finding))
|
|
2256
|
-
if field == "surfaces" and isinstance(item, dict):
|
|
2257
|
-
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
2258
|
-
if isinstance(item, dict) and "id" not in item:
|
|
2259
|
-
semantic_item = dict(item)
|
|
2260
|
-
existing = next(
|
|
2261
|
-
(
|
|
2262
|
-
row
|
|
2263
|
-
for row in output
|
|
2264
|
-
if isinstance(row, dict)
|
|
2265
|
-
and {key: value for key, value in row.items() if key != "id"}
|
|
2266
|
-
== semantic_item
|
|
2267
|
-
),
|
|
2268
|
-
None,
|
|
2269
|
-
)
|
|
2270
|
-
if existing is not None:
|
|
2271
|
-
if project_worker_coverage and worker_id is not None:
|
|
2272
|
-
worker_row_owners[id(existing)] = worker_id
|
|
2273
|
-
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2274
|
-
linked_surface_outputs[id(source_item)] = existing
|
|
2275
|
-
continue
|
|
2276
|
-
retained = append_coverage_row(output, item, worker_id, field)
|
|
2277
|
-
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2278
|
-
linked_surface_outputs[id(source_item)] = retained
|
|
2279
|
-
|
|
2280
|
-
identities: dict[str, str] = {}
|
|
2281
|
-
for finding in findings:
|
|
2282
|
-
if not valid_finding(finding):
|
|
2283
|
-
continue
|
|
2284
|
-
identity = finding.get("identity")
|
|
2285
|
-
if not isinstance(identity, dict):
|
|
2286
|
-
continue
|
|
2287
|
-
key = _encoded([finding.get("ruleId"), identity]).decode()
|
|
2288
|
-
variant = _finding_key(finding)
|
|
2289
|
-
if key in identities and identities[key] != variant:
|
|
2290
|
-
finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)
|
|
2291
|
-
identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"
|
|
2292
|
-
identities[key] = variant
|
|
2293
|
-
# Bind links to retained output objects before their IDs become unique.
|
|
2294
|
-
# Candidate metadata can give an anonymous row the same preliminary ID as
|
|
2295
|
-
# another surface, so only the final surface ID can be used for its links.
|
|
2296
|
-
retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
|
|
2297
|
-
output_tasks = coverage.get("deferred", [])
|
|
2298
|
-
task_candidates = [
|
|
2299
|
-
(worker_row_owners.get(id(row)), row)
|
|
2300
|
-
for row in (output_tasks if isinstance(output_tasks, list) else [])
|
|
2301
|
-
]
|
|
2302
|
-
for task, published_id, surface, _, task_source in surface_links:
|
|
2303
|
-
retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
|
|
2304
|
-
if retained_surface is None or not isinstance(retained_surface.get("id"), str):
|
|
2305
|
-
continue
|
|
2306
|
-
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2307
|
-
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2308
|
-
links.setdefault(published_id, retained_surface)
|
|
2309
|
-
# A newer worker checkpoint may omit a surface still needed by a retained
|
|
2310
|
-
# task. Restore its parent copy only when no worker replacement survived.
|
|
2311
|
-
for task, published_id, surface, parent_surface, task_source in surface_links:
|
|
2312
|
-
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2313
|
-
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2314
|
-
if published_id not in links and isinstance(coverage.get("surfaces"), list):
|
|
2315
|
-
retained_surface = append_coverage_row(
|
|
2316
|
-
coverage["surfaces"], parent_surface, None, "surfaces"
|
|
2317
|
-
)
|
|
2318
|
-
record_coverage_owner(
|
|
2319
|
-
retained_surface, surface, surface.get("workerId"), "surfaces"
|
|
2320
|
-
)
|
|
2321
|
-
links[published_id] = retained_surface
|
|
2322
|
-
if project_worker_coverage:
|
|
2323
|
-
worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
|
|
2324
|
-
named_surface_outputs = {
|
|
2325
|
-
identity for outputs in worker_surface_outputs.values() for identity in outputs
|
|
2326
|
-
}
|
|
2327
|
-
surfaces = coverage.get("surfaces", [])
|
|
2328
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2329
|
-
if not isinstance(surface, dict):
|
|
2330
|
-
continue
|
|
2331
|
-
owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
|
|
2332
|
-
if isinstance(owner, str) and isinstance(identity, str):
|
|
2333
|
-
key = (owner, identity)
|
|
2334
|
-
existing = worker_surfaces.get(key)
|
|
2335
|
-
# An anonymous candidate ID can collide with an explicit surface ID.
|
|
2336
|
-
# Bind a named worker link to the explicitly named source.
|
|
2337
|
-
if existing is None or (
|
|
2338
|
-
id(surface) in named_surface_outputs
|
|
2339
|
-
and id(existing) not in named_surface_outputs
|
|
2340
|
-
):
|
|
2341
|
-
worker_surfaces[key] = surface
|
|
2342
|
-
for task in output_tasks if isinstance(output_tasks, list) else []:
|
|
2343
|
-
if (
|
|
2344
|
-
not isinstance(task, dict)
|
|
2345
|
-
or (owner := worker_row_owners.get(id(task))) is None
|
|
2346
|
-
or not isinstance(task.get("surfaceIds"), list)
|
|
2347
|
-
):
|
|
2348
|
-
continue
|
|
2349
|
-
for identity in task["surfaceIds"]:
|
|
2350
|
-
if (
|
|
2351
|
-
isinstance(identity, str)
|
|
2352
|
-
and (surface := worker_surfaces.get((owner, identity))) is not None
|
|
2353
|
-
):
|
|
2354
|
-
_, links = retained_surface_links.setdefault(id(task), (task, {}))
|
|
2355
|
-
links.setdefault(identity, surface)
|
|
2356
|
-
for field in ("surfaces", "explicitExclusions", "deferred"):
|
|
2357
|
-
used: set[str] = set()
|
|
2358
|
-
items = coverage.setdefault(field, [])
|
|
2359
|
-
reserved = {
|
|
2360
|
-
row["id"]
|
|
2361
|
-
for row in (items if isinstance(items, list) else [])
|
|
2362
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
444
|
+
|
|
445
|
+
@contextmanager
|
|
446
|
+
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
447
|
+
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
448
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
449
|
+
previous = None
|
|
450
|
+
try:
|
|
451
|
+
head_path.lstat()
|
|
452
|
+
except FileNotFoundError:
|
|
453
|
+
pass
|
|
454
|
+
else:
|
|
455
|
+
descriptor = open_scan_local_file_descriptor(
|
|
456
|
+
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
457
|
+
)
|
|
458
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
459
|
+
metadata = os.fstat(handle.fileno())
|
|
460
|
+
previous = (handle.read(), metadata)
|
|
461
|
+
directories = ("checkpoints", "checkpoint-heads")
|
|
462
|
+
previous_files = {
|
|
463
|
+
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
464
|
+
}
|
|
465
|
+
try:
|
|
466
|
+
yield
|
|
467
|
+
except BaseException:
|
|
468
|
+
if previous is None:
|
|
469
|
+
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
470
|
+
else:
|
|
471
|
+
payload, metadata = previous
|
|
472
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
473
|
+
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
474
|
+
current_files = {
|
|
475
|
+
relative
|
|
476
|
+
for directory in directories
|
|
477
|
+
for relative in _checkpoint_paths(scan_dir, directory)
|
|
2363
478
|
}
|
|
2364
|
-
for
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
if id(item) in canonical_rows:
|
|
2368
|
-
if isinstance(item.get("id"), str):
|
|
2369
|
-
used.add(item["id"])
|
|
2370
|
-
continue
|
|
2371
|
-
item.setdefault("id", _saved_coverage_id(item))
|
|
2372
|
-
if not isinstance(item["id"], str):
|
|
2373
|
-
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
2374
|
-
continue
|
|
2375
|
-
if item["id"] in used:
|
|
2376
|
-
if project_worker_coverage:
|
|
2377
|
-
base = item["id"]
|
|
2378
|
-
suffix = 2
|
|
2379
|
-
while item["id"] in used or item["id"] in reserved:
|
|
2380
|
-
item["id"] = f"{base}-{suffix}"
|
|
2381
|
-
suffix += 1
|
|
2382
|
-
else:
|
|
2383
|
-
item["id"] = f"{item['id']}-{_digest(item)[:16]}"
|
|
2384
|
-
used.add(item["id"])
|
|
2385
|
-
if field == "surfaces":
|
|
2386
|
-
item.setdefault("receiptRefs", [])
|
|
2387
|
-
for task, links in retained_surface_links.values():
|
|
2388
|
-
task["surfaceIds"] = [
|
|
2389
|
-
links[identity]["id"] if isinstance(identity, str) and identity in links else identity
|
|
2390
|
-
for identity in task["surfaceIds"]
|
|
2391
|
-
]
|
|
2392
|
-
coverage.pop("completeness", None)
|
|
2393
|
-
if stopped:
|
|
2394
|
-
if not isinstance(coverage.get("deferred"), list):
|
|
2395
|
-
coverage["deferred"] = []
|
|
2396
|
-
item = {"id": "scan-stopped", "reason": reason}
|
|
2397
|
-
if item not in coverage["deferred"]:
|
|
2398
|
-
coverage["deferred"].append(item)
|
|
2399
|
-
return manifest, {"findings": findings}, coverage
|
|
479
|
+
for relative in current_files - previous_files:
|
|
480
|
+
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
481
|
+
raise
|
|
2400
482
|
|
|
2401
483
|
|
|
2402
484
|
def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
|
|
@@ -2413,107 +495,142 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
|
|
|
2413
495
|
return prepared.coverage
|
|
2414
496
|
|
|
2415
497
|
|
|
2416
|
-
def
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
descriptor = -1
|
|
2426
|
-
metadata = os.fstat(handle.fileno())
|
|
2427
|
-
snapshots[relative] = (handle.read(), metadata)
|
|
2428
|
-
except ContractError:
|
|
2429
|
-
path = scan_dir / relative
|
|
2430
|
-
if path.exists() or path.is_symlink():
|
|
2431
|
-
if relative == "threatmodel.md":
|
|
2432
|
-
# This optional projection will not replace an unsafe destination.
|
|
2433
|
-
continue
|
|
2434
|
-
raise
|
|
2435
|
-
snapshots[relative] = None
|
|
2436
|
-
finally:
|
|
2437
|
-
if descriptor >= 0:
|
|
2438
|
-
os.close(descriptor)
|
|
2439
|
-
return snapshots
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
def _restore_published_outputs(
|
|
2443
|
-
scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
|
|
2444
|
-
) -> None:
|
|
2445
|
-
for relative, snapshot in snapshots.items():
|
|
2446
|
-
if snapshot is None:
|
|
2447
|
-
path = scan_dir / relative
|
|
2448
|
-
if path.exists() or path.is_symlink():
|
|
2449
|
-
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
2450
|
-
else:
|
|
2451
|
-
contents, metadata = snapshot
|
|
2452
|
-
write_scan_local_bytes(scan_dir, relative, contents)
|
|
2453
|
-
# Saved-result selection uses mtime; rollback must preserve its ordering.
|
|
2454
|
-
os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
@contextmanager
|
|
2458
|
-
def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
|
|
2459
|
-
"""Recover a settled, zero-accepted Deep deadline without sealing the parent."""
|
|
498
|
+
def preserve_deep_deadline_draft(
|
|
499
|
+
db: Any,
|
|
500
|
+
connection: Any,
|
|
501
|
+
scan: Any,
|
|
502
|
+
*,
|
|
503
|
+
preserve_reduction: bool = False,
|
|
504
|
+
allow_empty: bool = False,
|
|
505
|
+
) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:
|
|
506
|
+
"""Recover unfinished reviews at a settled Deep deadline without sealing the parent."""
|
|
2460
507
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2461
|
-
snapshots = _snapshot_published_outputs(scan_dir)
|
|
2462
508
|
warnings = json.loads(scan["completion_warnings_json"])
|
|
2463
509
|
binding = db.workbench_completion_binding(scan, db.now())
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
510
|
+
workers = connection.execute(
|
|
511
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
|
|
512
|
+
(scan["id"],),
|
|
513
|
+
).fetchall()
|
|
514
|
+
if preserve_reduction:
|
|
515
|
+
# Completed reviews already went through reduction. Only add evidence
|
|
516
|
+
# from unfinished reviews; raw accepted drafts must not undo that result.
|
|
517
|
+
workers = [
|
|
518
|
+
worker
|
|
519
|
+
for worker in workers
|
|
520
|
+
if worker["kind"] == "discovery" and worker["status"] == "canceled"
|
|
521
|
+
]
|
|
522
|
+
if not workers:
|
|
523
|
+
return None
|
|
524
|
+
_prepare_scan_finalization(
|
|
525
|
+
scan_dir,
|
|
526
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
527
|
+
completion_binding=binding,
|
|
528
|
+
allow_missing_writeups=True,
|
|
529
|
+
)
|
|
530
|
+
snapshots = _snapshot_published_outputs(
|
|
531
|
+
scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json")
|
|
532
|
+
)
|
|
533
|
+
created_paths: set[str] = set()
|
|
534
|
+
try:
|
|
535
|
+
documents = merge_saved_results(
|
|
536
|
+
scan_dir,
|
|
537
|
+
scan["id"],
|
|
538
|
+
binding,
|
|
539
|
+
workers,
|
|
540
|
+
warnings,
|
|
541
|
+
stopped=not preserve_reduction,
|
|
542
|
+
preserve_canonical_parent=preserve_reduction,
|
|
543
|
+
project_worker_coverage=True,
|
|
544
|
+
# Without reduction, the coordinator's empty draft cannot replace worker evidence.
|
|
545
|
+
scan_mode="deep" if preserve_reduction else None,
|
|
546
|
+
finding_scope=db.requested_finding_scope(scan),
|
|
547
|
+
created_paths=created_paths,
|
|
548
|
+
current_contract=True,
|
|
549
|
+
reason=(
|
|
550
|
+
"The configured discovery time limit elapsed before any source review completed. "
|
|
551
|
+
"Saved findings and pending review were preserved without a completed reduction."
|
|
552
|
+
),
|
|
553
|
+
)
|
|
554
|
+
if documents is None:
|
|
555
|
+
if not allow_empty:
|
|
2483
556
|
raise ContractError("Deep Scan deadline has no saved parent draft.")
|
|
2484
|
-
|
|
2485
|
-
# Validate recovered evidence against the current finding contract,
|
|
2486
|
-
# scope, source lineage, and receipts before admitting this draft.
|
|
2487
|
-
prepared = _prepare_scan_finalization(
|
|
2488
|
-
scan_dir,
|
|
2489
|
-
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
2490
|
-
completion_binding=binding,
|
|
2491
|
-
recover_unsealed=True,
|
|
2492
|
-
draft_documents=documents,
|
|
2493
|
-
)
|
|
2494
|
-
warnings.extend(prepared.warnings)
|
|
2495
|
-
manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
|
|
2496
|
-
manifest["scan"].pop("sealedAt", None)
|
|
2497
|
-
manifest["scan"].pop("artifacts", None)
|
|
2498
|
-
checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
|
|
2499
|
-
contents = _encoded(checkpoint)
|
|
2500
|
-
name = f"{hashlib.sha256(contents).hexdigest()}.json"
|
|
2501
|
-
write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
|
|
2502
|
-
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
|
|
2503
|
-
for filename, document in (
|
|
2504
|
-
("findings.json", findings),
|
|
2505
|
-
("coverage.json", coverage),
|
|
2506
|
-
("scan-manifest.json", manifest),
|
|
2507
|
-
):
|
|
2508
|
-
write_scan_local_bytes(scan_dir, filename, _encoded(document))
|
|
557
|
+
_restore_published_outputs(scan_dir, snapshots, created_paths)
|
|
2509
558
|
connection.execute(
|
|
2510
559
|
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
2511
560
|
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
|
|
2512
561
|
)
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
562
|
+
return None
|
|
563
|
+
if preserve_reduction:
|
|
564
|
+
marker = {
|
|
565
|
+
"id": "scan-deadline",
|
|
566
|
+
"reason": (
|
|
567
|
+
"The configured discovery time limit elapsed. Saved findings and "
|
|
568
|
+
"pending work from unfinished source reviews were preserved."
|
|
569
|
+
),
|
|
570
|
+
}
|
|
571
|
+
if not any(
|
|
572
|
+
marker["id"] in (row.get("id"), row.get("candidateId"))
|
|
573
|
+
for key in ("deferred", "resolvedDeferred")
|
|
574
|
+
for row in documents[2].get(key, [])
|
|
575
|
+
):
|
|
576
|
+
documents[2]["deferred"].append(marker)
|
|
577
|
+
documents[0]["scan"].pop("complete", None)
|
|
578
|
+
expected_coverage_mode = db.expected_coverage_mode(scan)
|
|
579
|
+
if not preserve_reduction:
|
|
580
|
+
# A capped deadline can retain interrupted records even though its
|
|
581
|
+
# completion binding requires strict validation of completed findings.
|
|
582
|
+
_populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)
|
|
583
|
+
_populate_unsealed_artifact_envelope(*documents, binding)
|
|
584
|
+
if expected_coverage_mode == "deep_repository":
|
|
585
|
+
documents[2]["inventoryStrategy"] = "repository"
|
|
586
|
+
elif expected_coverage_mode == "scoped_path":
|
|
587
|
+
documents[2]["inventoryStrategy"] = "scoped_path"
|
|
588
|
+
schema_dir = Path(__file__).resolve().parent.parent / "schemas"
|
|
589
|
+
discarded_findings = _recover_unsealed_findings(
|
|
590
|
+
documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True
|
|
591
|
+
)
|
|
592
|
+
_recover_unsealed_coverage(
|
|
593
|
+
documents[2], schema_dir, scan_dir, warnings, discarded_findings
|
|
594
|
+
)
|
|
595
|
+
# Reuse stopped-result recovery's per-record validation, scope checks,
|
|
596
|
+
# source lineage, and receipt validation before admitting this draft.
|
|
597
|
+
prepared = _prepare_scan_finalization(
|
|
598
|
+
scan_dir,
|
|
599
|
+
expected_coverage_mode=expected_coverage_mode,
|
|
600
|
+
completion_binding=binding,
|
|
601
|
+
recover_unsealed=not preserve_reduction,
|
|
602
|
+
allow_missing_writeups=preserve_reduction,
|
|
603
|
+
draft_documents=documents,
|
|
604
|
+
finding_scope=db.requested_finding_scope(scan),
|
|
605
|
+
)
|
|
606
|
+
warnings.extend(prepared.warnings)
|
|
607
|
+
manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
|
|
608
|
+
manifest["scan"].pop("sealedAt", None)
|
|
609
|
+
manifest["scan"].pop("artifacts", None)
|
|
610
|
+
manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
|
|
611
|
+
checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
|
|
612
|
+
contents = _encoded(checkpoint)
|
|
613
|
+
name = f"{hashlib.sha256(contents).hexdigest()}.json"
|
|
614
|
+
checkpoint_path = f"checkpoints/{name}"
|
|
615
|
+
# This draft is recoverable parent evidence too. Restore its prior state
|
|
616
|
+
# on local publication failure or a later caller transaction rollback.
|
|
617
|
+
snapshots.update(_snapshot_published_outputs(scan_dir, (checkpoint_path,)))
|
|
618
|
+
write_scan_local_bytes(scan_dir, checkpoint_path, contents)
|
|
619
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
|
|
620
|
+
for filename, document in (
|
|
621
|
+
("findings.json", findings),
|
|
622
|
+
*([("coverage.json", coverage)] if "coverageRef" in manifest["scan"] else []),
|
|
623
|
+
("scan-manifest.json", manifest),
|
|
624
|
+
):
|
|
625
|
+
write_scan_local_bytes(scan_dir, filename, _encoded(document))
|
|
626
|
+
connection.execute(
|
|
627
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
628
|
+
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
|
|
629
|
+
)
|
|
630
|
+
except BaseException:
|
|
631
|
+
_restore_published_outputs(scan_dir, snapshots, created_paths)
|
|
632
|
+
raise
|
|
633
|
+
return scan_dir, snapshots, created_paths
|
|
2517
634
|
|
|
2518
635
|
|
|
2519
636
|
def preserve_scan_results_locked(
|
|
@@ -2522,25 +639,44 @@ def preserve_scan_results_locked(
|
|
|
2522
639
|
scan_id: str,
|
|
2523
640
|
*,
|
|
2524
641
|
recovery_source_digests: dict[str, str] | None = None,
|
|
642
|
+
recovery_checkpoint_heads: dict[str, str | None] | None = None,
|
|
2525
643
|
include_parent_with_recovery: bool = False,
|
|
644
|
+
recovery_snapshot_error: _CheckpointSnapshotError | None = None,
|
|
2526
645
|
) -> bool:
|
|
2527
646
|
"""Publish or verify retained terminal results through the workbench host."""
|
|
2528
647
|
scan = db.require_scan(connection, scan_id)
|
|
2529
648
|
if scan["status"] != "failed":
|
|
2530
649
|
return False
|
|
2531
650
|
frozen_source_digests: dict[str, str] | None = None
|
|
651
|
+
checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
|
|
2532
652
|
model_source: list[str] = []
|
|
2533
653
|
saved_model_source: str | None = None
|
|
2534
654
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
655
|
+
frozen_source_times = None
|
|
2535
656
|
if raw_frozen_sources is not None:
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
)
|
|
657
|
+
retained = json.loads(raw_frozen_sources)
|
|
658
|
+
frozen_source_digests, saved_model_source = _retained_source_state(retained)
|
|
659
|
+
if isinstance(retained.get("sources"), dict):
|
|
660
|
+
frozen_source_times = retained.get("sourceTimes")
|
|
2539
661
|
if saved_model_source is not None:
|
|
2540
662
|
model_source.append(saved_model_source)
|
|
663
|
+
canonical_sources = frozen_source_digests
|
|
2541
664
|
if recovery_source_digests is not None:
|
|
2542
665
|
frozen_source_digests = recovery_source_digests
|
|
666
|
+
checkpoint_heads = recovery_checkpoint_heads
|
|
2543
667
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
668
|
+
if frozen_source_digests is not None:
|
|
669
|
+
_restore_frozen_model_capture(
|
|
670
|
+
scan_dir, scan_id, frozen_source_digests, frozen_source_times, saved_model_source
|
|
671
|
+
)
|
|
672
|
+
if frozen_source_times is not None and recovery_source_digests is None:
|
|
673
|
+
_freeze_source_times(
|
|
674
|
+
scan_dir,
|
|
675
|
+
scan_id,
|
|
676
|
+
frozen_source_digests,
|
|
677
|
+
frozen_source_times,
|
|
678
|
+
retained.get("parentContext"),
|
|
679
|
+
)
|
|
2544
680
|
deep_run = connection.execute(
|
|
2545
681
|
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
2546
682
|
).fetchone()
|
|
@@ -2591,11 +727,15 @@ def preserve_scan_results_locked(
|
|
|
2591
727
|
db.index_findings(connection, scan_id, findings, scan["completed_at"])
|
|
2592
728
|
connection.execute(
|
|
2593
729
|
"UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
|
|
730
|
+
"retained_checkpoint_heads_json = ?, "
|
|
2594
731
|
"completion_warnings_json = ?, "
|
|
2595
732
|
"updated_at = ? WHERE id = ? AND status = 'failed'",
|
|
2596
733
|
(
|
|
2597
734
|
digest,
|
|
2598
735
|
_encode_retained_sources(retained_sources, model_source),
|
|
736
|
+
json.dumps(
|
|
737
|
+
manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True
|
|
738
|
+
),
|
|
2599
739
|
json.dumps(list(dict.fromkeys(warnings))),
|
|
2600
740
|
timestamp,
|
|
2601
741
|
scan_id,
|
|
@@ -2611,10 +751,7 @@ def preserve_scan_results_locked(
|
|
|
2611
751
|
existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
|
|
2612
752
|
existing = None
|
|
2613
753
|
if scan["seal_manifest_digest"] is not None or (
|
|
2614
|
-
isinstance(existing_scan, dict)
|
|
2615
|
-
and (
|
|
2616
|
-
existing_scan.get("sealedAt") is not None or existing_scan.get("artifacts") is not None
|
|
2617
|
-
)
|
|
754
|
+
isinstance(existing_scan, dict) and manifest_is_sealed(existing_scan)
|
|
2618
755
|
):
|
|
2619
756
|
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
2620
757
|
existing, existing_findings, _ = finalize_scan(
|
|
@@ -2625,11 +762,23 @@ def preserve_scan_results_locked(
|
|
|
2625
762
|
db.verify_manifest_binding(scan, existing)
|
|
2626
763
|
if existing_scan.get("status") == outcome:
|
|
2627
764
|
existing_sources = existing_scan.get("preservedSources")
|
|
765
|
+
existing_heads = existing_scan.get("preservedCheckpointHeads")
|
|
2628
766
|
if frozen_source_digests is None:
|
|
2629
767
|
frozen_source_digests = _source_digests(
|
|
2630
768
|
existing_sources, "Stopped scan source digests could not be frozen."
|
|
2631
769
|
)
|
|
2632
|
-
|
|
770
|
+
checkpoint_heads = existing_heads
|
|
771
|
+
unchanged = existing_sources == frozen_source_digests and (
|
|
772
|
+
existing_heads == checkpoint_heads
|
|
773
|
+
or (
|
|
774
|
+
"preservedCheckpointHeads" not in existing_scan
|
|
775
|
+
and recovery_source_digests is None
|
|
776
|
+
and not pending_recovery
|
|
777
|
+
)
|
|
778
|
+
)
|
|
779
|
+
if unchanged and not any(
|
|
780
|
+
Path(path).name == "checkpoint-head.json" for path in frozen_source_digests
|
|
781
|
+
):
|
|
2633
782
|
if (
|
|
2634
783
|
raw_frozen_sources is not None
|
|
2635
784
|
and scan["seal_manifest_digest"] is not None
|
|
@@ -2639,43 +788,116 @@ def preserve_scan_results_locked(
|
|
|
2639
788
|
return True
|
|
2640
789
|
record_publication(existing, existing_findings)
|
|
2641
790
|
return True
|
|
2642
|
-
if recovery_source_digests is None:
|
|
791
|
+
if not unchanged and recovery_source_digests is None and not pending_recovery:
|
|
2643
792
|
raise ContractError("Stopped scan sources changed after terminal publication.")
|
|
2644
793
|
binding = {
|
|
2645
794
|
**db.workbench_completion_binding(scan, scan["completed_at"], existing),
|
|
2646
795
|
"status": outcome,
|
|
2647
796
|
}
|
|
2648
|
-
|
|
797
|
+
canonical_model_source = saved_model_source
|
|
798
|
+
canonical_model = existing_scan.get("threatModel") if isinstance(existing_scan, dict) else None
|
|
799
|
+
if existing is None and frozen_source_digests is not None:
|
|
800
|
+
# An unsealed live draft is a fallback only when its complete identity
|
|
801
|
+
# belongs to the frozen set. Otherwise recover from the captured sources.
|
|
802
|
+
canonical_model = None
|
|
803
|
+
try:
|
|
804
|
+
_, canonical_draft = _read_saved_parent_result(scan_dir, scan_id)
|
|
805
|
+
digest = _digest(canonical_draft)
|
|
806
|
+
path = f"checkpoints/{digest}.json"
|
|
807
|
+
if frozen_source_digests.get(path) == digest:
|
|
808
|
+
canonical_model_source = path
|
|
809
|
+
canonical_model = canonical_draft.get("threatModel")
|
|
810
|
+
except (ContractError, OSError, ValueError):
|
|
811
|
+
# This live fallback is optional.
|
|
812
|
+
pass
|
|
813
|
+
if recovery_source_digests is not None and not pending_recovery:
|
|
2649
814
|
model_source.clear()
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
f"Scan {outcome}; saved findings and pending review were preserved. "
|
|
2662
|
-
f"{scan['failure_message'] or ''}"
|
|
2663
|
-
).strip(),
|
|
2664
|
-
frozen_source_digests=frozen_source_digests,
|
|
2665
|
-
frozen_model_source=model_source[0] if model_source else None,
|
|
2666
|
-
selected_model_source=model_source,
|
|
2667
|
-
project_worker_coverage=scan["mode"] == "deep",
|
|
2668
|
-
allow_frozen_legacy_parent=(
|
|
2669
|
-
include_parent_with_recovery
|
|
2670
|
-
or (
|
|
2671
|
-
recovery_source_digests is None
|
|
2672
|
-
and frozen_source_digests == {}
|
|
2673
|
-
and isinstance(existing_scan, dict)
|
|
2674
|
-
and existing_scan.get("sealedAt") is not None
|
|
2675
|
-
and "preservedSources" not in existing_scan
|
|
2676
|
-
)
|
|
2677
|
-
),
|
|
815
|
+
# Older pending recoveries retained the parent union without recording the mode.
|
|
816
|
+
legacy_parent = (
|
|
817
|
+
include_parent_with_recovery
|
|
818
|
+
or (pending_recovery is not None and pending_recovery.get("legacyParent", True) is True)
|
|
819
|
+
or (
|
|
820
|
+
recovery_source_digests is None
|
|
821
|
+
and frozen_source_digests == {}
|
|
822
|
+
and isinstance(existing_scan, dict)
|
|
823
|
+
and existing_scan.get("sealedAt") is not None
|
|
824
|
+
and "preservedSources" not in existing_scan
|
|
825
|
+
)
|
|
2678
826
|
)
|
|
827
|
+
|
|
828
|
+
def pending_selection(heads: dict[str, str | None]) -> dict[str, Any]:
|
|
829
|
+
return {
|
|
830
|
+
"heads": heads,
|
|
831
|
+
"recoveryBaseDigest": scan["seal_manifest_digest"],
|
|
832
|
+
"legacyParent": legacy_parent,
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
try:
|
|
836
|
+
documents = merge_saved_results(
|
|
837
|
+
scan_dir,
|
|
838
|
+
scan_id,
|
|
839
|
+
binding,
|
|
840
|
+
connection.execute(
|
|
841
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
|
|
842
|
+
(scan_id,),
|
|
843
|
+
).fetchall(),
|
|
844
|
+
warnings,
|
|
845
|
+
stopped=True,
|
|
846
|
+
scan_mode=scan["mode"],
|
|
847
|
+
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
|
|
848
|
+
reason=(
|
|
849
|
+
f"Scan {outcome}; saved findings and pending review were preserved. "
|
|
850
|
+
f"{scan['failure_message'] or ''}"
|
|
851
|
+
).strip(),
|
|
852
|
+
frozen_source_digests=frozen_source_digests,
|
|
853
|
+
checkpoint_heads=checkpoint_heads,
|
|
854
|
+
frozen_model_source=model_source[0] if model_source else None,
|
|
855
|
+
selected_model_source=model_source,
|
|
856
|
+
project_worker_coverage=scan["mode"] == "deep",
|
|
857
|
+
canonical_model=canonical_model,
|
|
858
|
+
canonical_model_source=canonical_model_source,
|
|
859
|
+
canonical_sources=canonical_sources,
|
|
860
|
+
recovery_snapshot_error=recovery_snapshot_error,
|
|
861
|
+
allow_frozen_legacy_parent=legacy_parent,
|
|
862
|
+
parent_already_captured=pending_recovery is not None,
|
|
863
|
+
)
|
|
864
|
+
except _CheckpointSnapshotError as exc:
|
|
865
|
+
if frozen_source_digests is None or recovery_snapshot_error is not None:
|
|
866
|
+
# The stop has already committed. Keep the source set and times in
|
|
867
|
+
# the database even when auxiliary snapshot files cannot be written.
|
|
868
|
+
# Changed raw heads remain rejected on retry.
|
|
869
|
+
captured = exc.captured
|
|
870
|
+
retained_sources = {path: value[0] for path, value in captured.items()}
|
|
871
|
+
source_times = {path: value[1] for path, value in captured.items()}
|
|
872
|
+
retained_heads = exc.heads
|
|
873
|
+
if recovery_snapshot_error is not None:
|
|
874
|
+
_prepare_scan_finalization(
|
|
875
|
+
scan_dir,
|
|
876
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
877
|
+
completion_binding=binding,
|
|
878
|
+
recover_unsealed=True,
|
|
879
|
+
draft_documents=exc.documents,
|
|
880
|
+
finding_scope=db.requested_finding_scope(scan)
|
|
881
|
+
if scan["mode"] == "deep"
|
|
882
|
+
else None,
|
|
883
|
+
)
|
|
884
|
+
retained_heads = pending_selection(retained_heads)
|
|
885
|
+
with connection:
|
|
886
|
+
connection.execute(
|
|
887
|
+
"UPDATE scans SET retained_source_digests_json = ?, "
|
|
888
|
+
"retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
|
|
889
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
890
|
+
(
|
|
891
|
+
_encode_retained_sources(
|
|
892
|
+
retained_sources, exc.model_source, source_times, exc.parent_context
|
|
893
|
+
),
|
|
894
|
+
json.dumps(retained_heads, sort_keys=True),
|
|
895
|
+
json.dumps(list(dict.fromkeys(warnings))),
|
|
896
|
+
scan_id,
|
|
897
|
+
raw_frozen_sources,
|
|
898
|
+
),
|
|
899
|
+
)
|
|
900
|
+
raise
|
|
2679
901
|
if documents is None:
|
|
2680
902
|
unpublished_warnings = list(dict.fromkeys([*warnings, *publication_follow_up_warnings]))
|
|
2681
903
|
if unpublished_warnings != stored_warnings:
|
|
@@ -2686,23 +908,46 @@ def preserve_scan_results_locked(
|
|
|
2686
908
|
(json.dumps(unpublished_warnings), db.now(), scan_id),
|
|
2687
909
|
)
|
|
2688
910
|
return False
|
|
2689
|
-
|
|
2690
|
-
|
|
911
|
+
retained_state = None
|
|
912
|
+
if (
|
|
913
|
+
frozen_source_digests is None
|
|
914
|
+
or scan["retained_checkpoint_heads_json"] is None
|
|
915
|
+
or recovery_source_digests is not None
|
|
916
|
+
or documents[0]["scan"]["preservedSources"] != frozen_source_digests
|
|
917
|
+
or model_source != ([saved_model_source] if saved_model_source is not None else [])
|
|
2691
918
|
):
|
|
2692
919
|
retained_sources = _source_digests(
|
|
2693
920
|
documents[0].get("scan", {}).get("preservedSources"),
|
|
2694
921
|
"Stopped scan source digests could not be frozen.",
|
|
2695
922
|
)
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
)
|
|
923
|
+
retained_heads = documents[0]["scan"]["preservedCheckpointHeads"]
|
|
924
|
+
if (
|
|
925
|
+
recovery_source_digests is not None
|
|
926
|
+
or pending_recovery
|
|
927
|
+
or (existing is not None and existing_scan.get("status") == outcome)
|
|
928
|
+
):
|
|
929
|
+
retained_heads = pending_selection(retained_heads)
|
|
930
|
+
retained_state = (
|
|
931
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
932
|
+
json.dumps(retained_heads, sort_keys=True),
|
|
933
|
+
json.dumps(list(dict.fromkeys(warnings))),
|
|
934
|
+
scan_id,
|
|
935
|
+
raw_frozen_sources,
|
|
936
|
+
)
|
|
937
|
+
|
|
938
|
+
def retain_sources() -> None:
|
|
939
|
+
if retained_state is not None:
|
|
940
|
+
with connection:
|
|
941
|
+
connection.execute(
|
|
942
|
+
"UPDATE scans SET retained_source_digests_json = ?, "
|
|
943
|
+
"retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
|
|
944
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
945
|
+
retained_state,
|
|
946
|
+
)
|
|
947
|
+
|
|
948
|
+
first_preservation = recovery_source_digests is None and not pending_recovery
|
|
949
|
+
if first_preservation:
|
|
950
|
+
retain_sources()
|
|
2706
951
|
prepared = _prepare_scan_finalization(
|
|
2707
952
|
scan_dir,
|
|
2708
953
|
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
@@ -2711,6 +956,8 @@ def preserve_scan_results_locked(
|
|
|
2711
956
|
draft_documents=documents,
|
|
2712
957
|
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
|
|
2713
958
|
)
|
|
959
|
+
if not first_preservation:
|
|
960
|
+
retain_sources()
|
|
2714
961
|
warnings.extend(prepared.warnings)
|
|
2715
962
|
snapshots = _snapshot_published_outputs(scan_dir)
|
|
2716
963
|
try:
|
|
@@ -2733,13 +980,24 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2733
980
|
raise SystemExit("Only a stopped scan can recover terminal results.")
|
|
2734
981
|
if scan["canceled_at"] is not None:
|
|
2735
982
|
raise SystemExit("Canceled scans cannot recover terminal results.")
|
|
2736
|
-
|
|
983
|
+
recovery_snapshot_error = None
|
|
984
|
+
try:
|
|
985
|
+
recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests(
|
|
986
|
+
db, connection, scan
|
|
987
|
+
)
|
|
988
|
+
except _CheckpointSnapshotError as exc:
|
|
989
|
+
recovery_snapshot_error = exc
|
|
990
|
+
recovery_source_digests = {path: value[0] for path, value in exc.captured.items()}
|
|
991
|
+
include_parent = exc.include_parent
|
|
992
|
+
checkpoint_heads = exc.heads
|
|
2737
993
|
if not preserve_scan_results_locked(
|
|
2738
994
|
db,
|
|
2739
995
|
connection,
|
|
2740
996
|
scan_id,
|
|
2741
997
|
recovery_source_digests=recovery_source_digests,
|
|
998
|
+
recovery_checkpoint_heads=checkpoint_heads,
|
|
2742
999
|
include_parent_with_recovery=include_parent,
|
|
1000
|
+
recovery_snapshot_error=recovery_snapshot_error,
|
|
2743
1001
|
):
|
|
2744
1002
|
raise SystemExit("No saved stopped-scan results were available to recover.")
|
|
2745
1003
|
db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
|
|
@@ -2807,7 +1065,7 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2807
1065
|
manifest_path = db.artifact_path(scan_dir, "scan-manifest.json", required=False)
|
|
2808
1066
|
if manifest_path is not None:
|
|
2809
1067
|
manifest = db.read_json_object(manifest_path).get("scan", {})
|
|
2810
|
-
if
|
|
1068
|
+
if manifest_is_sealed(manifest):
|
|
2811
1069
|
raise SystemExit("The scan is sealed; its artifacts cannot be modified.")
|
|
2812
1070
|
output = args.artifact_path
|
|
2813
1071
|
key = output.lower()
|
|
@@ -2823,66 +1081,6 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2823
1081
|
return {"scanId": scan_id, "path": str(scan_dir / output)}
|
|
2824
1082
|
|
|
2825
1083
|
|
|
2826
|
-
def _require_current_deep_publication(
|
|
2827
|
-
db: Any, connection: Any, scan: Any, draft: dict[str, Any]
|
|
2828
|
-
) -> None:
|
|
2829
|
-
scan_id = scan["id"]
|
|
2830
|
-
run = connection.execute(
|
|
2831
|
-
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
2832
|
-
).fetchone()
|
|
2833
|
-
publication = draft.get("deepScanPublication")
|
|
2834
|
-
if (
|
|
2835
|
-
publication is None
|
|
2836
|
-
and draft["manifest"]["scan"].get("complete") is False
|
|
2837
|
-
and (run is None or run["status"] == "running")
|
|
2838
|
-
):
|
|
2839
|
-
return
|
|
2840
|
-
if run is None:
|
|
2841
|
-
run = db.deep_scan.require_deep_scan_run(connection, scan_id)
|
|
2842
|
-
# Finished discovery-only runs hand remaining publication back to the parent.
|
|
2843
|
-
if (
|
|
2844
|
-
publication is None
|
|
2845
|
-
and run["status"] != "running"
|
|
2846
|
-
and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
2847
|
-
):
|
|
2848
|
-
return
|
|
2849
|
-
db.deep_scan.require_current_coordinator(
|
|
2850
|
-
run,
|
|
2851
|
-
argparse.Namespace(
|
|
2852
|
-
coordinator_generation=publication.get("coordinatorGeneration") if publication else None
|
|
2853
|
-
),
|
|
2854
|
-
)
|
|
2855
|
-
# Once finish selects the canonical parent, only completion may replay it.
|
|
2856
|
-
if publication is None:
|
|
2857
|
-
if run["status"] != "running" and run["manifest_path"] == str(
|
|
2858
|
-
Path(scan["scan_dir"]) / "scan-manifest.json"
|
|
2859
|
-
):
|
|
2860
|
-
raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
|
|
2861
|
-
return
|
|
2862
|
-
|
|
2863
|
-
# Match the durable reducer sequence used by coordinator recovery.
|
|
2864
|
-
def reducer_order(worker: Any) -> tuple[int, str]:
|
|
2865
|
-
match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
|
|
2866
|
-
return (int(match[1]) if match else 0, worker["id"])
|
|
2867
|
-
|
|
2868
|
-
reducer = max(
|
|
2869
|
-
(
|
|
2870
|
-
worker
|
|
2871
|
-
for worker in connection.execute(
|
|
2872
|
-
"SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
|
|
2873
|
-
)
|
|
2874
|
-
if worker["kind"] == "dedup"
|
|
2875
|
-
and worker["status"] == "succeeded"
|
|
2876
|
-
and worker["result_manifest_path"]
|
|
2877
|
-
),
|
|
2878
|
-
key=reducer_order,
|
|
2879
|
-
default=None,
|
|
2880
|
-
)
|
|
2881
|
-
selected_result = reducer["result_manifest_path"] if reducer is not None else None
|
|
2882
|
-
if publication["resultPath"] != selected_result:
|
|
2883
|
-
raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
1084
|
def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
2887
1085
|
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
2888
1086
|
with db.scan_completion_lock(scan_id):
|
|
@@ -2895,13 +1093,6 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2895
1093
|
"The scan stopped; its saved checkpoint was retained without replacing sealed results."
|
|
2896
1094
|
)
|
|
2897
1095
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2898
|
-
if (
|
|
2899
|
-
args.expected_draft_digest is not None
|
|
2900
|
-
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
2901
|
-
):
|
|
2902
|
-
raise SystemExit(
|
|
2903
|
-
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
2904
|
-
)
|
|
2905
1096
|
try:
|
|
2906
1097
|
relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
|
|
2907
1098
|
except ValueError as exc:
|
|
@@ -2911,12 +1102,24 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2911
1102
|
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
2912
1103
|
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
2913
1104
|
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
2914
|
-
|
|
1105
|
+
accepted_input = copy.deepcopy(draft)
|
|
1106
|
+
accepted_checkpoint = None
|
|
1107
|
+
manifest, findings, coverage = (
|
|
1108
|
+
draft["manifest"],
|
|
1109
|
+
draft["findings"],
|
|
1110
|
+
draft.get("reviewState", draft.get("coverage")),
|
|
1111
|
+
)
|
|
1112
|
+
manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
|
|
1113
|
+
manifest["scan"].pop("coverageRef", None)
|
|
2915
1114
|
binding = db.workbench_completion_binding(scan, db.now())
|
|
2916
1115
|
# Save scan IDs without sealing the draft.
|
|
2917
1116
|
_populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
|
|
2918
1117
|
_populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
|
|
2919
1118
|
_validate_completion_binding(manifest, findings, coverage, binding)
|
|
1119
|
+
manifest["scan"]["mode"] = coverage["mode"]
|
|
1120
|
+
manifest["scan"]["review"] = review_summary(coverage)
|
|
1121
|
+
projected = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
|
|
1122
|
+
admitted_input = None
|
|
2920
1123
|
checkpoint = manifest["scan"]
|
|
2921
1124
|
checkpoint_contents = None
|
|
2922
1125
|
if args.checkpoint_path is not None:
|
|
@@ -2933,8 +1136,13 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2933
1136
|
checkpoint, checkpoint_contents = _read_scan_local_json_bytes(
|
|
2934
1137
|
scan_dir, checkpoint_relative, "Staged scan checkpoint"
|
|
2935
1138
|
)
|
|
1139
|
+
accepted_checkpoint = checkpoint
|
|
2936
1140
|
if checkpoint.get("scanId") != scan_id:
|
|
2937
1141
|
raise SystemExit("Staged scan checkpoint belongs to another scan.")
|
|
1142
|
+
if scan["mode"] == "deep":
|
|
1143
|
+
checkpoint_contents = _admit_parent_checkpoint(
|
|
1144
|
+
scan_dir, scan_id, checkpoint, projected
|
|
1145
|
+
)
|
|
2938
1146
|
checkpoint_only = (
|
|
2939
1147
|
scan["mode"] == "deep"
|
|
2940
1148
|
and draft.get("deepScanPublication") is None
|
|
@@ -2948,8 +1156,40 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2948
1156
|
)
|
|
2949
1157
|
if scan["mode"] == "deep" and not checkpoint_only:
|
|
2950
1158
|
_require_current_deep_publication(db, connection, scan, draft)
|
|
1159
|
+
acceptance_input = {**accepted_input, "checkpoint": accepted_checkpoint}
|
|
1160
|
+
acceptance_relative = Path(relative).with_suffix(".accepted.json").as_posix()
|
|
1161
|
+
acknowledge = (
|
|
1162
|
+
scan["mode"] == "deep"
|
|
1163
|
+
and draft.get("deepScanPublication") is not None
|
|
1164
|
+
and db.deep_scan.require_deep_scan_run(connection, scan_id)["workflow_version"]
|
|
1165
|
+
in {"deep-scan-mcp/v1", "deep-security-scan/v1"}
|
|
1166
|
+
)
|
|
1167
|
+
if acknowledge and (scan_dir / acceptance_relative).exists():
|
|
1168
|
+
accepted = _read_scan_local_json(scan_dir, acceptance_relative, "Accepted scan draft")
|
|
1169
|
+
if (
|
|
1170
|
+
accepted.get("status") == "draft_written"
|
|
1171
|
+
and accepted.get("input") == acceptance_input
|
|
1172
|
+
):
|
|
1173
|
+
if accepted.get("draftDigest") != _scan_draft_digest(scan_dir):
|
|
1174
|
+
raise SystemExit(
|
|
1175
|
+
"scan_draft_conflict: canonical scan results changed after publication."
|
|
1176
|
+
)
|
|
1177
|
+
return {
|
|
1178
|
+
"scanId": scan_id,
|
|
1179
|
+
"status": "draft_written",
|
|
1180
|
+
**({"warnings": accepted["warnings"]} if accepted.get("warnings") else {}),
|
|
1181
|
+
}
|
|
1182
|
+
if (
|
|
1183
|
+
args.expected_draft_digest is not None
|
|
1184
|
+
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
1185
|
+
):
|
|
1186
|
+
raise SystemExit(
|
|
1187
|
+
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
1188
|
+
)
|
|
2951
1189
|
if checkpoint_contents is not None:
|
|
2952
1190
|
checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
|
|
1191
|
+
if scan["mode"] == "deep":
|
|
1192
|
+
admitted_input = f"{checkpoint_digest}.json"
|
|
2953
1193
|
write_scan_local_bytes(
|
|
2954
1194
|
scan_dir,
|
|
2955
1195
|
f"checkpoints/{checkpoint_digest}.json",
|
|
@@ -2957,27 +1197,27 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2957
1197
|
)
|
|
2958
1198
|
if checkpoint_only:
|
|
2959
1199
|
return {"scanId": scan_id, "status": "draft_written"}
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
|
|
1200
|
+
projected_contents = _encoded(projected)
|
|
1201
|
+
checkpoint_name = f"{hashlib.sha256(projected_contents).hexdigest()}.json"
|
|
2963
1202
|
checkpoint_relative = f"checkpoints/{checkpoint_name}"
|
|
2964
1203
|
if not (scan_dir / checkpoint_relative).exists():
|
|
2965
|
-
write_scan_local_bytes(scan_dir, checkpoint_relative,
|
|
1204
|
+
write_scan_local_bytes(scan_dir, checkpoint_relative, projected_contents)
|
|
2966
1205
|
if checkpoint_only:
|
|
2967
1206
|
return {"scanId": scan_id, "status": "draft_written"}
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
(
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
scan_dir,
|
|
2978
|
-
filename,
|
|
2979
|
-
(json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
|
|
1207
|
+
head = {"checkpoint": checkpoint_name}
|
|
1208
|
+
if admitted_input is not None:
|
|
1209
|
+
head["inputCheckpoint"] = admitted_input
|
|
1210
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded(head))
|
|
1211
|
+
documents = {
|
|
1212
|
+
filename: (json.dumps(document, allow_nan=False, indent=2) + "\n").encode()
|
|
1213
|
+
for filename, document in (
|
|
1214
|
+
("findings.json", findings),
|
|
1215
|
+
("scan-manifest.json", manifest),
|
|
2980
1216
|
)
|
|
1217
|
+
}
|
|
1218
|
+
for filename, contents in documents.items():
|
|
1219
|
+
write_scan_local_bytes(scan_dir, filename, contents)
|
|
1220
|
+
_remove_scan_local_file_if_exists(scan_dir, "coverage.json")
|
|
2981
1221
|
if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
|
|
2982
1222
|
# Select the accepted final publication before releasing the write lock.
|
|
2983
1223
|
with connection:
|
|
@@ -2987,6 +1227,19 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2987
1227
|
(str(scan_dir / "scan-manifest.json"), scan_id),
|
|
2988
1228
|
)
|
|
2989
1229
|
model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
|
|
1230
|
+
if acknowledge:
|
|
1231
|
+
# Optional recovery evidence is written only after the publication succeeds.
|
|
1232
|
+
try:
|
|
1233
|
+
acceptance = {
|
|
1234
|
+
"status": "draft_written",
|
|
1235
|
+
"input": acceptance_input,
|
|
1236
|
+
"draftDigest": _scan_draft_contents_digest(documents),
|
|
1237
|
+
**({"warnings": [model_warning]} if model_warning else {}),
|
|
1238
|
+
}
|
|
1239
|
+
write_scan_local_bytes(scan_dir, acceptance_relative, _encoded(acceptance))
|
|
1240
|
+
except (OSError, ScanLocalIOError):
|
|
1241
|
+
# A missing receipt prevents recovery but does not undo accepted results.
|
|
1242
|
+
pass
|
|
2990
1243
|
# Accepted Standard drafts are evidence of review or report assembly,
|
|
2991
1244
|
# even when the parent omitted its explicit progress call.
|
|
2992
1245
|
model_only_checkpoint = (
|
|
@@ -3025,20 +1278,31 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
3025
1278
|
|
|
3026
1279
|
|
|
3027
1280
|
def _scan_draft_digest(scan_dir: Path) -> str:
|
|
3028
|
-
|
|
1281
|
+
documents: dict[str, bytes | None] = {}
|
|
3029
1282
|
for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
3030
|
-
digest.update(filename.encode())
|
|
3031
|
-
digest.update(b"\0")
|
|
3032
1283
|
try:
|
|
3033
1284
|
(scan_dir / filename).lstat()
|
|
3034
1285
|
except FileNotFoundError:
|
|
3035
|
-
|
|
1286
|
+
documents[filename] = None
|
|
3036
1287
|
continue
|
|
3037
|
-
digest.update(b"present\0")
|
|
3038
1288
|
descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
|
|
3039
1289
|
with os.fdopen(descriptor, "rb") as handle:
|
|
3040
|
-
|
|
1290
|
+
documents[filename] = handle.read()
|
|
1291
|
+
return _scan_draft_contents_digest(documents)
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
def _scan_draft_contents_digest(documents: dict[str, bytes | None]) -> str:
|
|
1295
|
+
digest = hashlib.sha256()
|
|
1296
|
+
for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
1297
|
+
digest.update(filename.encode())
|
|
3041
1298
|
digest.update(b"\0")
|
|
1299
|
+
contents = documents.get(filename)
|
|
1300
|
+
if contents is None:
|
|
1301
|
+
digest.update(b"missing\0")
|
|
1302
|
+
else:
|
|
1303
|
+
digest.update(b"present\0")
|
|
1304
|
+
digest.update(contents)
|
|
1305
|
+
digest.update(b"\0")
|
|
3042
1306
|
return digest.hexdigest()
|
|
3043
1307
|
|
|
3044
1308
|
|
|
@@ -3064,7 +1328,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
3064
1328
|
args.claim_token,
|
|
3065
1329
|
error_message="Scan failure is owned by another continuation.",
|
|
3066
1330
|
)
|
|
3067
|
-
message = db.optional_text(args.message
|
|
1331
|
+
message = db.optional_text(args.message)
|
|
3068
1332
|
updated = connection.execute(
|
|
3069
1333
|
"""
|
|
3070
1334
|
UPDATE scans
|
|
@@ -3148,3 +1412,158 @@ def preserve_stopped_results_after_transition(db: Any, connection: Any, scan_id:
|
|
|
3148
1412
|
|
|
3149
1413
|
if __name__ == "__main__":
|
|
3150
1414
|
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
def budget_exhausted_draft(
|
|
1418
|
+
db: Any,
|
|
1419
|
+
scan: sqlite3.Row,
|
|
1420
|
+
scan_dir: Path,
|
|
1421
|
+
candidates: list[dict[str, Any]],
|
|
1422
|
+
warning: str,
|
|
1423
|
+
) -> None:
|
|
1424
|
+
documents: dict[str, dict[str, Any]] = {}
|
|
1425
|
+
for name in ("scan-manifest.json", "findings.json"):
|
|
1426
|
+
path = db.artifact_path(scan_dir, name, required=False)
|
|
1427
|
+
if path is not None:
|
|
1428
|
+
documents[name] = db.read_json_object(path)
|
|
1429
|
+
if documents and len(documents) != 2:
|
|
1430
|
+
raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.")
|
|
1431
|
+
|
|
1432
|
+
if documents:
|
|
1433
|
+
manifest = documents["scan-manifest.json"]
|
|
1434
|
+
findings = documents["findings.json"]
|
|
1435
|
+
coverage = (
|
|
1436
|
+
db.read_json_object(db.artifact_path(scan_dir, "coverage.json", required=True))
|
|
1437
|
+
if uses_legacy_review(manifest["scan"])
|
|
1438
|
+
else review_state(manifest["scan"])
|
|
1439
|
+
)
|
|
1440
|
+
if not isinstance(manifest.get("scan"), dict) or not isinstance(
|
|
1441
|
+
findings.get("findings"), list
|
|
1442
|
+
):
|
|
1443
|
+
raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.")
|
|
1444
|
+
for key in ("surfaces", "explicitExclusions", "deferred"):
|
|
1445
|
+
if not isinstance(coverage.get(key), list):
|
|
1446
|
+
raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.")
|
|
1447
|
+
if manifest_is_sealed(manifest["scan"]):
|
|
1448
|
+
raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.")
|
|
1449
|
+
else:
|
|
1450
|
+
contract = db.scan_contract(scan)
|
|
1451
|
+
target_contract = contract["target"]
|
|
1452
|
+
target: dict[str, Any] = {
|
|
1453
|
+
"kind": target_contract["allowedKinds"][0],
|
|
1454
|
+
"targetId": target_contract["targetId"],
|
|
1455
|
+
"displayName": target_contract["displayName"],
|
|
1456
|
+
}
|
|
1457
|
+
if scan["target_revision"] != "unversioned":
|
|
1458
|
+
target["revision"] = scan["target_revision"]
|
|
1459
|
+
if "requiredSnapshotDigest" in target_contract:
|
|
1460
|
+
target["snapshotDigest"] = target_contract["requiredSnapshotDigest"]
|
|
1461
|
+
manifest = {
|
|
1462
|
+
"scan": {
|
|
1463
|
+
"target": target,
|
|
1464
|
+
"scope": {"limitations": [warning]},
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
findings = {"findings": []}
|
|
1468
|
+
coverage = {
|
|
1469
|
+
"inventoryStrategy": (
|
|
1470
|
+
"scoped_path" if db.expected_coverage_mode(scan) == "scoped_path" else "repository"
|
|
1471
|
+
),
|
|
1472
|
+
"surfaces": [],
|
|
1473
|
+
"explicitExclusions": [],
|
|
1474
|
+
"deferred": [],
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
findings_by_candidate = {
|
|
1478
|
+
candidate_id
|
|
1479
|
+
for finding in findings["findings"]
|
|
1480
|
+
if isinstance(finding, dict)
|
|
1481
|
+
and isinstance(candidate_id := finding_candidate_id(finding), str)
|
|
1482
|
+
}
|
|
1483
|
+
existing_deferred = {
|
|
1484
|
+
item.get("candidateId", item.get("id"))
|
|
1485
|
+
for item in coverage["deferred"]
|
|
1486
|
+
if isinstance(item, dict) and isinstance(item.get("candidateId", item.get("id")), str)
|
|
1487
|
+
}
|
|
1488
|
+
existing_surfaces = {
|
|
1489
|
+
item.get("id")
|
|
1490
|
+
for item in coverage["surfaces"]
|
|
1491
|
+
if isinstance(item, dict) and isinstance(item.get("id"), str)
|
|
1492
|
+
}
|
|
1493
|
+
for candidate in candidates:
|
|
1494
|
+
candidate_id = candidate["candidate_id"]
|
|
1495
|
+
if candidate_id in findings_by_candidate or candidate_id in existing_deferred:
|
|
1496
|
+
continue
|
|
1497
|
+
paths = list(dict.fromkeys(location["path"] for location in candidate["locations"]))
|
|
1498
|
+
surface_id = f"candidate-{candidate_id}"
|
|
1499
|
+
validation = candidate.get("validation")
|
|
1500
|
+
validation = validation.get("disposition") if isinstance(validation, dict) else None
|
|
1501
|
+
attack = candidate.get("attack_path")
|
|
1502
|
+
attack = attack.get("decision") if isinstance(attack, dict) else None
|
|
1503
|
+
disposition = (
|
|
1504
|
+
"needs_follow_up"
|
|
1505
|
+
if validation == "deferred" or attack == "deferred"
|
|
1506
|
+
else "not_applicable"
|
|
1507
|
+
if validation == "not_applicable"
|
|
1508
|
+
else "rejected"
|
|
1509
|
+
if validation == "suppressed" or attack == "ignore"
|
|
1510
|
+
else "needs_follow_up"
|
|
1511
|
+
)
|
|
1512
|
+
if surface_id not in existing_surfaces:
|
|
1513
|
+
coverage["surfaces"].append(
|
|
1514
|
+
{
|
|
1515
|
+
"id": surface_id,
|
|
1516
|
+
"label": candidate["summary"],
|
|
1517
|
+
"disposition": disposition,
|
|
1518
|
+
"notes": candidate["evidence"],
|
|
1519
|
+
"receiptRefs": [],
|
|
1520
|
+
}
|
|
1521
|
+
)
|
|
1522
|
+
existing_surfaces.add(surface_id)
|
|
1523
|
+
if disposition != "needs_follow_up":
|
|
1524
|
+
continue
|
|
1525
|
+
coverage["deferred"].append(
|
|
1526
|
+
{
|
|
1527
|
+
"id": candidate_id,
|
|
1528
|
+
"candidateId": candidate_id,
|
|
1529
|
+
"reason": (
|
|
1530
|
+
"Validation was deferred because the scan reached its cost limit: "
|
|
1531
|
+
f"{candidate['summary']}. Evidence: {candidate['evidence']}"
|
|
1532
|
+
),
|
|
1533
|
+
"paths": paths,
|
|
1534
|
+
"surfaceIds": [surface_id],
|
|
1535
|
+
}
|
|
1536
|
+
)
|
|
1537
|
+
if not any(
|
|
1538
|
+
isinstance(item, dict)
|
|
1539
|
+
and isinstance(reason := item.get("reason"), str)
|
|
1540
|
+
and (
|
|
1541
|
+
reason == "Validation was deferred because the scan reached its cost limit."
|
|
1542
|
+
or reason.startswith(
|
|
1543
|
+
"Validation was deferred because the scan reached its cost limit: "
|
|
1544
|
+
)
|
|
1545
|
+
)
|
|
1546
|
+
for item in coverage["deferred"]
|
|
1547
|
+
):
|
|
1548
|
+
coverage["deferred"].append(
|
|
1549
|
+
{
|
|
1550
|
+
"id": "scan-cost-limit",
|
|
1551
|
+
"reason": "Validation was deferred because the scan reached its cost limit.",
|
|
1552
|
+
}
|
|
1553
|
+
)
|
|
1554
|
+
coverage.pop("completeness", None)
|
|
1555
|
+
manifest["scan"].pop("coverageRef", None)
|
|
1556
|
+
manifest["scan"]["mode"] = db.expected_coverage_mode(scan)
|
|
1557
|
+
manifest["scan"]["reviewState"] = checkpoint_review_state(coverage)
|
|
1558
|
+
for name, payload in (
|
|
1559
|
+
("findings.json", findings),
|
|
1560
|
+
("scan-manifest.json", manifest),
|
|
1561
|
+
):
|
|
1562
|
+
try:
|
|
1563
|
+
write_scan_local_bytes(
|
|
1564
|
+
scan_dir,
|
|
1565
|
+
name,
|
|
1566
|
+
(json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(),
|
|
1567
|
+
)
|
|
1568
|
+
except (ContractError, OSError, TypeError, ValueError) as exc:
|
|
1569
|
+
raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc
|