@openai/codex-security 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -37
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +45 -189
- package/_bundled_plugin/references/scan-contract.md +2 -0
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
- package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
- package/_bundled_plugin/scripts/report_projection.py +44 -15
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +51 -22
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
- package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +28 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +28 -24
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +5 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +341 -37
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +674 -419
- package/dist/cli.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +67 -33
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +30 -4
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +7 -5
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/endor-findings.d.ts +4 -3
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +16 -11
- package/dist/endor-findings.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +13 -5
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +8 -1
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +1 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +11 -3
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +11 -1
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +15 -6
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-import-contract.d.ts +3 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +33 -25
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +22 -5
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +21 -9
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +3 -1
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-findings.d.ts +4 -3
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +16 -14
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +19 -2
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +0 -5
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +1 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +17 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/models.d.ts +1 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +1 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -24
- package/dist/remediation.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +7 -7
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +8 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +1 -1
- package/dist/scan-owner-suggestions.d.ts.map +1 -1
- package/dist/scan-owner-suggestions.js +4 -4
- package/dist/scan-owner-suggestions.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +1 -1
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +7 -3
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +1 -20
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +4 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +9 -9
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +5 -4
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +18 -17
- package/dist/snyk-findings.js.map +1 -1
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +4 -1
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -1
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +3 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -1
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +27 -23
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +3 -4
- package/docs/cli.md +155 -38
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +19 -17
- package/docs/hosted-scans.md +37 -11
- package/package.json +8 -8
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -10,9 +10,8 @@ import json
|
|
|
10
10
|
import os
|
|
11
11
|
import re
|
|
12
12
|
import sqlite3
|
|
13
|
-
import stat
|
|
14
13
|
import sys
|
|
15
|
-
from collections.abc import
|
|
14
|
+
from collections.abc import Iterator
|
|
16
15
|
from contextlib import contextmanager
|
|
17
16
|
from pathlib import Path
|
|
18
17
|
from typing import Any
|
|
@@ -20,47 +19,127 @@ from typing import Any
|
|
|
20
19
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
21
20
|
from finalize_scan_contract import (
|
|
22
21
|
ContractError,
|
|
23
|
-
|
|
22
|
+
ScanLocalIOError,
|
|
24
23
|
_populate_unsealed_artifact_envelope,
|
|
25
24
|
_populate_unsealed_manifest_envelope,
|
|
26
25
|
_prepare_scan_finalization,
|
|
27
|
-
_read_json,
|
|
28
26
|
_read_saved_threat_model,
|
|
29
27
|
_read_scan_local_json,
|
|
30
28
|
_read_scan_local_json_bytes,
|
|
31
|
-
|
|
29
|
+
_recover_unsealed_coverage,
|
|
32
30
|
_recover_unsealed_findings,
|
|
33
31
|
_remove_scan_local_file_if_exists,
|
|
34
|
-
_require_portable_relative_path,
|
|
35
|
-
_require_str,
|
|
36
32
|
_validate_completion_binding,
|
|
37
|
-
_validate_resolved_deferred,
|
|
38
|
-
_validate_schema_node,
|
|
39
33
|
_write_prepared_scan_finalization,
|
|
40
34
|
finalize_scan,
|
|
41
|
-
|
|
35
|
+
manifest_is_sealed,
|
|
42
36
|
open_scan_local_file_descriptor,
|
|
43
37
|
write_scan_local_bytes,
|
|
44
38
|
write_threat_model_projection_if_possible,
|
|
45
39
|
)
|
|
40
|
+
from finalize_scan_contract import finding_candidate_id as finding_candidate_id
|
|
41
|
+
from workbench.retained_coverage import (
|
|
42
|
+
_append_previous_finding as _append_previous_finding,
|
|
43
|
+
)
|
|
44
|
+
from workbench.retained_coverage import (
|
|
45
|
+
_coverage_rows as _coverage_rows,
|
|
46
|
+
)
|
|
47
|
+
from workbench.retained_coverage import (
|
|
48
|
+
_ensure_finding_identities as _ensure_finding_identities,
|
|
49
|
+
)
|
|
50
|
+
from workbench.retained_coverage import (
|
|
51
|
+
_ensure_finding_identity as _ensure_finding_identity,
|
|
52
|
+
)
|
|
53
|
+
from workbench.retained_coverage import (
|
|
54
|
+
_finding_content as _finding_content,
|
|
55
|
+
)
|
|
56
|
+
from workbench.retained_coverage import (
|
|
57
|
+
_finding_identity as _finding_identity,
|
|
58
|
+
)
|
|
59
|
+
from workbench.retained_coverage import (
|
|
60
|
+
_finding_key as _finding_key,
|
|
61
|
+
)
|
|
62
|
+
from workbench.retained_coverage import (
|
|
63
|
+
_has_candidate_metadata as _has_candidate_metadata,
|
|
64
|
+
)
|
|
65
|
+
from workbench.retained_coverage import (
|
|
66
|
+
_identity_candidate as _identity_candidate,
|
|
67
|
+
)
|
|
68
|
+
from workbench.retained_coverage import (
|
|
69
|
+
_inherit_coverage_sources as _inherit_coverage_sources,
|
|
70
|
+
)
|
|
71
|
+
from workbench.retained_coverage import (
|
|
72
|
+
_retained_findings as _retained_findings,
|
|
73
|
+
)
|
|
74
|
+
from workbench.retained_coverage import (
|
|
75
|
+
_semantic_identity_key as _semantic_identity_key,
|
|
76
|
+
)
|
|
77
|
+
from workbench.retained_coverage import (
|
|
78
|
+
_worker_candidate_key as _worker_candidate_key,
|
|
79
|
+
)
|
|
80
|
+
from workbench.retained_coverage import merge_saved_results as merge_saved_results
|
|
81
|
+
from workbench.saved_coverage import (
|
|
82
|
+
_deferred_rows as _deferred_rows,
|
|
83
|
+
)
|
|
84
|
+
from workbench.saved_coverage import (
|
|
85
|
+
_merge_tied_parent_observations as _merge_tied_parent_observations,
|
|
86
|
+
)
|
|
87
|
+
from workbench.saved_coverage import (
|
|
88
|
+
_resolved_deferred_rows as _resolved_deferred_rows,
|
|
89
|
+
)
|
|
46
90
|
from workbench.saved_coverage import (
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
91
|
+
_saved_coverage_id as _saved_coverage_id,
|
|
92
|
+
)
|
|
93
|
+
from workbench.saved_outputs import _restore_published_outputs, _snapshot_published_outputs
|
|
94
|
+
from workbench.saved_sources import (
|
|
95
|
+
_admit_parent_checkpoint,
|
|
96
|
+
_capture_saved_source,
|
|
97
|
+
_checkpoint_paths,
|
|
98
|
+
_CheckpointSnapshotError,
|
|
99
|
+
_encode_retained_sources,
|
|
100
|
+
_encoded,
|
|
101
|
+
_freeze_source_times,
|
|
102
|
+
_frozen_checkpoint_heads,
|
|
103
|
+
_frozen_source_times,
|
|
104
|
+
_is_source_order_snapshot,
|
|
105
|
+
_legacy_checkpoint_observed_at,
|
|
106
|
+
_parent_scan_draft,
|
|
107
|
+
_read_saved_parent_result,
|
|
108
|
+
_read_saved_result,
|
|
109
|
+
_require_current_deep_publication,
|
|
110
|
+
_restore_checkpoint_head,
|
|
111
|
+
_restore_frozen_model_capture,
|
|
112
|
+
_retained_checkpoint_state,
|
|
113
|
+
_retained_source_state,
|
|
114
|
+
_saved_result_paths,
|
|
115
|
+
_source_digests,
|
|
116
|
+
)
|
|
117
|
+
from workbench.saved_sources import (
|
|
118
|
+
_capture_parent_observation as _capture_parent_observation,
|
|
119
|
+
)
|
|
120
|
+
from workbench.saved_sources import (
|
|
121
|
+
_checkpoint_head_directory as _checkpoint_head_directory,
|
|
122
|
+
)
|
|
123
|
+
from workbench.saved_sources import (
|
|
124
|
+
_digest as _digest,
|
|
125
|
+
)
|
|
126
|
+
from workbench.saved_sources import (
|
|
127
|
+
_latest_successful_reducer as _latest_successful_reducer,
|
|
128
|
+
)
|
|
129
|
+
from workbench.saved_sources import (
|
|
130
|
+
_parent_publication_ancestors as _parent_publication_ancestors,
|
|
131
|
+
)
|
|
132
|
+
from workbench.saved_sources import (
|
|
133
|
+
_worker_outputs as _worker_outputs,
|
|
50
134
|
)
|
|
51
135
|
from workbench_constants import PHASES
|
|
52
|
-
from
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
_PUBLISHED_OUTPUTS = (
|
|
56
|
-
"findings.json",
|
|
57
|
-
"coverage.json",
|
|
58
|
-
"scan-manifest.json",
|
|
59
|
-
"report.md",
|
|
60
|
-
"threatmodel.md",
|
|
61
|
-
"report.html",
|
|
62
|
-
"exports/results.sarif",
|
|
136
|
+
from workbench_coverage_receipts import (
|
|
137
|
+
_deferred_candidate_id as _deferred_candidate_id,
|
|
63
138
|
)
|
|
139
|
+
from workbench_coverage_receipts import (
|
|
140
|
+
_generic_surface_updates as _generic_surface_updates,
|
|
141
|
+
)
|
|
142
|
+
|
|
64
143
|
_PUBLICATION_FOLLOW_UP_WARNING = (
|
|
65
144
|
"Saved scan evidence remains on disk; result publication needs follow-up:"
|
|
66
145
|
)
|
|
@@ -124,284 +203,6 @@ def refresh_completed_scan(
|
|
|
124
203
|
return db.scan_context(connection, scan["id"])
|
|
125
204
|
|
|
126
205
|
|
|
127
|
-
def _encoded(value: Any) -> bytes:
|
|
128
|
-
return json.dumps(
|
|
129
|
-
value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
|
|
130
|
-
).encode()
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def _digest(value: Any) -> str:
|
|
134
|
-
return hashlib.sha256(_encoded(value)).hexdigest()
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def _children(scan_dir: Path, relative: str) -> list[str]:
|
|
138
|
-
cursor = scan_dir
|
|
139
|
-
for part in Path(relative).parts:
|
|
140
|
-
if part in {"..", "."}:
|
|
141
|
-
return []
|
|
142
|
-
cursor = cursor / part
|
|
143
|
-
try:
|
|
144
|
-
if not stat.S_ISDIR(cursor.lstat().st_mode):
|
|
145
|
-
return []
|
|
146
|
-
except FileNotFoundError:
|
|
147
|
-
return []
|
|
148
|
-
return sorted(child.name for child in cursor.iterdir())
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
152
|
-
return max(
|
|
153
|
-
(
|
|
154
|
-
worker
|
|
155
|
-
for worker in workers
|
|
156
|
-
if worker["kind"] == "dedup"
|
|
157
|
-
and worker["status"] == "succeeded"
|
|
158
|
-
and worker["result_manifest_path"]
|
|
159
|
-
),
|
|
160
|
-
key=lambda worker: (worker["completed_at"] or "", worker["id"]),
|
|
161
|
-
default=None,
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
|
|
166
|
-
return [
|
|
167
|
-
f"{directory}/{name}"
|
|
168
|
-
for name in _children(scan_dir, directory)
|
|
169
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
170
|
-
]
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
|
|
174
|
-
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
175
|
-
attempts = (output.parent if output.name == "output" else output) / "attempts"
|
|
176
|
-
archived = [
|
|
177
|
-
((attempts / name).as_posix(), int(name.split("-")[1]))
|
|
178
|
-
for name in _children(scan_dir, attempts.as_posix())
|
|
179
|
-
if re.fullmatch(r"attempt-\d+", name)
|
|
180
|
-
]
|
|
181
|
-
attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
|
|
182
|
-
if worker["kind"] == "discovery" and not attempt:
|
|
183
|
-
attempt = max((attempt for _, attempt in archived), default=0) + 1
|
|
184
|
-
return [(output.as_posix(), attempt), *archived]
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
188
|
-
latest_reducer = _latest_successful_reducer(workers)
|
|
189
|
-
yield "checkpoint-head.json", None
|
|
190
|
-
for directory in ("checkpoint-heads", "checkpoints"):
|
|
191
|
-
yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
|
|
192
|
-
for worker in workers:
|
|
193
|
-
if worker["kind"] not in {"dedup", "discovery"}:
|
|
194
|
-
continue
|
|
195
|
-
try:
|
|
196
|
-
outputs = _worker_outputs(scan_dir, worker)
|
|
197
|
-
except (TypeError, ValueError):
|
|
198
|
-
continue
|
|
199
|
-
for directory, _ in outputs:
|
|
200
|
-
if worker["kind"] == "discovery":
|
|
201
|
-
yield f"{directory}/checkpoint-head.json", worker["kind"]
|
|
202
|
-
yield from (
|
|
203
|
-
(path, worker["kind"])
|
|
204
|
-
for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
|
|
205
|
-
)
|
|
206
|
-
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
207
|
-
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
208
|
-
yield f"{directory}/result.json", worker["kind"]
|
|
209
|
-
yield from ((path, worker["kind"]) for path in checkpoint_paths)
|
|
210
|
-
if worker["result_manifest_path"] and (
|
|
211
|
-
worker["kind"] == "discovery"
|
|
212
|
-
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
213
|
-
):
|
|
214
|
-
try:
|
|
215
|
-
yield (
|
|
216
|
-
Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
|
|
217
|
-
worker["kind"],
|
|
218
|
-
)
|
|
219
|
-
except ValueError:
|
|
220
|
-
continue
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
224
|
-
path = Path(relative)
|
|
225
|
-
if path.name == "checkpoint-head.json":
|
|
226
|
-
return path.parent
|
|
227
|
-
if path.parent.name == "checkpoint-heads":
|
|
228
|
-
return path.parent.parent
|
|
229
|
-
return None
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
def _capture_saved_source(
|
|
233
|
-
scan_dir: Path,
|
|
234
|
-
relative: str,
|
|
235
|
-
scan_id: str,
|
|
236
|
-
*,
|
|
237
|
-
kind: str | None = None,
|
|
238
|
-
snapshot_head: bool = True,
|
|
239
|
-
write: bool = True,
|
|
240
|
-
) -> dict[str, tuple[str, int]]:
|
|
241
|
-
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
|
|
242
|
-
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
|
|
243
|
-
return {relative: (digest, observed)}
|
|
244
|
-
head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
245
|
-
observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
|
|
246
|
-
directory = Path(relative).parent
|
|
247
|
-
selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
|
|
248
|
-
_, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
|
|
249
|
-
digest = _digest(observation)
|
|
250
|
-
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
|
|
251
|
-
# Capture the selected file even if the worker created it after directory enumeration.
|
|
252
|
-
if write and not (scan_dir / snapshot).exists():
|
|
253
|
-
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
254
|
-
return {
|
|
255
|
-
snapshot: (digest, int(observation["observedAtNs"])),
|
|
256
|
-
selected: (selected_digest, selected_time),
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def _is_source_order_snapshot(relative: str) -> bool:
|
|
261
|
-
path = Path(relative)
|
|
262
|
-
return path.parent == Path("source-order") and bool(
|
|
263
|
-
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
264
|
-
)
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _read_saved_result(
|
|
268
|
-
scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
|
|
269
|
-
) -> tuple[dict[str, Any], str, int]:
|
|
270
|
-
draft, _, metadata = _read_scan_local_json_with_metadata(
|
|
271
|
-
scan_dir, relative, "Saved scan checkpoint"
|
|
272
|
-
)
|
|
273
|
-
directory = _checkpoint_head_directory(relative)
|
|
274
|
-
if directory is not None:
|
|
275
|
-
checkpoint = draft.get("checkpoint")
|
|
276
|
-
if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
|
|
277
|
-
raise ContractError("checkpoint head does not name a saved checkpoint")
|
|
278
|
-
_read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
|
|
279
|
-
if Path(relative).name == "checkpoint-head.json":
|
|
280
|
-
return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
|
|
281
|
-
observed = draft.get("observedAtNs")
|
|
282
|
-
if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
|
|
283
|
-
raise ContractError("checkpoint head has no observation time")
|
|
284
|
-
return draft, _digest(draft), int(observed)
|
|
285
|
-
if draft.get("scanId") != scan_id:
|
|
286
|
-
raise ContractError("checkpoint belongs to a different scan")
|
|
287
|
-
if not _is_source_order_snapshot(relative) and (
|
|
288
|
-
not isinstance(draft.get("findings"), list)
|
|
289
|
-
or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
|
|
290
|
-
):
|
|
291
|
-
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
292
|
-
return draft, _digest(draft), metadata.st_mtime_ns
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
|
|
296
|
-
times: dict[str, int] = {}
|
|
297
|
-
snapshots = [path for path in sources if _is_source_order_snapshot(path)]
|
|
298
|
-
for path in snapshots:
|
|
299
|
-
record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
|
|
300
|
-
if digest != sources[path] or not isinstance(record.get("sources"), dict):
|
|
301
|
-
raise ContractError("saved source ordering changed after the scan stopped")
|
|
302
|
-
for relative, observation in record["sources"].items():
|
|
303
|
-
if (
|
|
304
|
-
not isinstance(observation, dict)
|
|
305
|
-
or relative not in sources
|
|
306
|
-
or observation.get("digest") != sources[relative]
|
|
307
|
-
or not isinstance(observation.get("observedAtNs"), str)
|
|
308
|
-
or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
|
|
309
|
-
):
|
|
310
|
-
raise ContractError("saved source ordering does not match its frozen sources")
|
|
311
|
-
observed = int(observation["observedAtNs"])
|
|
312
|
-
if relative in times and times[relative] != observed:
|
|
313
|
-
raise ContractError("saved source ordering has conflicting observations")
|
|
314
|
-
times[relative] = observed
|
|
315
|
-
if snapshots and sources.keys() - set(snapshots) - times.keys():
|
|
316
|
-
raise ContractError("saved source ordering is incomplete")
|
|
317
|
-
return times
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
def _freeze_source_times(
|
|
321
|
-
scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
|
|
322
|
-
) -> None:
|
|
323
|
-
# Identical result rewrites must not change the order of frozen review evidence.
|
|
324
|
-
observations = {
|
|
325
|
-
path: {"digest": digest, "observedAtNs": str(times[path])}
|
|
326
|
-
for path, digest in sources.items()
|
|
327
|
-
if not _is_source_order_snapshot(path)
|
|
328
|
-
}
|
|
329
|
-
if not observations:
|
|
330
|
-
return
|
|
331
|
-
record = {"scanId": scan_id, "sources": observations}
|
|
332
|
-
digest = _digest(record)
|
|
333
|
-
path = f"source-order/{digest}.json"
|
|
334
|
-
if not (scan_dir / path).exists():
|
|
335
|
-
write_scan_local_bytes(scan_dir, path, _encoded(record))
|
|
336
|
-
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
|
|
337
|
-
raise ContractError("saved source ordering does not match its digest")
|
|
338
|
-
sources[path] = digest
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
def _parent_scan_draft(
|
|
342
|
-
scan_id: str,
|
|
343
|
-
parent_scan: dict[str, Any],
|
|
344
|
-
findings: dict[str, Any],
|
|
345
|
-
coverage: dict[str, Any],
|
|
346
|
-
) -> dict[str, Any]:
|
|
347
|
-
parent = {
|
|
348
|
-
"scanId": scan_id,
|
|
349
|
-
"findings": findings.get("findings"),
|
|
350
|
-
"coverage": coverage,
|
|
351
|
-
**{
|
|
352
|
-
key: parent_scan[key]
|
|
353
|
-
for key in ("scope", "threatModel", "complete")
|
|
354
|
-
if key in parent_scan
|
|
355
|
-
},
|
|
356
|
-
}
|
|
357
|
-
if not isinstance(parent["findings"], list):
|
|
358
|
-
raise ContractError("Saved parent draft has no findings array")
|
|
359
|
-
return parent
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
def _read_saved_parent_result(
|
|
363
|
-
scan_dir: Path, scan_id: str
|
|
364
|
-
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
365
|
-
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
|
|
366
|
-
findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
|
|
367
|
-
coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
|
|
368
|
-
parent_scan = manifest.get("scan")
|
|
369
|
-
if not isinstance(parent_scan, dict):
|
|
370
|
-
raise ContractError("Saved parent manifest has no scan object")
|
|
371
|
-
if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
|
|
372
|
-
parent_scan.get("id", scan_id) != scan_id
|
|
373
|
-
or findings.get("scanId", scan_id) != scan_id
|
|
374
|
-
or coverage.get("scanId", scan_id) != scan_id
|
|
375
|
-
):
|
|
376
|
-
raise ContractError("Saved parent documents belong to a different scan")
|
|
377
|
-
return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
def _source_digests(value: Any, error: str) -> dict[str, str]:
|
|
381
|
-
if not isinstance(value, dict) or not all(
|
|
382
|
-
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
383
|
-
):
|
|
384
|
-
raise ContractError(error)
|
|
385
|
-
return value
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
|
|
389
|
-
if isinstance(value, dict) and isinstance(value.get("sources"), dict):
|
|
390
|
-
sources = _source_digests(
|
|
391
|
-
value["sources"], "Saved stopped-scan source digests are malformed."
|
|
392
|
-
)
|
|
393
|
-
model_source = value.get("threatModelSource")
|
|
394
|
-
if not isinstance(model_source, str) or model_source not in sources:
|
|
395
|
-
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
|
|
396
|
-
return sources, model_source
|
|
397
|
-
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
|
|
401
|
-
state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
|
|
402
|
-
return json.dumps(state, sort_keys=True)
|
|
403
|
-
|
|
404
|
-
|
|
405
206
|
def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
406
207
|
try:
|
|
407
208
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
@@ -446,6 +247,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
446
247
|
manifest_scan.get("preservedSources", {}),
|
|
447
248
|
"Published scan source digests are malformed.",
|
|
448
249
|
)
|
|
250
|
+
_, pending_recovery = _retained_checkpoint_state(scan)
|
|
251
|
+
if pending_recovery:
|
|
252
|
+
return True
|
|
253
|
+
published_heads = manifest_scan.get("preservedCheckpointHeads")
|
|
254
|
+
if published_heads is None:
|
|
255
|
+
published_heads = _frozen_checkpoint_heads(
|
|
256
|
+
scan_dir, scan["id"], published_sources, skip_unreadable=True
|
|
257
|
+
)
|
|
258
|
+
elif "." not in published_heads:
|
|
259
|
+
# Older publications recorded root observations without selecting a root head.
|
|
260
|
+
legacy_heads = _frozen_checkpoint_heads(
|
|
261
|
+
scan_dir, scan["id"], published_sources, skip_unreadable=True
|
|
262
|
+
)
|
|
263
|
+
if "." in legacy_heads:
|
|
264
|
+
published_heads = {**published_heads, ".": legacy_heads["."]}
|
|
265
|
+
checkpoint_heads = dict(published_heads)
|
|
449
266
|
current_sources = dict(published_sources)
|
|
450
267
|
paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
|
|
451
268
|
for path in paths:
|
|
@@ -457,23 +274,53 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
457
274
|
kind=paths[path],
|
|
458
275
|
snapshot_head=path not in published_sources,
|
|
459
276
|
write=False,
|
|
277
|
+
checkpoint_heads=checkpoint_heads,
|
|
460
278
|
)
|
|
461
279
|
current_sources.update({path: value[0] for path, value in captured.items()})
|
|
280
|
+
directory = Path(path).parent.as_posix()
|
|
281
|
+
previous_head = published_heads.get(directory)
|
|
282
|
+
if (
|
|
283
|
+
Path(path).name == "checkpoint-head.json"
|
|
284
|
+
and previous_head in captured
|
|
285
|
+
and Path(previous_head).parent.name == "checkpoints"
|
|
286
|
+
):
|
|
287
|
+
observed = captured[checkpoint_heads[directory]][1]
|
|
288
|
+
if observed == _legacy_checkpoint_observed_at(
|
|
289
|
+
scan_dir, scan["id"], published_sources, previous_head
|
|
290
|
+
):
|
|
291
|
+
checkpoint_heads[directory] = previous_head
|
|
462
292
|
except (ContractError, OSError, ValueError):
|
|
463
|
-
|
|
464
|
-
return current_sources != published_sources
|
|
293
|
+
_restore_checkpoint_head(checkpoint_heads, published_heads, published_sources, path)
|
|
294
|
+
return current_sources != published_sources or checkpoint_heads != published_heads
|
|
465
295
|
except (ContractError, OSError, SystemExit, ValueError):
|
|
466
296
|
return False
|
|
467
297
|
|
|
468
298
|
|
|
469
|
-
def _recovery_source_digests(
|
|
299
|
+
def _recovery_source_digests(
|
|
300
|
+
db: Any, connection: Any, scan: Any
|
|
301
|
+
) -> tuple[dict[str, str], bool, dict[str, str | None]]:
|
|
470
302
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
471
303
|
frozen_sources: dict[str, str] | None = None
|
|
472
304
|
include_parent = True
|
|
473
305
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
474
306
|
if raw_frozen_sources is not None:
|
|
475
|
-
|
|
307
|
+
retained = json.loads(raw_frozen_sources)
|
|
308
|
+
frozen_sources, model_source = _retained_source_state(retained)
|
|
309
|
+
if isinstance(retained.get("sources"), dict) and retained.get("sourceTimes") is not None:
|
|
310
|
+
_restore_frozen_model_capture(
|
|
311
|
+
scan_dir, scan["id"], frozen_sources, retained["sourceTimes"], model_source
|
|
312
|
+
)
|
|
313
|
+
_freeze_source_times(
|
|
314
|
+
scan_dir,
|
|
315
|
+
scan["id"],
|
|
316
|
+
frozen_sources,
|
|
317
|
+
retained["sourceTimes"],
|
|
318
|
+
retained.get("parentContext"),
|
|
319
|
+
)
|
|
476
320
|
include_parent = False
|
|
321
|
+
checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
|
|
322
|
+
if pending_recovery:
|
|
323
|
+
return frozen_sources, False, checkpoint_heads
|
|
477
324
|
|
|
478
325
|
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
479
326
|
if manifest_path is not None:
|
|
@@ -485,9 +332,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
485
332
|
manifest_scan = manifest.get("scan")
|
|
486
333
|
if not isinstance(manifest_scan, dict):
|
|
487
334
|
raise ContractError("Saved scan manifest has no scan object")
|
|
488
|
-
if scan["seal_manifest_digest"] is not None or (
|
|
489
|
-
manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
|
|
490
|
-
):
|
|
335
|
+
if scan["seal_manifest_digest"] is not None or manifest_is_sealed(manifest_scan):
|
|
491
336
|
if "preservedSources" in manifest_scan:
|
|
492
337
|
published_sources = _source_digests(
|
|
493
338
|
manifest_scan["preservedSources"],
|
|
@@ -510,9 +355,14 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
510
355
|
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
511
356
|
(scan["id"],),
|
|
512
357
|
).fetchall()
|
|
358
|
+
previous_heads = checkpoint_heads
|
|
359
|
+
if previous_heads is None:
|
|
360
|
+
previous_heads = _frozen_checkpoint_heads(scan_dir, scan["id"], frozen_sources or {})
|
|
361
|
+
checkpoint_heads = dict(previous_heads)
|
|
513
362
|
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
514
363
|
recovery_sources = dict(frozen_sources or {})
|
|
515
|
-
|
|
364
|
+
parent_context: dict[str, str] = {}
|
|
365
|
+
source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources, parent_context)
|
|
516
366
|
for relative, expected_digest in recovery_sources.items():
|
|
517
367
|
try:
|
|
518
368
|
_, digest, observed = _read_saved_result(
|
|
@@ -525,18 +375,52 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
525
375
|
if not _is_source_order_snapshot(relative):
|
|
526
376
|
source_times.setdefault(relative, observed)
|
|
527
377
|
|
|
378
|
+
snapshot_error = None
|
|
379
|
+
captured_records = {}
|
|
528
380
|
for relative in paths.keys() - recovery_sources.keys():
|
|
529
381
|
try:
|
|
530
|
-
captured = _capture_saved_source(
|
|
382
|
+
captured = _capture_saved_source(
|
|
383
|
+
scan_dir,
|
|
384
|
+
relative,
|
|
385
|
+
scan["id"],
|
|
386
|
+
kind=paths[relative],
|
|
387
|
+
checkpoint_heads=checkpoint_heads,
|
|
388
|
+
)
|
|
389
|
+
except _CheckpointSnapshotError as exc:
|
|
390
|
+
snapshot_error = snapshot_error or exc
|
|
391
|
+
captured = exc.captured
|
|
392
|
+
captured_records.update(exc.records)
|
|
531
393
|
except (ContractError, OSError, ValueError):
|
|
394
|
+
_restore_checkpoint_head(
|
|
395
|
+
checkpoint_heads, previous_heads, frozen_sources or {}, relative
|
|
396
|
+
)
|
|
532
397
|
continue
|
|
398
|
+
if relative == "checkpoint-head.json":
|
|
399
|
+
parent_context["head"] = next(iter(captured))
|
|
533
400
|
for path, (digest, observed) in captured.items():
|
|
534
401
|
if path in recovery_sources and recovery_sources[path] != digest:
|
|
535
402
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
536
403
|
recovery_sources[path] = digest
|
|
537
404
|
source_times.setdefault(path, observed)
|
|
538
|
-
|
|
539
|
-
|
|
405
|
+
try:
|
|
406
|
+
if snapshot_error is not None:
|
|
407
|
+
raise snapshot_error
|
|
408
|
+
_freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times, parent_context)
|
|
409
|
+
except (ContractError, OSError) as exc:
|
|
410
|
+
failure = _CheckpointSnapshotError(
|
|
411
|
+
str(exc),
|
|
412
|
+
{
|
|
413
|
+
path: (digest, source_times[path])
|
|
414
|
+
for path, digest in recovery_sources.items()
|
|
415
|
+
if not _is_source_order_snapshot(path)
|
|
416
|
+
},
|
|
417
|
+
)
|
|
418
|
+
failure.records = captured_records
|
|
419
|
+
failure.heads = checkpoint_heads
|
|
420
|
+
failure.include_parent = include_parent
|
|
421
|
+
failure.parent_context = parent_context
|
|
422
|
+
raise failure from exc
|
|
423
|
+
return recovery_sources, include_parent, checkpoint_heads
|
|
540
424
|
|
|
541
425
|
|
|
542
426
|
def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
|
|
@@ -554,1849 +438,46 @@ def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
554
438
|
).fetchone()
|
|
555
439
|
if publication_error is not None and publication_error["publication_error_message"]:
|
|
556
440
|
return True
|
|
557
|
-
return _saved_results_changed(db, connection, scan)
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
def _finding_key(finding: dict[str, Any]) -> str:
|
|
561
|
-
# Wording and evidence may improve between checkpoints; distinct source locations
|
|
562
|
-
# must not collide merely because two workers chose the same semantic identity.
|
|
563
|
-
provenance = finding.get("provenance")
|
|
564
|
-
identity = (
|
|
565
|
-
provenance.get("preservedIdentity", finding.get("identity"))
|
|
566
|
-
if isinstance(provenance, dict)
|
|
567
|
-
else finding.get("identity")
|
|
568
|
-
)
|
|
569
|
-
if not isinstance(identity, dict):
|
|
570
|
-
normalized = dict(finding)
|
|
571
|
-
normalized.pop("identity", None)
|
|
572
|
-
_ensure_finding_identity(normalized)
|
|
573
|
-
identity = normalized["identity"]
|
|
574
|
-
locations = finding.get("locations", [])
|
|
575
|
-
if not isinstance(locations, list):
|
|
576
|
-
locations = []
|
|
577
|
-
return _digest(
|
|
578
|
-
[
|
|
579
|
-
finding.get("ruleId"),
|
|
580
|
-
identity,
|
|
581
|
-
sorted(
|
|
582
|
-
(
|
|
583
|
-
(
|
|
584
|
-
location.get("path"),
|
|
585
|
-
location.get("startLine"),
|
|
586
|
-
location.get("endLine", location.get("startLine")),
|
|
587
|
-
)
|
|
588
|
-
for location in locations
|
|
589
|
-
if isinstance(location, dict)
|
|
590
|
-
),
|
|
591
|
-
key=_encoded,
|
|
592
|
-
),
|
|
593
|
-
]
|
|
594
|
-
)
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
def _worker_candidate_key(
|
|
598
|
-
worker_id: str, candidate_id: str, finding: dict[str, Any]
|
|
599
|
-
) -> tuple[str, str, Any, Any, Any]:
|
|
600
|
-
"""Identify one worker-local candidate without merging unrelated locations."""
|
|
601
|
-
provenance = finding.get("provenance")
|
|
602
|
-
identity = (
|
|
603
|
-
provenance.get("preservedIdentity", finding.get("identity"))
|
|
604
|
-
if isinstance(provenance, dict)
|
|
605
|
-
else finding.get("identity")
|
|
606
|
-
)
|
|
607
|
-
if not isinstance(identity, dict):
|
|
608
|
-
normalized = dict(finding)
|
|
609
|
-
_ensure_finding_identity(normalized)
|
|
610
|
-
identity = normalized.get("identity")
|
|
611
|
-
anchor = identity.get("anchor") if isinstance(identity, dict) else None
|
|
612
|
-
instance = identity.get("instance") if isinstance(identity, dict) else None
|
|
613
|
-
return worker_id, candidate_id, finding.get("ruleId"), anchor, instance
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
def _finding_content(finding: dict[str, Any]) -> dict[str, Any]:
|
|
617
|
-
"""Return substantive finding content without generated identity or provenance."""
|
|
618
|
-
return {
|
|
619
|
-
key: value
|
|
620
|
-
for key, value in finding.items()
|
|
621
|
-
if key not in {"findingId", "occurrenceId", "fingerprints", "identity", "provenance"}
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
def _ensure_finding_identity(finding: Any, *, candidate_only: bool = False) -> None:
|
|
626
|
-
if not isinstance(finding, dict) or "identity" in finding:
|
|
627
|
-
return
|
|
628
|
-
if candidate_only and not finding_candidate_id(finding):
|
|
629
|
-
return
|
|
630
|
-
extensions = finding.get("extensions")
|
|
631
|
-
source = str(
|
|
632
|
-
(extensions.get("candidateId") if isinstance(extensions, dict) else None)
|
|
633
|
-
or finding.get("title")
|
|
634
|
-
or "finding"
|
|
635
|
-
)
|
|
636
|
-
anchor = re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
|
|
637
|
-
finding["identity"] = {"anchor": anchor}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
|
|
641
|
-
"""Yield canonical and historical findings without trusting candidate IDs."""
|
|
642
|
-
pending = [finding]
|
|
643
|
-
seen: set[int] = set()
|
|
644
|
-
while pending:
|
|
645
|
-
current = pending.pop()
|
|
646
|
-
marker = id(current)
|
|
647
|
-
if marker in seen:
|
|
648
|
-
continue
|
|
649
|
-
seen.add(marker)
|
|
650
|
-
yield current
|
|
651
|
-
provenance = current.get("provenance")
|
|
652
|
-
if not isinstance(provenance, dict):
|
|
653
|
-
continue
|
|
654
|
-
previous = provenance.get("previousFindings")
|
|
655
|
-
if isinstance(previous, list):
|
|
656
|
-
pending.extend(item for item in reversed(previous) if isinstance(item, dict))
|
|
657
|
-
sources = provenance.get("sourceFindings")
|
|
658
|
-
if isinstance(sources, list):
|
|
659
|
-
pending.extend(
|
|
660
|
-
source["finding"]
|
|
661
|
-
for source in reversed(sources)
|
|
662
|
-
if isinstance(source, dict) and isinstance(source.get("finding"), dict)
|
|
663
|
-
)
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
|
|
667
|
-
rows = coverage.get("deferred", [])
|
|
668
|
-
return rows if isinstance(rows, list) else []
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
672
|
-
if draft.get("complete") is False:
|
|
673
|
-
return []
|
|
674
|
-
coverage = draft["coverage"]
|
|
675
|
-
rows = coverage.get("resolvedDeferred", [])
|
|
676
|
-
try:
|
|
677
|
-
# Invalid closure metadata cannot discard the evidence it names.
|
|
678
|
-
_validate_schema_node(rows, schema, "coverage.resolvedDeferred")
|
|
679
|
-
_validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
|
|
680
|
-
except ContractError:
|
|
681
|
-
return []
|
|
682
|
-
return rows
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
def _merge_tied_parent_observations(
|
|
686
|
-
current: dict[str, Any], previous: dict[str, Any]
|
|
687
|
-
) -> dict[str, Any]:
|
|
688
|
-
merged = copy.deepcopy(current)
|
|
689
|
-
for finding in previous["findings"]:
|
|
690
|
-
if finding not in merged["findings"]:
|
|
691
|
-
merged["findings"].append(copy.deepcopy(finding))
|
|
692
|
-
coverage = merged["coverage"]
|
|
693
|
-
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
694
|
-
rows = previous["coverage"].get(field, [])
|
|
695
|
-
output = coverage.setdefault(field, [])
|
|
696
|
-
if isinstance(rows, list) and isinstance(output, list):
|
|
697
|
-
for row in rows:
|
|
698
|
-
if row not in output:
|
|
699
|
-
output.append(copy.deepcopy(row))
|
|
700
|
-
pending_ids = {
|
|
701
|
-
identity
|
|
702
|
-
for row in _deferred_rows(coverage)
|
|
703
|
-
if isinstance(row, dict)
|
|
704
|
-
for identity in (row.get("id"), row.get("candidateId"))
|
|
705
|
-
if isinstance(identity, str)
|
|
706
|
-
}
|
|
707
|
-
closure_schema = _read_json(
|
|
708
|
-
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
709
|
-
)["properties"]["resolvedDeferred"]
|
|
710
|
-
closures = {}
|
|
711
|
-
for observed in (current, previous):
|
|
712
|
-
for row in _resolved_deferred_rows(observed, closure_schema):
|
|
713
|
-
if row["id"] not in pending_ids:
|
|
714
|
-
closures.setdefault(row["id"], copy.deepcopy(row))
|
|
715
|
-
coverage.pop("resolvedDeferred", None)
|
|
716
|
-
if closures:
|
|
717
|
-
coverage["resolvedDeferred"] = list(closures.values())
|
|
718
|
-
if current.get("complete") is False or previous.get("complete") is False:
|
|
719
|
-
merged["complete"] = False
|
|
720
|
-
coverage.pop("completeness", None)
|
|
721
|
-
return merged
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
def _deferred_candidate_id(
|
|
725
|
-
row: dict[str, Any],
|
|
726
|
-
owner: str | None,
|
|
727
|
-
ambiguous_deferred: set[tuple[str | None, str]],
|
|
728
|
-
) -> str | None:
|
|
729
|
-
identity = row.get("candidateId") or row.get("id")
|
|
730
|
-
if not isinstance(identity, str):
|
|
731
|
-
return None
|
|
732
|
-
if (owner, identity) in ambiguous_deferred and not any(
|
|
733
|
-
key in row for key in ("candidateId", "candidate", "finding")
|
|
734
|
-
):
|
|
735
|
-
return None
|
|
736
|
-
return identity
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
def _generic_surface_updates(
|
|
740
|
-
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
741
|
-
source_order: dict[str, tuple[int, int]],
|
|
742
|
-
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
743
|
-
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
744
|
-
resolved_candidates: dict[tuple[str | None, str], str],
|
|
745
|
-
reopened_generic: set[tuple[str | None, str]],
|
|
746
|
-
surface_schema: dict[str, Any],
|
|
747
|
-
deferred_rows: dict[str, list[Any]],
|
|
748
|
-
ambiguous_deferred: set[tuple[str | None, str]],
|
|
749
|
-
*,
|
|
750
|
-
surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
|
|
751
|
-
eligible_receipt_sources: set[str],
|
|
752
|
-
receipt_root: Path | None = None,
|
|
753
|
-
receipt_warnings: list[str] | None = None,
|
|
754
|
-
fallback_surfaces: list[dict[str, Any]] | None = None,
|
|
755
|
-
) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
|
|
756
|
-
if not closed_deferred and not reopened_generic and receipt_root is None:
|
|
757
|
-
return set(), []
|
|
758
|
-
|
|
759
|
-
def receipt_error(ref: Any) -> ContractError | None:
|
|
760
|
-
if receipt_root is None or not isinstance(ref, str):
|
|
761
|
-
return ContractError("Saved coverage receipt is unavailable.")
|
|
762
|
-
try:
|
|
763
|
-
descriptor = open_scan_local_file_descriptor(
|
|
764
|
-
receipt_root, ref, "Saved coverage receipt"
|
|
765
|
-
)
|
|
766
|
-
os.close(descriptor)
|
|
767
|
-
except ContractError as exc:
|
|
768
|
-
return exc
|
|
769
|
-
return None
|
|
770
|
-
|
|
771
|
-
def linked(row: dict[str, Any], identity: str | None) -> bool:
|
|
772
|
-
surface_ids = row.get("surfaceIds", [])
|
|
773
|
-
return identity is not None and (
|
|
774
|
-
row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
|
|
775
|
-
)
|
|
776
|
-
|
|
777
|
-
saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
|
|
778
|
-
for relative, draft, owner in sources:
|
|
779
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
780
|
-
for row in rows if isinstance(rows, list) else []:
|
|
781
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
782
|
-
saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
|
|
783
|
-
replaced: set[int] = set()
|
|
784
|
-
updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
|
|
785
|
-
# Current selections across all surfaces take precedence over historical refs.
|
|
786
|
-
receipt_paths: dict[str, str] = {}
|
|
787
|
-
retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
|
|
788
|
-
if receipt_root is not None:
|
|
789
|
-
receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
|
|
790
|
-
for key, matches in saved_surfaces.items():
|
|
791
|
-
retained = [
|
|
792
|
-
match for match in matches if surface_is_retained(match[0], key[0], match[1])
|
|
793
|
-
]
|
|
794
|
-
if not retained:
|
|
795
|
-
continue
|
|
796
|
-
latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
|
|
797
|
-
retained_surfaces[key] = latest_surface
|
|
798
|
-
receipt_surfaces.extend(
|
|
799
|
-
(key[0], row)
|
|
800
|
-
for relative, row in retained
|
|
801
|
-
if source_order[relative] == source_order[latest_source]
|
|
802
|
-
)
|
|
803
|
-
for relative, draft, owner in sources:
|
|
804
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
805
|
-
for row in rows if isinstance(rows, list) else []:
|
|
806
|
-
if (
|
|
807
|
-
isinstance(row, dict)
|
|
808
|
-
and "id" not in row
|
|
809
|
-
and surface_is_retained(relative, owner, row)
|
|
810
|
-
):
|
|
811
|
-
receipt_surfaces.append((owner, row))
|
|
812
|
-
receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
|
|
813
|
-
for owner, latest_surface in receipt_surfaces:
|
|
814
|
-
# Recovery repairs disposition and receipts before validating row
|
|
815
|
-
# metadata. A row discarded there cannot reserve a receipt spelling.
|
|
816
|
-
metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
|
|
817
|
-
metadata.setdefault("id", _saved_coverage_id(metadata))
|
|
818
|
-
metadata.update(disposition="needs_follow_up", receiptRefs=[])
|
|
819
|
-
try:
|
|
820
|
-
_require_str(metadata, "id", "coverage.surfaces")
|
|
821
|
-
_validate_schema_node(metadata, surface_schema, "coverage.surfaces")
|
|
822
|
-
except ContractError:
|
|
823
|
-
continue
|
|
824
|
-
refs = latest_surface.get("receiptRefs", [])
|
|
825
|
-
for ref in refs if isinstance(refs, list) else []:
|
|
826
|
-
if receipt_error(ref) is None:
|
|
827
|
-
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
828
|
-
receipt_paths.setdefault(normalized.lower(), normalized)
|
|
829
|
-
for relative, draft, owner in sources:
|
|
830
|
-
closed_ids = {
|
|
831
|
-
identity
|
|
832
|
-
for (saved_owner, identity), (_, _, source) in closed_deferred.items()
|
|
833
|
-
if saved_owner == owner and source == relative
|
|
834
|
-
}
|
|
835
|
-
reopened_ids = {
|
|
836
|
-
identity
|
|
837
|
-
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
838
|
-
if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
|
|
839
|
-
}
|
|
840
|
-
if not closed_ids and not reopened_ids and receipt_root is None:
|
|
841
|
-
continue
|
|
842
|
-
current = draft["coverage"].get("surfaces", [])
|
|
843
|
-
for surface in current if isinstance(current, list) else []:
|
|
844
|
-
if not isinstance(surface, dict):
|
|
845
|
-
continue
|
|
846
|
-
reopening = surface.get("disposition") == "needs_follow_up"
|
|
847
|
-
work_ids = reopened_ids if reopening else closed_ids
|
|
848
|
-
repair_only = not work_ids
|
|
849
|
-
if repair_only and receipt_root is None:
|
|
850
|
-
continue
|
|
851
|
-
identity = surface.get("id")
|
|
852
|
-
if not isinstance(identity, str):
|
|
853
|
-
continue
|
|
854
|
-
matches = saved_surfaces[(owner, identity)]
|
|
855
|
-
# Receipt repair alone cannot close or reopen a saved surface.
|
|
856
|
-
if repair_only and any(
|
|
857
|
-
row.get("disposition") != surface.get("disposition") for _, row in matches
|
|
858
|
-
):
|
|
859
|
-
continue
|
|
860
|
-
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
861
|
-
# A closure cannot identify which of those observations it replaces.
|
|
862
|
-
by_source = dict(matches)
|
|
863
|
-
if any(row != by_source[saved_path] for saved_path, row in matches):
|
|
864
|
-
continue
|
|
865
|
-
if any(
|
|
866
|
-
"candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
|
|
867
|
-
):
|
|
868
|
-
continue
|
|
869
|
-
if any(
|
|
870
|
-
row is not surface
|
|
871
|
-
and source_order[saved_path] >= source_order[relative]
|
|
872
|
-
and row.get("disposition") != surface.get("disposition")
|
|
873
|
-
for saved_path, row in matches
|
|
874
|
-
):
|
|
875
|
-
continue
|
|
876
|
-
|
|
877
|
-
if not reopening and any(
|
|
878
|
-
saved_owner == owner
|
|
879
|
-
and (
|
|
880
|
-
not isinstance(row.get("id") or row.get("candidateId"), str)
|
|
881
|
-
or (
|
|
882
|
-
isinstance(row.get("id"), str)
|
|
883
|
-
and (owner, row["id"]) in ambiguous_deferred
|
|
884
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
885
|
-
)
|
|
886
|
-
)
|
|
887
|
-
and linked(row, identity)
|
|
888
|
-
for saved_path, _, saved_owner in sources
|
|
889
|
-
for row in deferred_rows[saved_path]
|
|
890
|
-
if isinstance(row, dict)
|
|
891
|
-
):
|
|
892
|
-
continue
|
|
893
|
-
if not reopening and any(
|
|
894
|
-
saved_owner == owner
|
|
895
|
-
and (owner, deferred_id) not in closed_deferred
|
|
896
|
-
and (
|
|
897
|
-
(candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
|
|
898
|
-
or (owner, candidate_id) not in resolved_candidates
|
|
899
|
-
)
|
|
900
|
-
and linked(row, identity)
|
|
901
|
-
for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
|
|
902
|
-
):
|
|
903
|
-
continue
|
|
904
|
-
# An accepted checkpoint can update a saved surface by ID without
|
|
905
|
-
# optional surfaceIds links on its generic task.
|
|
906
|
-
receipt_matches = [
|
|
907
|
-
(saved_path, row)
|
|
908
|
-
for saved_path, row in matches
|
|
909
|
-
if receipt_root is None
|
|
910
|
-
or saved_path in eligible_receipt_sources
|
|
911
|
-
or surface_is_retained(saved_path, owner, row)
|
|
912
|
-
]
|
|
913
|
-
if not receipt_matches:
|
|
914
|
-
continue
|
|
915
|
-
latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
|
|
916
|
-
target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
|
|
917
|
-
if target is None:
|
|
918
|
-
continue
|
|
919
|
-
update = copy.deepcopy(target)
|
|
920
|
-
# Historical evidence can repair a retained row without promoting
|
|
921
|
-
# an unselected observation's label, notes, or other metadata.
|
|
922
|
-
refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
|
|
923
|
-
update["receiptRefs"] = refs
|
|
924
|
-
replaced_receipt = False
|
|
925
|
-
replacement_paths = dict(receipt_paths)
|
|
926
|
-
if isinstance(refs, list):
|
|
927
|
-
repaired = (
|
|
928
|
-
not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
|
|
929
|
-
)
|
|
930
|
-
if repaired:
|
|
931
|
-
for ref in refs:
|
|
932
|
-
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
933
|
-
key = normalized.lower()
|
|
934
|
-
if key in replacement_paths and replacement_paths[key] != normalized:
|
|
935
|
-
repaired = False
|
|
936
|
-
break
|
|
937
|
-
replacement_paths[key] = normalized
|
|
938
|
-
for _, row in receipt_matches:
|
|
939
|
-
previous_refs = row.get("receiptRefs", [])
|
|
940
|
-
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
941
|
-
if ref not in refs:
|
|
942
|
-
failure = receipt_error(ref) if repaired else None
|
|
943
|
-
replaceable = failure is not None and isinstance(
|
|
944
|
-
failure.__cause__, FileNotFoundError
|
|
945
|
-
)
|
|
946
|
-
if repaired and isinstance(ref, str):
|
|
947
|
-
try:
|
|
948
|
-
normalized = _require_portable_relative_path(
|
|
949
|
-
ref, "Saved coverage receipt"
|
|
950
|
-
)
|
|
951
|
-
except ContractError as error:
|
|
952
|
-
failure = error
|
|
953
|
-
replaceable = True
|
|
954
|
-
else:
|
|
955
|
-
key = normalized.lower()
|
|
956
|
-
previous = replacement_paths.get(key)
|
|
957
|
-
if previous is not None and previous != normalized:
|
|
958
|
-
failure = ContractError(
|
|
959
|
-
f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
|
|
960
|
-
)
|
|
961
|
-
replaceable = True
|
|
962
|
-
elif not replaceable:
|
|
963
|
-
replacement_paths[key] = normalized
|
|
964
|
-
if replaceable:
|
|
965
|
-
replaced_receipt = True
|
|
966
|
-
if receipt_warnings is not None:
|
|
967
|
-
receipt_warnings.append(
|
|
968
|
-
f"Replaced invalid saved coverage receipt {ref}: {failure}"
|
|
969
|
-
)
|
|
970
|
-
continue
|
|
971
|
-
refs.append(ref)
|
|
972
|
-
if repair_only and not replaced_receipt:
|
|
973
|
-
continue
|
|
974
|
-
try:
|
|
975
|
-
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
976
|
-
except ContractError:
|
|
977
|
-
continue
|
|
978
|
-
replaced_sources = {
|
|
979
|
-
id(row)
|
|
980
|
-
for _, row in matches
|
|
981
|
-
if reopening
|
|
982
|
-
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
983
|
-
}
|
|
984
|
-
replaced.update(replaced_sources)
|
|
985
|
-
existing = next(
|
|
986
|
-
(entry for entry in updates if entry[0] == owner and entry[1] == update), None
|
|
987
|
-
)
|
|
988
|
-
if existing is None:
|
|
989
|
-
receipt_paths.update(replacement_paths)
|
|
990
|
-
updates.append((owner, update, replaced_sources))
|
|
991
|
-
else:
|
|
992
|
-
existing[2].update(replaced_sources)
|
|
993
|
-
return replaced, updates
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
@contextmanager
|
|
997
|
-
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
998
|
-
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
999
|
-
head_path = scan_dir / "checkpoint-head.json"
|
|
1000
|
-
previous = None
|
|
1001
|
-
try:
|
|
1002
|
-
head_path.lstat()
|
|
1003
|
-
except FileNotFoundError:
|
|
1004
|
-
pass
|
|
1005
|
-
else:
|
|
1006
|
-
descriptor = open_scan_local_file_descriptor(
|
|
1007
|
-
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
1008
|
-
)
|
|
1009
|
-
with os.fdopen(descriptor, "rb") as handle:
|
|
1010
|
-
metadata = os.fstat(handle.fileno())
|
|
1011
|
-
previous = (handle.read(), metadata)
|
|
1012
|
-
directories = ("checkpoints", "checkpoint-heads")
|
|
1013
|
-
previous_files = {
|
|
1014
|
-
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
1015
|
-
}
|
|
1016
|
-
try:
|
|
1017
|
-
yield
|
|
1018
|
-
except BaseException:
|
|
1019
|
-
if previous is None:
|
|
1020
|
-
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
1021
|
-
else:
|
|
1022
|
-
payload, metadata = previous
|
|
1023
|
-
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
1024
|
-
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
1025
|
-
current_files = {
|
|
1026
|
-
relative
|
|
1027
|
-
for directory in directories
|
|
1028
|
-
for relative in _checkpoint_paths(scan_dir, directory)
|
|
1029
|
-
}
|
|
1030
|
-
for relative in current_files - previous_files:
|
|
1031
|
-
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
1032
|
-
raise
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
def merge_saved_results(
|
|
1036
|
-
scan_dir: Path,
|
|
1037
|
-
scan_id: str,
|
|
1038
|
-
binding: dict[str, Any],
|
|
1039
|
-
workers: list[Any],
|
|
1040
|
-
warnings: list[str],
|
|
1041
|
-
*,
|
|
1042
|
-
stopped: bool,
|
|
1043
|
-
reason: str,
|
|
1044
|
-
frozen_source_digests: dict[str, str] | None = None,
|
|
1045
|
-
allow_frozen_legacy_parent: bool = False,
|
|
1046
|
-
frozen_model_source: str | None = None,
|
|
1047
|
-
selected_model_source: list[str] | None = None,
|
|
1048
|
-
project_worker_coverage: bool = False,
|
|
1049
|
-
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
1050
|
-
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
1051
|
-
initial_warnings = set(warnings)
|
|
1052
|
-
try:
|
|
1053
|
-
source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
|
|
1054
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1055
|
-
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
1056
|
-
parent: dict[str, Any] | None = None
|
|
1057
|
-
parent_manifest: dict[str, Any] | None = None
|
|
1058
|
-
parent_is_canonical = False
|
|
1059
|
-
parent_modified = 0
|
|
1060
|
-
if frozen_source_digests is None or allow_frozen_legacy_parent:
|
|
1061
|
-
try:
|
|
1062
|
-
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
1063
|
-
# Without an accepted head, file-authored coverage is a full replacement.
|
|
1064
|
-
parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
|
|
1065
|
-
parent_is_canonical = True
|
|
1066
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1067
|
-
if not stopped:
|
|
1068
|
-
raise
|
|
1069
|
-
if (scan_dir / "scan-manifest.json").exists():
|
|
1070
|
-
warnings.append(f"Could not read the saved parent draft: {exc}")
|
|
1071
|
-
parent_manifest = None
|
|
1072
|
-
parent = None
|
|
1073
|
-
if parent_manifest is not None and parent is not None:
|
|
1074
|
-
parent_scan = parent_manifest["scan"]
|
|
1075
|
-
try:
|
|
1076
|
-
previous_head, _, head_modified = _read_saved_result(
|
|
1077
|
-
scan_dir, "checkpoint-head.json", scan_id
|
|
1078
|
-
)
|
|
1079
|
-
except (ContractError, OSError, ValueError):
|
|
1080
|
-
head_modified = None
|
|
1081
|
-
if head_modified is not None:
|
|
1082
|
-
# A partial tool publication must not outrank its accepted head.
|
|
1083
|
-
parent_modified = min(
|
|
1084
|
-
(scan_dir / name).lstat().st_mtime_ns
|
|
1085
|
-
for name in ("findings.json", "coverage.json", "scan-manifest.json")
|
|
1086
|
-
)
|
|
1087
|
-
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
1088
|
-
head_path = scan_dir / "checkpoint-head.json"
|
|
1089
|
-
tied_observations = False
|
|
1090
|
-
if head_modified == parent_modified:
|
|
1091
|
-
previous_parent, _, _ = _read_saved_result(
|
|
1092
|
-
scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
|
|
1093
|
-
)
|
|
1094
|
-
if previous_parent != parent:
|
|
1095
|
-
# Tied observations cannot decide which pending work came last.
|
|
1096
|
-
parent = _merge_tied_parent_observations(parent, previous_parent)
|
|
1097
|
-
parent_is_canonical = False
|
|
1098
|
-
tied_observations = True
|
|
1099
|
-
payload = _encoded(parent)
|
|
1100
|
-
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
1101
|
-
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
1102
|
-
checkpoint_path = scan_dir / parent_checkpoint
|
|
1103
|
-
if not checkpoint_path.exists():
|
|
1104
|
-
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
1105
|
-
# A recovery copy must not appear newer than the review it copies.
|
|
1106
|
-
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
|
|
1107
|
-
if head_modified is None or head_modified < parent_modified or tied_observations:
|
|
1108
|
-
write_scan_local_bytes(
|
|
1109
|
-
scan_dir,
|
|
1110
|
-
"checkpoint-head.json",
|
|
1111
|
-
_encoded({"checkpoint": checkpoint_path.name}),
|
|
1112
|
-
)
|
|
1113
|
-
os.utime(head_path, ns=(parent_modified, parent_modified))
|
|
1114
|
-
if frozen_source_digests is not None:
|
|
1115
|
-
captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
|
|
1116
|
-
frozen_source_digests = {
|
|
1117
|
-
**frozen_source_digests,
|
|
1118
|
-
parent_checkpoint: parent_digest,
|
|
1119
|
-
**{path: value[0] for path, value in captured.items()},
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
1123
|
-
parent_preserved_sources: dict[str, str] = {}
|
|
1124
|
-
source_digests: dict[str, str] = {}
|
|
1125
|
-
source_order: dict[str, tuple[int, int]] = {}
|
|
1126
|
-
worker_attempts: dict[str, tuple[str, int]] = {}
|
|
1127
|
-
saved_heads: dict[str, str] = {}
|
|
1128
|
-
if parent_manifest:
|
|
1129
|
-
recorded = parent_manifest["scan"].get("preservedSources", {})
|
|
1130
|
-
if isinstance(recorded, dict):
|
|
1131
|
-
parent_preserved_sources = recorded
|
|
1132
|
-
source_digests.update(parent_preserved_sources)
|
|
1133
|
-
if frozen_source_digests is None:
|
|
1134
|
-
source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
|
|
1135
|
-
paths: dict[str, str | None] = {}
|
|
1136
|
-
reducer_paths: set[str] = set()
|
|
1137
|
-
current_results: set[str] = set()
|
|
1138
|
-
reducer_outputs: list[tuple[Any, str, list[str], int]] = []
|
|
1139
|
-
reducer = _latest_successful_reducer(workers)
|
|
1140
|
-
latest_reducer: str | None = None
|
|
1141
|
-
if reducer is not None:
|
|
1142
|
-
try:
|
|
1143
|
-
latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
1144
|
-
paths[latest_reducer] = None
|
|
1145
|
-
reducer_paths.add(latest_reducer)
|
|
1146
|
-
except ValueError:
|
|
1147
|
-
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
1148
|
-
|
|
1149
|
-
def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
|
|
1150
|
-
if worker_id is not None:
|
|
1151
|
-
root = Path(directory).parent.as_posix()
|
|
1152
|
-
worker_attempts[root] = (worker_id, attempt)
|
|
1153
|
-
paths[f"{root}/checkpoint-head.json"] = worker_id
|
|
1154
|
-
head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
|
|
1155
|
-
for saved_directory in (directory, head_snapshots):
|
|
1156
|
-
paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
|
|
1157
|
-
|
|
1158
|
-
paths["checkpoint-head.json"] = None
|
|
1159
|
-
checkpoints("checkpoints", None)
|
|
1160
|
-
for worker in workers:
|
|
1161
|
-
try:
|
|
1162
|
-
outputs = _worker_outputs(scan_dir, worker)
|
|
1163
|
-
except (TypeError, ValueError):
|
|
1164
|
-
warnings.append("Skipped a worker checkpoint outside the scan directory.")
|
|
1165
|
-
continue
|
|
1166
|
-
if worker["kind"] == "dedup":
|
|
1167
|
-
for directory, attempt in outputs:
|
|
1168
|
-
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
1169
|
-
if not checkpoint_paths:
|
|
1170
|
-
continue
|
|
1171
|
-
result_path = f"{directory}/result.json"
|
|
1172
|
-
retained_paths = [result_path, *checkpoint_paths]
|
|
1173
|
-
paths.update(dict.fromkeys(retained_paths))
|
|
1174
|
-
reducer_paths.update(retained_paths)
|
|
1175
|
-
reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
|
|
1176
|
-
continue
|
|
1177
|
-
if worker["kind"] != "discovery":
|
|
1178
|
-
continue
|
|
1179
|
-
output, attempt = outputs[0]
|
|
1180
|
-
paths[f"{output}/result.json"] = worker["id"]
|
|
1181
|
-
current_results.add(f"{output}/result.json")
|
|
1182
|
-
for archived, archived_attempt in outputs[1:]:
|
|
1183
|
-
paths[f"{archived}/result.json"] = worker["id"]
|
|
1184
|
-
checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
|
|
1185
|
-
checkpoints(f"{output}/checkpoints", worker["id"], attempt)
|
|
1186
|
-
if worker["result_manifest_path"]:
|
|
1187
|
-
try:
|
|
1188
|
-
current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
1189
|
-
paths[current_path] = worker["id"]
|
|
1190
|
-
current_results.add(current_path)
|
|
1191
|
-
except ValueError:
|
|
1192
|
-
warnings.append("Skipped a worker result outside the scan directory.")
|
|
1193
|
-
|
|
1194
|
-
if frozen_source_digests is None:
|
|
1195
|
-
for relative, worker_id in list(paths.items()):
|
|
1196
|
-
if Path(relative).name != "checkpoint-head.json":
|
|
1197
|
-
continue
|
|
1198
|
-
del paths[relative]
|
|
1199
|
-
try:
|
|
1200
|
-
captured = _capture_saved_source(scan_dir, relative, scan_id)
|
|
1201
|
-
paths.update({path: worker_id for path in captured})
|
|
1202
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1203
|
-
if (scan_dir / relative).exists():
|
|
1204
|
-
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1205
|
-
|
|
1206
|
-
if frozen_source_digests is not None:
|
|
1207
|
-
source_digests.update(
|
|
1208
|
-
{
|
|
1209
|
-
path: digest
|
|
1210
|
-
for path, digest in frozen_source_digests.items()
|
|
1211
|
-
if _is_source_order_snapshot(path)
|
|
1212
|
-
}
|
|
1213
|
-
)
|
|
1214
|
-
paths = {
|
|
1215
|
-
relative: worker_id
|
|
1216
|
-
for relative, worker_id in paths.items()
|
|
1217
|
-
if relative in frozen_source_digests
|
|
1218
|
-
}
|
|
1219
|
-
current_results.intersection_update(frozen_source_digests)
|
|
1220
|
-
if latest_reducer not in frozen_source_digests:
|
|
1221
|
-
latest_reducer = None
|
|
1222
|
-
|
|
1223
|
-
for relative, worker_id in paths.items():
|
|
1224
|
-
try:
|
|
1225
|
-
draft, digest, observed = _read_saved_result(
|
|
1226
|
-
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
1227
|
-
)
|
|
1228
|
-
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
1229
|
-
raise ContractError("checkpoint changed after the scan stopped")
|
|
1230
|
-
source_digests[relative] = digest
|
|
1231
|
-
source_times.setdefault(relative, observed)
|
|
1232
|
-
source_order[relative] = (0, source_times[relative])
|
|
1233
|
-
if _checkpoint_head_directory(relative) is not None:
|
|
1234
|
-
saved_heads[relative] = draft["checkpoint"]
|
|
1235
|
-
continue
|
|
1236
|
-
# Recovery expects coverage, but reducer results only contain findings
|
|
1237
|
-
# and context. Add an empty value after hashing the original result.
|
|
1238
|
-
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
1239
|
-
except (ContractError, OSError, ValueError) as exc:
|
|
1240
|
-
if (scan_dir / relative).exists():
|
|
1241
|
-
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1242
|
-
if frozen_source_digests is not None:
|
|
1243
|
-
if frozen_source_digests.keys() - source_digests.keys():
|
|
1244
|
-
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
|
|
1245
|
-
|
|
1246
|
-
# Frozen observations retain checkpoint selection even if a worker moves its head.
|
|
1247
|
-
headed_workers = {
|
|
1248
|
-
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
|
|
1249
|
-
for head in saved_heads
|
|
1250
|
-
if _checkpoint_head_directory(head) != Path(".")
|
|
1251
|
-
}
|
|
1252
|
-
for relative, _, worker_id in sources:
|
|
1253
|
-
if worker_id not in headed_workers:
|
|
1254
|
-
continue
|
|
1255
|
-
directory = Path(relative).parent
|
|
1256
|
-
if directory.name == "checkpoints":
|
|
1257
|
-
directory = directory.parent
|
|
1258
|
-
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
|
|
1259
|
-
source_order[relative] = (attempt, source_order[relative][1])
|
|
1260
|
-
selected_observations: dict[str, tuple[int, int]] = {}
|
|
1261
|
-
parent_heads: list[tuple[int, str]] = []
|
|
1262
|
-
for head, checkpoint in saved_heads.items():
|
|
1263
|
-
directory = _checkpoint_head_directory(head)
|
|
1264
|
-
selected = (directory / "checkpoints" / checkpoint).as_posix()
|
|
1265
|
-
if selected not in source_order:
|
|
1266
|
-
raise ContractError("Checkpoint head is outside the saved source set.")
|
|
1267
|
-
observed = source_order[head][1]
|
|
1268
|
-
if directory == Path("."):
|
|
1269
|
-
order = (0, max(source_order[selected][1], observed))
|
|
1270
|
-
parent_heads.append((observed, selected))
|
|
1271
|
-
else:
|
|
1272
|
-
_, attempt = worker_attempts[directory.as_posix()]
|
|
1273
|
-
order = (attempt, observed)
|
|
1274
|
-
selected_observations[selected] = max(selected_observations.get(selected, order), order)
|
|
1275
|
-
source_order.update(selected_observations)
|
|
1276
|
-
|
|
1277
|
-
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
1278
|
-
latest_reducer_key = (
|
|
1279
|
-
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
1280
|
-
if reducer is not None and latest_reducer in drafts_by_path
|
|
1281
|
-
else None
|
|
1282
|
-
)
|
|
1283
|
-
if latest_reducer_key is None:
|
|
1284
|
-
latest_reducer = None
|
|
1285
|
-
for worker, result_path, checkpoint_paths, attempt in reducer_outputs:
|
|
1286
|
-
result = drafts_by_path.get(result_path)
|
|
1287
|
-
if result is None or not any(
|
|
1288
|
-
drafts_by_path.get(checkpoint_path) == result for checkpoint_path in checkpoint_paths
|
|
1289
|
-
):
|
|
1290
|
-
continue
|
|
1291
|
-
current_results.add(result_path)
|
|
1292
|
-
candidate_key = (worker["completed_at"] or "", worker["id"], attempt)
|
|
1293
|
-
if latest_reducer_key is None or candidate_key > latest_reducer_key:
|
|
1294
|
-
latest_reducer_key = candidate_key
|
|
1295
|
-
latest_reducer = result_path
|
|
1296
|
-
|
|
1297
|
-
if parent_heads:
|
|
1298
|
-
latest_observation = max(observed for observed, _ in parent_heads)
|
|
1299
|
-
for observed, parent_path in parent_heads:
|
|
1300
|
-
if observed != latest_observation:
|
|
1301
|
-
continue
|
|
1302
|
-
draft = drafts_by_path[parent_path]
|
|
1303
|
-
modified = source_order[parent_path][1]
|
|
1304
|
-
if parent is None or modified > parent_modified:
|
|
1305
|
-
parent = draft
|
|
1306
|
-
parent_modified = modified
|
|
1307
|
-
parent_is_canonical = False
|
|
1308
|
-
elif modified == parent_modified and draft != parent:
|
|
1309
|
-
parent = _merge_tied_parent_observations(parent, draft)
|
|
1310
|
-
parent_is_canonical = False
|
|
1311
|
-
if parent is None and latest_reducer is not None:
|
|
1312
|
-
parent = drafts_by_path[latest_reducer]
|
|
1313
|
-
|
|
1314
|
-
project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
|
|
1315
|
-
sources, parent, source_order, enabled=project_worker_coverage
|
|
1316
|
-
)
|
|
1317
|
-
# Keep the actual retained row even when recovery adds previous findings.
|
|
1318
|
-
linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
|
|
1319
|
-
linked_surface_outputs: dict[int, dict[str, Any]] = {}
|
|
1320
|
-
worker_row_owners: dict[int, str] = {}
|
|
1321
|
-
worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
|
|
1322
|
-
|
|
1323
|
-
def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
|
|
1324
|
-
# Parent aggregates can carry worker metadata without originating in
|
|
1325
|
-
# that worker's retained source. Track the actual source of output rows.
|
|
1326
|
-
if project_worker_coverage and owner is not None and isinstance(retained, dict):
|
|
1327
|
-
worker_row_owners[id(retained)] = owner
|
|
1328
|
-
if field == "surfaces" and isinstance(item.get("id"), str):
|
|
1329
|
-
worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
|
|
1330
|
-
|
|
1331
|
-
def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
|
|
1332
|
-
retained = _project_worker_coverage_row(
|
|
1333
|
-
item, owner if project_worker_coverage else None, field
|
|
1334
|
-
)
|
|
1335
|
-
if retained not in output:
|
|
1336
|
-
output.append(retained)
|
|
1337
|
-
retained = output[output.index(retained)]
|
|
1338
|
-
record_coverage_owner(retained, item, owner, field)
|
|
1339
|
-
return retained
|
|
1340
|
-
|
|
1341
|
-
def linked_tasks(
|
|
1342
|
-
task: dict[str, Any],
|
|
1343
|
-
published_id: str,
|
|
1344
|
-
source_task: dict[str, Any] | None,
|
|
1345
|
-
candidates: list[tuple[str | None, Any]],
|
|
1346
|
-
) -> list[Any]:
|
|
1347
|
-
expected = (
|
|
1348
|
-
_project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
|
|
1349
|
-
if source_task is not None
|
|
1350
|
-
else None
|
|
1351
|
-
)
|
|
1352
|
-
return [
|
|
1353
|
-
retained
|
|
1354
|
-
for owner, retained in candidates
|
|
1355
|
-
if retained == task
|
|
1356
|
-
or (
|
|
1357
|
-
expected is not None
|
|
1358
|
-
and owner == expected["workerId"]
|
|
1359
|
-
and isinstance(retained, dict)
|
|
1360
|
-
and retained.get("id") == expected.get("id")
|
|
1361
|
-
and isinstance(retained.get("surfaceIds"), list)
|
|
1362
|
-
and published_id in retained["surfaceIds"]
|
|
1363
|
-
)
|
|
1364
|
-
]
|
|
1365
|
-
|
|
1366
|
-
def linked_surface(surface, outputs, named_outputs):
|
|
1367
|
-
retained = outputs.get(id(surface))
|
|
1368
|
-
if (
|
|
1369
|
-
retained is None
|
|
1370
|
-
and isinstance(owner := surface.get("workerId"), str)
|
|
1371
|
-
and isinstance(identity := surface.get("id"), str)
|
|
1372
|
-
):
|
|
1373
|
-
matches = named_outputs.get((owner, identity), {})
|
|
1374
|
-
if len(matches) == 1:
|
|
1375
|
-
retained = next(iter(matches.values()))
|
|
1376
|
-
return retained
|
|
1377
|
-
|
|
1378
|
-
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
1379
|
-
# Older checkpoints can omit IDs already assigned in their published output.
|
|
1380
|
-
for field in ("deferred", "surfaces"):
|
|
1381
|
-
named_rows: dict[str | None, list[dict[str, Any]]] = {}
|
|
1382
|
-
for _, draft, owner in all_sources:
|
|
1383
|
-
rows = draft["coverage"].get(field, [])
|
|
1384
|
-
named_rows.setdefault(owner, []).extend(
|
|
1385
|
-
row
|
|
1386
|
-
for row in (rows if isinstance(rows, list) else [])
|
|
1387
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1388
|
-
)
|
|
1389
|
-
for _, draft, owner in all_sources:
|
|
1390
|
-
rows = draft["coverage"].get(field, [])
|
|
1391
|
-
if not isinstance(rows, list):
|
|
1392
|
-
continue
|
|
1393
|
-
reserved = {
|
|
1394
|
-
row["id"]
|
|
1395
|
-
for row in rows
|
|
1396
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1397
|
-
}
|
|
1398
|
-
for row in rows:
|
|
1399
|
-
if not isinstance(row, dict) or "id" in row:
|
|
1400
|
-
continue
|
|
1401
|
-
if field == "deferred" and any(
|
|
1402
|
-
key in row for key in ("candidateId", "candidate", "finding")
|
|
1403
|
-
):
|
|
1404
|
-
continue
|
|
1405
|
-
content = {"receiptRefs": [], **row} if field == "surfaces" else row
|
|
1406
|
-
identity = next(
|
|
1407
|
-
(
|
|
1408
|
-
named["id"]
|
|
1409
|
-
for named in named_rows[owner]
|
|
1410
|
-
if named["id"] not in reserved
|
|
1411
|
-
and {
|
|
1412
|
-
**({"receiptRefs": []} if field == "surfaces" else {}),
|
|
1413
|
-
**{key: value for key, value in named.items() if key != "id"},
|
|
1414
|
-
}
|
|
1415
|
-
== content
|
|
1416
|
-
),
|
|
1417
|
-
None,
|
|
1418
|
-
)
|
|
1419
|
-
if identity is not None:
|
|
1420
|
-
row["id"] = identity
|
|
1421
|
-
reserved.add(identity)
|
|
1422
|
-
|
|
1423
|
-
if parent is None and not sources:
|
|
1424
|
-
return None
|
|
1425
|
-
if (
|
|
1426
|
-
parent_manifest
|
|
1427
|
-
and parent_manifest["scan"].get("sealedAt")
|
|
1428
|
-
and parent_manifest["scan"].get("status") == binding["status"]
|
|
1429
|
-
and parent_manifest["scan"].get("preservedSources") == source_digests
|
|
1430
|
-
and all(warning in initial_warnings for warning in warnings)
|
|
1431
|
-
):
|
|
1432
|
-
return None
|
|
1433
|
-
|
|
1434
|
-
if (
|
|
1435
|
-
frozen_source_digests is None
|
|
1436
|
-
or any(_is_source_order_snapshot(path) for path in source_digests)
|
|
1437
|
-
or allow_frozen_legacy_parent
|
|
1438
|
-
):
|
|
1439
|
-
_freeze_source_times(scan_dir, scan_id, source_digests, source_times)
|
|
1440
|
-
|
|
1441
|
-
target_kind = binding["allowedTargetKinds"][0]
|
|
1442
|
-
if (
|
|
1443
|
-
target_kind == "git_worktree"
|
|
1444
|
-
and "snapshotDigest" not in binding["target"]
|
|
1445
|
-
and "git_revision" in binding["allowedTargetKinds"]
|
|
1446
|
-
):
|
|
1447
|
-
target_kind = "git_revision"
|
|
1448
|
-
target = {"kind": target_kind, **binding["target"]}
|
|
1449
|
-
if target["kind"] == "git_diff" and "snapshotDigest" not in target:
|
|
1450
|
-
diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
|
|
1451
|
-
target["snapshotDigest"] = committed_diff_snapshot_digest(
|
|
1452
|
-
diff_kind, target["baseRevision"], target["headRevision"]
|
|
1453
|
-
)
|
|
1454
|
-
manifest = (
|
|
1455
|
-
copy.deepcopy(parent_manifest)
|
|
1456
|
-
if parent_manifest
|
|
1457
|
-
else {"scan": {"target": target, "scope": binding["scope"]}}
|
|
1458
|
-
)
|
|
1459
|
-
for key in ("sealedAt", "artifacts"):
|
|
1460
|
-
manifest["scan"].pop(key, None)
|
|
1461
|
-
manifest["scan"]["preservedSources"] = source_digests
|
|
1462
|
-
# Completion follows the selected parent, including legacy terminal drafts
|
|
1463
|
-
# that omit the optional marker, rather than an older canonical manifest.
|
|
1464
|
-
if parent is not None:
|
|
1465
|
-
manifest["scan"].pop("complete", None)
|
|
1466
|
-
if "complete" in parent:
|
|
1467
|
-
manifest["scan"]["complete"] = parent["complete"]
|
|
1468
|
-
for key in ("scope", "threatModel"):
|
|
1469
|
-
if isinstance(parent.get(key), dict):
|
|
1470
|
-
manifest["scan"][key] = copy.deepcopy(parent[key])
|
|
1471
|
-
if isinstance(manifest["scan"].get("scope"), dict):
|
|
1472
|
-
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
|
|
1473
|
-
if frozen_model_source is not None:
|
|
1474
|
-
# Publication retries retain the choice made with the frozen source set.
|
|
1475
|
-
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
|
|
1476
|
-
if not isinstance(model, dict):
|
|
1477
|
-
raise ContractError("Frozen stopped-scan model source is unavailable.")
|
|
1478
|
-
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1479
|
-
if paths[frozen_model_source] is not None:
|
|
1480
|
-
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1481
|
-
if selected_model_source is not None:
|
|
1482
|
-
selected_model_source[:] = [frozen_model_source]
|
|
1483
|
-
coverage = (
|
|
1484
|
-
copy.deepcopy(parent["coverage"])
|
|
1485
|
-
if parent and parent["coverage"]
|
|
1486
|
-
else {
|
|
1487
|
-
"mode": binding["coverageMode"],
|
|
1488
|
-
"inventoryStrategy": "diff"
|
|
1489
|
-
if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
|
|
1490
|
-
else "scoped_path"
|
|
1491
|
-
if binding["coverageMode"] == "scoped_path"
|
|
1492
|
-
else "repository",
|
|
1493
|
-
**binding["scope"],
|
|
1494
|
-
"surfaces": [],
|
|
1495
|
-
"explicitExclusions": [],
|
|
1496
|
-
"deferred": [],
|
|
1497
|
-
}
|
|
1498
|
-
)
|
|
1499
|
-
if isinstance(coverage.get("openQuestions"), list):
|
|
1500
|
-
coverage["openQuestions"] = [
|
|
1501
|
-
{"question": item.strip()} if isinstance(item, str) else item
|
|
1502
|
-
for item in coverage["openQuestions"]
|
|
1503
|
-
]
|
|
1504
|
-
canonical_rows = (
|
|
1505
|
-
{
|
|
1506
|
-
id(item)
|
|
1507
|
-
for field in ("surfaces", "explicitExclusions", "deferred")
|
|
1508
|
-
for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
|
|
1509
|
-
}
|
|
1510
|
-
if parent_is_canonical
|
|
1511
|
-
else set()
|
|
1512
|
-
)
|
|
1513
|
-
findings: list[dict[str, Any]] = []
|
|
1514
|
-
finding_positions: dict[str, int] = {}
|
|
1515
|
-
represented: dict[str, str | None] = {}
|
|
1516
|
-
represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}
|
|
1517
|
-
represented_history: dict[str, set[str]] = {}
|
|
1518
|
-
represented_candidate_history: dict[tuple[str, str, Any, Any, Any], set[str]] = {}
|
|
1519
|
-
rejected_history: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
1520
|
-
stopped_parent_seal = bool(
|
|
1521
|
-
stopped and parent_manifest and parent_manifest["scan"].get("sealedAt")
|
|
1522
|
-
)
|
|
1523
|
-
|
|
1524
|
-
def valid_finding(value: Any) -> bool:
|
|
1525
|
-
# Use the finalizer's own per-record recovery before a draft can suppress
|
|
1526
|
-
# an earlier checkpoint. Invalid latest records must not hide valid history.
|
|
1527
|
-
document = {"scanId": scan_id, "findings": [copy.deepcopy(value)]}
|
|
1528
|
-
if isinstance(document["findings"][0], dict):
|
|
1529
|
-
_ensure_finding_identity(document["findings"][0], candidate_only=True)
|
|
1530
|
-
_recover_unsealed_findings(
|
|
1531
|
-
{"scan": {"id": scan_id, "target": binding["target"]}},
|
|
1532
|
-
document,
|
|
1533
|
-
Path(__file__).resolve().parent.parent / "schemas",
|
|
1534
|
-
scan_dir,
|
|
1535
|
-
[],
|
|
1536
|
-
)
|
|
1537
|
-
return bool(document["findings"])
|
|
1538
|
-
|
|
1539
|
-
source_order["parent"] = (0, parent_modified)
|
|
1540
|
-
deferred_rows = {
|
|
1541
|
-
relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
|
|
1542
|
-
}
|
|
1543
|
-
# A legacy source can contain independent tasks with the same explicit ID.
|
|
1544
|
-
# Later rewrites cannot make an ID-only closure identify one of those tasks.
|
|
1545
|
-
ambiguous_deferred: set[tuple[str | None, str]] = set()
|
|
1546
|
-
for relative, _, owner in all_sources:
|
|
1547
|
-
by_id: dict[str, dict[str, Any]] = {}
|
|
1548
|
-
candidate_aliases = {
|
|
1549
|
-
identity
|
|
1550
|
-
for row in deferred_rows[relative]
|
|
1551
|
-
if isinstance(row, dict)
|
|
1552
|
-
and any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1553
|
-
for identity in (row.get("id"), row.get("candidateId"))
|
|
1554
|
-
if isinstance(identity, str)
|
|
1555
|
-
}
|
|
1556
|
-
for row in deferred_rows[relative]:
|
|
1557
|
-
if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
|
|
1558
|
-
continue
|
|
1559
|
-
if any(key in row for key in ("candidateId", "candidate", "finding")):
|
|
1560
|
-
continue
|
|
1561
|
-
if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
|
|
1562
|
-
ambiguous_deferred.add((owner, identity))
|
|
1563
|
-
by_id[identity] = row
|
|
1564
|
-
current_drafts = ([("parent", parent, None)] if parent else []) + [
|
|
1565
|
-
source for source in sources if source[0] in current_results | selected_observations.keys()
|
|
1566
|
-
]
|
|
1567
|
-
# Generic closures belong to one logical scan or worker, just like candidates.
|
|
1568
|
-
# Keep them when recovering a terminal checkpoint without its canonical write.
|
|
1569
|
-
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1570
|
-
|
|
1571
|
-
candidate_ids: set[tuple[str | None, str]] = set()
|
|
1572
|
-
accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1573
|
-
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1574
|
-
coverage_schema = _read_json(
|
|
1575
|
-
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
1576
|
-
)["properties"]
|
|
1577
|
-
for relative, draft, owner in all_sources:
|
|
1578
|
-
order = source_order[relative]
|
|
1579
|
-
for item in deferred_rows[relative]:
|
|
1580
|
-
if not isinstance(item, dict):
|
|
1581
|
-
continue
|
|
1582
|
-
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
|
|
1583
|
-
candidate_ids.update(
|
|
1584
|
-
(owner, identity)
|
|
1585
|
-
for identity in (item.get("id"), item.get("candidateId"))
|
|
1586
|
-
if isinstance(identity, str)
|
|
1587
|
-
)
|
|
1588
|
-
identity = item.get("id") or item.get("candidateId")
|
|
1589
|
-
if isinstance(identity, str):
|
|
1590
|
-
key = (owner, identity)
|
|
1591
|
-
previous = active_deferred.get(key)
|
|
1592
|
-
if previous is None or order > previous[0]:
|
|
1593
|
-
active_deferred[key] = (order, item, relative)
|
|
1594
|
-
if (
|
|
1595
|
-
relative == "parent"
|
|
1596
|
-
or relative in current_results
|
|
1597
|
-
or relative in selected_observations
|
|
1598
|
-
):
|
|
1599
|
-
try:
|
|
1600
|
-
# A malformed update cannot discard valid saved evidence.
|
|
1601
|
-
_validate_schema_node(
|
|
1602
|
-
item, coverage_schema["deferred"]["items"], "coverage.deferred"
|
|
1603
|
-
)
|
|
1604
|
-
except ContractError:
|
|
1605
|
-
continue
|
|
1606
|
-
accepted_deferred_orders[key] = max(
|
|
1607
|
-
accepted_deferred_orders.get(key, order), order
|
|
1608
|
-
)
|
|
1609
|
-
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
|
|
1610
|
-
key = (owner, closure["id"])
|
|
1611
|
-
previous = closed_deferred.get(key)
|
|
1612
|
-
if previous is None or order > previous[0]:
|
|
1613
|
-
# The parent comes first, preserving its reason on equal timestamps.
|
|
1614
|
-
closed_deferred[key] = (order, closure, relative)
|
|
1615
|
-
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
|
|
1616
|
-
reopened_generic: set[tuple[str | None, str]] = set()
|
|
1617
|
-
for key, (order, _, _) in list(closed_deferred.items()):
|
|
1618
|
-
# Equal timestamps cannot distinguish closure from reopened work.
|
|
1619
|
-
reopened = [
|
|
1620
|
-
active
|
|
1621
|
-
for (owner, identity), active in active_deferred.items()
|
|
1622
|
-
if owner == key[0]
|
|
1623
|
-
and (identity == key[1] or active[1].get("candidateId") == key[1])
|
|
1624
|
-
and active[0] >= order
|
|
1625
|
-
]
|
|
1626
|
-
if key in candidate_ids or key in ambiguous_deferred or reopened:
|
|
1627
|
-
del closed_deferred[key]
|
|
1628
|
-
for _, item, _ in reopened:
|
|
1629
|
-
reopened_rows.append((key[0], item))
|
|
1630
|
-
if key not in candidate_ids:
|
|
1631
|
-
reopened_generic.add(key)
|
|
1632
|
-
# Retain every distinct ambiguous task even when a terminal result supersedes
|
|
1633
|
-
# its source. Existing output ID normalization keeps each row distinct.
|
|
1634
|
-
for relative, _, owner in all_sources:
|
|
1635
|
-
for row in deferred_rows[relative]:
|
|
1636
|
-
if (
|
|
1637
|
-
isinstance(row, dict)
|
|
1638
|
-
and isinstance(identity := row.get("id"), str)
|
|
1639
|
-
and (owner, identity) in ambiguous_deferred
|
|
1640
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1641
|
-
and (owner, row) not in reopened_rows
|
|
1642
|
-
):
|
|
1643
|
-
reopened_rows.append((owner, row))
|
|
1644
|
-
parent_closures = [
|
|
1645
|
-
closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
|
|
1646
|
-
]
|
|
1647
|
-
coverage.pop("resolvedDeferred", None)
|
|
1648
|
-
if parent_closures:
|
|
1649
|
-
coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
|
|
1650
|
-
if parent:
|
|
1651
|
-
coverage["deferred"] = [
|
|
1652
|
-
row
|
|
1653
|
-
for row in deferred_rows["parent"]
|
|
1654
|
-
if not isinstance(row, dict)
|
|
1655
|
-
or not isinstance(row.get("id"), str)
|
|
1656
|
-
or (None, row["id"]) not in closed_deferred
|
|
1657
|
-
]
|
|
1658
|
-
resolved: dict[tuple[str | None, str], str] = {}
|
|
1659
|
-
|
|
1660
|
-
ordered_candidates = {
|
|
1661
|
-
(owner, identity)
|
|
1662
|
-
for owner, row in reopened_rows
|
|
1663
|
-
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
|
|
1664
|
-
and (owner, identity) in candidate_ids
|
|
1665
|
-
}
|
|
1666
|
-
ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
|
|
1667
|
-
|
|
1668
|
-
outcomes: list[tuple[str, str | None, str, str]] = []
|
|
1669
|
-
for relative, draft, owner in current_drafts:
|
|
1670
|
-
for finding in draft["findings"]:
|
|
1671
|
-
if (
|
|
1672
|
-
isinstance(finding, dict)
|
|
1673
|
-
and valid_finding(finding)
|
|
1674
|
-
and (candidate_id := finding_candidate_id(finding))
|
|
1675
|
-
):
|
|
1676
|
-
outcomes.append((relative, owner, candidate_id, "reported"))
|
|
1677
|
-
for field in ("surfaces", "explicitExclusions"):
|
|
1678
|
-
items = draft["coverage"].get(field, [])
|
|
1679
|
-
for item in items if isinstance(items, list) else []:
|
|
1680
|
-
if (
|
|
1681
|
-
isinstance(item, dict)
|
|
1682
|
-
and isinstance(item.get("candidateId"), str)
|
|
1683
|
-
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
1684
|
-
):
|
|
1685
|
-
outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
|
|
1686
|
-
ordered_candidates.update(
|
|
1687
|
-
(owner, candidate_id)
|
|
1688
|
-
for relative, owner, candidate_id, _ in outcomes
|
|
1689
|
-
if owner is not None and relative in selected_observations
|
|
1690
|
-
)
|
|
1691
|
-
# Reopened work and selected checkpoint outcomes follow the saved source order.
|
|
1692
|
-
for relative, owner, candidate_id, disposition in outcomes:
|
|
1693
|
-
key = (owner, candidate_id)
|
|
1694
|
-
if key not in ordered_candidates:
|
|
1695
|
-
if relative == "parent" or relative in current_results:
|
|
1696
|
-
resolved.setdefault(key, disposition)
|
|
1697
|
-
continue
|
|
1698
|
-
order = source_order[relative]
|
|
1699
|
-
if any(
|
|
1700
|
-
saved_owner == owner
|
|
1701
|
-
and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
|
|
1702
|
-
and modified >= order
|
|
1703
|
-
for (saved_owner, _), (modified, row, _) in active_deferred.items()
|
|
1704
|
-
):
|
|
1705
|
-
continue
|
|
1706
|
-
if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
|
|
1707
|
-
resolved[key] = disposition
|
|
1708
|
-
ordered_outcomes[key] = (order, relative)
|
|
1709
|
-
# Only the current parent may claim that another worker finding was absorbed.
|
|
1710
|
-
# A superseded checkpoint must not suppress a newer independent result.
|
|
1711
|
-
if parent:
|
|
1712
|
-
for finding in parent["findings"]:
|
|
1713
|
-
if valid_finding(finding):
|
|
1714
|
-
canonical_key = _finding_key(finding)
|
|
1715
|
-
for retained in _retained_findings(finding):
|
|
1716
|
-
retained_key = _finding_key(retained)
|
|
1717
|
-
if retained is not finding:
|
|
1718
|
-
represented_history.setdefault(retained_key, set()).add(
|
|
1719
|
-
_digest(_finding_content(retained))
|
|
1720
|
-
)
|
|
1721
|
-
previous_key = represented.get(retained_key)
|
|
1722
|
-
if retained_key not in represented:
|
|
1723
|
-
represented[retained_key] = canonical_key
|
|
1724
|
-
elif previous_key != canonical_key:
|
|
1725
|
-
# Ambiguous history cannot suppress an independent source.
|
|
1726
|
-
represented[retained_key] = None
|
|
1727
|
-
originals = finding["provenance"].get("sourceFindings", [])
|
|
1728
|
-
for original in originals if isinstance(originals, list) else []:
|
|
1729
|
-
if isinstance(original, dict) and isinstance(original.get("finding"), dict):
|
|
1730
|
-
source_id = original.get("id")
|
|
1731
|
-
candidate_id = finding_candidate_id(original["finding"])
|
|
1732
|
-
if isinstance(source_id, str) and ":" in source_id and candidate_id:
|
|
1733
|
-
candidate_key = _worker_candidate_key(
|
|
1734
|
-
source_id.rsplit(":", 1)[0],
|
|
1735
|
-
candidate_id,
|
|
1736
|
-
original["finding"],
|
|
1737
|
-
)
|
|
1738
|
-
previous_key = represented_candidates.get(candidate_key)
|
|
1739
|
-
if candidate_key not in represented_candidates:
|
|
1740
|
-
represented_candidates[candidate_key] = canonical_key
|
|
1741
|
-
elif previous_key != canonical_key:
|
|
1742
|
-
# Candidate ids are only authoritative within one
|
|
1743
|
-
# logical worker. Multiple canonical owners make
|
|
1744
|
-
# that worker-local identity ambiguous.
|
|
1745
|
-
represented_candidates[candidate_key] = None
|
|
1746
|
-
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
1747
|
-
_digest(_finding_content(original["finding"]))
|
|
1748
|
-
)
|
|
1749
|
-
selected_terminal_orders: dict[str, tuple[int, int]] = {}
|
|
1750
|
-
for relative, draft, worker_id in sources:
|
|
1751
|
-
if (
|
|
1752
|
-
worker_id is not None
|
|
1753
|
-
and relative in selected_observations
|
|
1754
|
-
and draft.get("complete") is not False
|
|
1755
|
-
):
|
|
1756
|
-
order = source_order[relative]
|
|
1757
|
-
selected_terminal_orders[worker_id] = max(
|
|
1758
|
-
selected_terminal_orders.get(worker_id, order), order
|
|
1759
|
-
)
|
|
1760
|
-
terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
|
|
1761
|
-
for relative, draft, worker_id in sources:
|
|
1762
|
-
if relative in current_results and draft.get("complete") is not False:
|
|
1763
|
-
order = source_order[relative]
|
|
1764
|
-
terminal_worker_orders[worker_id] = max(
|
|
1765
|
-
terminal_worker_orders.get(worker_id, order), order
|
|
1766
|
-
)
|
|
1767
|
-
|
|
1768
|
-
def source_coverage_selection(
|
|
1769
|
-
relative: str,
|
|
1770
|
-
worker_id: str | None,
|
|
1771
|
-
) -> tuple[bool, set[str], bool, bool]:
|
|
1772
|
-
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
1773
|
-
selected_coverage_superseded = worker_id in selected_terminal_orders and (
|
|
1774
|
-
source_order[relative] < selected_terminal_orders[worker_id]
|
|
1775
|
-
or (
|
|
1776
|
-
relative in current_results
|
|
1777
|
-
and source_order[relative] <= selected_terminal_orders[worker_id]
|
|
1778
|
-
)
|
|
1779
|
-
)
|
|
1780
|
-
selected_candidates = {
|
|
1781
|
-
candidate_id
|
|
1782
|
-
for (owner, candidate_id), (_, source) in ordered_outcomes.items()
|
|
1783
|
-
if owner == worker_id and source == relative
|
|
1784
|
-
}
|
|
1785
|
-
superseded = (
|
|
1786
|
-
worker_id is None
|
|
1787
|
-
and parent is not None
|
|
1788
|
-
and parent.get("complete") is not False
|
|
1789
|
-
and relative != "parent"
|
|
1790
|
-
and source_order[relative] <= (0, parent_modified)
|
|
1791
|
-
and (not stopped_parent_seal or relative in parent_preserved_sources)
|
|
1792
|
-
) or (
|
|
1793
|
-
relative not in current_results
|
|
1794
|
-
and worker_result_order is not None
|
|
1795
|
-
and (
|
|
1796
|
-
relative not in selected_observations
|
|
1797
|
-
or source_order[relative] < worker_result_order
|
|
1798
|
-
)
|
|
1799
|
-
)
|
|
1800
|
-
# A failed result write can leave pending work outside the accepted result.
|
|
1801
|
-
accepted_order = (
|
|
1802
|
-
(0, parent_modified)
|
|
1803
|
-
if worker_id is None and parent is not None
|
|
1804
|
-
else worker_result_order
|
|
1805
|
-
)
|
|
1806
|
-
retain_pending = (
|
|
1807
|
-
superseded
|
|
1808
|
-
and (worker_id in headed_workers or (worker_id is None and parent_heads))
|
|
1809
|
-
and accepted_order is not None
|
|
1810
|
-
and source_order[relative] >= accepted_order
|
|
1811
|
-
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
1812
|
-
)
|
|
1813
|
-
return selected_coverage_superseded, selected_candidates, superseded, retain_pending
|
|
1814
|
-
|
|
1815
|
-
def coverage_row_is_retained(
|
|
1816
|
-
field: str,
|
|
1817
|
-
item: Any,
|
|
1818
|
-
worker_id: str | None,
|
|
1819
|
-
selection: tuple[bool, set[str], bool, bool],
|
|
1820
|
-
) -> bool:
|
|
1821
|
-
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
1822
|
-
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
1823
|
-
# candidate evidence and pending tasks retain their own reconciliation.
|
|
1824
|
-
if (
|
|
1825
|
-
selected_coverage_superseded
|
|
1826
|
-
and field != "deferred"
|
|
1827
|
-
and not (
|
|
1828
|
-
isinstance(item, dict)
|
|
1829
|
-
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1830
|
-
)
|
|
1831
|
-
):
|
|
1832
|
-
return False
|
|
1833
|
-
# A selected outcome must retain its evidence even if its result write failed.
|
|
1834
|
-
if superseded and not (
|
|
1835
|
-
isinstance(item, dict)
|
|
1836
|
-
and (
|
|
1837
|
-
(field == "deferred" and retain_pending)
|
|
1838
|
-
or (
|
|
1839
|
-
isinstance(item.get("candidateId"), str)
|
|
1840
|
-
and item["candidateId"] in selected_candidates
|
|
1841
|
-
and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
|
|
1842
|
-
)
|
|
1843
|
-
)
|
|
1844
|
-
):
|
|
1845
|
-
return False
|
|
1846
|
-
if (
|
|
1847
|
-
isinstance(item, dict)
|
|
1848
|
-
and isinstance(
|
|
1849
|
-
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1850
|
-
if field == "deferred"
|
|
1851
|
-
else item.get("candidateId"),
|
|
1852
|
-
str,
|
|
1853
|
-
)
|
|
1854
|
-
and (worker_id, identity) in resolved
|
|
1855
|
-
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1856
|
-
):
|
|
1857
|
-
return False
|
|
1858
|
-
return True
|
|
1859
|
-
|
|
1860
|
-
ambiguous_surface_ids = {
|
|
1861
|
-
(owner, identity)
|
|
1862
|
-
for owner, row in reopened_rows
|
|
1863
|
-
if isinstance(row.get("id"), str)
|
|
1864
|
-
and (owner, row["id"]) in ambiguous_deferred
|
|
1865
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1866
|
-
for identity in [
|
|
1867
|
-
row["id"],
|
|
1868
|
-
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1869
|
-
]
|
|
1870
|
-
if isinstance(identity, str)
|
|
1871
|
-
}
|
|
1872
|
-
for _, draft, owner in all_sources:
|
|
1873
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
1874
|
-
by_id: dict[str, dict[str, Any]] = {}
|
|
1875
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1876
|
-
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1877
|
-
continue
|
|
1878
|
-
if identity in by_id and surface != by_id[identity]:
|
|
1879
|
-
ambiguous_surface_ids.add((owner, identity))
|
|
1880
|
-
by_id[identity] = surface
|
|
1881
|
-
|
|
1882
|
-
replaced_deferred = {
|
|
1883
|
-
id(row)
|
|
1884
|
-
for relative, _, owner in all_sources
|
|
1885
|
-
for row in deferred_rows[relative]
|
|
1886
|
-
if isinstance(row, dict)
|
|
1887
|
-
and isinstance(identity := row.get("id"), str)
|
|
1888
|
-
and (key := (owner, identity)) not in candidate_ids
|
|
1889
|
-
and key not in ambiguous_deferred
|
|
1890
|
-
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1891
|
-
and updated > source_order[relative]
|
|
1892
|
-
}
|
|
1893
|
-
fallback_surfaces = []
|
|
1894
|
-
if surface_links and isinstance(coverage.get("surfaces"), list):
|
|
1895
|
-
# Plan the surviving associations before historical receipts can claim
|
|
1896
|
-
# a spelling needed by a surface that must be restored for its task.
|
|
1897
|
-
prospective_tasks = []
|
|
1898
|
-
prospective_surfaces = {}
|
|
1899
|
-
prospective_named = {}
|
|
1900
|
-
for relative, draft, owner in all_sources:
|
|
1901
|
-
selection = source_coverage_selection(relative, owner)
|
|
1902
|
-
for row in deferred_rows[relative]:
|
|
1903
|
-
if (
|
|
1904
|
-
coverage_row_is_retained("deferred", row, owner, selection)
|
|
1905
|
-
and id(row) not in replaced_deferred
|
|
1906
|
-
and not (
|
|
1907
|
-
isinstance(row, dict)
|
|
1908
|
-
and isinstance(row.get("id"), str)
|
|
1909
|
-
and (owner, row["id"]) in closed_deferred
|
|
1910
|
-
)
|
|
1911
|
-
):
|
|
1912
|
-
prospective_tasks.append(
|
|
1913
|
-
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1914
|
-
)
|
|
1915
|
-
rows = draft["coverage"].get("surfaces", [])
|
|
1916
|
-
for row in rows if isinstance(rows, list) else []:
|
|
1917
|
-
if owner is None or not isinstance(row, dict):
|
|
1918
|
-
continue
|
|
1919
|
-
emitted = coverage_row_is_retained("surfaces", row, owner, selection)
|
|
1920
|
-
ambiguous = (
|
|
1921
|
-
isinstance(row.get("id"), str)
|
|
1922
|
-
and (owner, row["id"]) in ambiguous_surface_ids
|
|
1923
|
-
and row.get("disposition") == "needs_follow_up"
|
|
1924
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1925
|
-
)
|
|
1926
|
-
if not emitted and not ambiguous:
|
|
1927
|
-
continue
|
|
1928
|
-
projected = _project_worker_coverage_row(row, owner, "surfaces")
|
|
1929
|
-
if isinstance(row.get("id"), str):
|
|
1930
|
-
named = prospective_named.setdefault((owner, row["id"]), {})
|
|
1931
|
-
projected = next(
|
|
1932
|
-
(existing for existing in named.values() if existing == projected),
|
|
1933
|
-
projected,
|
|
1934
|
-
)
|
|
1935
|
-
named[id(projected)] = projected
|
|
1936
|
-
if emitted:
|
|
1937
|
-
prospective_surfaces[id(row)] = projected
|
|
1938
|
-
prospective_tasks.extend(
|
|
1939
|
-
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1940
|
-
for owner, row in reopened_rows
|
|
1941
|
-
if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
|
|
1942
|
-
)
|
|
1943
|
-
bound = {
|
|
1944
|
-
(id(retained), published_id)
|
|
1945
|
-
for task, published_id, surface, _, task_source in surface_links
|
|
1946
|
-
if linked_surface(surface, prospective_surfaces, prospective_named) is not None
|
|
1947
|
-
for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
|
|
1948
|
-
}
|
|
1949
|
-
for task, published_id, _, parent_surface, task_source in surface_links:
|
|
1950
|
-
for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
|
|
1951
|
-
key = (id(retained), published_id)
|
|
1952
|
-
if key not in bound:
|
|
1953
|
-
fallback_surfaces.append(parent_surface)
|
|
1954
|
-
bound.add(key)
|
|
441
|
+
return _saved_results_changed(db, connection, scan)
|
|
1955
442
|
|
|
1956
|
-
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1957
|
-
all_sources,
|
|
1958
|
-
source_order,
|
|
1959
|
-
closed_deferred,
|
|
1960
|
-
active_deferred,
|
|
1961
|
-
resolved,
|
|
1962
|
-
reopened_generic,
|
|
1963
|
-
coverage_schema["surfaces"]["items"],
|
|
1964
|
-
deferred_rows,
|
|
1965
|
-
ambiguous_deferred,
|
|
1966
|
-
receipt_root=scan_dir if project_worker_coverage else None,
|
|
1967
|
-
receipt_warnings=warnings,
|
|
1968
|
-
fallback_surfaces=fallback_surfaces,
|
|
1969
|
-
surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
|
|
1970
|
-
"surfaces", row, owner, source_coverage_selection(relative, owner)
|
|
1971
|
-
),
|
|
1972
|
-
eligible_receipt_sources={
|
|
1973
|
-
relative
|
|
1974
|
-
for relative, draft, _ in all_sources
|
|
1975
|
-
if draft.get("complete") is not False or relative in selected_observations
|
|
1976
|
-
},
|
|
1977
|
-
)
|
|
1978
|
-
replaced_rows = {
|
|
1979
|
-
"surfaces": replaced_surfaces,
|
|
1980
|
-
"deferred": replaced_deferred,
|
|
1981
|
-
}
|
|
1982
|
-
for field, replaced in replaced_rows.items():
|
|
1983
|
-
if parent and isinstance(coverage.get(field), list):
|
|
1984
|
-
previous = parent["coverage"].get(field, [])
|
|
1985
|
-
if isinstance(previous, list):
|
|
1986
|
-
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1987
|
-
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1988
|
-
if isinstance(coverage.get("surfaces"), list):
|
|
1989
|
-
for owner, surface, replaced_sources in surface_updates:
|
|
1990
|
-
retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
|
|
1991
|
-
for source_id in replaced_sources & linked_surface_sources:
|
|
1992
|
-
linked_surface_outputs[source_id] = retained
|
|
1993
|
-
for surface in coverage["surfaces"]:
|
|
1994
|
-
if isinstance(surface, dict):
|
|
1995
|
-
surface.setdefault("receiptRefs", [])
|
|
1996
443
|
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
)
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
2010
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2011
|
-
if (
|
|
2012
|
-
isinstance(surface, dict)
|
|
2013
|
-
and isinstance(surface.get("id"), str)
|
|
2014
|
-
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
2015
|
-
and surface.get("disposition") == "needs_follow_up"
|
|
2016
|
-
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
2017
|
-
and isinstance(coverage.get("surfaces"), list)
|
|
2018
|
-
):
|
|
2019
|
-
append_coverage_row(
|
|
2020
|
-
coverage["surfaces"],
|
|
2021
|
-
{**surface, "receiptRefs": surface.get("receiptRefs", [])},
|
|
2022
|
-
owner,
|
|
2023
|
-
"surfaces",
|
|
2024
|
-
)
|
|
2025
|
-
for relative, draft, worker_id in all_sources:
|
|
2026
|
-
selection = source_coverage_selection(relative, worker_id)
|
|
2027
|
-
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
2028
|
-
skip_superseded_findings = (
|
|
2029
|
-
superseded
|
|
2030
|
-
and not stopped
|
|
2031
|
-
and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
|
|
444
|
+
@contextmanager
|
|
445
|
+
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
446
|
+
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
447
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
448
|
+
previous = None
|
|
449
|
+
try:
|
|
450
|
+
head_path.lstat()
|
|
451
|
+
except FileNotFoundError:
|
|
452
|
+
pass
|
|
453
|
+
else:
|
|
454
|
+
descriptor = open_scan_local_file_descriptor(
|
|
455
|
+
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
2032
456
|
)
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
2054
|
-
]
|
|
2055
|
-
if selected_models:
|
|
2056
|
-
head_path = max(selected_models, key=source_order.__getitem__)
|
|
2057
|
-
current = drafts_by_path[head_path]
|
|
2058
|
-
# A terminal checkpoint is committed before result.json is replaced.
|
|
2059
|
-
# Use the admitted observation, including its frozen ordering on retries.
|
|
2060
|
-
if not prefer_worker_head and current.get("complete") is not False:
|
|
2061
|
-
prefer_worker_head = source_order[head_path] >= source_order[relative]
|
|
2062
|
-
if prefer_worker_head:
|
|
2063
|
-
model = current["threatModel"]
|
|
2064
|
-
model_path = head_path
|
|
2065
|
-
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
2066
|
-
if worker_id is not None:
|
|
2067
|
-
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
2068
|
-
if selected_model_source is not None and worker_id is not None:
|
|
2069
|
-
selected_model_source[:] = [model_path]
|
|
2070
|
-
for value in draft["findings"]:
|
|
2071
|
-
if skip_superseded_findings and not (
|
|
2072
|
-
isinstance(value, dict)
|
|
2073
|
-
and (candidate_id := finding_candidate_id(value)) in selected_candidates
|
|
2074
|
-
and resolved.get((worker_id, candidate_id)) == "reported"
|
|
2075
|
-
):
|
|
2076
|
-
continue
|
|
2077
|
-
if relative == "parent" and parent_is_canonical:
|
|
2078
|
-
finding = copy.deepcopy(value)
|
|
2079
|
-
_ensure_finding_identity(finding, candidate_only=True)
|
|
2080
|
-
provenance = finding.get("provenance") if isinstance(finding, dict) else None
|
|
2081
|
-
owner = provenance.get("workerId") if isinstance(provenance, dict) else None
|
|
2082
|
-
candidate_id = finding_candidate_id(finding) if isinstance(finding, dict) else None
|
|
2083
|
-
if (
|
|
2084
|
-
stopped_parent_seal
|
|
2085
|
-
and isinstance(owner, str)
|
|
2086
|
-
and candidate_id
|
|
2087
|
-
and resolved.get((owner, candidate_id)) in {"rejected", "not_applicable"}
|
|
2088
|
-
):
|
|
2089
|
-
rejected_history.setdefault((owner, candidate_id), []).append(finding)
|
|
2090
|
-
continue
|
|
2091
|
-
if valid_finding(finding):
|
|
2092
|
-
finding_positions.setdefault(_finding_key(finding), len(findings))
|
|
2093
|
-
findings.append(finding)
|
|
2094
|
-
continue
|
|
2095
|
-
if relative != "parent" and parent and value in parent["findings"]:
|
|
2096
|
-
continue
|
|
2097
|
-
if not isinstance(value, dict):
|
|
2098
|
-
warnings.append(f"Retained malformed finding evidence in {relative}.")
|
|
2099
|
-
continue
|
|
2100
|
-
finding = copy.deepcopy(value)
|
|
2101
|
-
candidate_id = finding_candidate_id(finding)
|
|
2102
|
-
if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
|
|
2103
|
-
"rejected",
|
|
2104
|
-
"not_applicable",
|
|
2105
|
-
}:
|
|
2106
|
-
surfaces = coverage.get("surfaces")
|
|
2107
|
-
for item in surfaces if isinstance(surfaces, list) else []:
|
|
2108
|
-
if isinstance(item, dict) and item.get("candidateId") == candidate_id:
|
|
2109
|
-
if not isinstance(item.get("previousFindings"), list):
|
|
2110
|
-
item["previousFindings"] = []
|
|
2111
|
-
history = item["previousFindings"]
|
|
2112
|
-
if not any(
|
|
2113
|
-
isinstance(previous, dict)
|
|
2114
|
-
and _finding_key(previous) == _finding_key(finding)
|
|
2115
|
-
and _finding_content(previous) == _finding_content(finding)
|
|
2116
|
-
for previous in history
|
|
2117
|
-
):
|
|
2118
|
-
history.append(finding)
|
|
2119
|
-
continue
|
|
2120
|
-
for key in ("findingId", "occurrenceId", "fingerprints"):
|
|
2121
|
-
finding.pop(key, None)
|
|
2122
|
-
locations = finding.get("locations", [])
|
|
2123
|
-
if not isinstance(locations, list) or not any(
|
|
2124
|
-
isinstance(location, dict)
|
|
2125
|
-
and isinstance(location.get("path"), str)
|
|
2126
|
-
and any(
|
|
2127
|
-
path_within_scope(location["path"], path)
|
|
2128
|
-
for path in binding["scope"]["includePaths"]
|
|
2129
|
-
)
|
|
2130
|
-
for location in locations
|
|
2131
|
-
):
|
|
2132
|
-
warnings.append(f"Skipped out-of-scope finding from {relative}.")
|
|
2133
|
-
continue
|
|
2134
|
-
provenance = finding.setdefault("provenance", {"source": "local_plugin"})
|
|
2135
|
-
if not isinstance(provenance, dict):
|
|
2136
|
-
findings.append(finding)
|
|
2137
|
-
continue
|
|
2138
|
-
if worker_id:
|
|
2139
|
-
provenance.setdefault("workerId", worker_id)
|
|
2140
|
-
_ensure_finding_identity(finding)
|
|
2141
|
-
if not valid_finding(finding):
|
|
2142
|
-
findings.append(finding)
|
|
2143
|
-
continue
|
|
2144
|
-
key = _finding_key(finding)
|
|
2145
|
-
represented_by_parent = False
|
|
2146
|
-
if relative != "parent":
|
|
2147
|
-
if key in represented:
|
|
2148
|
-
mapped_key = represented[key]
|
|
2149
|
-
historical_contents = represented_history.get(key, set())
|
|
2150
|
-
elif worker_id and candidate_id:
|
|
2151
|
-
candidate_key = _worker_candidate_key(worker_id, candidate_id, finding)
|
|
2152
|
-
if candidate_key not in represented_candidates:
|
|
2153
|
-
represented_candidates[candidate_key] = key
|
|
2154
|
-
mapped_key = represented_candidates[candidate_key]
|
|
2155
|
-
historical_contents = represented_candidate_history.get(candidate_key, set())
|
|
2156
|
-
else:
|
|
2157
|
-
mapped_key = None
|
|
2158
|
-
historical_contents = set()
|
|
2159
|
-
if mapped_key is not None:
|
|
2160
|
-
key = mapped_key
|
|
2161
|
-
represented_by_parent = _digest(_finding_content(value)) in historical_contents
|
|
2162
|
-
if key in finding_positions:
|
|
2163
|
-
retained = findings[finding_positions[key]]
|
|
2164
|
-
if finding != retained:
|
|
2165
|
-
if not represented_by_parent and _finding_strength(finding) > _finding_strength(
|
|
2166
|
-
retained
|
|
2167
|
-
):
|
|
2168
|
-
previous = copy.deepcopy(retained)
|
|
2169
|
-
previous_history = previous["provenance"].pop("previousFindings", [])
|
|
2170
|
-
retained = finding
|
|
2171
|
-
findings[finding_positions[key]] = retained
|
|
2172
|
-
else:
|
|
2173
|
-
previous = copy.deepcopy(value)
|
|
2174
|
-
previous_history = previous.get("provenance", {}).pop(
|
|
2175
|
-
"previousFindings", []
|
|
2176
|
-
)
|
|
2177
|
-
retained_provenance = retained["provenance"]
|
|
2178
|
-
retained_history = retained_provenance.get("previousFindings")
|
|
2179
|
-
history = (
|
|
2180
|
-
[item for item in retained_history if isinstance(item, dict)]
|
|
2181
|
-
if isinstance(retained_history, list)
|
|
2182
|
-
else []
|
|
2183
|
-
)
|
|
2184
|
-
retained_provenance["previousFindings"] = history
|
|
2185
|
-
for original in [
|
|
2186
|
-
*(previous_history if isinstance(previous_history, list) else []),
|
|
2187
|
-
previous,
|
|
2188
|
-
]:
|
|
2189
|
-
if not isinstance(original, dict):
|
|
2190
|
-
continue
|
|
2191
|
-
source_key = _finding_key(original)
|
|
2192
|
-
source_content = _finding_content(original)
|
|
2193
|
-
already_retained = any(
|
|
2194
|
-
source_key == _finding_key(historical)
|
|
2195
|
-
and source_content == _finding_content(historical)
|
|
2196
|
-
for historical in _retained_findings(retained)
|
|
2197
|
-
)
|
|
2198
|
-
if (
|
|
2199
|
-
not already_retained
|
|
2200
|
-
and original not in history
|
|
2201
|
-
and original != retained
|
|
2202
|
-
):
|
|
2203
|
-
history.append(original)
|
|
2204
|
-
continue
|
|
2205
|
-
finding_positions[key] = len(findings)
|
|
2206
|
-
findings.append(finding)
|
|
2207
|
-
if superseded and not selected_candidates and not retain_pending:
|
|
2208
|
-
continue
|
|
2209
|
-
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
2210
|
-
if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
|
|
2211
|
-
continue
|
|
2212
|
-
items = draft["coverage"].get(field, [])
|
|
2213
|
-
if field == "deferred":
|
|
2214
|
-
items = deferred_rows[relative]
|
|
2215
|
-
if not isinstance(items, list):
|
|
2216
|
-
continue
|
|
2217
|
-
output = coverage.setdefault(field, [])
|
|
2218
|
-
if not isinstance(output, list):
|
|
2219
|
-
# Keep malformed canonical collections for the existing finalizer's
|
|
2220
|
-
# recovery and warnings rather than silently changing its contract.
|
|
2221
|
-
continue
|
|
2222
|
-
for item in items:
|
|
2223
|
-
source_item = item
|
|
2224
|
-
if not coverage_row_is_retained(field, item, worker_id, selection):
|
|
2225
|
-
continue
|
|
2226
|
-
if id(item) in replaced_rows.get(field, ()):
|
|
2227
|
-
continue
|
|
2228
|
-
if (
|
|
2229
|
-
field == "deferred"
|
|
2230
|
-
and isinstance(item, dict)
|
|
2231
|
-
and isinstance(item.get("id"), str)
|
|
2232
|
-
and (worker_id, item["id"]) in closed_deferred
|
|
2233
|
-
):
|
|
2234
|
-
continue
|
|
2235
|
-
if field == "openQuestions" and isinstance(item, str):
|
|
2236
|
-
item = {"question": item.strip()}
|
|
2237
|
-
if (
|
|
2238
|
-
field == "surfaces"
|
|
2239
|
-
and isinstance(item, dict)
|
|
2240
|
-
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
2241
|
-
and isinstance(item.get("candidateId"), str)
|
|
2242
|
-
and (history_findings := rejected_history.get((worker_id, item["candidateId"])))
|
|
2243
|
-
):
|
|
2244
|
-
item = copy.deepcopy(item)
|
|
2245
|
-
if not isinstance(item.get("previousFindings"), list):
|
|
2246
|
-
item["previousFindings"] = []
|
|
2247
|
-
history = item["previousFindings"]
|
|
2248
|
-
for finding in history_findings:
|
|
2249
|
-
if not any(
|
|
2250
|
-
isinstance(previous, dict)
|
|
2251
|
-
and _finding_key(previous) == _finding_key(finding)
|
|
2252
|
-
and _finding_content(previous) == _finding_content(finding)
|
|
2253
|
-
for previous in history
|
|
2254
|
-
):
|
|
2255
|
-
history.append(copy.deepcopy(finding))
|
|
2256
|
-
if field == "surfaces" and isinstance(item, dict):
|
|
2257
|
-
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
2258
|
-
if isinstance(item, dict) and "id" not in item:
|
|
2259
|
-
semantic_item = dict(item)
|
|
2260
|
-
existing = next(
|
|
2261
|
-
(
|
|
2262
|
-
row
|
|
2263
|
-
for row in output
|
|
2264
|
-
if isinstance(row, dict)
|
|
2265
|
-
and {key: value for key, value in row.items() if key != "id"}
|
|
2266
|
-
== semantic_item
|
|
2267
|
-
),
|
|
2268
|
-
None,
|
|
2269
|
-
)
|
|
2270
|
-
if existing is not None:
|
|
2271
|
-
if project_worker_coverage and worker_id is not None:
|
|
2272
|
-
worker_row_owners[id(existing)] = worker_id
|
|
2273
|
-
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2274
|
-
linked_surface_outputs[id(source_item)] = existing
|
|
2275
|
-
continue
|
|
2276
|
-
retained = append_coverage_row(output, item, worker_id, field)
|
|
2277
|
-
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2278
|
-
linked_surface_outputs[id(source_item)] = retained
|
|
2279
|
-
|
|
2280
|
-
identities: dict[str, str] = {}
|
|
2281
|
-
for finding in findings:
|
|
2282
|
-
if not valid_finding(finding):
|
|
2283
|
-
continue
|
|
2284
|
-
identity = finding.get("identity")
|
|
2285
|
-
if not isinstance(identity, dict):
|
|
2286
|
-
continue
|
|
2287
|
-
key = _encoded([finding.get("ruleId"), identity]).decode()
|
|
2288
|
-
variant = _finding_key(finding)
|
|
2289
|
-
if key in identities and identities[key] != variant:
|
|
2290
|
-
finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)
|
|
2291
|
-
identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"
|
|
2292
|
-
identities[key] = variant
|
|
2293
|
-
# Bind links to retained output objects before their IDs become unique.
|
|
2294
|
-
# Candidate metadata can give an anonymous row the same preliminary ID as
|
|
2295
|
-
# another surface, so only the final surface ID can be used for its links.
|
|
2296
|
-
retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
|
|
2297
|
-
output_tasks = coverage.get("deferred", [])
|
|
2298
|
-
task_candidates = [
|
|
2299
|
-
(worker_row_owners.get(id(row)), row)
|
|
2300
|
-
for row in (output_tasks if isinstance(output_tasks, list) else [])
|
|
2301
|
-
]
|
|
2302
|
-
for task, published_id, surface, _, task_source in surface_links:
|
|
2303
|
-
retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
|
|
2304
|
-
if retained_surface is None or not isinstance(retained_surface.get("id"), str):
|
|
2305
|
-
continue
|
|
2306
|
-
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2307
|
-
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2308
|
-
links.setdefault(published_id, retained_surface)
|
|
2309
|
-
# A newer worker checkpoint may omit a surface still needed by a retained
|
|
2310
|
-
# task. Restore its parent copy only when no worker replacement survived.
|
|
2311
|
-
for task, published_id, surface, parent_surface, task_source in surface_links:
|
|
2312
|
-
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2313
|
-
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2314
|
-
if published_id not in links and isinstance(coverage.get("surfaces"), list):
|
|
2315
|
-
retained_surface = append_coverage_row(
|
|
2316
|
-
coverage["surfaces"], parent_surface, None, "surfaces"
|
|
2317
|
-
)
|
|
2318
|
-
record_coverage_owner(
|
|
2319
|
-
retained_surface, surface, surface.get("workerId"), "surfaces"
|
|
2320
|
-
)
|
|
2321
|
-
links[published_id] = retained_surface
|
|
2322
|
-
if project_worker_coverage:
|
|
2323
|
-
worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
|
|
2324
|
-
named_surface_outputs = {
|
|
2325
|
-
identity for outputs in worker_surface_outputs.values() for identity in outputs
|
|
2326
|
-
}
|
|
2327
|
-
surfaces = coverage.get("surfaces", [])
|
|
2328
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2329
|
-
if not isinstance(surface, dict):
|
|
2330
|
-
continue
|
|
2331
|
-
owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
|
|
2332
|
-
if isinstance(owner, str) and isinstance(identity, str):
|
|
2333
|
-
key = (owner, identity)
|
|
2334
|
-
existing = worker_surfaces.get(key)
|
|
2335
|
-
# An anonymous candidate ID can collide with an explicit surface ID.
|
|
2336
|
-
# Bind a named worker link to the explicitly named source.
|
|
2337
|
-
if existing is None or (
|
|
2338
|
-
id(surface) in named_surface_outputs
|
|
2339
|
-
and id(existing) not in named_surface_outputs
|
|
2340
|
-
):
|
|
2341
|
-
worker_surfaces[key] = surface
|
|
2342
|
-
for task in output_tasks if isinstance(output_tasks, list) else []:
|
|
2343
|
-
if (
|
|
2344
|
-
not isinstance(task, dict)
|
|
2345
|
-
or (owner := worker_row_owners.get(id(task))) is None
|
|
2346
|
-
or not isinstance(task.get("surfaceIds"), list)
|
|
2347
|
-
):
|
|
2348
|
-
continue
|
|
2349
|
-
for identity in task["surfaceIds"]:
|
|
2350
|
-
if (
|
|
2351
|
-
isinstance(identity, str)
|
|
2352
|
-
and (surface := worker_surfaces.get((owner, identity))) is not None
|
|
2353
|
-
):
|
|
2354
|
-
_, links = retained_surface_links.setdefault(id(task), (task, {}))
|
|
2355
|
-
links.setdefault(identity, surface)
|
|
2356
|
-
for field in ("surfaces", "explicitExclusions", "deferred"):
|
|
2357
|
-
used: set[str] = set()
|
|
2358
|
-
items = coverage.setdefault(field, [])
|
|
2359
|
-
reserved = {
|
|
2360
|
-
row["id"]
|
|
2361
|
-
for row in (items if isinstance(items, list) else [])
|
|
2362
|
-
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
457
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
458
|
+
metadata = os.fstat(handle.fileno())
|
|
459
|
+
previous = (handle.read(), metadata)
|
|
460
|
+
directories = ("checkpoints", "checkpoint-heads")
|
|
461
|
+
previous_files = {
|
|
462
|
+
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
463
|
+
}
|
|
464
|
+
try:
|
|
465
|
+
yield
|
|
466
|
+
except BaseException:
|
|
467
|
+
if previous is None:
|
|
468
|
+
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
469
|
+
else:
|
|
470
|
+
payload, metadata = previous
|
|
471
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
472
|
+
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
473
|
+
current_files = {
|
|
474
|
+
relative
|
|
475
|
+
for directory in directories
|
|
476
|
+
for relative in _checkpoint_paths(scan_dir, directory)
|
|
2363
477
|
}
|
|
2364
|
-
for
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
if id(item) in canonical_rows:
|
|
2368
|
-
if isinstance(item.get("id"), str):
|
|
2369
|
-
used.add(item["id"])
|
|
2370
|
-
continue
|
|
2371
|
-
item.setdefault("id", _saved_coverage_id(item))
|
|
2372
|
-
if not isinstance(item["id"], str):
|
|
2373
|
-
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
2374
|
-
continue
|
|
2375
|
-
if item["id"] in used:
|
|
2376
|
-
if project_worker_coverage:
|
|
2377
|
-
base = item["id"]
|
|
2378
|
-
suffix = 2
|
|
2379
|
-
while item["id"] in used or item["id"] in reserved:
|
|
2380
|
-
item["id"] = f"{base}-{suffix}"
|
|
2381
|
-
suffix += 1
|
|
2382
|
-
else:
|
|
2383
|
-
item["id"] = f"{item['id']}-{_digest(item)[:16]}"
|
|
2384
|
-
used.add(item["id"])
|
|
2385
|
-
if field == "surfaces":
|
|
2386
|
-
item.setdefault("receiptRefs", [])
|
|
2387
|
-
for task, links in retained_surface_links.values():
|
|
2388
|
-
task["surfaceIds"] = [
|
|
2389
|
-
links[identity]["id"] if isinstance(identity, str) and identity in links else identity
|
|
2390
|
-
for identity in task["surfaceIds"]
|
|
2391
|
-
]
|
|
2392
|
-
coverage.pop("completeness", None)
|
|
2393
|
-
if stopped:
|
|
2394
|
-
if not isinstance(coverage.get("deferred"), list):
|
|
2395
|
-
coverage["deferred"] = []
|
|
2396
|
-
item = {"id": "scan-stopped", "reason": reason}
|
|
2397
|
-
if item not in coverage["deferred"]:
|
|
2398
|
-
coverage["deferred"].append(item)
|
|
2399
|
-
return manifest, {"findings": findings}, coverage
|
|
478
|
+
for relative in current_files - previous_files:
|
|
479
|
+
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
480
|
+
raise
|
|
2400
481
|
|
|
2401
482
|
|
|
2402
483
|
def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
|
|
@@ -2413,107 +494,141 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
|
|
|
2413
494
|
return prepared.coverage
|
|
2414
495
|
|
|
2415
496
|
|
|
2416
|
-
def
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
descriptor = -1
|
|
2426
|
-
metadata = os.fstat(handle.fileno())
|
|
2427
|
-
snapshots[relative] = (handle.read(), metadata)
|
|
2428
|
-
except ContractError:
|
|
2429
|
-
path = scan_dir / relative
|
|
2430
|
-
if path.exists() or path.is_symlink():
|
|
2431
|
-
if relative == "threatmodel.md":
|
|
2432
|
-
# This optional projection will not replace an unsafe destination.
|
|
2433
|
-
continue
|
|
2434
|
-
raise
|
|
2435
|
-
snapshots[relative] = None
|
|
2436
|
-
finally:
|
|
2437
|
-
if descriptor >= 0:
|
|
2438
|
-
os.close(descriptor)
|
|
2439
|
-
return snapshots
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
def _restore_published_outputs(
|
|
2443
|
-
scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
|
|
2444
|
-
) -> None:
|
|
2445
|
-
for relative, snapshot in snapshots.items():
|
|
2446
|
-
if snapshot is None:
|
|
2447
|
-
path = scan_dir / relative
|
|
2448
|
-
if path.exists() or path.is_symlink():
|
|
2449
|
-
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
2450
|
-
else:
|
|
2451
|
-
contents, metadata = snapshot
|
|
2452
|
-
write_scan_local_bytes(scan_dir, relative, contents)
|
|
2453
|
-
# Saved-result selection uses mtime; rollback must preserve its ordering.
|
|
2454
|
-
os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
@contextmanager
|
|
2458
|
-
def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
|
|
2459
|
-
"""Recover a settled, zero-accepted Deep deadline without sealing the parent."""
|
|
497
|
+
def preserve_deep_deadline_draft(
|
|
498
|
+
db: Any,
|
|
499
|
+
connection: Any,
|
|
500
|
+
scan: Any,
|
|
501
|
+
*,
|
|
502
|
+
preserve_reduction: bool = False,
|
|
503
|
+
allow_empty: bool = False,
|
|
504
|
+
) -> tuple[Path, dict[str, tuple[bytes, os.stat_result] | None], set[str]] | None:
|
|
505
|
+
"""Recover unfinished reviews at a settled Deep deadline without sealing the parent."""
|
|
2460
506
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2461
|
-
snapshots = _snapshot_published_outputs(scan_dir)
|
|
2462
507
|
warnings = json.loads(scan["completion_warnings_json"])
|
|
2463
508
|
binding = db.workbench_completion_binding(scan, db.now())
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
509
|
+
workers = connection.execute(
|
|
510
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
|
|
511
|
+
(scan["id"],),
|
|
512
|
+
).fetchall()
|
|
513
|
+
if preserve_reduction:
|
|
514
|
+
# Completed reviews already went through reduction. Only add evidence
|
|
515
|
+
# from unfinished reviews; raw accepted drafts must not undo that result.
|
|
516
|
+
workers = [
|
|
517
|
+
worker
|
|
518
|
+
for worker in workers
|
|
519
|
+
if worker["kind"] == "discovery" and worker["status"] == "canceled"
|
|
520
|
+
]
|
|
521
|
+
if not workers:
|
|
522
|
+
return None
|
|
523
|
+
_prepare_scan_finalization(
|
|
524
|
+
scan_dir,
|
|
525
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
526
|
+
completion_binding=binding,
|
|
527
|
+
allow_missing_writeups=True,
|
|
528
|
+
)
|
|
529
|
+
snapshots = _snapshot_published_outputs(
|
|
530
|
+
scan_dir, ("findings.json", "coverage.json", "scan-manifest.json", "checkpoint-head.json")
|
|
531
|
+
)
|
|
532
|
+
created_paths: set[str] = set()
|
|
533
|
+
try:
|
|
534
|
+
documents = merge_saved_results(
|
|
535
|
+
scan_dir,
|
|
536
|
+
scan["id"],
|
|
537
|
+
binding,
|
|
538
|
+
workers,
|
|
539
|
+
warnings,
|
|
540
|
+
stopped=not preserve_reduction,
|
|
541
|
+
preserve_canonical_parent=preserve_reduction,
|
|
542
|
+
project_worker_coverage=True,
|
|
543
|
+
# Without reduction, the coordinator's empty draft cannot replace worker evidence.
|
|
544
|
+
scan_mode="deep" if preserve_reduction else None,
|
|
545
|
+
finding_scope=db.requested_finding_scope(scan),
|
|
546
|
+
created_paths=created_paths,
|
|
547
|
+
current_contract=True,
|
|
548
|
+
reason=(
|
|
549
|
+
"The configured discovery time limit elapsed before any source review completed. "
|
|
550
|
+
"Saved findings and pending review were preserved without a completed reduction."
|
|
551
|
+
),
|
|
552
|
+
)
|
|
553
|
+
if documents is None:
|
|
554
|
+
if not allow_empty:
|
|
2483
555
|
raise ContractError("Deep Scan deadline has no saved parent draft.")
|
|
2484
|
-
|
|
2485
|
-
# Validate recovered evidence against the current finding contract,
|
|
2486
|
-
# scope, source lineage, and receipts before admitting this draft.
|
|
2487
|
-
prepared = _prepare_scan_finalization(
|
|
2488
|
-
scan_dir,
|
|
2489
|
-
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
2490
|
-
completion_binding=binding,
|
|
2491
|
-
recover_unsealed=True,
|
|
2492
|
-
draft_documents=documents,
|
|
2493
|
-
)
|
|
2494
|
-
warnings.extend(prepared.warnings)
|
|
2495
|
-
manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
|
|
2496
|
-
manifest["scan"].pop("sealedAt", None)
|
|
2497
|
-
manifest["scan"].pop("artifacts", None)
|
|
2498
|
-
checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
|
|
2499
|
-
contents = _encoded(checkpoint)
|
|
2500
|
-
name = f"{hashlib.sha256(contents).hexdigest()}.json"
|
|
2501
|
-
write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
|
|
2502
|
-
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
|
|
2503
|
-
for filename, document in (
|
|
2504
|
-
("findings.json", findings),
|
|
2505
|
-
("coverage.json", coverage),
|
|
2506
|
-
("scan-manifest.json", manifest),
|
|
2507
|
-
):
|
|
2508
|
-
write_scan_local_bytes(scan_dir, filename, _encoded(document))
|
|
556
|
+
_restore_published_outputs(scan_dir, snapshots, created_paths)
|
|
2509
557
|
connection.execute(
|
|
2510
558
|
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
2511
559
|
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
|
|
2512
560
|
)
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
561
|
+
return None
|
|
562
|
+
if preserve_reduction:
|
|
563
|
+
marker = {
|
|
564
|
+
"id": "scan-deadline",
|
|
565
|
+
"reason": (
|
|
566
|
+
"The configured discovery time limit elapsed. Saved findings and "
|
|
567
|
+
"pending work from unfinished source reviews were preserved."
|
|
568
|
+
),
|
|
569
|
+
}
|
|
570
|
+
if not any(
|
|
571
|
+
marker["id"] in (row.get("id"), row.get("candidateId"))
|
|
572
|
+
for key in ("deferred", "resolvedDeferred")
|
|
573
|
+
for row in documents[2].get(key, [])
|
|
574
|
+
):
|
|
575
|
+
documents[2]["deferred"].append(marker)
|
|
576
|
+
documents[0]["scan"].pop("complete", None)
|
|
577
|
+
expected_coverage_mode = db.expected_coverage_mode(scan)
|
|
578
|
+
if not preserve_reduction:
|
|
579
|
+
# A capped deadline can retain interrupted records even though its
|
|
580
|
+
# completion binding requires strict validation of completed findings.
|
|
581
|
+
_populate_unsealed_manifest_envelope(documents[0], documents[0]["scan"], binding)
|
|
582
|
+
_populate_unsealed_artifact_envelope(*documents, binding)
|
|
583
|
+
if expected_coverage_mode == "deep_repository":
|
|
584
|
+
documents[2]["inventoryStrategy"] = "repository"
|
|
585
|
+
elif expected_coverage_mode == "scoped_path":
|
|
586
|
+
documents[2]["inventoryStrategy"] = "scoped_path"
|
|
587
|
+
schema_dir = Path(__file__).resolve().parent.parent / "schemas"
|
|
588
|
+
discarded_findings = _recover_unsealed_findings(
|
|
589
|
+
documents[0], documents[1], schema_dir, scan_dir, warnings, current_contract=True
|
|
590
|
+
)
|
|
591
|
+
_recover_unsealed_coverage(
|
|
592
|
+
documents[2], schema_dir, scan_dir, warnings, discarded_findings
|
|
593
|
+
)
|
|
594
|
+
# Reuse stopped-result recovery's per-record validation, scope checks,
|
|
595
|
+
# source lineage, and receipt validation before admitting this draft.
|
|
596
|
+
prepared = _prepare_scan_finalization(
|
|
597
|
+
scan_dir,
|
|
598
|
+
expected_coverage_mode=expected_coverage_mode,
|
|
599
|
+
completion_binding=binding,
|
|
600
|
+
recover_unsealed=not preserve_reduction,
|
|
601
|
+
allow_missing_writeups=preserve_reduction,
|
|
602
|
+
draft_documents=documents,
|
|
603
|
+
finding_scope=db.requested_finding_scope(scan),
|
|
604
|
+
)
|
|
605
|
+
warnings.extend(prepared.warnings)
|
|
606
|
+
manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
|
|
607
|
+
manifest["scan"].pop("sealedAt", None)
|
|
608
|
+
manifest["scan"].pop("artifacts", None)
|
|
609
|
+
checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
|
|
610
|
+
contents = _encoded(checkpoint)
|
|
611
|
+
name = f"{hashlib.sha256(contents).hexdigest()}.json"
|
|
612
|
+
checkpoint_path = f"checkpoints/{name}"
|
|
613
|
+
# This draft is recoverable parent evidence too. Restore its prior state
|
|
614
|
+
# on local publication failure or a later caller transaction rollback.
|
|
615
|
+
snapshots.update(_snapshot_published_outputs(scan_dir, (checkpoint_path,)))
|
|
616
|
+
write_scan_local_bytes(scan_dir, checkpoint_path, contents)
|
|
617
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
|
|
618
|
+
for filename, document in (
|
|
619
|
+
("findings.json", findings),
|
|
620
|
+
("coverage.json", coverage),
|
|
621
|
+
("scan-manifest.json", manifest),
|
|
622
|
+
):
|
|
623
|
+
write_scan_local_bytes(scan_dir, filename, _encoded(document))
|
|
624
|
+
connection.execute(
|
|
625
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
626
|
+
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
|
|
627
|
+
)
|
|
628
|
+
except BaseException:
|
|
629
|
+
_restore_published_outputs(scan_dir, snapshots, created_paths)
|
|
630
|
+
raise
|
|
631
|
+
return scan_dir, snapshots, created_paths
|
|
2517
632
|
|
|
2518
633
|
|
|
2519
634
|
def preserve_scan_results_locked(
|
|
@@ -2522,25 +637,44 @@ def preserve_scan_results_locked(
|
|
|
2522
637
|
scan_id: str,
|
|
2523
638
|
*,
|
|
2524
639
|
recovery_source_digests: dict[str, str] | None = None,
|
|
640
|
+
recovery_checkpoint_heads: dict[str, str | None] | None = None,
|
|
2525
641
|
include_parent_with_recovery: bool = False,
|
|
642
|
+
recovery_snapshot_error: _CheckpointSnapshotError | None = None,
|
|
2526
643
|
) -> bool:
|
|
2527
644
|
"""Publish or verify retained terminal results through the workbench host."""
|
|
2528
645
|
scan = db.require_scan(connection, scan_id)
|
|
2529
646
|
if scan["status"] != "failed":
|
|
2530
647
|
return False
|
|
2531
648
|
frozen_source_digests: dict[str, str] | None = None
|
|
649
|
+
checkpoint_heads, pending_recovery = _retained_checkpoint_state(scan)
|
|
2532
650
|
model_source: list[str] = []
|
|
2533
651
|
saved_model_source: str | None = None
|
|
2534
652
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
653
|
+
frozen_source_times = None
|
|
2535
654
|
if raw_frozen_sources is not None:
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
)
|
|
655
|
+
retained = json.loads(raw_frozen_sources)
|
|
656
|
+
frozen_source_digests, saved_model_source = _retained_source_state(retained)
|
|
657
|
+
if isinstance(retained.get("sources"), dict):
|
|
658
|
+
frozen_source_times = retained.get("sourceTimes")
|
|
2539
659
|
if saved_model_source is not None:
|
|
2540
660
|
model_source.append(saved_model_source)
|
|
661
|
+
canonical_sources = frozen_source_digests
|
|
2541
662
|
if recovery_source_digests is not None:
|
|
2542
663
|
frozen_source_digests = recovery_source_digests
|
|
664
|
+
checkpoint_heads = recovery_checkpoint_heads
|
|
2543
665
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
666
|
+
if frozen_source_digests is not None:
|
|
667
|
+
_restore_frozen_model_capture(
|
|
668
|
+
scan_dir, scan_id, frozen_source_digests, frozen_source_times, saved_model_source
|
|
669
|
+
)
|
|
670
|
+
if frozen_source_times is not None and recovery_source_digests is None:
|
|
671
|
+
_freeze_source_times(
|
|
672
|
+
scan_dir,
|
|
673
|
+
scan_id,
|
|
674
|
+
frozen_source_digests,
|
|
675
|
+
frozen_source_times,
|
|
676
|
+
retained.get("parentContext"),
|
|
677
|
+
)
|
|
2544
678
|
deep_run = connection.execute(
|
|
2545
679
|
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
2546
680
|
).fetchone()
|
|
@@ -2591,11 +725,15 @@ def preserve_scan_results_locked(
|
|
|
2591
725
|
db.index_findings(connection, scan_id, findings, scan["completed_at"])
|
|
2592
726
|
connection.execute(
|
|
2593
727
|
"UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
|
|
728
|
+
"retained_checkpoint_heads_json = ?, "
|
|
2594
729
|
"completion_warnings_json = ?, "
|
|
2595
730
|
"updated_at = ? WHERE id = ? AND status = 'failed'",
|
|
2596
731
|
(
|
|
2597
732
|
digest,
|
|
2598
733
|
_encode_retained_sources(retained_sources, model_source),
|
|
734
|
+
json.dumps(
|
|
735
|
+
manifest["scan"].get("preservedCheckpointHeads", {}), sort_keys=True
|
|
736
|
+
),
|
|
2599
737
|
json.dumps(list(dict.fromkeys(warnings))),
|
|
2600
738
|
timestamp,
|
|
2601
739
|
scan_id,
|
|
@@ -2611,10 +749,7 @@ def preserve_scan_results_locked(
|
|
|
2611
749
|
existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
|
|
2612
750
|
existing = None
|
|
2613
751
|
if scan["seal_manifest_digest"] is not None or (
|
|
2614
|
-
isinstance(existing_scan, dict)
|
|
2615
|
-
and (
|
|
2616
|
-
existing_scan.get("sealedAt") is not None or existing_scan.get("artifacts") is not None
|
|
2617
|
-
)
|
|
752
|
+
isinstance(existing_scan, dict) and manifest_is_sealed(existing_scan)
|
|
2618
753
|
):
|
|
2619
754
|
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
2620
755
|
existing, existing_findings, _ = finalize_scan(
|
|
@@ -2625,11 +760,23 @@ def preserve_scan_results_locked(
|
|
|
2625
760
|
db.verify_manifest_binding(scan, existing)
|
|
2626
761
|
if existing_scan.get("status") == outcome:
|
|
2627
762
|
existing_sources = existing_scan.get("preservedSources")
|
|
763
|
+
existing_heads = existing_scan.get("preservedCheckpointHeads")
|
|
2628
764
|
if frozen_source_digests is None:
|
|
2629
765
|
frozen_source_digests = _source_digests(
|
|
2630
766
|
existing_sources, "Stopped scan source digests could not be frozen."
|
|
2631
767
|
)
|
|
2632
|
-
|
|
768
|
+
checkpoint_heads = existing_heads
|
|
769
|
+
unchanged = existing_sources == frozen_source_digests and (
|
|
770
|
+
existing_heads == checkpoint_heads
|
|
771
|
+
or (
|
|
772
|
+
"preservedCheckpointHeads" not in existing_scan
|
|
773
|
+
and recovery_source_digests is None
|
|
774
|
+
and not pending_recovery
|
|
775
|
+
)
|
|
776
|
+
)
|
|
777
|
+
if unchanged and not any(
|
|
778
|
+
Path(path).name == "checkpoint-head.json" for path in frozen_source_digests
|
|
779
|
+
):
|
|
2633
780
|
if (
|
|
2634
781
|
raw_frozen_sources is not None
|
|
2635
782
|
and scan["seal_manifest_digest"] is not None
|
|
@@ -2639,43 +786,116 @@ def preserve_scan_results_locked(
|
|
|
2639
786
|
return True
|
|
2640
787
|
record_publication(existing, existing_findings)
|
|
2641
788
|
return True
|
|
2642
|
-
if recovery_source_digests is None:
|
|
789
|
+
if not unchanged and recovery_source_digests is None and not pending_recovery:
|
|
2643
790
|
raise ContractError("Stopped scan sources changed after terminal publication.")
|
|
2644
791
|
binding = {
|
|
2645
792
|
**db.workbench_completion_binding(scan, scan["completed_at"], existing),
|
|
2646
793
|
"status": outcome,
|
|
2647
794
|
}
|
|
2648
|
-
|
|
795
|
+
canonical_model_source = saved_model_source
|
|
796
|
+
canonical_model = existing_scan.get("threatModel") if isinstance(existing_scan, dict) else None
|
|
797
|
+
if existing is None and frozen_source_digests is not None:
|
|
798
|
+
# An unsealed live draft is a fallback only when its complete identity
|
|
799
|
+
# belongs to the frozen set. Otherwise recover from the captured sources.
|
|
800
|
+
canonical_model = None
|
|
801
|
+
try:
|
|
802
|
+
_, canonical_draft = _read_saved_parent_result(scan_dir, scan_id)
|
|
803
|
+
digest = _digest(canonical_draft)
|
|
804
|
+
path = f"checkpoints/{digest}.json"
|
|
805
|
+
if frozen_source_digests.get(path) == digest:
|
|
806
|
+
canonical_model_source = path
|
|
807
|
+
canonical_model = canonical_draft.get("threatModel")
|
|
808
|
+
except (ContractError, OSError, ValueError):
|
|
809
|
+
# This live fallback is optional.
|
|
810
|
+
pass
|
|
811
|
+
if recovery_source_digests is not None and not pending_recovery:
|
|
2649
812
|
model_source.clear()
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
f"Scan {outcome}; saved findings and pending review were preserved. "
|
|
2662
|
-
f"{scan['failure_message'] or ''}"
|
|
2663
|
-
).strip(),
|
|
2664
|
-
frozen_source_digests=frozen_source_digests,
|
|
2665
|
-
frozen_model_source=model_source[0] if model_source else None,
|
|
2666
|
-
selected_model_source=model_source,
|
|
2667
|
-
project_worker_coverage=scan["mode"] == "deep",
|
|
2668
|
-
allow_frozen_legacy_parent=(
|
|
2669
|
-
include_parent_with_recovery
|
|
2670
|
-
or (
|
|
2671
|
-
recovery_source_digests is None
|
|
2672
|
-
and frozen_source_digests == {}
|
|
2673
|
-
and isinstance(existing_scan, dict)
|
|
2674
|
-
and existing_scan.get("sealedAt") is not None
|
|
2675
|
-
and "preservedSources" not in existing_scan
|
|
2676
|
-
)
|
|
2677
|
-
),
|
|
813
|
+
# Older pending recoveries retained the parent union without recording the mode.
|
|
814
|
+
legacy_parent = (
|
|
815
|
+
include_parent_with_recovery
|
|
816
|
+
or (pending_recovery is not None and pending_recovery.get("legacyParent", True) is True)
|
|
817
|
+
or (
|
|
818
|
+
recovery_source_digests is None
|
|
819
|
+
and frozen_source_digests == {}
|
|
820
|
+
and isinstance(existing_scan, dict)
|
|
821
|
+
and existing_scan.get("sealedAt") is not None
|
|
822
|
+
and "preservedSources" not in existing_scan
|
|
823
|
+
)
|
|
2678
824
|
)
|
|
825
|
+
|
|
826
|
+
def pending_selection(heads: dict[str, str | None]) -> dict[str, Any]:
|
|
827
|
+
return {
|
|
828
|
+
"heads": heads,
|
|
829
|
+
"recoveryBaseDigest": scan["seal_manifest_digest"],
|
|
830
|
+
"legacyParent": legacy_parent,
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
try:
|
|
834
|
+
documents = merge_saved_results(
|
|
835
|
+
scan_dir,
|
|
836
|
+
scan_id,
|
|
837
|
+
binding,
|
|
838
|
+
connection.execute(
|
|
839
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
|
|
840
|
+
(scan_id,),
|
|
841
|
+
).fetchall(),
|
|
842
|
+
warnings,
|
|
843
|
+
stopped=True,
|
|
844
|
+
scan_mode=scan["mode"],
|
|
845
|
+
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
|
|
846
|
+
reason=(
|
|
847
|
+
f"Scan {outcome}; saved findings and pending review were preserved. "
|
|
848
|
+
f"{scan['failure_message'] or ''}"
|
|
849
|
+
).strip(),
|
|
850
|
+
frozen_source_digests=frozen_source_digests,
|
|
851
|
+
checkpoint_heads=checkpoint_heads,
|
|
852
|
+
frozen_model_source=model_source[0] if model_source else None,
|
|
853
|
+
selected_model_source=model_source,
|
|
854
|
+
project_worker_coverage=scan["mode"] == "deep",
|
|
855
|
+
canonical_model=canonical_model,
|
|
856
|
+
canonical_model_source=canonical_model_source,
|
|
857
|
+
canonical_sources=canonical_sources,
|
|
858
|
+
recovery_snapshot_error=recovery_snapshot_error,
|
|
859
|
+
allow_frozen_legacy_parent=legacy_parent,
|
|
860
|
+
parent_already_captured=pending_recovery is not None,
|
|
861
|
+
)
|
|
862
|
+
except _CheckpointSnapshotError as exc:
|
|
863
|
+
if frozen_source_digests is None or recovery_snapshot_error is not None:
|
|
864
|
+
# The stop has already committed. Keep the source set and times in
|
|
865
|
+
# the database even when auxiliary snapshot files cannot be written.
|
|
866
|
+
# Changed raw heads remain rejected on retry.
|
|
867
|
+
captured = exc.captured
|
|
868
|
+
retained_sources = {path: value[0] for path, value in captured.items()}
|
|
869
|
+
source_times = {path: value[1] for path, value in captured.items()}
|
|
870
|
+
retained_heads = exc.heads
|
|
871
|
+
if recovery_snapshot_error is not None:
|
|
872
|
+
_prepare_scan_finalization(
|
|
873
|
+
scan_dir,
|
|
874
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
875
|
+
completion_binding=binding,
|
|
876
|
+
recover_unsealed=True,
|
|
877
|
+
draft_documents=exc.documents,
|
|
878
|
+
finding_scope=db.requested_finding_scope(scan)
|
|
879
|
+
if scan["mode"] == "deep"
|
|
880
|
+
else None,
|
|
881
|
+
)
|
|
882
|
+
retained_heads = pending_selection(retained_heads)
|
|
883
|
+
with connection:
|
|
884
|
+
connection.execute(
|
|
885
|
+
"UPDATE scans SET retained_source_digests_json = ?, "
|
|
886
|
+
"retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
|
|
887
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
888
|
+
(
|
|
889
|
+
_encode_retained_sources(
|
|
890
|
+
retained_sources, exc.model_source, source_times, exc.parent_context
|
|
891
|
+
),
|
|
892
|
+
json.dumps(retained_heads, sort_keys=True),
|
|
893
|
+
json.dumps(list(dict.fromkeys(warnings))),
|
|
894
|
+
scan_id,
|
|
895
|
+
raw_frozen_sources,
|
|
896
|
+
),
|
|
897
|
+
)
|
|
898
|
+
raise
|
|
2679
899
|
if documents is None:
|
|
2680
900
|
unpublished_warnings = list(dict.fromkeys([*warnings, *publication_follow_up_warnings]))
|
|
2681
901
|
if unpublished_warnings != stored_warnings:
|
|
@@ -2686,23 +906,46 @@ def preserve_scan_results_locked(
|
|
|
2686
906
|
(json.dumps(unpublished_warnings), db.now(), scan_id),
|
|
2687
907
|
)
|
|
2688
908
|
return False
|
|
2689
|
-
|
|
2690
|
-
|
|
909
|
+
retained_state = None
|
|
910
|
+
if (
|
|
911
|
+
frozen_source_digests is None
|
|
912
|
+
or scan["retained_checkpoint_heads_json"] is None
|
|
913
|
+
or recovery_source_digests is not None
|
|
914
|
+
or documents[0]["scan"]["preservedSources"] != frozen_source_digests
|
|
915
|
+
or model_source != ([saved_model_source] if saved_model_source is not None else [])
|
|
2691
916
|
):
|
|
2692
917
|
retained_sources = _source_digests(
|
|
2693
918
|
documents[0].get("scan", {}).get("preservedSources"),
|
|
2694
919
|
"Stopped scan source digests could not be frozen.",
|
|
2695
920
|
)
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
)
|
|
921
|
+
retained_heads = documents[0]["scan"]["preservedCheckpointHeads"]
|
|
922
|
+
if (
|
|
923
|
+
recovery_source_digests is not None
|
|
924
|
+
or pending_recovery
|
|
925
|
+
or (existing is not None and existing_scan.get("status") == outcome)
|
|
926
|
+
):
|
|
927
|
+
retained_heads = pending_selection(retained_heads)
|
|
928
|
+
retained_state = (
|
|
929
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
930
|
+
json.dumps(retained_heads, sort_keys=True),
|
|
931
|
+
json.dumps(list(dict.fromkeys(warnings))),
|
|
932
|
+
scan_id,
|
|
933
|
+
raw_frozen_sources,
|
|
934
|
+
)
|
|
935
|
+
|
|
936
|
+
def retain_sources() -> None:
|
|
937
|
+
if retained_state is not None:
|
|
938
|
+
with connection:
|
|
939
|
+
connection.execute(
|
|
940
|
+
"UPDATE scans SET retained_source_digests_json = ?, "
|
|
941
|
+
"retained_checkpoint_heads_json = ?, completion_warnings_json = ? "
|
|
942
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
943
|
+
retained_state,
|
|
944
|
+
)
|
|
945
|
+
|
|
946
|
+
first_preservation = recovery_source_digests is None and not pending_recovery
|
|
947
|
+
if first_preservation:
|
|
948
|
+
retain_sources()
|
|
2706
949
|
prepared = _prepare_scan_finalization(
|
|
2707
950
|
scan_dir,
|
|
2708
951
|
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
@@ -2711,6 +954,8 @@ def preserve_scan_results_locked(
|
|
|
2711
954
|
draft_documents=documents,
|
|
2712
955
|
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
|
|
2713
956
|
)
|
|
957
|
+
if not first_preservation:
|
|
958
|
+
retain_sources()
|
|
2714
959
|
warnings.extend(prepared.warnings)
|
|
2715
960
|
snapshots = _snapshot_published_outputs(scan_dir)
|
|
2716
961
|
try:
|
|
@@ -2733,13 +978,24 @@ def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2733
978
|
raise SystemExit("Only a stopped scan can recover terminal results.")
|
|
2734
979
|
if scan["canceled_at"] is not None:
|
|
2735
980
|
raise SystemExit("Canceled scans cannot recover terminal results.")
|
|
2736
|
-
|
|
981
|
+
recovery_snapshot_error = None
|
|
982
|
+
try:
|
|
983
|
+
recovery_source_digests, include_parent, checkpoint_heads = _recovery_source_digests(
|
|
984
|
+
db, connection, scan
|
|
985
|
+
)
|
|
986
|
+
except _CheckpointSnapshotError as exc:
|
|
987
|
+
recovery_snapshot_error = exc
|
|
988
|
+
recovery_source_digests = {path: value[0] for path, value in exc.captured.items()}
|
|
989
|
+
include_parent = exc.include_parent
|
|
990
|
+
checkpoint_heads = exc.heads
|
|
2737
991
|
if not preserve_scan_results_locked(
|
|
2738
992
|
db,
|
|
2739
993
|
connection,
|
|
2740
994
|
scan_id,
|
|
2741
995
|
recovery_source_digests=recovery_source_digests,
|
|
996
|
+
recovery_checkpoint_heads=checkpoint_heads,
|
|
2742
997
|
include_parent_with_recovery=include_parent,
|
|
998
|
+
recovery_snapshot_error=recovery_snapshot_error,
|
|
2743
999
|
):
|
|
2744
1000
|
raise SystemExit("No saved stopped-scan results were available to recover.")
|
|
2745
1001
|
db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
|
|
@@ -2807,7 +1063,7 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2807
1063
|
manifest_path = db.artifact_path(scan_dir, "scan-manifest.json", required=False)
|
|
2808
1064
|
if manifest_path is not None:
|
|
2809
1065
|
manifest = db.read_json_object(manifest_path).get("scan", {})
|
|
2810
|
-
if
|
|
1066
|
+
if manifest_is_sealed(manifest):
|
|
2811
1067
|
raise SystemExit("The scan is sealed; its artifacts cannot be modified.")
|
|
2812
1068
|
output = args.artifact_path
|
|
2813
1069
|
key = output.lower()
|
|
@@ -2823,66 +1079,6 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2823
1079
|
return {"scanId": scan_id, "path": str(scan_dir / output)}
|
|
2824
1080
|
|
|
2825
1081
|
|
|
2826
|
-
def _require_current_deep_publication(
|
|
2827
|
-
db: Any, connection: Any, scan: Any, draft: dict[str, Any]
|
|
2828
|
-
) -> None:
|
|
2829
|
-
scan_id = scan["id"]
|
|
2830
|
-
run = connection.execute(
|
|
2831
|
-
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
2832
|
-
).fetchone()
|
|
2833
|
-
publication = draft.get("deepScanPublication")
|
|
2834
|
-
if (
|
|
2835
|
-
publication is None
|
|
2836
|
-
and draft["manifest"]["scan"].get("complete") is False
|
|
2837
|
-
and (run is None or run["status"] == "running")
|
|
2838
|
-
):
|
|
2839
|
-
return
|
|
2840
|
-
if run is None:
|
|
2841
|
-
run = db.deep_scan.require_deep_scan_run(connection, scan_id)
|
|
2842
|
-
# Finished discovery-only runs hand remaining publication back to the parent.
|
|
2843
|
-
if (
|
|
2844
|
-
publication is None
|
|
2845
|
-
and run["status"] != "running"
|
|
2846
|
-
and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
2847
|
-
):
|
|
2848
|
-
return
|
|
2849
|
-
db.deep_scan.require_current_coordinator(
|
|
2850
|
-
run,
|
|
2851
|
-
argparse.Namespace(
|
|
2852
|
-
coordinator_generation=publication.get("coordinatorGeneration") if publication else None
|
|
2853
|
-
),
|
|
2854
|
-
)
|
|
2855
|
-
# Once finish selects the canonical parent, only completion may replay it.
|
|
2856
|
-
if publication is None:
|
|
2857
|
-
if run["status"] != "running" and run["manifest_path"] == str(
|
|
2858
|
-
Path(scan["scan_dir"]) / "scan-manifest.json"
|
|
2859
|
-
):
|
|
2860
|
-
raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
|
|
2861
|
-
return
|
|
2862
|
-
|
|
2863
|
-
# Match the durable reducer sequence used by coordinator recovery.
|
|
2864
|
-
def reducer_order(worker: Any) -> tuple[int, str]:
|
|
2865
|
-
match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
|
|
2866
|
-
return (int(match[1]) if match else 0, worker["id"])
|
|
2867
|
-
|
|
2868
|
-
reducer = max(
|
|
2869
|
-
(
|
|
2870
|
-
worker
|
|
2871
|
-
for worker in connection.execute(
|
|
2872
|
-
"SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
|
|
2873
|
-
)
|
|
2874
|
-
if worker["kind"] == "dedup"
|
|
2875
|
-
and worker["status"] == "succeeded"
|
|
2876
|
-
and worker["result_manifest_path"]
|
|
2877
|
-
),
|
|
2878
|
-
key=reducer_order,
|
|
2879
|
-
default=None,
|
|
2880
|
-
)
|
|
2881
|
-
selected_result = reducer["result_manifest_path"] if reducer is not None else None
|
|
2882
|
-
if publication["resultPath"] != selected_result:
|
|
2883
|
-
raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
1082
|
def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
2887
1083
|
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
2888
1084
|
with db.scan_completion_lock(scan_id):
|
|
@@ -2895,13 +1091,6 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2895
1091
|
"The scan stopped; its saved checkpoint was retained without replacing sealed results."
|
|
2896
1092
|
)
|
|
2897
1093
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2898
|
-
if (
|
|
2899
|
-
args.expected_draft_digest is not None
|
|
2900
|
-
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
2901
|
-
):
|
|
2902
|
-
raise SystemExit(
|
|
2903
|
-
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
2904
|
-
)
|
|
2905
1094
|
try:
|
|
2906
1095
|
relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
|
|
2907
1096
|
except ValueError as exc:
|
|
@@ -2911,12 +1100,16 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2911
1100
|
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
2912
1101
|
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
2913
1102
|
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
1103
|
+
accepted_input = copy.deepcopy(draft)
|
|
1104
|
+
accepted_checkpoint = None
|
|
2914
1105
|
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
|
|
2915
1106
|
binding = db.workbench_completion_binding(scan, db.now())
|
|
2916
1107
|
# Save scan IDs without sealing the draft.
|
|
2917
1108
|
_populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
|
|
2918
1109
|
_populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
|
|
2919
1110
|
_validate_completion_binding(manifest, findings, coverage, binding)
|
|
1111
|
+
projected = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
|
|
1112
|
+
admitted_input = None
|
|
2920
1113
|
checkpoint = manifest["scan"]
|
|
2921
1114
|
checkpoint_contents = None
|
|
2922
1115
|
if args.checkpoint_path is not None:
|
|
@@ -2933,8 +1126,13 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2933
1126
|
checkpoint, checkpoint_contents = _read_scan_local_json_bytes(
|
|
2934
1127
|
scan_dir, checkpoint_relative, "Staged scan checkpoint"
|
|
2935
1128
|
)
|
|
1129
|
+
accepted_checkpoint = checkpoint
|
|
2936
1130
|
if checkpoint.get("scanId") != scan_id:
|
|
2937
1131
|
raise SystemExit("Staged scan checkpoint belongs to another scan.")
|
|
1132
|
+
if scan["mode"] == "deep":
|
|
1133
|
+
checkpoint_contents = _admit_parent_checkpoint(
|
|
1134
|
+
scan_dir, scan_id, checkpoint, projected
|
|
1135
|
+
)
|
|
2938
1136
|
checkpoint_only = (
|
|
2939
1137
|
scan["mode"] == "deep"
|
|
2940
1138
|
and draft.get("deepScanPublication") is None
|
|
@@ -2948,8 +1146,40 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2948
1146
|
)
|
|
2949
1147
|
if scan["mode"] == "deep" and not checkpoint_only:
|
|
2950
1148
|
_require_current_deep_publication(db, connection, scan, draft)
|
|
1149
|
+
acceptance_input = {**accepted_input, "checkpoint": accepted_checkpoint}
|
|
1150
|
+
acceptance_relative = Path(relative).with_suffix(".accepted.json").as_posix()
|
|
1151
|
+
acknowledge = (
|
|
1152
|
+
scan["mode"] == "deep"
|
|
1153
|
+
and draft.get("deepScanPublication") is not None
|
|
1154
|
+
and db.deep_scan.require_deep_scan_run(connection, scan_id)["workflow_version"]
|
|
1155
|
+
in {"deep-scan-mcp/v1", "deep-security-scan/v1"}
|
|
1156
|
+
)
|
|
1157
|
+
if acknowledge and (scan_dir / acceptance_relative).exists():
|
|
1158
|
+
accepted = _read_scan_local_json(scan_dir, acceptance_relative, "Accepted scan draft")
|
|
1159
|
+
if (
|
|
1160
|
+
accepted.get("status") == "draft_written"
|
|
1161
|
+
and accepted.get("input") == acceptance_input
|
|
1162
|
+
):
|
|
1163
|
+
if accepted.get("draftDigest") != _scan_draft_digest(scan_dir):
|
|
1164
|
+
raise SystemExit(
|
|
1165
|
+
"scan_draft_conflict: canonical scan results changed after publication."
|
|
1166
|
+
)
|
|
1167
|
+
return {
|
|
1168
|
+
"scanId": scan_id,
|
|
1169
|
+
"status": "draft_written",
|
|
1170
|
+
**({"warnings": accepted["warnings"]} if accepted.get("warnings") else {}),
|
|
1171
|
+
}
|
|
1172
|
+
if (
|
|
1173
|
+
args.expected_draft_digest is not None
|
|
1174
|
+
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
1175
|
+
):
|
|
1176
|
+
raise SystemExit(
|
|
1177
|
+
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
1178
|
+
)
|
|
2951
1179
|
if checkpoint_contents is not None:
|
|
2952
1180
|
checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
|
|
1181
|
+
if scan["mode"] == "deep":
|
|
1182
|
+
admitted_input = f"{checkpoint_digest}.json"
|
|
2953
1183
|
write_scan_local_bytes(
|
|
2954
1184
|
scan_dir,
|
|
2955
1185
|
f"checkpoints/{checkpoint_digest}.json",
|
|
@@ -2957,27 +1187,27 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2957
1187
|
)
|
|
2958
1188
|
if checkpoint_only:
|
|
2959
1189
|
return {"scanId": scan_id, "status": "draft_written"}
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
|
|
1190
|
+
projected_contents = _encoded(projected)
|
|
1191
|
+
checkpoint_name = f"{hashlib.sha256(projected_contents).hexdigest()}.json"
|
|
2963
1192
|
checkpoint_relative = f"checkpoints/{checkpoint_name}"
|
|
2964
1193
|
if not (scan_dir / checkpoint_relative).exists():
|
|
2965
|
-
write_scan_local_bytes(scan_dir, checkpoint_relative,
|
|
1194
|
+
write_scan_local_bytes(scan_dir, checkpoint_relative, projected_contents)
|
|
2966
1195
|
if checkpoint_only:
|
|
2967
1196
|
return {"scanId": scan_id, "status": "draft_written"}
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
(
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
filename,
|
|
2979
|
-
(json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
|
|
1197
|
+
head = {"checkpoint": checkpoint_name}
|
|
1198
|
+
if admitted_input is not None:
|
|
1199
|
+
head["inputCheckpoint"] = admitted_input
|
|
1200
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded(head))
|
|
1201
|
+
documents = {
|
|
1202
|
+
filename: (json.dumps(document, allow_nan=False, indent=2) + "\n").encode()
|
|
1203
|
+
for filename, document in (
|
|
1204
|
+
("findings.json", findings),
|
|
1205
|
+
("coverage.json", coverage),
|
|
1206
|
+
("scan-manifest.json", manifest),
|
|
2980
1207
|
)
|
|
1208
|
+
}
|
|
1209
|
+
for filename, contents in documents.items():
|
|
1210
|
+
write_scan_local_bytes(scan_dir, filename, contents)
|
|
2981
1211
|
if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
|
|
2982
1212
|
# Select the accepted final publication before releasing the write lock.
|
|
2983
1213
|
with connection:
|
|
@@ -2987,6 +1217,19 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
2987
1217
|
(str(scan_dir / "scan-manifest.json"), scan_id),
|
|
2988
1218
|
)
|
|
2989
1219
|
model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
|
|
1220
|
+
if acknowledge:
|
|
1221
|
+
# Optional recovery evidence is written only after the publication succeeds.
|
|
1222
|
+
try:
|
|
1223
|
+
acceptance = {
|
|
1224
|
+
"status": "draft_written",
|
|
1225
|
+
"input": acceptance_input,
|
|
1226
|
+
"draftDigest": _scan_draft_contents_digest(documents),
|
|
1227
|
+
**({"warnings": [model_warning]} if model_warning else {}),
|
|
1228
|
+
}
|
|
1229
|
+
write_scan_local_bytes(scan_dir, acceptance_relative, _encoded(acceptance))
|
|
1230
|
+
except (OSError, ScanLocalIOError):
|
|
1231
|
+
# A missing receipt prevents recovery but does not undo accepted results.
|
|
1232
|
+
pass
|
|
2990
1233
|
# Accepted Standard drafts are evidence of review or report assembly,
|
|
2991
1234
|
# even when the parent omitted its explicit progress call.
|
|
2992
1235
|
model_only_checkpoint = (
|
|
@@ -3025,20 +1268,31 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
3025
1268
|
|
|
3026
1269
|
|
|
3027
1270
|
def _scan_draft_digest(scan_dir: Path) -> str:
|
|
3028
|
-
|
|
1271
|
+
documents: dict[str, bytes | None] = {}
|
|
3029
1272
|
for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
3030
|
-
digest.update(filename.encode())
|
|
3031
|
-
digest.update(b"\0")
|
|
3032
1273
|
try:
|
|
3033
1274
|
(scan_dir / filename).lstat()
|
|
3034
1275
|
except FileNotFoundError:
|
|
3035
|
-
|
|
1276
|
+
documents[filename] = None
|
|
3036
1277
|
continue
|
|
3037
|
-
digest.update(b"present\0")
|
|
3038
1278
|
descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
|
|
3039
1279
|
with os.fdopen(descriptor, "rb") as handle:
|
|
3040
|
-
|
|
1280
|
+
documents[filename] = handle.read()
|
|
1281
|
+
return _scan_draft_contents_digest(documents)
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _scan_draft_contents_digest(documents: dict[str, bytes | None]) -> str:
|
|
1285
|
+
digest = hashlib.sha256()
|
|
1286
|
+
for filename in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
1287
|
+
digest.update(filename.encode())
|
|
3041
1288
|
digest.update(b"\0")
|
|
1289
|
+
contents = documents[filename]
|
|
1290
|
+
if contents is None:
|
|
1291
|
+
digest.update(b"missing\0")
|
|
1292
|
+
else:
|
|
1293
|
+
digest.update(b"present\0")
|
|
1294
|
+
digest.update(contents)
|
|
1295
|
+
digest.update(b"\0")
|
|
3042
1296
|
return digest.hexdigest()
|
|
3043
1297
|
|
|
3044
1298
|
|
|
@@ -3064,7 +1318,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
3064
1318
|
args.claim_token,
|
|
3065
1319
|
error_message="Scan failure is owned by another continuation.",
|
|
3066
1320
|
)
|
|
3067
|
-
message = db.optional_text(args.message
|
|
1321
|
+
message = db.optional_text(args.message)
|
|
3068
1322
|
updated = connection.execute(
|
|
3069
1323
|
"""
|
|
3070
1324
|
UPDATE scans
|