@openai/codex-security 0.1.30 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +386 -37
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +4 -4
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/artifact-storage.md +4 -4
- package/_bundled_plugin/references/config-preflight.md +1 -1
- package/_bundled_plugin/references/core-scan.md +14 -2
- package/_bundled_plugin/references/final-report.md +4 -2
- package/_bundled_plugin/references/scan-artifacts.md +2 -2
- package/_bundled_plugin/references/scan-contract.md +2 -2
- package/_bundled_plugin/references/threat-model.md +2 -2
- package/_bundled_plugin/schemas/coverage.schema.json +22 -0
- package/_bundled_plugin/schemas/scan-manifest.schema.json +100 -85
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +19 -4
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +116 -102
- package/_bundled_plugin/scripts/config_preflight.py +4 -8
- package/_bundled_plugin/scripts/deep_scan_workbench.py +83 -217
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
- package/_bundled_plugin/scripts/finalize_scan_contract.py +441 -166
- package/_bundled_plugin/scripts/finding_preview.py +42 -82
- package/_bundled_plugin/scripts/generate_in_scope_files.py +30 -34
- package/_bundled_plugin/scripts/generate_rank_input.py +58 -99
- package/_bundled_plugin/scripts/launch_codex_security_mcp +5 -1
- package/_bundled_plugin/scripts/rank_preview.py +86 -68
- package/_bundled_plugin/scripts/report_projection.py +37 -58
- package/_bundled_plugin/scripts/threat_model_projection.py +164 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +11 -6
- package/_bundled_plugin/scripts/workbench/storage.py +11 -0
- package/_bundled_plugin/scripts/workbench_cli.py +11 -17
- package/_bundled_plugin/scripts/workbench_constants.py +6 -14
- package/_bundled_plugin/scripts/workbench_db.py +116 -228
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -6
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -4
- package/_bundled_plugin/scripts/workbench_progress.py +12 -34
- package/_bundled_plugin/scripts/workbench_publication.py +47 -62
- package/_bundled_plugin/scripts/workbench_remediation.py +2 -20
- package/_bundled_plugin/scripts/workbench_saved_results.py +1246 -230
- package/_bundled_plugin/scripts/workbench_scan_start.py +4 -11
- package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -15
- package/_bundled_plugin/scripts/workbench_schema.py +86 -271
- package/_bundled_plugin/scripts/workbench_severity.py +3 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +5 -6
- package/_bundled_plugin/scripts/workbench_target.py +98 -10
- package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
- package/_bundled_plugin/scripts/workbench_validation.py +6 -20
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/security-scan/SKILL.md +2 -2
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
- package/dist/api.d.ts +3 -5
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +201 -207
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +70 -0
- package/dist/artifact-export.d.ts.map +1 -0
- package/dist/artifact-export.js +277 -0
- package/dist/artifact-export.js.map +1 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +9 -4
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +8 -9
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +2 -5
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity.d.ts +2 -8
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -5
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts +3 -0
- package/dist/cli-help.d.ts.map +1 -0
- package/dist/cli-help.js +281 -0
- package/dist/cli-help.js.map +1 -0
- package/dist/cli.d.ts +21 -22
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +673 -701
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +1 -8
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +1 -2
- package/dist/codex-prompt.js.map +1 -1
- package/dist/codex-sdk-environment.d.ts +2 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -0
- package/dist/codex-sdk-environment.js +30 -0
- package/dist/codex-sdk-environment.js.map +1 -0
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +2 -6
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts +1 -0
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +31 -9
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +32 -33
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +2 -1
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +63 -122
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +65 -34
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +15 -31
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.js +4 -4
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +17 -16
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +10 -11
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +2 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +3 -3
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +2 -0
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.js +40 -62
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +1 -4
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.d.ts +2 -1
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +24 -6
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/review.d.ts +7 -0
- package/dist/deduplication/review.d.ts.map +1 -1
- package/dist/deduplication/review.js.map +1 -1
- package/dist/deduplication/scan.js +3 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +3 -19
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +0 -1
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +22 -40
- package/dist/deep-progress.js.map +1 -1
- package/dist/errors.d.ts +3 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +5 -7
- package/dist/errors.js.map +1 -1
- package/dist/finding-catalogue.d.ts.map +1 -1
- package/dist/finding-catalogue.js +22 -22
- package/dist/finding-catalogue.js.map +1 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +4 -7
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +7 -17
- package/dist/findings-import.js.map +1 -1
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +2 -3
- package/dist/github.js.map +1 -1
- package/dist/import-scan.js +2 -2
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.js +7 -3
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js +2 -3
- package/dist/linear.js.map +1 -1
- package/dist/models.d.ts +24 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +36 -31
- package/dist/multiscan.js.map +1 -1
- package/dist/owner-evidence.d.ts +29 -0
- package/dist/owner-evidence.d.ts.map +1 -0
- package/dist/owner-evidence.js +192 -0
- package/dist/owner-evidence.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +2 -4
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +10 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -0
- package/dist/project-config.js.map +1 -1
- package/dist/prompt-files.d.ts +0 -1
- package/dist/prompt-files.d.ts.map +1 -1
- package/dist/prompt-files.js +2 -4
- package/dist/prompt-files.js.map +1 -1
- package/dist/publication-events.d.ts.map +1 -1
- package/dist/publication-events.js +15 -29
- package/dist/publication-events.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +9 -13
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +11 -14
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +31 -41
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +2 -0
- package/dist/record.d.ts.map +1 -0
- package/dist/record.js +4 -0
- package/dist/record.js.map +1 -0
- package/dist/result.d.ts +4 -1
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +7 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +10 -13
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +230 -145
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan.d.ts.map +1 -1
- package/dist/saved-scan.js +10 -20
- package/dist/saved-scan.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +8 -22
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +4 -4
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +38 -73
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +0 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +10 -19
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +1 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +2 -6
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +1 -3
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +5 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/security-policy-cli.d.ts.map +1 -1
- package/dist/security-policy-cli.js +8 -6
- package/dist/security-policy-cli.js.map +1 -1
- package/dist/security-policy.d.ts +8 -4
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +98 -84
- package/dist/security-policy.js.map +1 -1
- package/dist/server/dashboard/app.js +5 -5
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +1 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/routes.d.ts +3 -2
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +19 -7
- package/dist/server/routes.js.map +1 -1
- package/dist/server/server.d.ts.map +1 -1
- package/dist/server/server.js +2 -3
- package/dist/server/server.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +4 -14
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-store.js +8 -15
- package/dist/severity-store.js.map +1 -1
- package/dist/suggest-owners.d.ts +37 -0
- package/dist/suggest-owners.d.ts.map +1 -0
- package/dist/suggest-owners.js +111 -0
- package/dist/suggest-owners.js.map +1 -0
- package/dist/targets.d.ts +7 -2
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +29 -25
- package/dist/targets.js.map +1 -1
- package/dist/thread-source.d.ts +1 -0
- package/dist/thread-source.d.ts.map +1 -1
- package/dist/thread-source.js +1 -0
- package/dist/thread-source.js.map +1 -1
- package/dist/trusted-executable.d.ts +5 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +22 -5
- package/dist/trusted-executable.js.map +1 -1
- package/dist/value.d.ts +2 -0
- package/dist/value.d.ts.map +1 -0
- package/dist/value.js +20 -0
- package/dist/value.js.map +1 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +3 -6
- package/dist/version.js.map +1 -1
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +1 -2
- package/dist/windows-path.js.map +1 -1
- package/dist/worker-progress.d.ts +1 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +13 -34
- package/dist/worker-progress.js.map +1 -1
- package/docs/dedupe-records.md +24 -0
- package/package.json +15 -13
- package/schemas/project-config.schema.json +5 -0
- package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
- package/dist/server/findings-service.d.ts +0 -20
- package/dist/server/findings-service.d.ts.map +0 -1
- package/dist/server/findings-service.js +0 -31
- package/dist/server/findings-service.js.map +0 -1
|
@@ -12,10 +12,9 @@ import re
|
|
|
12
12
|
import sqlite3
|
|
13
13
|
import stat
|
|
14
14
|
import sys
|
|
15
|
-
from collections.abc import
|
|
16
|
-
from
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from contextlib import contextmanager
|
|
17
17
|
from pathlib import Path
|
|
18
|
-
from types import ModuleType
|
|
19
18
|
from typing import Any
|
|
20
19
|
|
|
21
20
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
@@ -25,18 +24,25 @@ from finalize_scan_contract import (
|
|
|
25
24
|
_populate_unsealed_artifact_envelope,
|
|
26
25
|
_populate_unsealed_manifest_envelope,
|
|
27
26
|
_prepare_scan_finalization,
|
|
27
|
+
_read_json,
|
|
28
|
+
_read_saved_threat_model,
|
|
28
29
|
_read_scan_local_json,
|
|
29
30
|
_read_scan_local_json_bytes,
|
|
31
|
+
_read_scan_local_json_with_metadata,
|
|
30
32
|
_recover_unsealed_findings,
|
|
31
33
|
_remove_scan_local_file_if_exists,
|
|
32
34
|
_validate_completion_binding,
|
|
35
|
+
_validate_resolved_deferred,
|
|
36
|
+
_validate_schema_node,
|
|
33
37
|
_write_prepared_scan_finalization,
|
|
34
38
|
finalize_scan,
|
|
35
39
|
finding_candidate_id,
|
|
36
40
|
open_scan_local_file_descriptor,
|
|
37
41
|
write_scan_local_bytes,
|
|
42
|
+
write_threat_model_projection_if_possible,
|
|
38
43
|
)
|
|
39
44
|
from workbench_constants import PHASES
|
|
45
|
+
from workbench_target import committed_diff_snapshot_digest
|
|
40
46
|
from workbench_validation import path_within_scope
|
|
41
47
|
|
|
42
48
|
_PUBLISHED_OUTPUTS = (
|
|
@@ -44,6 +50,7 @@ _PUBLISHED_OUTPUTS = (
|
|
|
44
50
|
"coverage.json",
|
|
45
51
|
"scan-manifest.json",
|
|
46
52
|
"report.md",
|
|
53
|
+
"threatmodel.md",
|
|
47
54
|
"report.html",
|
|
48
55
|
"exports/results.sarif",
|
|
49
56
|
)
|
|
@@ -55,29 +62,59 @@ _RESERVED_ARTIFACT_PATHS = json.loads(
|
|
|
55
62
|
)
|
|
56
63
|
|
|
57
64
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
65
|
+
def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]:
|
|
66
|
+
scan_dir = Path(scan["scan_dir"])
|
|
67
|
+
fields: dict[str, Any] = {"threatModelAvailable": False}
|
|
68
|
+
try:
|
|
69
|
+
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
70
|
+
db.verify_manifest_binding(
|
|
71
|
+
scan, _read_scan_local_json(scan_dir, db.ARTIFACTS["manifest"], "scan manifest")
|
|
72
|
+
)
|
|
73
|
+
saved_model = _read_saved_threat_model(scan_dir)
|
|
74
|
+
if saved_model is not None:
|
|
75
|
+
description = saved_model[0]
|
|
76
|
+
fields.update(
|
|
77
|
+
threatModelAvailable=True,
|
|
78
|
+
threatModelProvenance=description["provenance"],
|
|
79
|
+
)
|
|
80
|
+
if description["path"] is not None:
|
|
81
|
+
fields["threatModelPath"] = description["path"]
|
|
82
|
+
except (ContractError, OSError, SystemExit):
|
|
83
|
+
# Unavailable optional model data must not prevent reading the saved scan.
|
|
84
|
+
pass
|
|
85
|
+
return fields
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def refresh_completed_scan(
|
|
89
|
+
db: Any,
|
|
90
|
+
connection: sqlite3.Connection,
|
|
91
|
+
scan: sqlite3.Row,
|
|
92
|
+
cost_json: str | None,
|
|
93
|
+
) -> dict[str, Any]:
|
|
94
|
+
warnings = json.loads(scan["completion_warnings_json"])
|
|
95
|
+
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
96
|
+
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
97
|
+
db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"]))
|
|
98
|
+
try:
|
|
99
|
+
manifest, _, _ = finalize_scan(
|
|
100
|
+
scan_dir,
|
|
101
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
102
|
+
projection_warnings=warnings,
|
|
103
|
+
)
|
|
104
|
+
except ContractError as exc:
|
|
105
|
+
raise SystemExit(str(exc)) from exc
|
|
106
|
+
db.verify_manifest_binding(scan, manifest)
|
|
107
|
+
manifest_digest = db.published_manifest_digest(scan_dir, manifest)
|
|
108
|
+
db.pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
|
|
109
|
+
if cost_json is not None and scan["recipe_json"] is not None:
|
|
110
|
+
db.scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json)
|
|
111
|
+
if warnings != json.loads(scan["completion_warnings_json"]):
|
|
112
|
+
with connection:
|
|
113
|
+
connection.execute(
|
|
114
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
115
|
+
(json.dumps(warnings), scan["id"]),
|
|
116
|
+
)
|
|
117
|
+
return db.scan_context(connection, scan["id"])
|
|
81
118
|
|
|
82
119
|
|
|
83
120
|
def _encoded(value: Any) -> bytes:
|
|
@@ -118,35 +155,51 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
|
118
155
|
)
|
|
119
156
|
|
|
120
157
|
|
|
121
|
-
def
|
|
122
|
-
|
|
158
|
+
def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
|
|
159
|
+
return [
|
|
160
|
+
f"{directory}/{name}"
|
|
161
|
+
for name in _children(scan_dir, directory)
|
|
162
|
+
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
163
|
+
]
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
|
|
167
|
+
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
168
|
+
attempts = (output.parent if output.name == "output" else output) / "attempts"
|
|
169
|
+
archived = [
|
|
170
|
+
((attempts / name).as_posix(), int(name.split("-")[1]))
|
|
171
|
+
for name in _children(scan_dir, attempts.as_posix())
|
|
172
|
+
if re.fullmatch(r"attempt-\d+", name)
|
|
173
|
+
]
|
|
174
|
+
attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
|
|
175
|
+
if worker["kind"] == "discovery" and not attempt:
|
|
176
|
+
attempt = max((attempt for _, attempt in archived), default=0) + 1
|
|
177
|
+
return [(output.as_posix(), attempt), *archived]
|
|
123
178
|
|
|
124
|
-
def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
|
|
125
|
-
for name in _children(scan_dir, directory):
|
|
126
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name):
|
|
127
|
-
yield f"{directory}/{name}", kind
|
|
128
179
|
|
|
129
|
-
|
|
180
|
+
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
181
|
+
latest_reducer = _latest_successful_reducer(workers)
|
|
182
|
+
yield "checkpoint-head.json", None
|
|
183
|
+
for directory in ("checkpoint-heads", "checkpoints"):
|
|
184
|
+
yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
|
|
130
185
|
for worker in workers:
|
|
131
186
|
if worker["kind"] not in {"dedup", "discovery"}:
|
|
132
187
|
continue
|
|
133
188
|
try:
|
|
134
|
-
|
|
189
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
135
190
|
except (TypeError, ValueError):
|
|
136
191
|
continue
|
|
137
|
-
|
|
138
|
-
"
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
for directory in directories:
|
|
146
|
-
checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
|
|
192
|
+
for directory, _ in outputs:
|
|
193
|
+
if worker["kind"] == "discovery":
|
|
194
|
+
yield f"{directory}/checkpoint-head.json", worker["kind"]
|
|
195
|
+
yield from (
|
|
196
|
+
(path, worker["kind"])
|
|
197
|
+
for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
|
|
198
|
+
)
|
|
199
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
147
200
|
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
148
201
|
yield f"{directory}/result.json", worker["kind"]
|
|
149
|
-
yield from checkpoint_paths
|
|
202
|
+
yield from ((path, worker["kind"]) for path in checkpoint_paths)
|
|
150
203
|
if worker["result_manifest_path"] and (
|
|
151
204
|
worker["kind"] == "discovery"
|
|
152
205
|
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
@@ -160,17 +213,143 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st
|
|
|
160
213
|
continue
|
|
161
214
|
|
|
162
215
|
|
|
216
|
+
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
217
|
+
path = Path(relative)
|
|
218
|
+
if path.name == "checkpoint-head.json":
|
|
219
|
+
return path.parent
|
|
220
|
+
if path.parent.name == "checkpoint-heads":
|
|
221
|
+
return path.parent.parent
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _capture_saved_source(
|
|
226
|
+
scan_dir: Path,
|
|
227
|
+
relative: str,
|
|
228
|
+
scan_id: str,
|
|
229
|
+
*,
|
|
230
|
+
kind: str | None = None,
|
|
231
|
+
snapshot_head: bool = True,
|
|
232
|
+
write: bool = True,
|
|
233
|
+
) -> dict[str, tuple[str, int]]:
|
|
234
|
+
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
|
|
235
|
+
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
|
|
236
|
+
return {relative: (digest, observed)}
|
|
237
|
+
head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
238
|
+
observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
|
|
239
|
+
directory = Path(relative).parent
|
|
240
|
+
selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
|
|
241
|
+
_, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
|
|
242
|
+
digest = _digest(observation)
|
|
243
|
+
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
|
|
244
|
+
# Capture the selected file even if the worker created it after directory enumeration.
|
|
245
|
+
if write and not (scan_dir / snapshot).exists():
|
|
246
|
+
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
247
|
+
return {
|
|
248
|
+
snapshot: (digest, int(observation["observedAtNs"])),
|
|
249
|
+
selected: (selected_digest, selected_time),
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _is_source_order_snapshot(relative: str) -> bool:
|
|
254
|
+
path = Path(relative)
|
|
255
|
+
return path.parent == Path("source-order") and bool(
|
|
256
|
+
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
163
260
|
def _read_saved_result(
|
|
164
261
|
scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
|
|
165
|
-
) -> tuple[dict[str, Any], str]:
|
|
166
|
-
draft
|
|
262
|
+
) -> tuple[dict[str, Any], str, int]:
|
|
263
|
+
draft, _, metadata = _read_scan_local_json_with_metadata(
|
|
264
|
+
scan_dir, relative, "Saved scan checkpoint"
|
|
265
|
+
)
|
|
266
|
+
directory = _checkpoint_head_directory(relative)
|
|
267
|
+
if directory is not None:
|
|
268
|
+
checkpoint = draft.get("checkpoint")
|
|
269
|
+
if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
|
|
270
|
+
raise ContractError("checkpoint head does not name a saved checkpoint")
|
|
271
|
+
_read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
|
|
272
|
+
if Path(relative).name == "checkpoint-head.json":
|
|
273
|
+
return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
|
|
274
|
+
observed = draft.get("observedAtNs")
|
|
275
|
+
if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
|
|
276
|
+
raise ContractError("checkpoint head has no observation time")
|
|
277
|
+
return draft, _digest(draft), int(observed)
|
|
167
278
|
if draft.get("scanId") != scan_id:
|
|
168
279
|
raise ContractError("checkpoint belongs to a different scan")
|
|
169
|
-
if not
|
|
170
|
-
draft.get("
|
|
280
|
+
if not _is_source_order_snapshot(relative) and (
|
|
281
|
+
not isinstance(draft.get("findings"), list)
|
|
282
|
+
or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
|
|
171
283
|
):
|
|
172
284
|
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
173
|
-
return draft, _digest(draft)
|
|
285
|
+
return draft, _digest(draft), metadata.st_mtime_ns
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
|
|
289
|
+
times: dict[str, int] = {}
|
|
290
|
+
snapshots = [path for path in sources if _is_source_order_snapshot(path)]
|
|
291
|
+
for path in snapshots:
|
|
292
|
+
record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
|
|
293
|
+
if digest != sources[path] or not isinstance(record.get("sources"), dict):
|
|
294
|
+
raise ContractError("saved source ordering changed after the scan stopped")
|
|
295
|
+
for relative, observation in record["sources"].items():
|
|
296
|
+
if (
|
|
297
|
+
not isinstance(observation, dict)
|
|
298
|
+
or relative not in sources
|
|
299
|
+
or observation.get("digest") != sources[relative]
|
|
300
|
+
or not isinstance(observation.get("observedAtNs"), str)
|
|
301
|
+
or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
|
|
302
|
+
):
|
|
303
|
+
raise ContractError("saved source ordering does not match its frozen sources")
|
|
304
|
+
observed = int(observation["observedAtNs"])
|
|
305
|
+
if relative in times and times[relative] != observed:
|
|
306
|
+
raise ContractError("saved source ordering has conflicting observations")
|
|
307
|
+
times[relative] = observed
|
|
308
|
+
if snapshots and sources.keys() - set(snapshots) - times.keys():
|
|
309
|
+
raise ContractError("saved source ordering is incomplete")
|
|
310
|
+
return times
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _freeze_source_times(
|
|
314
|
+
scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
|
|
315
|
+
) -> None:
|
|
316
|
+
# Identical result rewrites must not change the order of frozen review evidence.
|
|
317
|
+
observations = {
|
|
318
|
+
path: {"digest": digest, "observedAtNs": str(times[path])}
|
|
319
|
+
for path, digest in sources.items()
|
|
320
|
+
if not _is_source_order_snapshot(path)
|
|
321
|
+
}
|
|
322
|
+
if not observations:
|
|
323
|
+
return
|
|
324
|
+
record = {"scanId": scan_id, "sources": observations}
|
|
325
|
+
digest = _digest(record)
|
|
326
|
+
path = f"source-order/{digest}.json"
|
|
327
|
+
if not (scan_dir / path).exists():
|
|
328
|
+
write_scan_local_bytes(scan_dir, path, _encoded(record))
|
|
329
|
+
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
|
|
330
|
+
raise ContractError("saved source ordering does not match its digest")
|
|
331
|
+
sources[path] = digest
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _parent_scan_draft(
|
|
335
|
+
scan_id: str,
|
|
336
|
+
parent_scan: dict[str, Any],
|
|
337
|
+
findings: dict[str, Any],
|
|
338
|
+
coverage: dict[str, Any],
|
|
339
|
+
) -> dict[str, Any]:
|
|
340
|
+
parent = {
|
|
341
|
+
"scanId": scan_id,
|
|
342
|
+
"findings": findings.get("findings"),
|
|
343
|
+
"coverage": coverage,
|
|
344
|
+
**{
|
|
345
|
+
key: parent_scan[key]
|
|
346
|
+
for key in ("scope", "threatModel", "complete")
|
|
347
|
+
if key in parent_scan
|
|
348
|
+
},
|
|
349
|
+
}
|
|
350
|
+
if not isinstance(parent["findings"], list):
|
|
351
|
+
raise ContractError("Saved parent draft has no findings array")
|
|
352
|
+
return parent
|
|
174
353
|
|
|
175
354
|
|
|
176
355
|
def _read_saved_parent_result(
|
|
@@ -188,29 +367,34 @@ def _read_saved_parent_result(
|
|
|
188
367
|
or coverage.get("scanId", scan_id) != scan_id
|
|
189
368
|
):
|
|
190
369
|
raise ContractError("Saved parent documents belong to a different scan")
|
|
191
|
-
|
|
192
|
-
"scanId": scan_id,
|
|
193
|
-
"findings": findings.get("findings"),
|
|
194
|
-
"coverage": coverage,
|
|
195
|
-
**{
|
|
196
|
-
key: parent_scan[key]
|
|
197
|
-
for key in ("scope", "threatModel", "complete")
|
|
198
|
-
if key in parent_scan
|
|
199
|
-
},
|
|
200
|
-
}
|
|
201
|
-
if not isinstance(parent["findings"], list):
|
|
202
|
-
raise ContractError("Saved parent draft has no findings array")
|
|
203
|
-
return manifest, parent
|
|
370
|
+
return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
|
|
204
371
|
|
|
205
372
|
|
|
206
|
-
def _source_digests(value: Any,
|
|
373
|
+
def _source_digests(value: Any, error: str) -> dict[str, str]:
|
|
207
374
|
if not isinstance(value, dict) or not all(
|
|
208
375
|
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
209
376
|
):
|
|
210
|
-
raise ContractError(
|
|
377
|
+
raise ContractError(error)
|
|
211
378
|
return value
|
|
212
379
|
|
|
213
380
|
|
|
381
|
+
def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
|
|
382
|
+
if isinstance(value, dict) and isinstance(value.get("sources"), dict):
|
|
383
|
+
sources = _source_digests(
|
|
384
|
+
value["sources"], "Saved stopped-scan source digests are malformed."
|
|
385
|
+
)
|
|
386
|
+
model_source = value.get("threatModelSource")
|
|
387
|
+
if not isinstance(model_source, str) or model_source not in sources:
|
|
388
|
+
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
|
|
389
|
+
return sources, model_source
|
|
390
|
+
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
|
|
394
|
+
state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
|
|
395
|
+
return json.dumps(state, sort_keys=True)
|
|
396
|
+
|
|
397
|
+
|
|
214
398
|
def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
215
399
|
try:
|
|
216
400
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
@@ -234,7 +418,7 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
234
418
|
|
|
235
419
|
if manifest_path is None:
|
|
236
420
|
if frozen_sources is not None:
|
|
237
|
-
return bool(
|
|
421
|
+
return bool(_retained_source_state(json.loads(frozen_sources))[0])
|
|
238
422
|
return has_saved_source()
|
|
239
423
|
if scan["seal_manifest_digest"] is None:
|
|
240
424
|
try:
|
|
@@ -252,14 +436,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
252
436
|
if not isinstance(manifest_scan, dict):
|
|
253
437
|
return True
|
|
254
438
|
published_sources = _source_digests(
|
|
255
|
-
manifest_scan.get("preservedSources", {}),
|
|
439
|
+
manifest_scan.get("preservedSources", {}),
|
|
440
|
+
"Published scan source digests are malformed.",
|
|
256
441
|
)
|
|
257
442
|
current_sources = dict(published_sources)
|
|
443
|
+
paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
|
|
258
444
|
for path in paths:
|
|
259
445
|
try:
|
|
260
|
-
|
|
261
|
-
scan_dir,
|
|
446
|
+
captured = _capture_saved_source(
|
|
447
|
+
scan_dir,
|
|
448
|
+
path,
|
|
449
|
+
scan["id"],
|
|
450
|
+
kind=paths[path],
|
|
451
|
+
snapshot_head=path not in published_sources,
|
|
452
|
+
write=False,
|
|
262
453
|
)
|
|
454
|
+
current_sources.update({path: value[0] for path, value in captured.items()})
|
|
263
455
|
except (ContractError, OSError, ValueError):
|
|
264
456
|
continue
|
|
265
457
|
return current_sources != published_sources
|
|
@@ -273,7 +465,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
273
465
|
include_parent = True
|
|
274
466
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
275
467
|
if raw_frozen_sources is not None:
|
|
276
|
-
frozen_sources =
|
|
468
|
+
frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
|
|
277
469
|
include_parent = False
|
|
278
470
|
|
|
279
471
|
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
@@ -291,7 +483,8 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
291
483
|
):
|
|
292
484
|
if "preservedSources" in manifest_scan:
|
|
293
485
|
published_sources = _source_digests(
|
|
294
|
-
manifest_scan["preservedSources"],
|
|
486
|
+
manifest_scan["preservedSources"],
|
|
487
|
+
"Published scan source digests are malformed.",
|
|
295
488
|
)
|
|
296
489
|
include_parent = not published_sources
|
|
297
490
|
if published_sources:
|
|
@@ -312,21 +505,30 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
312
505
|
).fetchall()
|
|
313
506
|
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
314
507
|
recovery_sources = dict(frozen_sources or {})
|
|
508
|
+
source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
|
|
315
509
|
for relative, expected_digest in recovery_sources.items():
|
|
316
510
|
try:
|
|
317
|
-
_, digest = _read_saved_result(
|
|
511
|
+
_, digest, observed = _read_saved_result(
|
|
512
|
+
scan_dir, relative, scan["id"], kind=paths.get(relative)
|
|
513
|
+
)
|
|
318
514
|
except (ContractError, OSError, ValueError) as exc:
|
|
319
515
|
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
320
516
|
if digest != expected_digest:
|
|
321
517
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
518
|
+
if not _is_source_order_snapshot(relative):
|
|
519
|
+
source_times.setdefault(relative, observed)
|
|
322
520
|
|
|
323
521
|
for relative in paths.keys() - recovery_sources.keys():
|
|
324
522
|
try:
|
|
325
|
-
|
|
326
|
-
scan_dir, relative, scan["id"], kind=paths[relative]
|
|
327
|
-
)
|
|
523
|
+
captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
|
|
328
524
|
except (ContractError, OSError, ValueError):
|
|
329
525
|
continue
|
|
526
|
+
for path, (digest, observed) in captured.items():
|
|
527
|
+
if path in recovery_sources and recovery_sources[path] != digest:
|
|
528
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
529
|
+
recovery_sources[path] = digest
|
|
530
|
+
source_times.setdefault(path, observed)
|
|
531
|
+
_freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
|
|
330
532
|
return recovery_sources, include_parent
|
|
331
533
|
|
|
332
534
|
|
|
@@ -358,15 +560,10 @@ def _finding_key(finding: dict[str, Any]) -> str:
|
|
|
358
560
|
else finding.get("identity")
|
|
359
561
|
)
|
|
360
562
|
if not isinstance(identity, dict):
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
or "finding"
|
|
366
|
-
)
|
|
367
|
-
identity = {
|
|
368
|
-
"anchor": re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
|
|
369
|
-
}
|
|
563
|
+
normalized = dict(finding)
|
|
564
|
+
normalized.pop("identity", None)
|
|
565
|
+
_ensure_finding_identity(normalized)
|
|
566
|
+
identity = normalized["identity"]
|
|
370
567
|
locations = finding.get("locations", [])
|
|
371
568
|
if not isinstance(locations, list):
|
|
372
569
|
locations = []
|
|
@@ -459,6 +656,257 @@ def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
|
|
|
459
656
|
)
|
|
460
657
|
|
|
461
658
|
|
|
659
|
+
def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
|
|
660
|
+
rows = coverage.get("deferred", [])
|
|
661
|
+
return rows if isinstance(rows, list) else []
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
665
|
+
if draft.get("complete") is False:
|
|
666
|
+
return []
|
|
667
|
+
coverage = draft["coverage"]
|
|
668
|
+
rows = coverage.get("resolvedDeferred", [])
|
|
669
|
+
try:
|
|
670
|
+
# Invalid closure metadata cannot discard the evidence it names.
|
|
671
|
+
_validate_schema_node(rows, schema, "coverage.resolvedDeferred")
|
|
672
|
+
_validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
|
|
673
|
+
except ContractError:
|
|
674
|
+
return []
|
|
675
|
+
return rows
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _merge_tied_parent_observations(
|
|
679
|
+
current: dict[str, Any], previous: dict[str, Any]
|
|
680
|
+
) -> dict[str, Any]:
|
|
681
|
+
merged = copy.deepcopy(current)
|
|
682
|
+
for finding in previous["findings"]:
|
|
683
|
+
if finding not in merged["findings"]:
|
|
684
|
+
merged["findings"].append(copy.deepcopy(finding))
|
|
685
|
+
coverage = merged["coverage"]
|
|
686
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
687
|
+
rows = previous["coverage"].get(field, [])
|
|
688
|
+
output = coverage.setdefault(field, [])
|
|
689
|
+
if isinstance(rows, list) and isinstance(output, list):
|
|
690
|
+
for row in rows:
|
|
691
|
+
if row not in output:
|
|
692
|
+
output.append(copy.deepcopy(row))
|
|
693
|
+
pending_ids = {
|
|
694
|
+
identity
|
|
695
|
+
for row in _deferred_rows(coverage)
|
|
696
|
+
if isinstance(row, dict)
|
|
697
|
+
for identity in (row.get("id"), row.get("candidateId"))
|
|
698
|
+
if isinstance(identity, str)
|
|
699
|
+
}
|
|
700
|
+
closure_schema = _read_json(
|
|
701
|
+
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
702
|
+
)["properties"]["resolvedDeferred"]
|
|
703
|
+
closures = {}
|
|
704
|
+
for observed in (current, previous):
|
|
705
|
+
for row in _resolved_deferred_rows(observed, closure_schema):
|
|
706
|
+
if row["id"] not in pending_ids:
|
|
707
|
+
closures.setdefault(row["id"], copy.deepcopy(row))
|
|
708
|
+
coverage.pop("resolvedDeferred", None)
|
|
709
|
+
if closures:
|
|
710
|
+
coverage["resolvedDeferred"] = list(closures.values())
|
|
711
|
+
if current.get("complete") is False or previous.get("complete") is False:
|
|
712
|
+
merged["complete"] = False
|
|
713
|
+
if (
|
|
714
|
+
coverage.get("deferred")
|
|
715
|
+
or merged.get("complete") is False
|
|
716
|
+
or (
|
|
717
|
+
isinstance(coverage.get("surfaces"), list)
|
|
718
|
+
and any(
|
|
719
|
+
isinstance(row, dict) and row.get("disposition") == "needs_follow_up"
|
|
720
|
+
for row in coverage["surfaces"]
|
|
721
|
+
)
|
|
722
|
+
)
|
|
723
|
+
or previous["coverage"].get("completeness") == "partial"
|
|
724
|
+
):
|
|
725
|
+
coverage["completeness"] = "partial"
|
|
726
|
+
return merged
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _saved_coverage_id(item: dict[str, Any]) -> str:
|
|
730
|
+
return item.get("candidateId") or f"saved-{_digest(item)[:16]}"
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _deferred_candidate_id(
|
|
734
|
+
row: dict[str, Any],
|
|
735
|
+
owner: str | None,
|
|
736
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
737
|
+
) -> str | None:
|
|
738
|
+
identity = row.get("candidateId") or row.get("id")
|
|
739
|
+
if not isinstance(identity, str):
|
|
740
|
+
return None
|
|
741
|
+
if (owner, identity) in ambiguous_deferred and not any(
|
|
742
|
+
key in row for key in ("candidateId", "candidate", "finding")
|
|
743
|
+
):
|
|
744
|
+
return None
|
|
745
|
+
return identity
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def _generic_surface_updates(
|
|
749
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
750
|
+
source_order: dict[str, tuple[int, int]],
|
|
751
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
752
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
753
|
+
resolved_candidates: dict[tuple[str | None, str], str],
|
|
754
|
+
reopened_generic: set[tuple[str | None, str]],
|
|
755
|
+
surface_schema: dict[str, Any],
|
|
756
|
+
deferred_rows: dict[str, list[Any]],
|
|
757
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
758
|
+
) -> tuple[set[int], list[dict[str, Any]]]:
|
|
759
|
+
if not closed_deferred and not reopened_generic:
|
|
760
|
+
return set(), []
|
|
761
|
+
|
|
762
|
+
def linked(row: dict[str, Any], identity: str | None) -> bool:
|
|
763
|
+
surface_ids = row.get("surfaceIds", [])
|
|
764
|
+
return identity is not None and (
|
|
765
|
+
row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
|
|
769
|
+
for relative, draft, owner in sources:
|
|
770
|
+
rows = draft["coverage"].get("surfaces", [])
|
|
771
|
+
for row in rows if isinstance(rows, list) else []:
|
|
772
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
773
|
+
saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
|
|
774
|
+
replaced: set[int] = set()
|
|
775
|
+
updates: list[dict[str, Any]] = []
|
|
776
|
+
for relative, draft, owner in sources:
|
|
777
|
+
closed_ids = {
|
|
778
|
+
identity
|
|
779
|
+
for (saved_owner, identity), (_, _, source) in closed_deferred.items()
|
|
780
|
+
if saved_owner == owner and source == relative
|
|
781
|
+
}
|
|
782
|
+
reopened_ids = {
|
|
783
|
+
identity
|
|
784
|
+
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
785
|
+
if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
|
|
786
|
+
}
|
|
787
|
+
if not closed_ids and not reopened_ids:
|
|
788
|
+
continue
|
|
789
|
+
current = draft["coverage"].get("surfaces", [])
|
|
790
|
+
for surface in current if isinstance(current, list) else []:
|
|
791
|
+
if not isinstance(surface, dict):
|
|
792
|
+
continue
|
|
793
|
+
reopening = surface.get("disposition") == "needs_follow_up"
|
|
794
|
+
work_ids = reopened_ids if reopening else closed_ids
|
|
795
|
+
if not work_ids:
|
|
796
|
+
continue
|
|
797
|
+
identity = surface.get("id")
|
|
798
|
+
if not isinstance(identity, str):
|
|
799
|
+
continue
|
|
800
|
+
matches = saved_surfaces[(owner, identity)]
|
|
801
|
+
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
802
|
+
# A closure cannot identify which of those observations it replaces.
|
|
803
|
+
by_source = dict(matches)
|
|
804
|
+
if any(row != by_source[saved_path] for saved_path, row in matches):
|
|
805
|
+
continue
|
|
806
|
+
if any(
|
|
807
|
+
"candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
|
|
808
|
+
):
|
|
809
|
+
continue
|
|
810
|
+
if any(
|
|
811
|
+
row is not surface
|
|
812
|
+
and source_order[saved_path] >= source_order[relative]
|
|
813
|
+
and row.get("disposition") != surface.get("disposition")
|
|
814
|
+
for saved_path, row in matches
|
|
815
|
+
):
|
|
816
|
+
continue
|
|
817
|
+
|
|
818
|
+
if not reopening and any(
|
|
819
|
+
saved_owner == owner
|
|
820
|
+
and (
|
|
821
|
+
not isinstance(row.get("id") or row.get("candidateId"), str)
|
|
822
|
+
or (
|
|
823
|
+
isinstance(row.get("id"), str)
|
|
824
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
825
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
826
|
+
)
|
|
827
|
+
)
|
|
828
|
+
and linked(row, identity)
|
|
829
|
+
for saved_path, _, saved_owner in sources
|
|
830
|
+
for row in deferred_rows[saved_path]
|
|
831
|
+
if isinstance(row, dict)
|
|
832
|
+
):
|
|
833
|
+
continue
|
|
834
|
+
if not reopening and any(
|
|
835
|
+
saved_owner == owner
|
|
836
|
+
and (owner, deferred_id) not in closed_deferred
|
|
837
|
+
and (
|
|
838
|
+
(candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
|
|
839
|
+
or (owner, candidate_id) not in resolved_candidates
|
|
840
|
+
)
|
|
841
|
+
and linked(row, identity)
|
|
842
|
+
for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
|
|
843
|
+
):
|
|
844
|
+
continue
|
|
845
|
+
# An accepted checkpoint can update a saved surface by ID without
|
|
846
|
+
# optional surfaceIds links on its generic task.
|
|
847
|
+
latest_surface = max(matches, key=lambda match: source_order[match[0]])[1]
|
|
848
|
+
update = copy.deepcopy(latest_surface)
|
|
849
|
+
refs = update.setdefault("receiptRefs", [])
|
|
850
|
+
if isinstance(refs, list):
|
|
851
|
+
for _, row in matches:
|
|
852
|
+
previous_refs = row.get("receiptRefs", [])
|
|
853
|
+
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
854
|
+
if ref not in refs:
|
|
855
|
+
refs.append(ref)
|
|
856
|
+
try:
|
|
857
|
+
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
858
|
+
except ContractError:
|
|
859
|
+
continue
|
|
860
|
+
replaced.update(
|
|
861
|
+
id(row)
|
|
862
|
+
for _, row in matches
|
|
863
|
+
if reopening
|
|
864
|
+
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
865
|
+
)
|
|
866
|
+
if update not in updates:
|
|
867
|
+
updates.append(update)
|
|
868
|
+
return replaced, updates
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
@contextmanager
|
|
872
|
+
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
873
|
+
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
874
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
875
|
+
previous = None
|
|
876
|
+
try:
|
|
877
|
+
head_path.lstat()
|
|
878
|
+
except FileNotFoundError:
|
|
879
|
+
pass
|
|
880
|
+
else:
|
|
881
|
+
descriptor = open_scan_local_file_descriptor(
|
|
882
|
+
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
883
|
+
)
|
|
884
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
885
|
+
metadata = os.fstat(handle.fileno())
|
|
886
|
+
previous = (handle.read(), metadata)
|
|
887
|
+
directories = ("checkpoints", "checkpoint-heads")
|
|
888
|
+
previous_files = {
|
|
889
|
+
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
890
|
+
}
|
|
891
|
+
try:
|
|
892
|
+
yield
|
|
893
|
+
except ContractError:
|
|
894
|
+
if previous is None:
|
|
895
|
+
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
896
|
+
else:
|
|
897
|
+
payload, metadata = previous
|
|
898
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
899
|
+
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
900
|
+
current_files = {
|
|
901
|
+
relative
|
|
902
|
+
for directory in directories
|
|
903
|
+
for relative in _checkpoint_paths(scan_dir, directory)
|
|
904
|
+
}
|
|
905
|
+
for relative in current_files - previous_files:
|
|
906
|
+
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
907
|
+
raise
|
|
908
|
+
|
|
909
|
+
|
|
462
910
|
def merge_saved_results(
|
|
463
911
|
scan_dir: Path,
|
|
464
912
|
scan_id: str,
|
|
@@ -470,14 +918,25 @@ def merge_saved_results(
|
|
|
470
918
|
reason: str,
|
|
471
919
|
frozen_source_digests: dict[str, str] | None = None,
|
|
472
920
|
allow_frozen_legacy_parent: bool = False,
|
|
921
|
+
frozen_model_source: str | None = None,
|
|
922
|
+
selected_model_source: list[str] | None = None,
|
|
473
923
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
474
924
|
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
475
925
|
initial_warnings = set(warnings)
|
|
926
|
+
try:
|
|
927
|
+
source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
|
|
928
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
929
|
+
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
476
930
|
parent: dict[str, Any] | None = None
|
|
477
931
|
parent_manifest: dict[str, Any] | None = None
|
|
932
|
+
parent_is_canonical = False
|
|
933
|
+
parent_modified = 0
|
|
478
934
|
if frozen_source_digests is None or allow_frozen_legacy_parent:
|
|
479
935
|
try:
|
|
480
936
|
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
937
|
+
# Without an accepted head, file-authored coverage is a full replacement.
|
|
938
|
+
parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
|
|
939
|
+
parent_is_canonical = True
|
|
481
940
|
except (ContractError, OSError, ValueError) as exc:
|
|
482
941
|
if not stopped:
|
|
483
942
|
raise
|
|
@@ -487,25 +946,66 @@ def merge_saved_results(
|
|
|
487
946
|
parent = None
|
|
488
947
|
if parent_manifest is not None and parent is not None:
|
|
489
948
|
parent_scan = parent_manifest["scan"]
|
|
949
|
+
try:
|
|
950
|
+
previous_head, _, head_modified = _read_saved_result(
|
|
951
|
+
scan_dir, "checkpoint-head.json", scan_id
|
|
952
|
+
)
|
|
953
|
+
except (ContractError, OSError, ValueError):
|
|
954
|
+
head_modified = None
|
|
955
|
+
if head_modified is not None:
|
|
956
|
+
# A partial tool publication must not outrank its accepted head.
|
|
957
|
+
parent_modified = min(
|
|
958
|
+
(scan_dir / name).lstat().st_mtime_ns
|
|
959
|
+
for name in ("findings.json", "coverage.json", "scan-manifest.json")
|
|
960
|
+
)
|
|
490
961
|
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
962
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
963
|
+
tied_observations = False
|
|
964
|
+
if head_modified == parent_modified:
|
|
965
|
+
previous_parent, _, _ = _read_saved_result(
|
|
966
|
+
scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
|
|
967
|
+
)
|
|
968
|
+
if previous_parent != parent:
|
|
969
|
+
# Tied observations cannot decide which pending work came last.
|
|
970
|
+
parent = _merge_tied_parent_observations(parent, previous_parent)
|
|
971
|
+
parent_is_canonical = False
|
|
972
|
+
tied_observations = True
|
|
491
973
|
payload = _encoded(parent)
|
|
492
974
|
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
493
975
|
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
494
|
-
|
|
976
|
+
checkpoint_path = scan_dir / parent_checkpoint
|
|
977
|
+
if not checkpoint_path.exists():
|
|
978
|
+
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
979
|
+
# A recovery copy must not appear newer than the review it copies.
|
|
980
|
+
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
|
|
981
|
+
if head_modified is None or head_modified < parent_modified or tied_observations:
|
|
982
|
+
write_scan_local_bytes(
|
|
983
|
+
scan_dir,
|
|
984
|
+
"checkpoint-head.json",
|
|
985
|
+
_encoded({"checkpoint": checkpoint_path.name}),
|
|
986
|
+
)
|
|
987
|
+
os.utime(head_path, ns=(parent_modified, parent_modified))
|
|
495
988
|
if frozen_source_digests is not None:
|
|
989
|
+
captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
|
|
496
990
|
frozen_source_digests = {
|
|
497
991
|
**frozen_source_digests,
|
|
498
992
|
parent_checkpoint: parent_digest,
|
|
993
|
+
**{path: value[0] for path, value in captured.items()},
|
|
499
994
|
}
|
|
500
995
|
|
|
501
996
|
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
502
997
|
parent_preserved_sources: dict[str, str] = {}
|
|
503
998
|
source_digests: dict[str, str] = {}
|
|
999
|
+
source_order: dict[str, tuple[int, int]] = {}
|
|
1000
|
+
worker_attempts: dict[str, tuple[str, int]] = {}
|
|
1001
|
+
saved_heads: dict[str, str] = {}
|
|
504
1002
|
if parent_manifest:
|
|
505
1003
|
recorded = parent_manifest["scan"].get("preservedSources", {})
|
|
506
1004
|
if isinstance(recorded, dict):
|
|
507
1005
|
parent_preserved_sources = recorded
|
|
508
1006
|
source_digests.update(parent_preserved_sources)
|
|
1007
|
+
if frozen_source_digests is None:
|
|
1008
|
+
source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
|
|
509
1009
|
paths: dict[str, str | None] = {}
|
|
510
1010
|
reducer_paths: set[str] = set()
|
|
511
1011
|
current_results: set[str] = set()
|
|
@@ -520,57 +1020,43 @@ def merge_saved_results(
|
|
|
520
1020
|
except ValueError:
|
|
521
1021
|
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
522
1022
|
|
|
523
|
-
def checkpoints(directory: str, worker_id: str | None) -> None:
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
1023
|
+
def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
|
|
1024
|
+
if worker_id is not None:
|
|
1025
|
+
root = Path(directory).parent.as_posix()
|
|
1026
|
+
worker_attempts[root] = (worker_id, attempt)
|
|
1027
|
+
paths[f"{root}/checkpoint-head.json"] = worker_id
|
|
1028
|
+
head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
|
|
1029
|
+
for saved_directory in (directory, head_snapshots):
|
|
1030
|
+
paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
|
|
527
1031
|
|
|
1032
|
+
paths["checkpoint-head.json"] = None
|
|
528
1033
|
checkpoints("checkpoints", None)
|
|
529
1034
|
for worker in workers:
|
|
530
1035
|
try:
|
|
531
|
-
|
|
1036
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
532
1037
|
except (TypeError, ValueError):
|
|
533
1038
|
warnings.append("Skipped a worker checkpoint outside the scan directory.")
|
|
534
1039
|
continue
|
|
535
1040
|
if worker["kind"] == "dedup":
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
result_path = f"{directory}/result.json"
|
|
539
|
-
checkpoint_paths = [
|
|
540
|
-
f"{directory}/checkpoints/{name}"
|
|
541
|
-
for name in _children(scan_dir, f"{directory}/checkpoints")
|
|
542
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
543
|
-
]
|
|
1041
|
+
for directory, attempt in outputs:
|
|
1042
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
544
1043
|
if not checkpoint_paths:
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
reducer_paths.update(
|
|
550
|
-
reducer_outputs.append((
|
|
551
|
-
|
|
552
|
-
reducer_output(output, int(worker["attempt"] or 0), worker)
|
|
553
|
-
attempts = (
|
|
554
|
-
Path(output).parent if Path(output).name == "output" else Path(output)
|
|
555
|
-
) / "attempts"
|
|
556
|
-
for name in _children(scan_dir, attempts.as_posix()):
|
|
557
|
-
match = re.fullmatch(r"attempt-(\d+)", name)
|
|
558
|
-
if match:
|
|
559
|
-
reducer_output((attempts / name).as_posix(), int(match.group(1)), worker)
|
|
1044
|
+
continue
|
|
1045
|
+
result_path = f"{directory}/result.json"
|
|
1046
|
+
retained_paths = [result_path, *checkpoint_paths]
|
|
1047
|
+
paths.update(dict.fromkeys(retained_paths))
|
|
1048
|
+
reducer_paths.update(retained_paths)
|
|
1049
|
+
reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
|
|
560
1050
|
continue
|
|
561
1051
|
if worker["kind"] != "discovery":
|
|
562
1052
|
continue
|
|
1053
|
+
output, attempt = outputs[0]
|
|
563
1054
|
paths[f"{output}/result.json"] = worker["id"]
|
|
564
1055
|
current_results.add(f"{output}/result.json")
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
for name in _children(scan_dir, attempts.as_posix()):
|
|
570
|
-
if re.fullmatch(r"attempt-\d+", name):
|
|
571
|
-
archived = (attempts / name).as_posix()
|
|
572
|
-
paths[f"{archived}/result.json"] = worker["id"]
|
|
573
|
-
checkpoints(f"{archived}/checkpoints", worker["id"])
|
|
1056
|
+
for archived, archived_attempt in outputs[1:]:
|
|
1057
|
+
paths[f"{archived}/result.json"] = worker["id"]
|
|
1058
|
+
checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
|
|
1059
|
+
checkpoints(f"{output}/checkpoints", worker["id"], attempt)
|
|
574
1060
|
if worker["result_manifest_path"]:
|
|
575
1061
|
try:
|
|
576
1062
|
current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
@@ -579,7 +1065,26 @@ def merge_saved_results(
|
|
|
579
1065
|
except ValueError:
|
|
580
1066
|
warnings.append("Skipped a worker result outside the scan directory.")
|
|
581
1067
|
|
|
1068
|
+
if frozen_source_digests is None:
|
|
1069
|
+
for relative, worker_id in list(paths.items()):
|
|
1070
|
+
if Path(relative).name != "checkpoint-head.json":
|
|
1071
|
+
continue
|
|
1072
|
+
del paths[relative]
|
|
1073
|
+
try:
|
|
1074
|
+
captured = _capture_saved_source(scan_dir, relative, scan_id)
|
|
1075
|
+
paths.update({path: worker_id for path in captured})
|
|
1076
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
1077
|
+
if (scan_dir / relative).exists():
|
|
1078
|
+
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1079
|
+
|
|
582
1080
|
if frozen_source_digests is not None:
|
|
1081
|
+
source_digests.update(
|
|
1082
|
+
{
|
|
1083
|
+
path: digest
|
|
1084
|
+
for path, digest in frozen_source_digests.items()
|
|
1085
|
+
if _is_source_order_snapshot(path)
|
|
1086
|
+
}
|
|
1087
|
+
)
|
|
583
1088
|
paths = {
|
|
584
1089
|
relative: worker_id
|
|
585
1090
|
for relative, worker_id in paths.items()
|
|
@@ -591,12 +1096,17 @@ def merge_saved_results(
|
|
|
591
1096
|
|
|
592
1097
|
for relative, worker_id in paths.items():
|
|
593
1098
|
try:
|
|
594
|
-
draft, digest = _read_saved_result(
|
|
1099
|
+
draft, digest, observed = _read_saved_result(
|
|
595
1100
|
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
596
1101
|
)
|
|
597
1102
|
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
598
1103
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
599
1104
|
source_digests[relative] = digest
|
|
1105
|
+
source_times.setdefault(relative, observed)
|
|
1106
|
+
source_order[relative] = (0, source_times[relative])
|
|
1107
|
+
if _checkpoint_head_directory(relative) is not None:
|
|
1108
|
+
saved_heads[relative] = draft["checkpoint"]
|
|
1109
|
+
continue
|
|
600
1110
|
# Recovery expects coverage, but reducer results only contain findings
|
|
601
1111
|
# and context. Add an empty value after hashing the original result.
|
|
602
1112
|
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
@@ -607,6 +1117,37 @@ def merge_saved_results(
|
|
|
607
1117
|
if frozen_source_digests.keys() - source_digests.keys():
|
|
608
1118
|
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
|
|
609
1119
|
|
|
1120
|
+
# Frozen observations retain checkpoint selection even if a worker moves its head.
|
|
1121
|
+
headed_workers = {
|
|
1122
|
+
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
|
|
1123
|
+
for head in saved_heads
|
|
1124
|
+
if _checkpoint_head_directory(head) != Path(".")
|
|
1125
|
+
}
|
|
1126
|
+
for relative, _, worker_id in sources:
|
|
1127
|
+
if worker_id not in headed_workers:
|
|
1128
|
+
continue
|
|
1129
|
+
directory = Path(relative).parent
|
|
1130
|
+
if directory.name == "checkpoints":
|
|
1131
|
+
directory = directory.parent
|
|
1132
|
+
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
|
|
1133
|
+
source_order[relative] = (attempt, source_order[relative][1])
|
|
1134
|
+
selected_observations: dict[str, tuple[int, int]] = {}
|
|
1135
|
+
parent_heads: list[tuple[int, str]] = []
|
|
1136
|
+
for head, checkpoint in saved_heads.items():
|
|
1137
|
+
directory = _checkpoint_head_directory(head)
|
|
1138
|
+
selected = (directory / "checkpoints" / checkpoint).as_posix()
|
|
1139
|
+
if selected not in source_order:
|
|
1140
|
+
raise ContractError("Checkpoint head is outside the saved source set.")
|
|
1141
|
+
observed = source_order[head][1]
|
|
1142
|
+
if directory == Path("."):
|
|
1143
|
+
order = (0, max(source_order[selected][1], observed))
|
|
1144
|
+
parent_heads.append((observed, selected))
|
|
1145
|
+
else:
|
|
1146
|
+
_, attempt = worker_attempts[directory.as_posix()]
|
|
1147
|
+
order = (attempt, observed)
|
|
1148
|
+
selected_observations[selected] = max(selected_observations.get(selected, order), order)
|
|
1149
|
+
source_order.update(selected_observations)
|
|
1150
|
+
|
|
610
1151
|
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
611
1152
|
latest_reducer_key = (
|
|
612
1153
|
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
@@ -627,8 +1168,67 @@ def merge_saved_results(
|
|
|
627
1168
|
latest_reducer_key = candidate_key
|
|
628
1169
|
latest_reducer = result_path
|
|
629
1170
|
|
|
1171
|
+
if parent_heads:
|
|
1172
|
+
latest_observation = max(observed for observed, _ in parent_heads)
|
|
1173
|
+
for observed, parent_path in parent_heads:
|
|
1174
|
+
if observed != latest_observation:
|
|
1175
|
+
continue
|
|
1176
|
+
draft = drafts_by_path[parent_path]
|
|
1177
|
+
modified = source_order[parent_path][1]
|
|
1178
|
+
if parent is None or modified > parent_modified:
|
|
1179
|
+
parent = draft
|
|
1180
|
+
parent_modified = modified
|
|
1181
|
+
parent_is_canonical = False
|
|
1182
|
+
elif modified == parent_modified and draft != parent:
|
|
1183
|
+
parent = _merge_tied_parent_observations(parent, draft)
|
|
1184
|
+
parent_is_canonical = False
|
|
630
1185
|
if parent is None and latest_reducer is not None:
|
|
631
|
-
parent =
|
|
1186
|
+
parent = drafts_by_path[latest_reducer]
|
|
1187
|
+
|
|
1188
|
+
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
1189
|
+
# Older checkpoints can omit IDs already assigned in their published output.
|
|
1190
|
+
for field in ("deferred", "surfaces"):
|
|
1191
|
+
named_rows: dict[str | None, list[dict[str, Any]]] = {}
|
|
1192
|
+
for _, draft, owner in all_sources:
|
|
1193
|
+
rows = draft["coverage"].get(field, [])
|
|
1194
|
+
named_rows.setdefault(owner, []).extend(
|
|
1195
|
+
row
|
|
1196
|
+
for row in (rows if isinstance(rows, list) else [])
|
|
1197
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1198
|
+
)
|
|
1199
|
+
for _, draft, owner in all_sources:
|
|
1200
|
+
rows = draft["coverage"].get(field, [])
|
|
1201
|
+
if not isinstance(rows, list):
|
|
1202
|
+
continue
|
|
1203
|
+
reserved = {
|
|
1204
|
+
row["id"]
|
|
1205
|
+
for row in rows
|
|
1206
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1207
|
+
}
|
|
1208
|
+
for row in rows:
|
|
1209
|
+
if not isinstance(row, dict) or "id" in row:
|
|
1210
|
+
continue
|
|
1211
|
+
if field == "deferred" and any(
|
|
1212
|
+
key in row for key in ("candidateId", "candidate", "finding")
|
|
1213
|
+
):
|
|
1214
|
+
continue
|
|
1215
|
+
content = {"receiptRefs": [], **row} if field == "surfaces" else row
|
|
1216
|
+
identity = next(
|
|
1217
|
+
(
|
|
1218
|
+
named["id"]
|
|
1219
|
+
for named in named_rows[owner]
|
|
1220
|
+
if named["id"] not in reserved
|
|
1221
|
+
and {
|
|
1222
|
+
**({"receiptRefs": []} if field == "surfaces" else {}),
|
|
1223
|
+
**{key: value for key, value in named.items() if key != "id"},
|
|
1224
|
+
}
|
|
1225
|
+
== content
|
|
1226
|
+
),
|
|
1227
|
+
None,
|
|
1228
|
+
)
|
|
1229
|
+
if identity is not None:
|
|
1230
|
+
row["id"] = identity
|
|
1231
|
+
reserved.add(identity)
|
|
632
1232
|
|
|
633
1233
|
if parent is None and not sources:
|
|
634
1234
|
return None
|
|
@@ -641,6 +1241,13 @@ def merge_saved_results(
|
|
|
641
1241
|
):
|
|
642
1242
|
return None
|
|
643
1243
|
|
|
1244
|
+
if (
|
|
1245
|
+
frozen_source_digests is None
|
|
1246
|
+
or any(_is_source_order_snapshot(path) for path in source_digests)
|
|
1247
|
+
or allow_frozen_legacy_parent
|
|
1248
|
+
):
|
|
1249
|
+
_freeze_source_times(scan_dir, scan_id, source_digests, source_times)
|
|
1250
|
+
|
|
644
1251
|
target_kind = binding["allowedTargetKinds"][0]
|
|
645
1252
|
if (
|
|
646
1253
|
target_kind == "git_worktree"
|
|
@@ -651,15 +1258,9 @@ def merge_saved_results(
|
|
|
651
1258
|
target = {"kind": target_kind, **binding["target"]}
|
|
652
1259
|
if target["kind"] == "git_diff" and "snapshotDigest" not in target:
|
|
653
1260
|
diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
+ b"\0"
|
|
658
|
-
+ target["baseRevision"].encode()
|
|
659
|
-
+ b"\0"
|
|
660
|
-
+ target["headRevision"].encode()
|
|
661
|
-
).hexdigest()
|
|
662
|
-
target["snapshotDigest"] = f"codex-security-snapshot/v1:sha256:{digest}"
|
|
1261
|
+
target["snapshotDigest"] = committed_diff_snapshot_digest(
|
|
1262
|
+
diff_kind, target["baseRevision"], target["headRevision"]
|
|
1263
|
+
)
|
|
663
1264
|
manifest = (
|
|
664
1265
|
copy.deepcopy(parent_manifest)
|
|
665
1266
|
if parent_manifest
|
|
@@ -668,6 +1269,27 @@ def merge_saved_results(
|
|
|
668
1269
|
for key in ("sealedAt", "artifacts"):
|
|
669
1270
|
manifest["scan"].pop(key, None)
|
|
670
1271
|
manifest["scan"]["preservedSources"] = source_digests
|
|
1272
|
+
# Completion follows the selected parent, including legacy terminal drafts
|
|
1273
|
+
# that omit the optional marker, rather than an older canonical manifest.
|
|
1274
|
+
if parent is not None:
|
|
1275
|
+
manifest["scan"].pop("complete", None)
|
|
1276
|
+
if "complete" in parent:
|
|
1277
|
+
manifest["scan"]["complete"] = parent["complete"]
|
|
1278
|
+
for key in ("scope", "threatModel"):
|
|
1279
|
+
if isinstance(parent.get(key), dict):
|
|
1280
|
+
manifest["scan"][key] = copy.deepcopy(parent[key])
|
|
1281
|
+
if isinstance(manifest["scan"].get("scope"), dict):
|
|
1282
|
+
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
|
|
1283
|
+
if frozen_model_source is not None:
|
|
1284
|
+
# Publication retries retain the choice made with the frozen source set.
|
|
1285
|
+
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
|
|
1286
|
+
if not isinstance(model, dict):
|
|
1287
|
+
raise ContractError("Frozen stopped-scan model source is unavailable.")
|
|
1288
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1289
|
+
if paths[frozen_model_source] is not None:
|
|
1290
|
+
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1291
|
+
if selected_model_source is not None:
|
|
1292
|
+
selected_model_source[:] = [frozen_model_source]
|
|
671
1293
|
coverage = (
|
|
672
1294
|
copy.deepcopy(parent["coverage"])
|
|
673
1295
|
if parent and parent["coverage"]
|
|
@@ -696,7 +1318,7 @@ def merge_saved_results(
|
|
|
696
1318
|
for field in ("surfaces", "explicitExclusions", "deferred")
|
|
697
1319
|
for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
|
|
698
1320
|
}
|
|
699
|
-
if
|
|
1321
|
+
if parent_is_canonical
|
|
700
1322
|
else set()
|
|
701
1323
|
)
|
|
702
1324
|
findings: list[dict[str, Any]] = []
|
|
@@ -725,32 +1347,180 @@ def merge_saved_results(
|
|
|
725
1347
|
)
|
|
726
1348
|
return bool(document["findings"])
|
|
727
1349
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
(
|
|
1350
|
+
source_order["parent"] = (0, parent_modified)
|
|
1351
|
+
deferred_rows = {
|
|
1352
|
+
relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
|
|
1353
|
+
}
|
|
1354
|
+
# A legacy source can contain independent tasks with the same explicit ID.
|
|
1355
|
+
# Later rewrites cannot make an ID-only closure identify one of those tasks.
|
|
1356
|
+
ambiguous_deferred: set[tuple[str | None, str]] = set()
|
|
1357
|
+
for relative, _, owner in all_sources:
|
|
1358
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1359
|
+
candidate_aliases = {
|
|
1360
|
+
identity
|
|
1361
|
+
for row in deferred_rows[relative]
|
|
1362
|
+
if isinstance(row, dict)
|
|
1363
|
+
and any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1364
|
+
for identity in (row.get("id"), row.get("candidateId"))
|
|
1365
|
+
if isinstance(identity, str)
|
|
1366
|
+
}
|
|
1367
|
+
for row in deferred_rows[relative]:
|
|
1368
|
+
if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
|
|
1369
|
+
continue
|
|
1370
|
+
if any(key in row for key in ("candidateId", "candidate", "finding")):
|
|
1371
|
+
continue
|
|
1372
|
+
if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
|
|
1373
|
+
ambiguous_deferred.add((owner, identity))
|
|
1374
|
+
by_id[identity] = row
|
|
1375
|
+
current_drafts = ([("parent", parent, None)] if parent else []) + [
|
|
1376
|
+
source for source in sources if source[0] in current_results | selected_observations.keys()
|
|
1377
|
+
]
|
|
1378
|
+
# Generic closures belong to one logical scan or worker, just like candidates.
|
|
1379
|
+
# Keep them when recovering a terminal checkpoint without its canonical write.
|
|
1380
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1381
|
+
|
|
1382
|
+
candidate_ids: set[tuple[str | None, str]] = set()
|
|
1383
|
+
accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1384
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1385
|
+
coverage_schema = _read_json(
|
|
1386
|
+
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
1387
|
+
)["properties"]
|
|
1388
|
+
for relative, draft, owner in all_sources:
|
|
1389
|
+
order = source_order[relative]
|
|
1390
|
+
for item in deferred_rows[relative]:
|
|
1391
|
+
if not isinstance(item, dict):
|
|
1392
|
+
continue
|
|
1393
|
+
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
|
|
1394
|
+
candidate_ids.update(
|
|
1395
|
+
(owner, identity)
|
|
1396
|
+
for identity in (item.get("id"), item.get("candidateId"))
|
|
1397
|
+
if isinstance(identity, str)
|
|
1398
|
+
)
|
|
1399
|
+
identity = item.get("id") or item.get("candidateId")
|
|
1400
|
+
if isinstance(identity, str):
|
|
1401
|
+
key = (owner, identity)
|
|
1402
|
+
previous = active_deferred.get(key)
|
|
1403
|
+
if previous is None or order > previous[0]:
|
|
1404
|
+
active_deferred[key] = (order, item, relative)
|
|
1405
|
+
if (
|
|
1406
|
+
relative == "parent"
|
|
1407
|
+
or relative in current_results
|
|
1408
|
+
or relative in selected_observations
|
|
1409
|
+
):
|
|
1410
|
+
try:
|
|
1411
|
+
# A malformed update cannot discard valid saved evidence.
|
|
1412
|
+
_validate_schema_node(
|
|
1413
|
+
item, coverage_schema["deferred"]["items"], "coverage.deferred"
|
|
1414
|
+
)
|
|
1415
|
+
except ContractError:
|
|
1416
|
+
continue
|
|
1417
|
+
accepted_deferred_orders[key] = max(
|
|
1418
|
+
accepted_deferred_orders.get(key, order), order
|
|
1419
|
+
)
|
|
1420
|
+
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
|
|
1421
|
+
key = (owner, closure["id"])
|
|
1422
|
+
previous = closed_deferred.get(key)
|
|
1423
|
+
if previous is None or order > previous[0]:
|
|
1424
|
+
# The parent comes first, preserving its reason on equal timestamps.
|
|
1425
|
+
closed_deferred[key] = (order, closure, relative)
|
|
1426
|
+
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
|
|
1427
|
+
reopened_generic: set[tuple[str | None, str]] = set()
|
|
1428
|
+
for key, (order, _, _) in list(closed_deferred.items()):
|
|
1429
|
+
# Equal timestamps cannot distinguish closure from reopened work.
|
|
1430
|
+
reopened = [
|
|
1431
|
+
active
|
|
1432
|
+
for (owner, identity), active in active_deferred.items()
|
|
1433
|
+
if owner == key[0]
|
|
1434
|
+
and (identity == key[1] or active[1].get("candidateId") == key[1])
|
|
1435
|
+
and active[0] >= order
|
|
1436
|
+
]
|
|
1437
|
+
if key in candidate_ids or key in ambiguous_deferred or reopened:
|
|
1438
|
+
del closed_deferred[key]
|
|
1439
|
+
for _, item, _ in reopened:
|
|
1440
|
+
reopened_rows.append((key[0], item))
|
|
1441
|
+
if key not in candidate_ids:
|
|
1442
|
+
reopened_generic.add(key)
|
|
1443
|
+
# Retain every distinct ambiguous task even when a terminal result supersedes
|
|
1444
|
+
# its source. Existing output ID normalization keeps each row distinct.
|
|
1445
|
+
for relative, _, owner in all_sources:
|
|
1446
|
+
for row in deferred_rows[relative]:
|
|
1447
|
+
if (
|
|
1448
|
+
isinstance(row, dict)
|
|
1449
|
+
and isinstance(identity := row.get("id"), str)
|
|
1450
|
+
and (owner, identity) in ambiguous_deferred
|
|
1451
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1452
|
+
and (owner, row) not in reopened_rows
|
|
1453
|
+
):
|
|
1454
|
+
reopened_rows.append((owner, row))
|
|
1455
|
+
parent_closures = [
|
|
1456
|
+
closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
|
|
731
1457
|
]
|
|
1458
|
+
coverage.pop("resolvedDeferred", None)
|
|
1459
|
+
if parent_closures:
|
|
1460
|
+
coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
|
|
1461
|
+
if parent:
|
|
1462
|
+
coverage["deferred"] = [
|
|
1463
|
+
row
|
|
1464
|
+
for row in deferred_rows["parent"]
|
|
1465
|
+
if not isinstance(row, dict)
|
|
1466
|
+
or not isinstance(row.get("id"), str)
|
|
1467
|
+
or (None, row["id"]) not in closed_deferred
|
|
1468
|
+
]
|
|
732
1469
|
resolved: dict[tuple[str | None, str], str] = {}
|
|
733
|
-
|
|
1470
|
+
|
|
1471
|
+
ordered_candidates = {
|
|
1472
|
+
(owner, identity)
|
|
1473
|
+
for owner, row in reopened_rows
|
|
1474
|
+
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
|
|
1475
|
+
and (owner, identity) in candidate_ids
|
|
1476
|
+
}
|
|
1477
|
+
ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
|
|
1478
|
+
|
|
1479
|
+
outcomes: list[tuple[str, str | None, str, str]] = []
|
|
1480
|
+
for relative, draft, owner in current_drafts:
|
|
734
1481
|
for finding in draft["findings"]:
|
|
735
1482
|
if (
|
|
736
1483
|
isinstance(finding, dict)
|
|
737
1484
|
and valid_finding(finding)
|
|
738
1485
|
and (candidate_id := finding_candidate_id(finding))
|
|
739
1486
|
):
|
|
740
|
-
|
|
1487
|
+
outcomes.append((relative, owner, candidate_id, "reported"))
|
|
741
1488
|
for field in ("surfaces", "explicitExclusions"):
|
|
742
1489
|
items = draft["coverage"].get(field, [])
|
|
743
1490
|
for item in items if isinstance(items, list) else []:
|
|
744
1491
|
if (
|
|
745
1492
|
isinstance(item, dict)
|
|
746
1493
|
and isinstance(item.get("candidateId"), str)
|
|
747
|
-
and item.get("disposition") in {"
|
|
1494
|
+
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
748
1495
|
):
|
|
749
|
-
|
|
1496
|
+
outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
|
|
1497
|
+
ordered_candidates.update(
|
|
1498
|
+
(owner, candidate_id)
|
|
1499
|
+
for relative, owner, candidate_id, _ in outcomes
|
|
1500
|
+
if owner is not None and relative in selected_observations
|
|
1501
|
+
)
|
|
1502
|
+
# Reopened work and selected checkpoint outcomes follow the saved source order.
|
|
1503
|
+
for relative, owner, candidate_id, disposition in outcomes:
|
|
1504
|
+
key = (owner, candidate_id)
|
|
1505
|
+
if key not in ordered_candidates:
|
|
1506
|
+
if relative == "parent" or relative in current_results:
|
|
1507
|
+
resolved.setdefault(key, disposition)
|
|
1508
|
+
continue
|
|
1509
|
+
order = source_order[relative]
|
|
1510
|
+
if any(
|
|
1511
|
+
saved_owner == owner
|
|
1512
|
+
and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
|
|
1513
|
+
and modified >= order
|
|
1514
|
+
for (saved_owner, _), (modified, row, _) in active_deferred.items()
|
|
1515
|
+
):
|
|
1516
|
+
continue
|
|
1517
|
+
if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
|
|
1518
|
+
resolved[key] = disposition
|
|
1519
|
+
ordered_outcomes[key] = (order, relative)
|
|
750
1520
|
# Only the current parent may claim that another worker finding was absorbed.
|
|
751
1521
|
# A superseded checkpoint must not suppress a newer independent result.
|
|
752
|
-
|
|
753
|
-
for finding in
|
|
1522
|
+
if parent:
|
|
1523
|
+
for finding in parent["findings"]:
|
|
754
1524
|
if valid_finding(finding):
|
|
755
1525
|
canonical_key = _finding_key(finding)
|
|
756
1526
|
for retained in _retained_findings(finding):
|
|
@@ -787,26 +1557,155 @@ def merge_saved_results(
|
|
|
787
1557
|
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
788
1558
|
_digest(_finding_content(original["finding"]))
|
|
789
1559
|
)
|
|
790
|
-
|
|
1560
|
+
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1561
|
+
all_sources,
|
|
1562
|
+
source_order,
|
|
1563
|
+
closed_deferred,
|
|
1564
|
+
active_deferred,
|
|
1565
|
+
resolved,
|
|
1566
|
+
reopened_generic,
|
|
1567
|
+
coverage_schema["surfaces"]["items"],
|
|
1568
|
+
deferred_rows,
|
|
1569
|
+
ambiguous_deferred,
|
|
1570
|
+
)
|
|
1571
|
+
replaced_rows = {
|
|
1572
|
+
"surfaces": replaced_surfaces,
|
|
1573
|
+
"deferred": {
|
|
1574
|
+
id(row)
|
|
1575
|
+
for relative, _, owner in all_sources
|
|
1576
|
+
for row in deferred_rows[relative]
|
|
1577
|
+
if isinstance(row, dict)
|
|
1578
|
+
and isinstance(identity := row.get("id"), str)
|
|
1579
|
+
and (key := (owner, identity)) not in candidate_ids
|
|
1580
|
+
and key not in ambiguous_deferred
|
|
1581
|
+
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1582
|
+
and updated > source_order[relative]
|
|
1583
|
+
},
|
|
1584
|
+
}
|
|
1585
|
+
for field, replaced in replaced_rows.items():
|
|
1586
|
+
if parent and isinstance(coverage.get(field), list):
|
|
1587
|
+
previous = parent["coverage"].get(field, [])
|
|
1588
|
+
if isinstance(previous, list):
|
|
1589
|
+
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1590
|
+
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1591
|
+
if isinstance(coverage.get("surfaces"), list):
|
|
1592
|
+
for surface in surface_updates:
|
|
1593
|
+
if surface not in coverage["surfaces"]:
|
|
1594
|
+
coverage["surfaces"].append(surface)
|
|
1595
|
+
for surface in coverage["surfaces"]:
|
|
1596
|
+
if isinstance(surface, dict):
|
|
1597
|
+
surface.setdefault("receiptRefs", [])
|
|
1598
|
+
|
|
1599
|
+
# Reopened work survives a superseded checkpoint, but current candidate
|
|
1600
|
+
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
1601
|
+
for owner, item in reopened_rows:
|
|
1602
|
+
if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
|
|
1603
|
+
owner,
|
|
1604
|
+
identity,
|
|
1605
|
+
) in resolved:
|
|
1606
|
+
continue
|
|
1607
|
+
pending = coverage.setdefault("deferred", [])
|
|
1608
|
+
if isinstance(pending, list) and item not in pending:
|
|
1609
|
+
pending.append(copy.deepcopy(item))
|
|
1610
|
+
ambiguous_surface_ids = {
|
|
1611
|
+
(owner, identity)
|
|
1612
|
+
for owner, row in reopened_rows
|
|
1613
|
+
if isinstance(row.get("id"), str)
|
|
1614
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
1615
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1616
|
+
for identity in [
|
|
1617
|
+
row["id"],
|
|
1618
|
+
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1619
|
+
]
|
|
1620
|
+
if isinstance(identity, str)
|
|
1621
|
+
}
|
|
1622
|
+
for _, draft, owner in all_sources:
|
|
1623
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1624
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1625
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1626
|
+
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1627
|
+
continue
|
|
1628
|
+
if identity in by_id and surface != by_id[identity]:
|
|
1629
|
+
ambiguous_surface_ids.add((owner, identity))
|
|
1630
|
+
by_id[identity] = surface
|
|
1631
|
+
for _, draft, owner in all_sources:
|
|
1632
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1633
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1634
|
+
if (
|
|
1635
|
+
isinstance(surface, dict)
|
|
1636
|
+
and isinstance(surface.get("id"), str)
|
|
1637
|
+
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
1638
|
+
and surface.get("disposition") == "needs_follow_up"
|
|
1639
|
+
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
1640
|
+
and isinstance(coverage.get("surfaces"), list)
|
|
1641
|
+
):
|
|
1642
|
+
retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
|
|
1643
|
+
if retained_surface not in coverage["surfaces"]:
|
|
1644
|
+
coverage["surfaces"].append(copy.deepcopy(retained_surface))
|
|
1645
|
+
selected_terminal_orders: dict[str, tuple[int, int]] = {}
|
|
1646
|
+
for relative, draft, worker_id in sources:
|
|
1647
|
+
if (
|
|
1648
|
+
worker_id is not None
|
|
1649
|
+
and relative in selected_observations
|
|
1650
|
+
and draft.get("complete") is not False
|
|
1651
|
+
):
|
|
1652
|
+
order = source_order[relative]
|
|
1653
|
+
selected_terminal_orders[worker_id] = max(
|
|
1654
|
+
selected_terminal_orders.get(worker_id, order), order
|
|
1655
|
+
)
|
|
1656
|
+
terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
|
|
1657
|
+
for relative, draft, worker_id in sources:
|
|
1658
|
+
if relative in current_results and draft.get("complete") is not False:
|
|
1659
|
+
order = source_order[relative]
|
|
1660
|
+
terminal_worker_orders[worker_id] = max(
|
|
1661
|
+
terminal_worker_orders.get(worker_id, order), order
|
|
1662
|
+
)
|
|
791
1663
|
for relative, draft, worker_id in all_sources:
|
|
1664
|
+
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
1665
|
+
selected_coverage_superseded = worker_id in selected_terminal_orders and (
|
|
1666
|
+
source_order[relative] < selected_terminal_orders[worker_id]
|
|
1667
|
+
or (
|
|
1668
|
+
relative in current_results
|
|
1669
|
+
and source_order[relative] <= selected_terminal_orders[worker_id]
|
|
1670
|
+
)
|
|
1671
|
+
)
|
|
1672
|
+
selected_candidates = {
|
|
1673
|
+
candidate_id
|
|
1674
|
+
for (owner, candidate_id), (_, source) in ordered_outcomes.items()
|
|
1675
|
+
if owner == worker_id and source == relative
|
|
1676
|
+
}
|
|
792
1677
|
superseded = (
|
|
793
1678
|
worker_id is None
|
|
794
1679
|
and parent is not None
|
|
795
1680
|
and parent.get("complete") is not False
|
|
796
1681
|
and relative != "parent"
|
|
1682
|
+
and source_order[relative] <= (0, parent_modified)
|
|
797
1683
|
and (not stopped_parent_seal or relative in parent_preserved_sources)
|
|
798
1684
|
) or (
|
|
799
1685
|
relative not in current_results
|
|
800
|
-
and
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
for saved_path, current, saved_worker in sources
|
|
1686
|
+
and worker_result_order is not None
|
|
1687
|
+
and (
|
|
1688
|
+
relative not in selected_observations
|
|
1689
|
+
or source_order[relative] < worker_result_order
|
|
805
1690
|
)
|
|
806
1691
|
)
|
|
1692
|
+
# A failed result write can leave pending work outside the accepted result.
|
|
1693
|
+
accepted_order = (
|
|
1694
|
+
(0, parent_modified)
|
|
1695
|
+
if worker_id is None and parent is not None
|
|
1696
|
+
else worker_result_order
|
|
1697
|
+
)
|
|
1698
|
+
retain_pending = (
|
|
1699
|
+
superseded
|
|
1700
|
+
and (worker_id in headed_workers or (worker_id is None and parent_heads))
|
|
1701
|
+
and accepted_order is not None
|
|
1702
|
+
and source_order[relative] >= accepted_order
|
|
1703
|
+
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
1704
|
+
)
|
|
807
1705
|
if (
|
|
808
|
-
(relative != "parent" or not
|
|
1706
|
+
(relative != "parent" or not parent_is_canonical)
|
|
809
1707
|
and not superseded
|
|
1708
|
+
and not selected_coverage_superseded
|
|
810
1709
|
and (
|
|
811
1710
|
draft.get("complete") is False
|
|
812
1711
|
or draft["coverage"].get("completeness") != "complete"
|
|
@@ -814,16 +1713,56 @@ def merge_saved_results(
|
|
|
814
1713
|
and coverage.get("completeness") in {"complete", "unknown"}
|
|
815
1714
|
):
|
|
816
1715
|
coverage["completeness"] = "partial"
|
|
817
|
-
|
|
1716
|
+
skip_superseded_findings = (
|
|
818
1717
|
superseded
|
|
819
1718
|
and not stopped
|
|
820
1719
|
and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
|
|
821
|
-
)
|
|
1720
|
+
)
|
|
1721
|
+
if skip_superseded_findings and not selected_candidates and not retain_pending:
|
|
822
1722
|
continue
|
|
823
|
-
if
|
|
824
|
-
|
|
1723
|
+
if (
|
|
1724
|
+
not skip_superseded_findings
|
|
1725
|
+
and "threatModel" not in manifest["scan"]
|
|
1726
|
+
and isinstance(draft.get("threatModel"), dict)
|
|
1727
|
+
):
|
|
1728
|
+
model = draft["threatModel"]
|
|
1729
|
+
model_path = relative
|
|
1730
|
+
checkpoint_dir = Path(relative).parent
|
|
1731
|
+
prefer_worker_head = worker_id is not None and (
|
|
1732
|
+
checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
|
|
1733
|
+
)
|
|
1734
|
+
if worker_id is not None:
|
|
1735
|
+
if checkpoint_dir.name != "checkpoints":
|
|
1736
|
+
checkpoint_dir /= "checkpoints"
|
|
1737
|
+
selected_models = [
|
|
1738
|
+
path
|
|
1739
|
+
for path in selected_observations
|
|
1740
|
+
if Path(path).parent == checkpoint_dir
|
|
1741
|
+
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
1742
|
+
]
|
|
1743
|
+
if selected_models:
|
|
1744
|
+
head_path = max(selected_models, key=source_order.__getitem__)
|
|
1745
|
+
current = drafts_by_path[head_path]
|
|
1746
|
+
# A terminal checkpoint is committed before result.json is replaced.
|
|
1747
|
+
# Use the admitted observation, including its frozen ordering on retries.
|
|
1748
|
+
if not prefer_worker_head and current.get("complete") is not False:
|
|
1749
|
+
prefer_worker_head = source_order[head_path] >= source_order[relative]
|
|
1750
|
+
if prefer_worker_head:
|
|
1751
|
+
model = current["threatModel"]
|
|
1752
|
+
model_path = head_path
|
|
1753
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1754
|
+
if worker_id is not None:
|
|
1755
|
+
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1756
|
+
if selected_model_source is not None and worker_id is not None:
|
|
1757
|
+
selected_model_source[:] = [model_path]
|
|
825
1758
|
for value in draft["findings"]:
|
|
826
|
-
if
|
|
1759
|
+
if skip_superseded_findings and not (
|
|
1760
|
+
isinstance(value, dict)
|
|
1761
|
+
and (candidate_id := finding_candidate_id(value)) in selected_candidates
|
|
1762
|
+
and resolved.get((worker_id, candidate_id)) == "reported"
|
|
1763
|
+
):
|
|
1764
|
+
continue
|
|
1765
|
+
if relative == "parent" and parent_is_canonical:
|
|
827
1766
|
finding = copy.deepcopy(value)
|
|
828
1767
|
_ensure_finding_identity(finding, candidate_only=True)
|
|
829
1768
|
provenance = finding.get("provenance") if isinstance(finding, dict) else None
|
|
@@ -846,7 +1785,6 @@ def merge_saved_results(
|
|
|
846
1785
|
if not isinstance(value, dict):
|
|
847
1786
|
warnings.append(f"Retained malformed finding evidence in {relative}.")
|
|
848
1787
|
continue
|
|
849
|
-
source_value = copy.deepcopy(value)
|
|
850
1788
|
finding = copy.deepcopy(value)
|
|
851
1789
|
candidate_id = finding_candidate_id(finding)
|
|
852
1790
|
if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
|
|
@@ -909,24 +1847,19 @@ def merge_saved_results(
|
|
|
909
1847
|
historical_contents = set()
|
|
910
1848
|
if mapped_key is not None:
|
|
911
1849
|
key = mapped_key
|
|
912
|
-
represented_by_parent = (
|
|
913
|
-
_digest(_finding_content(source_value)) in historical_contents
|
|
914
|
-
)
|
|
1850
|
+
represented_by_parent = _digest(_finding_content(value)) in historical_contents
|
|
915
1851
|
if key in finding_positions:
|
|
916
1852
|
retained = findings[finding_positions[key]]
|
|
917
1853
|
if finding != retained:
|
|
918
|
-
if represented_by_parent
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
"previousFindings", []
|
|
922
|
-
)
|
|
923
|
-
elif _finding_strength(finding) > _finding_strength(retained):
|
|
1854
|
+
if not represented_by_parent and _finding_strength(finding) > _finding_strength(
|
|
1855
|
+
retained
|
|
1856
|
+
):
|
|
924
1857
|
previous = copy.deepcopy(retained)
|
|
925
1858
|
previous_history = previous["provenance"].pop("previousFindings", [])
|
|
926
1859
|
retained = finding
|
|
927
1860
|
findings[finding_positions[key]] = retained
|
|
928
1861
|
else:
|
|
929
|
-
previous = copy.deepcopy(
|
|
1862
|
+
previous = copy.deepcopy(value)
|
|
930
1863
|
previous_history = previous.get("provenance", {}).pop(
|
|
931
1864
|
"previousFindings", []
|
|
932
1865
|
)
|
|
@@ -960,10 +1893,14 @@ def merge_saved_results(
|
|
|
960
1893
|
continue
|
|
961
1894
|
finding_positions[key] = len(findings)
|
|
962
1895
|
findings.append(finding)
|
|
963
|
-
if superseded:
|
|
1896
|
+
if superseded and not selected_candidates and not retain_pending:
|
|
964
1897
|
continue
|
|
965
1898
|
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
1899
|
+
if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
|
|
1900
|
+
continue
|
|
966
1901
|
items = draft["coverage"].get(field, [])
|
|
1902
|
+
if field == "deferred":
|
|
1903
|
+
items = deferred_rows[relative]
|
|
967
1904
|
if not isinstance(items, list):
|
|
968
1905
|
continue
|
|
969
1906
|
output = coverage.setdefault(field, [])
|
|
@@ -972,6 +1909,40 @@ def merge_saved_results(
|
|
|
972
1909
|
# recovery and warnings rather than silently changing its contract.
|
|
973
1910
|
continue
|
|
974
1911
|
for item in items:
|
|
1912
|
+
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
1913
|
+
# candidate evidence and pending tasks retain their own reconciliation.
|
|
1914
|
+
if (
|
|
1915
|
+
selected_coverage_superseded
|
|
1916
|
+
and field != "deferred"
|
|
1917
|
+
and not (
|
|
1918
|
+
isinstance(item, dict)
|
|
1919
|
+
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1920
|
+
)
|
|
1921
|
+
):
|
|
1922
|
+
continue
|
|
1923
|
+
# A selected outcome must retain its evidence even if its result write failed.
|
|
1924
|
+
if superseded and not (
|
|
1925
|
+
isinstance(item, dict)
|
|
1926
|
+
and (
|
|
1927
|
+
(field == "deferred" and retain_pending)
|
|
1928
|
+
or (
|
|
1929
|
+
isinstance(item.get("candidateId"), str)
|
|
1930
|
+
and item["candidateId"] in selected_candidates
|
|
1931
|
+
and item.get("disposition")
|
|
1932
|
+
== resolved.get((worker_id, item["candidateId"]))
|
|
1933
|
+
)
|
|
1934
|
+
)
|
|
1935
|
+
):
|
|
1936
|
+
continue
|
|
1937
|
+
if id(item) in replaced_rows.get(field, ()):
|
|
1938
|
+
continue
|
|
1939
|
+
if (
|
|
1940
|
+
field == "deferred"
|
|
1941
|
+
and isinstance(item, dict)
|
|
1942
|
+
and isinstance(item.get("id"), str)
|
|
1943
|
+
and (worker_id, item["id"]) in closed_deferred
|
|
1944
|
+
):
|
|
1945
|
+
continue
|
|
975
1946
|
if field == "openQuestions" and isinstance(item, str):
|
|
976
1947
|
item = {"question": item.strip()}
|
|
977
1948
|
if (
|
|
@@ -995,14 +1966,20 @@ def merge_saved_results(
|
|
|
995
1966
|
history.append(copy.deepcopy(finding))
|
|
996
1967
|
if (
|
|
997
1968
|
isinstance(item, dict)
|
|
998
|
-
and (
|
|
1969
|
+
and isinstance(
|
|
1970
|
+
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1971
|
+
if field == "deferred"
|
|
1972
|
+
else item.get("candidateId"),
|
|
1973
|
+
str,
|
|
1974
|
+
)
|
|
1975
|
+
and (worker_id, identity) in resolved
|
|
999
1976
|
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1000
1977
|
):
|
|
1001
1978
|
continue
|
|
1979
|
+
if field == "surfaces" and isinstance(item, dict):
|
|
1980
|
+
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
1002
1981
|
if isinstance(item, dict) and "id" not in item:
|
|
1003
1982
|
semantic_item = dict(item)
|
|
1004
|
-
if field == "surfaces":
|
|
1005
|
-
semantic_item.setdefault("receiptRefs", [])
|
|
1006
1983
|
if any(
|
|
1007
1984
|
isinstance(existing, dict)
|
|
1008
1985
|
and {key: value for key, value in existing.items() if key != "id"}
|
|
@@ -1036,7 +2013,10 @@ def merge_saved_results(
|
|
|
1036
2013
|
if isinstance(item.get("id"), str):
|
|
1037
2014
|
used.add(item["id"])
|
|
1038
2015
|
continue
|
|
1039
|
-
item.setdefault("id",
|
|
2016
|
+
item.setdefault("id", _saved_coverage_id(item))
|
|
2017
|
+
if not isinstance(item["id"], str):
|
|
2018
|
+
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
2019
|
+
continue
|
|
1040
2020
|
if item["id"] in used:
|
|
1041
2021
|
item["id"] = f"{item['id']}-{_digest(item)[:16]}"
|
|
1042
2022
|
used.add(item["id"])
|
|
@@ -1081,6 +2061,9 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
|
|
|
1081
2061
|
except ContractError:
|
|
1082
2062
|
path = scan_dir / relative
|
|
1083
2063
|
if path.exists() or path.is_symlink():
|
|
2064
|
+
if relative == "threatmodel.md":
|
|
2065
|
+
# This optional projection will not replace an unsafe destination.
|
|
2066
|
+
continue
|
|
1084
2067
|
raise
|
|
1085
2068
|
snapshots[relative] = None
|
|
1086
2069
|
finally:
|
|
@@ -1112,13 +2095,17 @@ def preserve_scan_results_locked(
|
|
|
1112
2095
|
if scan["status"] != "failed":
|
|
1113
2096
|
return False
|
|
1114
2097
|
frozen_source_digests: dict[str, str] | None = None
|
|
2098
|
+
model_source: list[str] = []
|
|
2099
|
+
saved_model_source: str | None = None
|
|
1115
2100
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
2101
|
+
if raw_frozen_sources is not None:
|
|
2102
|
+
frozen_source_digests, saved_model_source = _retained_source_state(
|
|
2103
|
+
json.loads(raw_frozen_sources)
|
|
2104
|
+
)
|
|
2105
|
+
if saved_model_source is not None:
|
|
2106
|
+
model_source.append(saved_model_source)
|
|
1116
2107
|
if recovery_source_digests is not None:
|
|
1117
2108
|
frozen_source_digests = recovery_source_digests
|
|
1118
|
-
elif raw_frozen_sources is not None:
|
|
1119
|
-
frozen_source_digests = _source_digests(
|
|
1120
|
-
json.loads(raw_frozen_sources), "Saved stopped-scan"
|
|
1121
|
-
)
|
|
1122
2109
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1123
2110
|
deep_run = connection.execute(
|
|
1124
2111
|
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
@@ -1141,12 +2128,10 @@ def preserve_scan_results_locked(
|
|
|
1141
2128
|
]
|
|
1142
2129
|
|
|
1143
2130
|
def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> None:
|
|
1144
|
-
retained_sources =
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
):
|
|
1149
|
-
raise ContractError("Stopped scan source digests could not be frozen.")
|
|
2131
|
+
retained_sources = _source_digests(
|
|
2132
|
+
manifest.get("scan", {}).get("preservedSources"),
|
|
2133
|
+
"Stopped scan source digests could not be frozen.",
|
|
2134
|
+
)
|
|
1150
2135
|
digest = db.published_manifest_digest(scan_dir, manifest)
|
|
1151
2136
|
timestamp = db.now()
|
|
1152
2137
|
with connection:
|
|
@@ -1176,7 +2161,7 @@ def preserve_scan_results_locked(
|
|
|
1176
2161
|
"updated_at = ? WHERE id = ? AND status = 'failed'",
|
|
1177
2162
|
(
|
|
1178
2163
|
digest,
|
|
1179
|
-
|
|
2164
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
1180
2165
|
json.dumps(list(dict.fromkeys(warnings))),
|
|
1181
2166
|
timestamp,
|
|
1182
2167
|
scan_id,
|
|
@@ -1190,6 +2175,7 @@ def preserve_scan_results_locked(
|
|
|
1190
2175
|
|
|
1191
2176
|
existing_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
1192
2177
|
existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
|
|
2178
|
+
existing = None
|
|
1193
2179
|
if scan["seal_manifest_digest"] is not None or (
|
|
1194
2180
|
isinstance(existing_scan, dict)
|
|
1195
2181
|
and (
|
|
@@ -1198,30 +2184,35 @@ def preserve_scan_results_locked(
|
|
|
1198
2184
|
):
|
|
1199
2185
|
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
1200
2186
|
existing, existing_findings, _ = finalize_scan(
|
|
1201
|
-
scan_dir,
|
|
2187
|
+
scan_dir,
|
|
2188
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
2189
|
+
projection_warnings=warnings,
|
|
1202
2190
|
)
|
|
1203
2191
|
db.verify_manifest_binding(scan, existing)
|
|
1204
2192
|
if existing_scan.get("status") == outcome:
|
|
1205
2193
|
existing_sources = existing_scan.get("preservedSources")
|
|
1206
2194
|
if frozen_source_digests is None:
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
):
|
|
1211
|
-
raise ContractError("Stopped scan source digests could not be frozen.")
|
|
1212
|
-
frozen_source_digests = existing_sources
|
|
2195
|
+
frozen_source_digests = _source_digests(
|
|
2196
|
+
existing_sources, "Stopped scan source digests could not be frozen."
|
|
2197
|
+
)
|
|
1213
2198
|
if existing_sources == frozen_source_digests:
|
|
1214
2199
|
if (
|
|
1215
2200
|
raw_frozen_sources is not None
|
|
1216
2201
|
and scan["seal_manifest_digest"] is not None
|
|
1217
2202
|
and not publication_follow_up_warnings
|
|
2203
|
+
and warnings == stored_warnings
|
|
1218
2204
|
):
|
|
1219
2205
|
return True
|
|
1220
2206
|
record_publication(existing, existing_findings)
|
|
1221
2207
|
return True
|
|
1222
2208
|
if recovery_source_digests is None:
|
|
1223
2209
|
raise ContractError("Stopped scan sources changed after terminal publication.")
|
|
1224
|
-
binding = {
|
|
2210
|
+
binding = {
|
|
2211
|
+
**db.workbench_completion_binding(scan, scan["completed_at"], existing),
|
|
2212
|
+
"status": outcome,
|
|
2213
|
+
}
|
|
2214
|
+
if recovery_source_digests is not None:
|
|
2215
|
+
model_source.clear()
|
|
1225
2216
|
documents = merge_saved_results(
|
|
1226
2217
|
scan_dir,
|
|
1227
2218
|
scan_id,
|
|
@@ -1237,6 +2228,8 @@ def preserve_scan_results_locked(
|
|
|
1237
2228
|
f"{scan['failure_message'] or ''}"
|
|
1238
2229
|
).strip(),
|
|
1239
2230
|
frozen_source_digests=frozen_source_digests,
|
|
2231
|
+
frozen_model_source=model_source[0] if model_source else None,
|
|
2232
|
+
selected_model_source=model_source,
|
|
1240
2233
|
allow_frozen_legacy_parent=(
|
|
1241
2234
|
include_parent_with_recovery
|
|
1242
2235
|
or (
|
|
@@ -1258,18 +2251,22 @@ def preserve_scan_results_locked(
|
|
|
1258
2251
|
(json.dumps(unpublished_warnings), db.now(), scan_id),
|
|
1259
2252
|
)
|
|
1260
2253
|
return False
|
|
1261
|
-
if frozen_source_digests is None
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
2254
|
+
if frozen_source_digests is None or (
|
|
2255
|
+
recovery_source_digests is None and saved_model_source is None and model_source
|
|
2256
|
+
):
|
|
2257
|
+
retained_sources = _source_digests(
|
|
2258
|
+
documents[0].get("scan", {}).get("preservedSources"),
|
|
2259
|
+
"Stopped scan source digests could not be frozen.",
|
|
2260
|
+
)
|
|
1268
2261
|
with connection:
|
|
1269
2262
|
connection.execute(
|
|
1270
2263
|
"UPDATE scans SET retained_source_digests_json = ? "
|
|
1271
|
-
"WHERE id = ? AND retained_source_digests_json IS
|
|
1272
|
-
(
|
|
2264
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
2265
|
+
(
|
|
2266
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
2267
|
+
scan_id,
|
|
2268
|
+
raw_frozen_sources,
|
|
2269
|
+
),
|
|
1273
2270
|
)
|
|
1274
2271
|
prepared = _prepare_scan_finalization(
|
|
1275
2272
|
scan_dir,
|
|
@@ -1280,7 +2277,9 @@ def preserve_scan_results_locked(
|
|
|
1280
2277
|
)
|
|
1281
2278
|
snapshots = _snapshot_published_outputs(scan_dir)
|
|
1282
2279
|
try:
|
|
1283
|
-
manifest, findings, _ = _write_prepared_scan_finalization(
|
|
2280
|
+
manifest, findings, _ = _write_prepared_scan_finalization(
|
|
2281
|
+
prepared, projection_warnings=warnings
|
|
2282
|
+
)
|
|
1284
2283
|
db.verify_manifest_binding(scan, manifest)
|
|
1285
2284
|
record_publication(manifest, findings)
|
|
1286
2285
|
except BaseException:
|
|
@@ -1399,6 +2398,28 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1399
2398
|
"The scan stopped; its saved checkpoint was retained without replacing sealed results."
|
|
1400
2399
|
)
|
|
1401
2400
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2401
|
+
if (
|
|
2402
|
+
args.expected_draft_digest is not None
|
|
2403
|
+
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
2404
|
+
):
|
|
2405
|
+
raise SystemExit(
|
|
2406
|
+
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
2407
|
+
)
|
|
2408
|
+
try:
|
|
2409
|
+
relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
|
|
2410
|
+
except ValueError as exc:
|
|
2411
|
+
raise SystemExit(
|
|
2412
|
+
"Scan draft must be inside the registered scan drafts directory."
|
|
2413
|
+
) from exc
|
|
2414
|
+
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
2415
|
+
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
2416
|
+
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
2417
|
+
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
|
|
2418
|
+
binding = db.workbench_completion_binding(scan, db.now())
|
|
2419
|
+
# Save scan IDs without sealing the draft.
|
|
2420
|
+
_populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
|
|
2421
|
+
_populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
|
|
2422
|
+
_validate_completion_binding(manifest, findings, coverage, binding)
|
|
1402
2423
|
if args.checkpoint_path is not None:
|
|
1403
2424
|
try:
|
|
1404
2425
|
checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix()
|
|
@@ -1421,33 +2442,15 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1421
2442
|
f"checkpoints/{checkpoint_digest}.json",
|
|
1422
2443
|
checkpoint_contents,
|
|
1423
2444
|
)
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
|
|
1433
|
-
except ValueError as exc:
|
|
1434
|
-
raise SystemExit(
|
|
1435
|
-
"Scan draft must be inside the registered scan drafts directory."
|
|
1436
|
-
) from exc
|
|
1437
|
-
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
1438
|
-
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
1439
|
-
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
1440
|
-
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
|
|
1441
|
-
binding = db.workbench_completion_binding(scan, db.now())
|
|
1442
|
-
# Validate on copies: saved canonical documents remain ordinary unsealed drafts.
|
|
1443
|
-
copied_manifest = copy.deepcopy(manifest)
|
|
1444
|
-
copied_findings = copy.deepcopy(findings)
|
|
1445
|
-
copied_coverage = copy.deepcopy(coverage)
|
|
1446
|
-
_populate_unsealed_manifest_envelope(copied_manifest, copied_manifest["scan"], binding)
|
|
1447
|
-
_populate_unsealed_artifact_envelope(
|
|
1448
|
-
copied_manifest, copied_findings, copied_coverage, binding
|
|
2445
|
+
checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
|
|
2446
|
+
checkpoint_contents = _encoded(checkpoint)
|
|
2447
|
+
checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
|
|
2448
|
+
checkpoint_relative = f"checkpoints/{checkpoint_name}"
|
|
2449
|
+
if not (scan_dir / checkpoint_relative).exists():
|
|
2450
|
+
write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
|
|
2451
|
+
write_scan_local_bytes(
|
|
2452
|
+
scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
|
|
1449
2453
|
)
|
|
1450
|
-
_validate_completion_binding(copied_manifest, copied_findings, copied_coverage, binding)
|
|
1451
2454
|
for filename, document in (
|
|
1452
2455
|
("findings.json", findings),
|
|
1453
2456
|
("coverage.json", coverage),
|
|
@@ -1458,9 +2461,17 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1458
2461
|
filename,
|
|
1459
2462
|
(json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
|
|
1460
2463
|
)
|
|
2464
|
+
model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
|
|
1461
2465
|
# Accepted Standard drafts are evidence of review or report assembly,
|
|
1462
2466
|
# even when the parent omitted its explicit progress call.
|
|
1463
|
-
|
|
2467
|
+
model_only_checkpoint = (
|
|
2468
|
+
manifest["scan"].get("complete") is False
|
|
2469
|
+
and isinstance(manifest["scan"].get("threatModel"), dict)
|
|
2470
|
+
and not findings.get("findings")
|
|
2471
|
+
and not coverage.get("surfaces")
|
|
2472
|
+
and not coverage.get("deferred")
|
|
2473
|
+
)
|
|
2474
|
+
if scan["mode"] == "standard" and not model_only_checkpoint:
|
|
1464
2475
|
phase = "discovery" if manifest["scan"].get("complete") is False else "reporting"
|
|
1465
2476
|
earlier = PHASES[: PHASES.index(phase)]
|
|
1466
2477
|
placeholders = ",".join("?" for _ in earlier)
|
|
@@ -1481,7 +2492,11 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1481
2492
|
)
|
|
1482
2493
|
except sqlite3.Error as exc:
|
|
1483
2494
|
print(f"Could not save scan progress: {exc}", file=sys.stderr)
|
|
1484
|
-
return {
|
|
2495
|
+
return {
|
|
2496
|
+
"scanId": scan_id,
|
|
2497
|
+
"status": "draft_written",
|
|
2498
|
+
**({"warnings": [model_warning]} if model_warning else {}),
|
|
2499
|
+
}
|
|
1485
2500
|
|
|
1486
2501
|
|
|
1487
2502
|
def _scan_draft_digest(scan_dir: Path) -> str:
|
|
@@ -1494,9 +2509,10 @@ def _scan_draft_digest(scan_dir: Path) -> str:
|
|
|
1494
2509
|
except FileNotFoundError:
|
|
1495
2510
|
digest.update(b"missing\0")
|
|
1496
2511
|
continue
|
|
1497
|
-
_, contents = _read_scan_local_json_bytes(scan_dir, filename, filename)
|
|
1498
2512
|
digest.update(b"present\0")
|
|
1499
|
-
|
|
2513
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
|
|
2514
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
2515
|
+
digest.update(handle.read())
|
|
1500
2516
|
digest.update(b"\0")
|
|
1501
2517
|
return digest.hexdigest()
|
|
1502
2518
|
|