@openai/codex-security 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -36
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +45 -189
- package/_bundled_plugin/references/scan-contract.md +2 -0
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
- package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
- package/_bundled_plugin/scripts/report_projection.py +44 -15
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +51 -22
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +53 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
- package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +44 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -30
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +134 -0
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +8 -4
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +8 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +414 -72
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/classify-severity-records.d.ts +5 -2
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +40 -34
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +5 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +20 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +1 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +802 -149
- package/dist/cli.js.map +1 -1
- package/dist/cloud-endpoint.d.ts +6 -0
- package/dist/cloud-endpoint.d.ts.map +1 -0
- package/dist/cloud-endpoint.js +51 -0
- package/dist/cloud-endpoint.js.map +1 -0
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +7 -5
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +67 -33
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +30 -4
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +7 -5
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +3 -0
- package/dist/default-severity-rubric.d.ts.map +1 -0
- package/dist/default-severity-rubric.js +67 -0
- package/dist/default-severity-rubric.js.map +1 -0
- package/dist/endor-findings.d.ts +8 -0
- package/dist/endor-findings.d.ts.map +1 -0
- package/dist/endor-findings.js +238 -0
- package/dist/endor-findings.js.map +1 -0
- package/dist/external-findings-import.d.ts +5 -0
- package/dist/external-findings-import.d.ts.map +1 -0
- package/dist/external-findings-import.js +46 -0
- package/dist/external-findings-import.js.map +1 -0
- package/dist/external-findings-local.d.ts +63 -0
- package/dist/external-findings-local.d.ts.map +1 -0
- package/dist/external-findings-local.js +344 -0
- package/dist/external-findings-local.js.map +1 -0
- package/dist/external-findings-publish.d.ts +73 -0
- package/dist/external-findings-publish.d.ts.map +1 -0
- package/dist/external-findings-publish.js +721 -0
- package/dist/external-findings-publish.js.map +1 -0
- package/dist/external-findings-read.d.ts +16 -0
- package/dist/external-findings-read.d.ts.map +1 -0
- package/dist/external-findings-read.js +97 -0
- package/dist/external-findings-read.js.map +1 -0
- package/dist/external-findings.d.ts +21 -0
- package/dist/external-findings.d.ts.map +1 -0
- package/dist/external-findings.js +2 -0
- package/dist/external-findings.js.map +1 -0
- package/dist/external-import-contract.d.ts +15 -0
- package/dist/external-import-contract.d.ts.map +1 -0
- package/dist/external-import-contract.js +262 -0
- package/dist/external-import-contract.js.map +1 -0
- package/dist/external-import-models.d.ts +362 -0
- package/dist/external-import-models.d.ts.map +1 -0
- package/dist/external-import-models.js +3 -0
- package/dist/external-import-models.js.map +1 -0
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +23 -6
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +23 -10
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts +386 -0
- package/dist/github-advisories.d.ts.map +1 -0
- package/dist/github-advisories.js +133 -0
- package/dist/github-advisories.js.map +1 -0
- package/dist/github-findings.d.ts +11 -0
- package/dist/github-findings.d.ts.map +1 -0
- package/dist/github-findings.js +258 -0
- package/dist/github-findings.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +2 -1
- package/dist/hosted-remediation-protocol.d.ts.map +1 -1
- package/dist/hosted-remediation-protocol.js +17 -11
- package/dist/hosted-remediation-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts +4 -0
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +22 -3
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +16 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +9 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +1 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +17 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/models.d.ts +1 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -0
- package/dist/project-config.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +7 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +18 -28
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +10 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +10 -8
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +8 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +8 -0
- package/dist/scan-owner-suggestions.d.ts.map +1 -0
- package/dist/scan-owner-suggestions.js +33 -0
- package/dist/scan-owner-suggestions.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +20 -0
- package/dist/scanner-import.d.ts.map +1 -0
- package/dist/scanner-import.js +208 -0
- package/dist/scanner-import.js.map +1 -0
- package/dist/security-policy-diff.d.ts +2 -0
- package/dist/security-policy-diff.d.ts.map +1 -0
- package/dist/security-policy-diff.js +197 -0
- package/dist/security-policy-diff.js.map +1 -0
- package/dist/security-policy.d.ts +7 -3
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +11 -76
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +10 -0
- package/dist/semgrep-findings.d.ts.map +1 -0
- package/dist/semgrep-findings.js +293 -0
- package/dist/semgrep-findings.js.map +1 -0
- package/dist/severity-review.d.ts +3 -0
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +19 -0
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-store.js +7 -2
- package/dist/severity-store.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +10 -0
- package/dist/snyk-findings.d.ts.map +1 -0
- package/dist/snyk-findings.js +350 -0
- package/dist/snyk-findings.js.map +1 -0
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +2 -2
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +16 -3
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +9 -0
- package/dist/wiz-csv.d.ts.map +1 -0
- package/dist/wiz-csv.js +116 -0
- package/dist/wiz-csv.js.map +1 -0
- package/dist/wiz-findings.d.ts +5 -0
- package/dist/wiz-findings.d.ts.map +1 -0
- package/dist/wiz-findings.js +464 -0
- package/dist/wiz-findings.js.map +1 -0
- package/docs/classify-severity-records.md +30 -13
- package/docs/cli.md +835 -47
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +43 -24
- package/docs/hosted-scans.md +38 -0
- package/package.json +11 -9
- package/schemas/external-findings.schema.json +1421 -0
- package/schemas/project-config.schema.json +4 -0
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -0,0 +1,3218 @@
|
|
|
1
|
+
"""Retain semantic scan drafts without rerunning analysis or changing run state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
8
|
+
from collections import Counter, deque
|
|
9
|
+
from functools import partial
|
|
10
|
+
from pathlib import Path, PurePosixPath
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from finalize_scan_contract import (
|
|
14
|
+
ContractError,
|
|
15
|
+
_finding_strength,
|
|
16
|
+
_read_json,
|
|
17
|
+
_recover_unsealed_findings,
|
|
18
|
+
_require_safe_json_value,
|
|
19
|
+
_validate_schema_node,
|
|
20
|
+
finding_candidate_id,
|
|
21
|
+
write_scan_local_bytes,
|
|
22
|
+
)
|
|
23
|
+
from workbench.retained_records import (
|
|
24
|
+
_append_previous_finding,
|
|
25
|
+
_competing_rejection_receipts,
|
|
26
|
+
_conflicting_outcome_receipts,
|
|
27
|
+
_ensure_finding_identities,
|
|
28
|
+
_ensure_finding_identity,
|
|
29
|
+
_finding_content,
|
|
30
|
+
_finding_identity,
|
|
31
|
+
_finding_key,
|
|
32
|
+
_identity_candidate,
|
|
33
|
+
_recover_preserved_worker_coverage,
|
|
34
|
+
_recover_preserved_worker_source,
|
|
35
|
+
_retained_findings,
|
|
36
|
+
_saved_candidate_outcomes,
|
|
37
|
+
_semantic_identity_key,
|
|
38
|
+
_worker_candidate_key,
|
|
39
|
+
contains_raw_value,
|
|
40
|
+
distinct_reports,
|
|
41
|
+
identifiers,
|
|
42
|
+
raw_owner,
|
|
43
|
+
raw_scope,
|
|
44
|
+
same_raw_content,
|
|
45
|
+
same_raw_finding,
|
|
46
|
+
same_raw_value,
|
|
47
|
+
)
|
|
48
|
+
from workbench.saved_coverage import (
|
|
49
|
+
_coverage_rows,
|
|
50
|
+
_has_candidate_metadata,
|
|
51
|
+
_inherit_coverage_sources,
|
|
52
|
+
_merge_tied_parent_observations,
|
|
53
|
+
_parent_observation_content,
|
|
54
|
+
_project_worker_closure,
|
|
55
|
+
_projected_worker_closure_sources,
|
|
56
|
+
_resolved_deferred_rows,
|
|
57
|
+
prepare_saved_coverage_projection,
|
|
58
|
+
)
|
|
59
|
+
from workbench.saved_sources import (
|
|
60
|
+
_capture_parent_observation,
|
|
61
|
+
_capture_saved_source,
|
|
62
|
+
_checkpoint_head_directory,
|
|
63
|
+
_checkpoint_paths,
|
|
64
|
+
_CheckpointSnapshotError,
|
|
65
|
+
_digest,
|
|
66
|
+
_encoded,
|
|
67
|
+
_freeze_source_times,
|
|
68
|
+
_frozen_checkpoint_heads,
|
|
69
|
+
_frozen_source_times,
|
|
70
|
+
_is_source_order_snapshot,
|
|
71
|
+
_latest_successful_reducer,
|
|
72
|
+
_model_capture,
|
|
73
|
+
_parent_publication_ancestors,
|
|
74
|
+
_read_saved_parent_result,
|
|
75
|
+
_read_saved_result,
|
|
76
|
+
_unselected_root_order,
|
|
77
|
+
_worker_outputs,
|
|
78
|
+
)
|
|
79
|
+
from workbench_coverage_receipts import (
|
|
80
|
+
_deferred_candidate_id,
|
|
81
|
+
_generic_surface_updates,
|
|
82
|
+
receipt_checkpoints,
|
|
83
|
+
resolve_coverage_receipts,
|
|
84
|
+
retain_recorded_receipt_ownership,
|
|
85
|
+
)
|
|
86
|
+
from workbench_target import committed_diff_snapshot_digest
|
|
87
|
+
from workbench_validation import javascript_utf8, path_within_scope
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def merge_saved_results(
|
|
91
|
+
scan_dir: Path,
|
|
92
|
+
scan_id: str,
|
|
93
|
+
binding: dict[str, Any],
|
|
94
|
+
workers: list[Any],
|
|
95
|
+
warnings: list[str],
|
|
96
|
+
*,
|
|
97
|
+
stopped: bool,
|
|
98
|
+
reason: str,
|
|
99
|
+
scan_mode: str | None = None,
|
|
100
|
+
frozen_source_digests: dict[str, str] | None = None,
|
|
101
|
+
checkpoint_heads: dict[str, str | None] | None = None,
|
|
102
|
+
allow_frozen_legacy_parent: bool = False,
|
|
103
|
+
parent_already_captured: bool = False,
|
|
104
|
+
frozen_model_source: str | None = None,
|
|
105
|
+
selected_model_source: list[str] | None = None,
|
|
106
|
+
canonical_model: dict[str, Any] | None = None,
|
|
107
|
+
canonical_model_source: str | None = None,
|
|
108
|
+
canonical_sources: dict[str, str] | None = None,
|
|
109
|
+
finding_scope: list[tuple[str, bool]] | None = None,
|
|
110
|
+
preserve_canonical_parent: bool = False,
|
|
111
|
+
project_worker_coverage: bool = False,
|
|
112
|
+
created_paths: set[str] | None = None,
|
|
113
|
+
current_contract: bool = False,
|
|
114
|
+
recovery_snapshot_error: _CheckpointSnapshotError | None = None,
|
|
115
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
116
|
+
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
117
|
+
initial_warnings = set(warnings)
|
|
118
|
+
if frozen_source_digests is None:
|
|
119
|
+
checkpoint_heads = {}
|
|
120
|
+
snapshot_error = recovery_snapshot_error
|
|
121
|
+
captured_records = dict(snapshot_error.records) if snapshot_error is not None else {}
|
|
122
|
+
|
|
123
|
+
def capture_head(relative: str, canonical: Path | None = None) -> dict[str, tuple[str, int]]:
|
|
124
|
+
nonlocal snapshot_error
|
|
125
|
+
try:
|
|
126
|
+
if canonical is not None:
|
|
127
|
+
return _capture_parent_observation(
|
|
128
|
+
scan_dir,
|
|
129
|
+
scan_id,
|
|
130
|
+
canonical.name,
|
|
131
|
+
max(parent_modified, canonical.lstat().st_mtime_ns),
|
|
132
|
+
allow_frozen_legacy_parent,
|
|
133
|
+
created_paths=created_paths,
|
|
134
|
+
)
|
|
135
|
+
return _capture_saved_source(
|
|
136
|
+
scan_dir,
|
|
137
|
+
relative,
|
|
138
|
+
scan_id,
|
|
139
|
+
checkpoint_heads=checkpoint_heads,
|
|
140
|
+
created_paths=created_paths,
|
|
141
|
+
)
|
|
142
|
+
except _CheckpointSnapshotError as exc:
|
|
143
|
+
if "status" not in binding:
|
|
144
|
+
# A running coordinator can continue from its remaining evidence.
|
|
145
|
+
if checkpoint_heads is not None:
|
|
146
|
+
checkpoint_heads.pop(Path(relative).parent.as_posix(), None)
|
|
147
|
+
raise
|
|
148
|
+
# Finish selection from captured bytes before freezing an I/O failure.
|
|
149
|
+
snapshot_error = snapshot_error or exc
|
|
150
|
+
captured_records.update(exc.records)
|
|
151
|
+
return exc.captured
|
|
152
|
+
|
|
153
|
+
parent_context = dict(recovery_snapshot_error.parent_context) if recovery_snapshot_error else {}
|
|
154
|
+
try:
|
|
155
|
+
source_times = _frozen_source_times(
|
|
156
|
+
scan_dir, scan_id, frozen_source_digests or {}, parent_context
|
|
157
|
+
)
|
|
158
|
+
if snapshot_error is not None:
|
|
159
|
+
source_times.update({path: item[1] for path, item in snapshot_error.captured.items()})
|
|
160
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
161
|
+
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
162
|
+
parent: dict[str, Any] | None = None
|
|
163
|
+
frozen_parent_path: str | None = None
|
|
164
|
+
parent_manifest: dict[str, Any] | None = None
|
|
165
|
+
parent_is_canonical = False
|
|
166
|
+
parent_modified = 0
|
|
167
|
+
parent_model_source: str | None = None
|
|
168
|
+
canonical_model_order: tuple[int, int] | None = None
|
|
169
|
+
canonical_model_digest: str | None = None
|
|
170
|
+
if frozen_source_digests is None or (
|
|
171
|
+
allow_frozen_legacy_parent and not parent_already_captured
|
|
172
|
+
):
|
|
173
|
+
try:
|
|
174
|
+
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
175
|
+
# Without an accepted head, file-authored coverage is a full replacement.
|
|
176
|
+
parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
|
|
177
|
+
parent_is_canonical = True
|
|
178
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
179
|
+
if not stopped:
|
|
180
|
+
raise
|
|
181
|
+
if (scan_dir / "scan-manifest.json").exists():
|
|
182
|
+
warnings.append(f"Could not read the saved parent draft: {exc}")
|
|
183
|
+
parent_manifest = None
|
|
184
|
+
parent = None
|
|
185
|
+
if parent_manifest is not None and parent is not None and not preserve_canonical_parent:
|
|
186
|
+
parent_scan = parent_manifest["scan"]
|
|
187
|
+
try:
|
|
188
|
+
_, _, head_modified = _read_saved_result(scan_dir, "checkpoint-head.json", scan_id)
|
|
189
|
+
except (ContractError, OSError, ValueError):
|
|
190
|
+
head_modified = None
|
|
191
|
+
if head_modified is not None:
|
|
192
|
+
# A partial tool publication must not outrank its accepted head.
|
|
193
|
+
parent_modified = min(
|
|
194
|
+
(scan_dir / name).lstat().st_mtime_ns
|
|
195
|
+
for name in ("findings.json", "coverage.json", "scan-manifest.json")
|
|
196
|
+
)
|
|
197
|
+
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
198
|
+
payload = _encoded(parent)
|
|
199
|
+
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
200
|
+
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
201
|
+
if isinstance(parent.get("threatModel"), dict):
|
|
202
|
+
canonical_model_source = parent_checkpoint
|
|
203
|
+
parent_model_source = parent_checkpoint
|
|
204
|
+
canonical_model_order = (0, parent_modified)
|
|
205
|
+
canonical_model_digest = parent_digest
|
|
206
|
+
parent_context["canonical"] = parent_checkpoint
|
|
207
|
+
parent_context["canonicalObservedAtNs"] = str(parent_modified)
|
|
208
|
+
checkpoint_path = scan_dir / parent_checkpoint
|
|
209
|
+
if not checkpoint_path.exists():
|
|
210
|
+
if created_paths is not None and not os.path.lexists(checkpoint_path):
|
|
211
|
+
created_paths.add(parent_checkpoint)
|
|
212
|
+
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
213
|
+
# A recovery copy must not appear newer than the review it copies.
|
|
214
|
+
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
|
|
215
|
+
captured = capture_head(
|
|
216
|
+
"checkpoint-head.json", checkpoint_path if head_modified is None else None
|
|
217
|
+
)
|
|
218
|
+
if captured and (head_modified is not None or allow_frozen_legacy_parent):
|
|
219
|
+
if checkpoint_heads is not None:
|
|
220
|
+
checkpoint_heads["."] = next(iter(captured))
|
|
221
|
+
parent_context["head"] = next(iter(captured))
|
|
222
|
+
if frozen_source_digests is not None:
|
|
223
|
+
frozen_source_digests = {
|
|
224
|
+
**frozen_source_digests,
|
|
225
|
+
parent_checkpoint: parent_digest,
|
|
226
|
+
**{path: value[0] for path, value in captured.items()},
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
230
|
+
parent_preserved_sources: dict[str, str] = {}
|
|
231
|
+
source_digests: dict[str, str] = {}
|
|
232
|
+
source_order: dict[str, tuple[int, int]] = {}
|
|
233
|
+
worker_attempts: dict[str, tuple[str, int]] = {}
|
|
234
|
+
saved_heads: dict[str, str] = {}
|
|
235
|
+
saved_inputs: dict[str, str] = {}
|
|
236
|
+
if parent_manifest:
|
|
237
|
+
recorded = parent_manifest["scan"].get("preservedSources", {})
|
|
238
|
+
if isinstance(recorded, dict):
|
|
239
|
+
parent_preserved_sources = recorded
|
|
240
|
+
source_digests.update(parent_preserved_sources)
|
|
241
|
+
if frozen_source_digests is None:
|
|
242
|
+
source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
|
|
243
|
+
paths: dict[str, str | None] = {}
|
|
244
|
+
reducer_paths: set[str] = set()
|
|
245
|
+
current_results: set[str] = set()
|
|
246
|
+
reducer_outputs: list[tuple[Any, str, list[str], int]] = []
|
|
247
|
+
reducer = _latest_successful_reducer(workers)
|
|
248
|
+
latest_reducer: str | None = None
|
|
249
|
+
if reducer is not None:
|
|
250
|
+
try:
|
|
251
|
+
latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
252
|
+
paths[latest_reducer] = None
|
|
253
|
+
reducer_paths.add(latest_reducer)
|
|
254
|
+
except ValueError:
|
|
255
|
+
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
256
|
+
|
|
257
|
+
def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
|
|
258
|
+
if worker_id is not None:
|
|
259
|
+
root = Path(directory).parent.as_posix()
|
|
260
|
+
worker_attempts[root] = (worker_id, attempt)
|
|
261
|
+
paths[f"{root}/checkpoint-head.json"] = worker_id
|
|
262
|
+
head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
|
|
263
|
+
for saved_directory in (directory, head_snapshots):
|
|
264
|
+
paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
|
|
265
|
+
|
|
266
|
+
if not preserve_canonical_parent:
|
|
267
|
+
paths["checkpoint-head.json"] = None
|
|
268
|
+
checkpoints("checkpoints", None)
|
|
269
|
+
for worker in workers:
|
|
270
|
+
try:
|
|
271
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
272
|
+
except (TypeError, ValueError):
|
|
273
|
+
warnings.append("Skipped a worker checkpoint outside the scan directory.")
|
|
274
|
+
continue
|
|
275
|
+
if worker["kind"] == "dedup":
|
|
276
|
+
for directory, attempt in outputs:
|
|
277
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
278
|
+
if not checkpoint_paths:
|
|
279
|
+
continue
|
|
280
|
+
result_path = f"{directory}/result.json"
|
|
281
|
+
retained_paths = [result_path, *checkpoint_paths]
|
|
282
|
+
paths.update(dict.fromkeys(retained_paths))
|
|
283
|
+
reducer_paths.update(retained_paths)
|
|
284
|
+
reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
|
|
285
|
+
continue
|
|
286
|
+
if worker["kind"] != "discovery":
|
|
287
|
+
continue
|
|
288
|
+
output, attempt = outputs[0]
|
|
289
|
+
paths[f"{output}/result.json"] = worker["id"]
|
|
290
|
+
current_results.add(f"{output}/result.json")
|
|
291
|
+
for archived, archived_attempt in outputs[1:]:
|
|
292
|
+
paths[f"{archived}/result.json"] = worker["id"]
|
|
293
|
+
checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
|
|
294
|
+
checkpoints(f"{output}/checkpoints", worker["id"], attempt)
|
|
295
|
+
if worker["result_manifest_path"]:
|
|
296
|
+
try:
|
|
297
|
+
current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
298
|
+
paths[current_path] = worker["id"]
|
|
299
|
+
current_results.add(current_path)
|
|
300
|
+
except ValueError:
|
|
301
|
+
warnings.append("Skipped a worker result outside the scan directory.")
|
|
302
|
+
|
|
303
|
+
if frozen_source_digests is None:
|
|
304
|
+
for relative, worker_id in list(paths.items()):
|
|
305
|
+
if Path(relative).name != "checkpoint-head.json":
|
|
306
|
+
continue
|
|
307
|
+
del paths[relative]
|
|
308
|
+
try:
|
|
309
|
+
captured = capture_head(relative)
|
|
310
|
+
if relative == "checkpoint-head.json" and frozen_source_digests is None:
|
|
311
|
+
parent_context["head"] = next(iter(captured))
|
|
312
|
+
paths.update({path: worker_id for path in captured})
|
|
313
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
314
|
+
if (scan_dir / relative).exists() and not (
|
|
315
|
+
relative == "checkpoint-head.json" and parent is not None
|
|
316
|
+
):
|
|
317
|
+
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
318
|
+
|
|
319
|
+
if frozen_source_digests is not None:
|
|
320
|
+
source_digests.update(
|
|
321
|
+
{
|
|
322
|
+
path: digest
|
|
323
|
+
for path, digest in frozen_source_digests.items()
|
|
324
|
+
if _is_source_order_snapshot(path)
|
|
325
|
+
}
|
|
326
|
+
)
|
|
327
|
+
paths = {
|
|
328
|
+
relative: worker_id
|
|
329
|
+
for relative, worker_id in paths.items()
|
|
330
|
+
if relative in frozen_source_digests
|
|
331
|
+
}
|
|
332
|
+
current_results.intersection_update(frozen_source_digests)
|
|
333
|
+
if latest_reducer not in frozen_source_digests:
|
|
334
|
+
latest_reducer = None
|
|
335
|
+
|
|
336
|
+
discarded_by_source: dict[str, list[str]] = {}
|
|
337
|
+
pending = deque(paths.items())
|
|
338
|
+
while pending:
|
|
339
|
+
relative, worker_id = pending.popleft()
|
|
340
|
+
try:
|
|
341
|
+
draft, digest, observed = captured_records.get(relative) or _read_saved_result(
|
|
342
|
+
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
343
|
+
)
|
|
344
|
+
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
345
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
346
|
+
source_digests[relative] = digest
|
|
347
|
+
source_times.setdefault(relative, observed)
|
|
348
|
+
source_order[relative] = (0, source_times[relative])
|
|
349
|
+
directory = Path(relative).parent
|
|
350
|
+
if directory.name in {"checkpoints", "checkpoint-heads"}:
|
|
351
|
+
directory = directory.parent
|
|
352
|
+
if _checkpoint_head_directory(relative) is not None:
|
|
353
|
+
saved_heads[relative] = draft["checkpoint"]
|
|
354
|
+
if isinstance(draft.get("inputCheckpoint"), str):
|
|
355
|
+
saved_inputs[relative] = draft["inputCheckpoint"]
|
|
356
|
+
continue
|
|
357
|
+
if (
|
|
358
|
+
checkpoint_heads is not None
|
|
359
|
+
and checkpoint_heads.get(directory.as_posix(), relative) is None
|
|
360
|
+
):
|
|
361
|
+
continue
|
|
362
|
+
if preserve_canonical_parent and worker_id is not None:
|
|
363
|
+
discarded = _recover_preserved_worker_source(
|
|
364
|
+
scan_dir,
|
|
365
|
+
relative,
|
|
366
|
+
scan_id,
|
|
367
|
+
binding,
|
|
368
|
+
worker_id,
|
|
369
|
+
next(worker for worker in workers if worker["id"] == worker_id),
|
|
370
|
+
draft,
|
|
371
|
+
warnings,
|
|
372
|
+
)
|
|
373
|
+
if discarded:
|
|
374
|
+
discarded_by_source[relative] = discarded
|
|
375
|
+
# Recovery expects coverage, but reducer results only contain findings
|
|
376
|
+
# and context. Add an empty value after hashing the original result.
|
|
377
|
+
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
378
|
+
if frozen_source_digests is None:
|
|
379
|
+
pending.extend(
|
|
380
|
+
receipt_checkpoints(
|
|
381
|
+
relative, worker_id, paths, partial(_checkpoint_paths, scan_dir)
|
|
382
|
+
).items()
|
|
383
|
+
)
|
|
384
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
385
|
+
if (scan_dir / relative).exists():
|
|
386
|
+
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
387
|
+
if frozen_source_digests is not None:
|
|
388
|
+
if frozen_source_digests.keys() - source_digests.keys():
|
|
389
|
+
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
|
|
390
|
+
|
|
391
|
+
if checkpoint_heads is None:
|
|
392
|
+
checkpoint_heads = _frozen_checkpoint_heads(scan_dir, scan_id, source_digests)
|
|
393
|
+
heads_snapshotted = False
|
|
394
|
+
if frozen_source_digests is not None and snapshot_error is None:
|
|
395
|
+
# Raw heads are a temporary fallback, not permanent publication sources.
|
|
396
|
+
for relative in list(saved_heads):
|
|
397
|
+
if Path(relative).name != "checkpoint-head.json":
|
|
398
|
+
continue
|
|
399
|
+
captured = _capture_saved_source(
|
|
400
|
+
scan_dir,
|
|
401
|
+
relative,
|
|
402
|
+
scan_id,
|
|
403
|
+
expected_digest=source_digests[relative],
|
|
404
|
+
created_paths=created_paths,
|
|
405
|
+
)
|
|
406
|
+
for path, (digest, observed) in captured.items():
|
|
407
|
+
if path in source_digests and source_digests[path] != digest:
|
|
408
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
409
|
+
source_digests[path] = digest
|
|
410
|
+
source_times.setdefault(path, observed)
|
|
411
|
+
if _checkpoint_head_directory(path) is not None:
|
|
412
|
+
saved_heads[path] = saved_heads[relative]
|
|
413
|
+
if relative in saved_inputs:
|
|
414
|
+
saved_inputs[path] = saved_inputs[relative]
|
|
415
|
+
if parent_context.get("head") == relative:
|
|
416
|
+
parent_context["head"] = path
|
|
417
|
+
source_order[path] = (0, observed)
|
|
418
|
+
key = Path(relative).parent.as_posix()
|
|
419
|
+
if checkpoint_heads.get(key) == relative:
|
|
420
|
+
checkpoint_heads[key] = path
|
|
421
|
+
saved_inputs.pop(relative, None)
|
|
422
|
+
del saved_heads[relative], source_digests[relative]
|
|
423
|
+
heads_snapshotted = True
|
|
424
|
+
if heads_snapshotted:
|
|
425
|
+
source_digests = {
|
|
426
|
+
path: digest
|
|
427
|
+
for path, digest in source_digests.items()
|
|
428
|
+
if not _is_source_order_snapshot(path)
|
|
429
|
+
}
|
|
430
|
+
headed_workers = {
|
|
431
|
+
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
|
|
432
|
+
for head in saved_heads
|
|
433
|
+
if _checkpoint_head_directory(head) != Path(".")
|
|
434
|
+
}
|
|
435
|
+
for relative, _, worker_id in sources:
|
|
436
|
+
if worker_id not in headed_workers:
|
|
437
|
+
continue
|
|
438
|
+
directory = Path(relative).parent
|
|
439
|
+
if directory.name == "checkpoints":
|
|
440
|
+
directory = directory.parent
|
|
441
|
+
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
|
|
442
|
+
source_order[relative] = (attempt, source_order[relative][1])
|
|
443
|
+
selected_observations: dict[str, tuple[int, int]] = {}
|
|
444
|
+
authoritative_heads: set[str] = set()
|
|
445
|
+
authoritative_observations: dict[str, tuple[int, int]] = {}
|
|
446
|
+
parent_heads: list[tuple[int, str]] = []
|
|
447
|
+
for head, checkpoint in saved_heads.items():
|
|
448
|
+
directory = _checkpoint_head_directory(head)
|
|
449
|
+
selected = (directory / "checkpoints" / checkpoint).as_posix()
|
|
450
|
+
if selected not in source_order:
|
|
451
|
+
raise ContractError("Checkpoint head is outside the saved source set.")
|
|
452
|
+
key = directory.as_posix()
|
|
453
|
+
if key in checkpoint_heads and checkpoint_heads[key] is None:
|
|
454
|
+
continue
|
|
455
|
+
observed = source_order[head][1]
|
|
456
|
+
attempt = 0 if directory == Path(".") else worker_attempts[key][1]
|
|
457
|
+
order = (attempt, observed)
|
|
458
|
+
if key not in checkpoint_heads or checkpoint_heads[key] in (head, selected):
|
|
459
|
+
authoritative_heads.add(selected)
|
|
460
|
+
if checkpoint_heads.get(key) == head:
|
|
461
|
+
authoritative_observations[selected] = order
|
|
462
|
+
if directory == Path("."):
|
|
463
|
+
parent_heads.append((observed, selected))
|
|
464
|
+
selected_observations[selected] = max(selected_observations.get(selected, order), order)
|
|
465
|
+
# A registered result independently retains its latest accepted observation.
|
|
466
|
+
# Restoring the head does not withdraw that result's publication authority.
|
|
467
|
+
registered_result_orders = {
|
|
468
|
+
path: order for path, order in selected_observations.items() if path in current_results
|
|
469
|
+
}
|
|
470
|
+
# A restored head selects one observation, not the newest time for its payload.
|
|
471
|
+
selected_observations.update(authoritative_observations)
|
|
472
|
+
source_order.update(selected_observations)
|
|
473
|
+
source_order.update(registered_result_orders)
|
|
474
|
+
|
|
475
|
+
# Keep frozen source digests over the original data, before discarding markers
|
|
476
|
+
# that never passed through the worker draft writer.
|
|
477
|
+
retain_recorded_receipt_ownership(
|
|
478
|
+
sources, _digest, binding["scope"]["includePaths"], finding_scope
|
|
479
|
+
)
|
|
480
|
+
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
481
|
+
# A frozen model capture supplies context without replacing earlier findings.
|
|
482
|
+
model_path = frozen_model_source or canonical_model_source
|
|
483
|
+
model_draft = drafts_by_path.get(model_path)
|
|
484
|
+
if (
|
|
485
|
+
model_draft is not None
|
|
486
|
+
and model_path not in selected_observations
|
|
487
|
+
and model_path != parent_context.get("canonical")
|
|
488
|
+
and isinstance(model := model_draft.get("threatModel"), dict)
|
|
489
|
+
and model_draft == _model_capture(scan_id, model)
|
|
490
|
+
):
|
|
491
|
+
model_draft["complete"] = False
|
|
492
|
+
if frozen_source_digests is not None and "canonical" in parent_context:
|
|
493
|
+
canonical = parent_context["canonical"]
|
|
494
|
+
parent = drafts_by_path[canonical]
|
|
495
|
+
parent_model_source = canonical
|
|
496
|
+
parent_modified = int(
|
|
497
|
+
parent_context.get("canonicalObservedAtNs", source_order[canonical][1])
|
|
498
|
+
)
|
|
499
|
+
parent_is_canonical = True
|
|
500
|
+
if "head" in parent_context:
|
|
501
|
+
head = parent_context["head"]
|
|
502
|
+
parent_heads = [(source_order[head][1], f"checkpoints/{saved_heads[head]}")]
|
|
503
|
+
|
|
504
|
+
if parent is None and frozen_source_digests is not None and "canonical" not in parent_context:
|
|
505
|
+
# Older host snapshots remain evidence; only the newest is the current parent.
|
|
506
|
+
saved_parents = {
|
|
507
|
+
relative: draft
|
|
508
|
+
for relative, draft, _ in sources
|
|
509
|
+
if relative.startswith("checkpoints/")
|
|
510
|
+
and isinstance(draft.get("previousParentCheckpoints"), list)
|
|
511
|
+
}
|
|
512
|
+
previous_parents = {
|
|
513
|
+
relative
|
|
514
|
+
for draft in saved_parents.values()
|
|
515
|
+
for relative in draft["previousParentCheckpoints"]
|
|
516
|
+
if isinstance(relative, str)
|
|
517
|
+
}
|
|
518
|
+
for relative, draft in saved_parents.items():
|
|
519
|
+
if relative not in previous_parents:
|
|
520
|
+
frozen_parent_path, parent = relative, draft
|
|
521
|
+
parent_model_source = relative
|
|
522
|
+
parent_is_canonical = draft.get("canonicalParent") is True
|
|
523
|
+
break
|
|
524
|
+
projected_coverage = parent["coverage"] if parent else {}
|
|
525
|
+
# V1 persists the review projection in the parent, without reducer sourceCoverage.
|
|
526
|
+
|
|
527
|
+
coverage_receipts = partial(resolve_coverage_receipts, scan_dir)
|
|
528
|
+
|
|
529
|
+
def canonical_coverage_record(field: str, item: dict[str, Any]) -> dict[str, Any]:
|
|
530
|
+
refs = item.get("receiptRefs", [])
|
|
531
|
+
if field != "surfaces" or not isinstance(refs, list):
|
|
532
|
+
return item
|
|
533
|
+
origin = projection_origins.get(id(item))
|
|
534
|
+
if origin is not None and (worker := workers_by_id.get(origin[0])) is not None:
|
|
535
|
+
refs = coverage_receipts(item, worker, origin[1])
|
|
536
|
+
result = {
|
|
537
|
+
**item,
|
|
538
|
+
"receiptRefs": [Path(ref).as_posix() if isinstance(ref, str) else ref for ref in refs],
|
|
539
|
+
}
|
|
540
|
+
if isinstance(provenance := result.get("provenance"), dict):
|
|
541
|
+
result["provenance"] = {
|
|
542
|
+
key: value for key, value in provenance.items() if key != "scanReceiptRefs"
|
|
543
|
+
}
|
|
544
|
+
return result
|
|
545
|
+
|
|
546
|
+
def retained_coverage_record(field: str, item: dict[str, Any]) -> dict[str, Any] | None:
|
|
547
|
+
return projection.retained_record(coverage, field, item, canonical_coverage_record)
|
|
548
|
+
|
|
549
|
+
reconciled_surface_ids: dict[tuple[str | None, str], str] = {}
|
|
550
|
+
|
|
551
|
+
def project_missing_record(
|
|
552
|
+
field: str,
|
|
553
|
+
item: dict[str, Any],
|
|
554
|
+
index: int,
|
|
555
|
+
worker: Any,
|
|
556
|
+
source: dict[str, Any],
|
|
557
|
+
*,
|
|
558
|
+
receipts_resolved: bool = False,
|
|
559
|
+
) -> dict[str, Any]:
|
|
560
|
+
# Retain projectDiscoveryCoverage worker ownership.
|
|
561
|
+
prefix = f"{worker['id']}-attempt-{worker['attempt']}"
|
|
562
|
+
|
|
563
|
+
def accepted_positions(surface: dict[str, Any]) -> list[int]:
|
|
564
|
+
return [
|
|
565
|
+
position
|
|
566
|
+
for position, accepted in accepted_surfaces.get((worker["id"], surface["id"]), [])
|
|
567
|
+
if canonical_coverage_record("surfaces", accepted)
|
|
568
|
+
== canonical_coverage_record("surfaces", surface)
|
|
569
|
+
]
|
|
570
|
+
|
|
571
|
+
def surface_id(surface: dict[str, Any], offset: int) -> str:
|
|
572
|
+
source_id = surface.get("id")
|
|
573
|
+
if id(source) not in accepted_coverage_sources and isinstance(source_id, str):
|
|
574
|
+
positions = accepted_positions(surface)
|
|
575
|
+
if len(positions) == 1:
|
|
576
|
+
offset = positions[0]
|
|
577
|
+
elif not positions:
|
|
578
|
+
return f"{prefix}-surface-{offset}-{_digest(surface)[:16]}"
|
|
579
|
+
return f"{prefix}-surface-{offset}"
|
|
580
|
+
|
|
581
|
+
result = copy.deepcopy(item)
|
|
582
|
+
provenance = result.get("provenance")
|
|
583
|
+
if not isinstance(provenance, dict):
|
|
584
|
+
provenance = {}
|
|
585
|
+
for key in ("workerId", "attempt", "sourceId", "candidateId"):
|
|
586
|
+
provenance.pop(key, None)
|
|
587
|
+
provenance.update(workerId=worker["id"], attempt=worker["attempt"])
|
|
588
|
+
result["provenance"] = provenance
|
|
589
|
+
source_rows = source.get(field, [])
|
|
590
|
+
inferred_id = (
|
|
591
|
+
isinstance(source_rows, list)
|
|
592
|
+
and index <= len(source_rows)
|
|
593
|
+
and id(source_rows[index - 1]) in inferred_coverage_ids
|
|
594
|
+
)
|
|
595
|
+
for key, name in (("id", "sourceId"), ("candidateId", "candidateId")):
|
|
596
|
+
if key == "id" and inferred_id:
|
|
597
|
+
continue
|
|
598
|
+
if key in item:
|
|
599
|
+
result["provenance"][name] = item[key]
|
|
600
|
+
if isinstance(item.get("candidateId"), str):
|
|
601
|
+
candidate_digest = hashlib.sha256(javascript_utf8(item["candidateId"])).hexdigest()
|
|
602
|
+
result["candidateId"] = f"{prefix}-candidate-{candidate_digest}"
|
|
603
|
+
if field == "surfaces":
|
|
604
|
+
original = source_rows[index - 1] if isinstance(source_rows, list) else item
|
|
605
|
+
result["id"] = surface_id(original, index)
|
|
606
|
+
if not receipts_resolved and (origin := projection_origins.get(id(original))):
|
|
607
|
+
result["receiptRefs"] = coverage_receipts(result, worker, origin[1])
|
|
608
|
+
previous = retained_coverage_record("surfaces", result)
|
|
609
|
+
if previous is not None and isinstance(previous.get("id"), str):
|
|
610
|
+
result["id"] = previous["id"]
|
|
611
|
+
else:
|
|
612
|
+
surfaces = coverage.get("surfaces", [])
|
|
613
|
+
if any(
|
|
614
|
+
isinstance(row, dict) and row.get("id") == result["id"]
|
|
615
|
+
for row in (surfaces if isinstance(surfaces, list) else [])
|
|
616
|
+
):
|
|
617
|
+
result["id"] = f"{result['id']}-{_digest(result)[:16]}"
|
|
618
|
+
elif field == "deferred":
|
|
619
|
+
result["id"] = f"{prefix}-deferred-{index}"
|
|
620
|
+
if isinstance(item.get("surfaceIds"), list):
|
|
621
|
+
surface_ids: dict[str, str] = {}
|
|
622
|
+
surfaces = projected_coverage.get("surfaces", [])
|
|
623
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
624
|
+
if not isinstance(surface, dict) or not isinstance(surface.get("id"), str):
|
|
625
|
+
continue
|
|
626
|
+
provenance = surface.get("provenance", {})
|
|
627
|
+
if (
|
|
628
|
+
isinstance(provenance, dict)
|
|
629
|
+
and provenance.get("workerId") == worker["id"]
|
|
630
|
+
and provenance.get("attempt") == worker["attempt"]
|
|
631
|
+
and isinstance(provenance.get("sourceId"), str)
|
|
632
|
+
):
|
|
633
|
+
surface_ids.setdefault(provenance["sourceId"], surface["id"])
|
|
634
|
+
surfaces = source.get("surfaces", [])
|
|
635
|
+
source_surface_ids: dict[str, tuple[int, dict[str, Any]]] = {}
|
|
636
|
+
for offset, surface in enumerate(surfaces if isinstance(surfaces, list) else [], 1):
|
|
637
|
+
if isinstance(surface, dict) and isinstance(surface.get("id"), str):
|
|
638
|
+
source_surface_ids.setdefault(surface["id"], (offset, surface))
|
|
639
|
+
for (owner, source_id), accepted in accepted_surfaces.items():
|
|
640
|
+
if owner == worker["id"] and accepted:
|
|
641
|
+
source_surface_ids.setdefault(source_id, accepted[0])
|
|
642
|
+
for source_id, (offset, surface) in source_surface_ids.items():
|
|
643
|
+
if source_id not in item["surfaceIds"]:
|
|
644
|
+
continue
|
|
645
|
+
if (worker["id"], source_id) in reconciled_surface_ids:
|
|
646
|
+
surface_ids[source_id] = reconciled_surface_ids[(worker["id"], source_id)]
|
|
647
|
+
continue
|
|
648
|
+
projected_id = surface_id(surface, offset)
|
|
649
|
+
if id(source) in accepted_coverage_sources or accepted_positions(surface):
|
|
650
|
+
origin = projection_origins.get(id(surface))
|
|
651
|
+
projected = project_missing_record(
|
|
652
|
+
"surfaces", surface, offset, worker, origin[4] if origin else source
|
|
653
|
+
)
|
|
654
|
+
if origin := projection_origins.get(id(surface)):
|
|
655
|
+
projected["receiptRefs"] = coverage_receipts(
|
|
656
|
+
projected, worker, origin[1]
|
|
657
|
+
)
|
|
658
|
+
projected_id = projected["id"]
|
|
659
|
+
previous = retained_coverage_record("surfaces", projected)
|
|
660
|
+
if previous is not None and isinstance(previous.get("id"), str):
|
|
661
|
+
projected_id = previous["id"]
|
|
662
|
+
surface_ids[source_id] = projected_id
|
|
663
|
+
result["surfaceIds"] = [
|
|
664
|
+
surface_ids.get(value, value) if isinstance(value, str) else value
|
|
665
|
+
for value in item["surfaceIds"]
|
|
666
|
+
]
|
|
667
|
+
return result
|
|
668
|
+
|
|
669
|
+
workers_by_id = {worker["id"]: worker for worker in workers}
|
|
670
|
+
latest_reducer_key = (
|
|
671
|
+
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
672
|
+
if reducer is not None and latest_reducer in drafts_by_path
|
|
673
|
+
else None
|
|
674
|
+
)
|
|
675
|
+
if latest_reducer_key is None:
|
|
676
|
+
latest_reducer = None
|
|
677
|
+
for worker, result_path, checkpoint_paths, attempt in reducer_outputs:
|
|
678
|
+
result = drafts_by_path.get(result_path)
|
|
679
|
+
if result is None or not any(
|
|
680
|
+
drafts_by_path.get(checkpoint_path) == result for checkpoint_path in checkpoint_paths
|
|
681
|
+
):
|
|
682
|
+
continue
|
|
683
|
+
current_results.add(result_path)
|
|
684
|
+
candidate_key = (worker["completed_at"] or "", worker["id"], attempt)
|
|
685
|
+
if latest_reducer_key is None or candidate_key > latest_reducer_key:
|
|
686
|
+
latest_reducer_key = candidate_key
|
|
687
|
+
latest_reducer = result_path
|
|
688
|
+
|
|
689
|
+
if parent_heads:
|
|
690
|
+
latest_observation = max(observed for observed, _ in parent_heads)
|
|
691
|
+
for observed, parent_path in parent_heads:
|
|
692
|
+
if observed != latest_observation:
|
|
693
|
+
continue
|
|
694
|
+
draft = drafts_by_path[parent_path]
|
|
695
|
+
modified = source_order[parent_path][1]
|
|
696
|
+
if parent is None or modified > parent_modified:
|
|
697
|
+
# Only a replacement head can replay without the canonical draft.
|
|
698
|
+
if parent_context.get("canonical") not in source_digests:
|
|
699
|
+
parent_context.pop("canonical", None)
|
|
700
|
+
parent_context.pop("canonicalObservedAtNs", None)
|
|
701
|
+
parent = draft
|
|
702
|
+
parent_model_source = parent_path
|
|
703
|
+
frozen_parent_path = (
|
|
704
|
+
parent_path
|
|
705
|
+
if isinstance(draft.get("previousParentCheckpoints"), list)
|
|
706
|
+
else None
|
|
707
|
+
)
|
|
708
|
+
parent_modified = modified
|
|
709
|
+
parent_is_canonical = draft.get("canonicalParent") is True
|
|
710
|
+
elif modified == parent_modified and _parent_observation_content(
|
|
711
|
+
draft
|
|
712
|
+
) != _parent_observation_content(parent):
|
|
713
|
+
parent = _merge_tied_parent_observations(parent, draft)
|
|
714
|
+
parent_is_canonical = False
|
|
715
|
+
# A reducer supplies findings but cannot supersede saved parent coverage.
|
|
716
|
+
saved_parent = parent
|
|
717
|
+
parent_is_root = parent is not None
|
|
718
|
+
if parent is None and latest_reducer is not None:
|
|
719
|
+
parent = drafts_by_path[latest_reducer]
|
|
720
|
+
parent_model_source = latest_reducer
|
|
721
|
+
|
|
722
|
+
projected_coverage = saved_parent["coverage"] if saved_parent else {}
|
|
723
|
+
reviews = projected_coverage.get("reviews")
|
|
724
|
+
reviewed_attempts = {
|
|
725
|
+
(review.get("workerId"), review.get("attempt"))
|
|
726
|
+
for review in (reviews if isinstance(reviews, list) else [])
|
|
727
|
+
if isinstance(review, dict)
|
|
728
|
+
and isinstance(review.get("workerId"), str)
|
|
729
|
+
and isinstance(review.get("attempt"), int)
|
|
730
|
+
}
|
|
731
|
+
# Reconcile raw source IDs and timestamps before projecting accepted outputs.
|
|
732
|
+
accepted_sources = {
|
|
733
|
+
relative
|
|
734
|
+
for relative, draft, worker_id in sources
|
|
735
|
+
if (worker := workers_by_id.get(worker_id)) is not None
|
|
736
|
+
and worker["status"] == "succeeded"
|
|
737
|
+
and worker["result_manifest_path"] == str(scan_dir / relative)
|
|
738
|
+
and draft.get("complete") is not False
|
|
739
|
+
}
|
|
740
|
+
reviewed_attempts.update(
|
|
741
|
+
(worker_id, workers_by_id[worker_id]["attempt"])
|
|
742
|
+
for relative, _, worker_id in sources
|
|
743
|
+
if relative in accepted_sources
|
|
744
|
+
)
|
|
745
|
+
projection = prepare_saved_coverage_projection(
|
|
746
|
+
sources,
|
|
747
|
+
parent,
|
|
748
|
+
source_order,
|
|
749
|
+
project_worker_coverage=project_worker_coverage,
|
|
750
|
+
preserve_canonical_parent=preserve_canonical_parent,
|
|
751
|
+
receipt_resolver=lambda relative, row, owner: resolve_coverage_receipts(
|
|
752
|
+
scan_dir, row, workers_by_id[owner], relative
|
|
753
|
+
),
|
|
754
|
+
)
|
|
755
|
+
project_worker_coverage = projection.project_worker_coverage
|
|
756
|
+
|
|
757
|
+
def retain_coverage_row(output, item, owner, field, original=None, qualified=False):
|
|
758
|
+
retained = projection.retain_coverage_row(output, item, owner, field, original, qualified)
|
|
759
|
+
source = item if original is None else original
|
|
760
|
+
_inherit_coverage_sources(retained, [item, source], coverage_sources)
|
|
761
|
+
if preserve_canonical_parent and (
|
|
762
|
+
id(item) in canonical_rows or id(source) in canonical_rows
|
|
763
|
+
):
|
|
764
|
+
canonical_rows.add(id(retained))
|
|
765
|
+
if field == "surfaces":
|
|
766
|
+
retained_surface_rows.add(id(source))
|
|
767
|
+
return retained
|
|
768
|
+
|
|
769
|
+
accepted_coverage_sources = {
|
|
770
|
+
id(draft["coverage"]) for relative, draft, _ in sources if relative in accepted_sources
|
|
771
|
+
}
|
|
772
|
+
accepted_surfaces: dict[tuple[str, str], list[tuple[int, dict[str, Any]]]] = {}
|
|
773
|
+
for relative, draft, worker_id in sources:
|
|
774
|
+
if relative not in accepted_sources:
|
|
775
|
+
continue
|
|
776
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
777
|
+
for index, surface in enumerate(surfaces if isinstance(surfaces, list) else [], 1):
|
|
778
|
+
if isinstance(surface, dict) and isinstance(surface.get("id"), str):
|
|
779
|
+
accepted_surfaces.setdefault((worker_id, surface["id"]), []).append(
|
|
780
|
+
(index, surface)
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
for _, draft, _ in sources:
|
|
784
|
+
questions = draft["coverage"].get("openQuestions")
|
|
785
|
+
if isinstance(questions, list):
|
|
786
|
+
draft["coverage"]["openQuestions"] = [
|
|
787
|
+
{"question": item.strip()} if isinstance(item, str) else item for item in questions
|
|
788
|
+
]
|
|
789
|
+
projection_origins = {
|
|
790
|
+
id(item): (worker_id, relative, field, index, draft["coverage"])
|
|
791
|
+
for relative, draft, worker_id in sources
|
|
792
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions")
|
|
793
|
+
for index, item in enumerate(
|
|
794
|
+
draft["coverage"].get(field, [])
|
|
795
|
+
if isinstance(draft["coverage"].get(field, []), list)
|
|
796
|
+
else [],
|
|
797
|
+
1,
|
|
798
|
+
)
|
|
799
|
+
if isinstance(item, dict)
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
def project_record(
|
|
803
|
+
field: str,
|
|
804
|
+
item: dict[str, Any],
|
|
805
|
+
source_item: dict[str, Any] | None = None,
|
|
806
|
+
*,
|
|
807
|
+
receipts_resolved: bool = False,
|
|
808
|
+
) -> dict[str, Any]:
|
|
809
|
+
original = item if source_item is None else source_item
|
|
810
|
+
origin = projection_origins.get(id(original))
|
|
811
|
+
if origin is None:
|
|
812
|
+
return item
|
|
813
|
+
worker_id, relative, _, index, source = origin
|
|
814
|
+
if relative not in accepted_sources:
|
|
815
|
+
directory = Path(relative).parent
|
|
816
|
+
if directory.name == "checkpoints":
|
|
817
|
+
directory = directory.parent
|
|
818
|
+
worker = workers_by_id.get(worker_id)
|
|
819
|
+
if worker is None or worker_attempts.get(directory.as_posix(), (None, None)) != (
|
|
820
|
+
worker_id,
|
|
821
|
+
worker["attempt"],
|
|
822
|
+
):
|
|
823
|
+
return item
|
|
824
|
+
# Reproduced outputs and links from omitted checkpoints use the
|
|
825
|
+
# accepted worker's surface namespace.
|
|
826
|
+
accepted = next(
|
|
827
|
+
(
|
|
828
|
+
(path, draft["coverage"])
|
|
829
|
+
for path, draft, owner in sources
|
|
830
|
+
if owner == worker_id and path in accepted_sources
|
|
831
|
+
),
|
|
832
|
+
None,
|
|
833
|
+
)
|
|
834
|
+
if accepted is not None and source == accepted[1]:
|
|
835
|
+
relative, source = accepted
|
|
836
|
+
elif relative not in selected_observations or accepted is None:
|
|
837
|
+
if field == "deferred" and accepted is not None and "surfaceIds" in item:
|
|
838
|
+
return {
|
|
839
|
+
**item,
|
|
840
|
+
"surfaceIds": project_missing_record(
|
|
841
|
+
field, item, index, worker, accepted[1]
|
|
842
|
+
)["surfaceIds"],
|
|
843
|
+
}
|
|
844
|
+
return item
|
|
845
|
+
worker = workers_by_id[worker_id]
|
|
846
|
+
result = project_missing_record(
|
|
847
|
+
field, item, index, worker, source, receipts_resolved=receipts_resolved
|
|
848
|
+
)
|
|
849
|
+
if field == "surfaces" and not receipts_resolved:
|
|
850
|
+
result["receiptRefs"] = coverage_receipts(result, worker, relative)
|
|
851
|
+
return result
|
|
852
|
+
|
|
853
|
+
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
854
|
+
source_owners = {relative: owner for relative, _, owner in all_sources}
|
|
855
|
+
inferred_coverage_ids: set[int] = set()
|
|
856
|
+
# Older checkpoints can omit IDs already assigned in their published output.
|
|
857
|
+
for field in ("deferred", "surfaces"):
|
|
858
|
+
named_rows: dict[str | None, list[dict[str, Any]]] = {}
|
|
859
|
+
for _, draft, owner in all_sources:
|
|
860
|
+
named_rows.setdefault(owner, []).extend(
|
|
861
|
+
row
|
|
862
|
+
for row in _coverage_rows(draft["coverage"], field)
|
|
863
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
864
|
+
)
|
|
865
|
+
for _, draft, owner in all_sources:
|
|
866
|
+
rows = _coverage_rows(draft["coverage"], field)
|
|
867
|
+
reserved = {
|
|
868
|
+
row["id"]
|
|
869
|
+
for row in rows
|
|
870
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
871
|
+
}
|
|
872
|
+
for row in rows:
|
|
873
|
+
if not isinstance(row, dict) or "id" in row:
|
|
874
|
+
continue
|
|
875
|
+
if field == "deferred" and _has_candidate_metadata(row):
|
|
876
|
+
continue
|
|
877
|
+
content = {"receiptRefs": [], **row} if field == "surfaces" else row
|
|
878
|
+
identity = next(
|
|
879
|
+
(
|
|
880
|
+
named["id"]
|
|
881
|
+
for named in named_rows[owner]
|
|
882
|
+
if named["id"] not in reserved
|
|
883
|
+
and {
|
|
884
|
+
**({"receiptRefs": []} if field == "surfaces" else {}),
|
|
885
|
+
**{key: value for key, value in named.items() if key != "id"},
|
|
886
|
+
}
|
|
887
|
+
== content
|
|
888
|
+
),
|
|
889
|
+
None,
|
|
890
|
+
)
|
|
891
|
+
if identity is not None:
|
|
892
|
+
inferred_coverage_ids.add(id(row))
|
|
893
|
+
row["id"] = identity
|
|
894
|
+
reserved.add(identity)
|
|
895
|
+
|
|
896
|
+
# A discarded local surface must not fall back to another source's reused ID.
|
|
897
|
+
surface_ids_by_source: dict[tuple[str, str], list[str]] = {
|
|
898
|
+
(relative, row["id"]): []
|
|
899
|
+
for relative, draft, _ in all_sources
|
|
900
|
+
for row in (
|
|
901
|
+
draft["coverage"].get("surfaces", [])
|
|
902
|
+
if isinstance(draft["coverage"].get("surfaces"), list)
|
|
903
|
+
else []
|
|
904
|
+
)
|
|
905
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
906
|
+
}
|
|
907
|
+
discarded_coverage_sources: set[str] = set()
|
|
908
|
+
if preserve_canonical_parent:
|
|
909
|
+
for relative, draft, worker_id in sources:
|
|
910
|
+
if worker_id is None:
|
|
911
|
+
continue
|
|
912
|
+
if _recover_preserved_worker_coverage(
|
|
913
|
+
scan_dir,
|
|
914
|
+
relative,
|
|
915
|
+
worker_id,
|
|
916
|
+
draft,
|
|
917
|
+
discarded_by_source.get(relative, []),
|
|
918
|
+
warnings,
|
|
919
|
+
):
|
|
920
|
+
discarded_coverage_sources.add(relative)
|
|
921
|
+
|
|
922
|
+
if parent is None and not sources:
|
|
923
|
+
return None
|
|
924
|
+
if (
|
|
925
|
+
parent_manifest
|
|
926
|
+
and parent_manifest["scan"].get("sealedAt")
|
|
927
|
+
and parent_manifest["scan"].get("status") == binding["status"]
|
|
928
|
+
and parent_manifest["scan"].get("preservedSources") == source_digests
|
|
929
|
+
and parent_manifest["scan"].get("preservedCheckpointHeads", {}) == checkpoint_heads
|
|
930
|
+
and all(warning in initial_warnings for warning in warnings)
|
|
931
|
+
):
|
|
932
|
+
return None
|
|
933
|
+
|
|
934
|
+
target_kind = binding["allowedTargetKinds"][0]
|
|
935
|
+
if (
|
|
936
|
+
target_kind == "git_worktree"
|
|
937
|
+
and "snapshotDigest" not in binding["target"]
|
|
938
|
+
and "git_revision" in binding["allowedTargetKinds"]
|
|
939
|
+
):
|
|
940
|
+
target_kind = "git_revision"
|
|
941
|
+
target = {"kind": target_kind, **binding["target"]}
|
|
942
|
+
if target["kind"] == "git_diff" and "snapshotDigest" not in target:
|
|
943
|
+
diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
|
|
944
|
+
target["snapshotDigest"] = committed_diff_snapshot_digest(
|
|
945
|
+
diff_kind, target["baseRevision"], target["headRevision"]
|
|
946
|
+
)
|
|
947
|
+
manifest = (
|
|
948
|
+
copy.deepcopy(parent_manifest)
|
|
949
|
+
if parent_manifest
|
|
950
|
+
else {"scan": {"target": target, "scope": binding["scope"]}}
|
|
951
|
+
)
|
|
952
|
+
if allow_frozen_legacy_parent:
|
|
953
|
+
manifest["scan"]["target"] = target
|
|
954
|
+
for key in ("sealedAt", "artifacts"):
|
|
955
|
+
manifest["scan"].pop(key, None)
|
|
956
|
+
manifest["scan"]["preservedSources"] = source_digests
|
|
957
|
+
manifest["scan"]["preservedCheckpointHeads"] = checkpoint_heads
|
|
958
|
+
# Completion follows the selected parent, including legacy terminal drafts
|
|
959
|
+
# that omit the optional marker, rather than an older canonical manifest.
|
|
960
|
+
if parent is not None:
|
|
961
|
+
manifest["scan"].pop("complete", None)
|
|
962
|
+
if "complete" in parent:
|
|
963
|
+
manifest["scan"]["complete"] = parent["complete"]
|
|
964
|
+
if isinstance(parent.get("scope"), dict):
|
|
965
|
+
manifest["scan"]["scope"] = copy.deepcopy(parent["scope"])
|
|
966
|
+
if isinstance(manifest["scan"].get("scope"), dict):
|
|
967
|
+
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
|
|
968
|
+
model_checks: dict[str, dict[str, Any] | None] = {}
|
|
969
|
+
|
|
970
|
+
def recovered_model(
|
|
971
|
+
relative: str, draft: dict[str, Any], owner: str | None
|
|
972
|
+
) -> dict[str, Any] | None:
|
|
973
|
+
if relative not in model_checks:
|
|
974
|
+
model = copy.deepcopy(draft.get("threatModel"))
|
|
975
|
+
if isinstance(model, dict) and owner is not None:
|
|
976
|
+
model["origin"] = "recovered"
|
|
977
|
+
schema = _read_json(
|
|
978
|
+
Path(__file__).resolve().parents[2] / "schemas/scan-manifest.schema.json"
|
|
979
|
+
)
|
|
980
|
+
try:
|
|
981
|
+
_require_safe_json_value(model, f"Saved threat model {relative}")
|
|
982
|
+
_validate_schema_node(
|
|
983
|
+
model,
|
|
984
|
+
schema["properties"]["scan"]["properties"]["threatModel"],
|
|
985
|
+
f"Saved threat model {relative}",
|
|
986
|
+
schema,
|
|
987
|
+
)
|
|
988
|
+
except ContractError as exc:
|
|
989
|
+
warnings.append(f"Skipped malformed saved threat model {relative}: {exc}")
|
|
990
|
+
model = None
|
|
991
|
+
model_checks[relative] = model
|
|
992
|
+
return model_checks[relative]
|
|
993
|
+
|
|
994
|
+
parent_model = (
|
|
995
|
+
recovered_model(parent_model_source or "parent", parent, None)
|
|
996
|
+
if parent is not None and "threatModel" in parent
|
|
997
|
+
else None
|
|
998
|
+
)
|
|
999
|
+
canonical_model = manifest["scan"].pop("threatModel", canonical_model)
|
|
1000
|
+
if parent_model is not None:
|
|
1001
|
+
manifest["scan"]["threatModel"] = parent_model
|
|
1002
|
+
if (
|
|
1003
|
+
selected_model_source is not None
|
|
1004
|
+
and parent_model_source in drafts_by_path
|
|
1005
|
+
and (
|
|
1006
|
+
parent_model_source != canonical_model_source
|
|
1007
|
+
or canonical_model_digest is None
|
|
1008
|
+
or source_digests[parent_model_source] == canonical_model_digest
|
|
1009
|
+
)
|
|
1010
|
+
):
|
|
1011
|
+
selected_model_source[:] = [parent_model_source]
|
|
1012
|
+
elif frozen_model_source is None:
|
|
1013
|
+
# Model fallback is independent of whether older findings are superseded.
|
|
1014
|
+
if canonical_model is not None:
|
|
1015
|
+
canonical_model = recovered_model("canonical", {"threatModel": canonical_model}, None)
|
|
1016
|
+
if (
|
|
1017
|
+
canonical_model_source is not None
|
|
1018
|
+
and Path(canonical_model_source).parent != Path("checkpoints")
|
|
1019
|
+
) or (
|
|
1020
|
+
canonical_model_source is None
|
|
1021
|
+
and isinstance(canonical_model, dict)
|
|
1022
|
+
and canonical_model.get("origin") == "recovered"
|
|
1023
|
+
):
|
|
1024
|
+
# Worker models follow their own selected head and independent result.
|
|
1025
|
+
canonical_model = None
|
|
1026
|
+
canonical_model_source = None
|
|
1027
|
+
if (
|
|
1028
|
+
canonical_model_source in drafts_by_path
|
|
1029
|
+
and recovered_model(
|
|
1030
|
+
canonical_model_source, drafts_by_path[canonical_model_source], None
|
|
1031
|
+
)
|
|
1032
|
+
is not None
|
|
1033
|
+
and (
|
|
1034
|
+
canonical_model_digest is None
|
|
1035
|
+
or source_digests[canonical_model_source] == canonical_model_digest
|
|
1036
|
+
)
|
|
1037
|
+
):
|
|
1038
|
+
canonical_model = drafts_by_path[canonical_model_source].get("threatModel")
|
|
1039
|
+
canonical_model_order = max(
|
|
1040
|
+
canonical_model_order or source_order[canonical_model_source],
|
|
1041
|
+
source_order[canonical_model_source],
|
|
1042
|
+
)
|
|
1043
|
+
else:
|
|
1044
|
+
# A skipped checkpoint cannot supply provenance for valid canonical data.
|
|
1045
|
+
canonical_model_source = None
|
|
1046
|
+
root_models = [
|
|
1047
|
+
path
|
|
1048
|
+
for path in drafts_by_path
|
|
1049
|
+
if path in selected_observations
|
|
1050
|
+
and Path(path).parent == Path("checkpoints")
|
|
1051
|
+
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
1052
|
+
and recovered_model(path, drafts_by_path[path], None) is not None
|
|
1053
|
+
]
|
|
1054
|
+
selected_root_models = [path for path in root_models if path in authoritative_heads]
|
|
1055
|
+
if isinstance(canonical_model, dict) and canonical_model_order is None:
|
|
1056
|
+
# Legacy publications have no model-source identity. Only newly admitted
|
|
1057
|
+
# observations can supersede that fallback without an explicit selection.
|
|
1058
|
+
new_models = {
|
|
1059
|
+
f"checkpoints/{checkpoint}"
|
|
1060
|
+
for head, checkpoint in saved_heads.items()
|
|
1061
|
+
if _checkpoint_head_directory(head) == Path(".")
|
|
1062
|
+
and head not in (canonical_sources or {})
|
|
1063
|
+
}
|
|
1064
|
+
root_models = [path for path in root_models if path in new_models]
|
|
1065
|
+
model_source = max(
|
|
1066
|
+
selected_root_models or root_models,
|
|
1067
|
+
key=lambda path: source_order[path],
|
|
1068
|
+
default=None,
|
|
1069
|
+
)
|
|
1070
|
+
if model_source is not None and (
|
|
1071
|
+
selected_root_models
|
|
1072
|
+
or not isinstance(canonical_model, dict)
|
|
1073
|
+
or canonical_model_order is None
|
|
1074
|
+
or source_order[model_source] > canonical_model_order
|
|
1075
|
+
):
|
|
1076
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(
|
|
1077
|
+
drafts_by_path[model_source]["threatModel"]
|
|
1078
|
+
)
|
|
1079
|
+
if selected_model_source is not None:
|
|
1080
|
+
selected_model_source[:] = [model_source]
|
|
1081
|
+
elif isinstance(canonical_model, dict):
|
|
1082
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(canonical_model)
|
|
1083
|
+
if stopped and frozen_source_digests is None and canonical_model_source is None:
|
|
1084
|
+
# An unreadable canonical checkpoint must not leave a valid
|
|
1085
|
+
# inherited model dependent on live files after the first stop.
|
|
1086
|
+
model_draft = _model_capture(scan_id, canonical_model)
|
|
1087
|
+
digest = _digest(model_draft)
|
|
1088
|
+
canonical_model_source = f"checkpoints/{digest}.json"
|
|
1089
|
+
observed = (canonical_model_order or (0, parent_modified))[1]
|
|
1090
|
+
source_digests[canonical_model_source] = digest
|
|
1091
|
+
source_times[canonical_model_source] = observed
|
|
1092
|
+
try:
|
|
1093
|
+
if not (scan_dir / canonical_model_source).exists():
|
|
1094
|
+
if created_paths is not None and not os.path.lexists(
|
|
1095
|
+
scan_dir / canonical_model_source
|
|
1096
|
+
):
|
|
1097
|
+
created_paths.add(canonical_model_source)
|
|
1098
|
+
write_scan_local_bytes(
|
|
1099
|
+
scan_dir, canonical_model_source, _encoded(model_draft)
|
|
1100
|
+
)
|
|
1101
|
+
os.utime(scan_dir / canonical_model_source, ns=(observed, observed))
|
|
1102
|
+
_, actual, _ = _read_saved_result(scan_dir, canonical_model_source, scan_id)
|
|
1103
|
+
if actual != digest:
|
|
1104
|
+
raise ContractError("Saved inherited model checkpoint changed.")
|
|
1105
|
+
except (ContractError, OSError) as exc:
|
|
1106
|
+
snapshot_error = snapshot_error or exc
|
|
1107
|
+
if selected_model_source is not None and canonical_model_source is not None:
|
|
1108
|
+
selected_model_source[:] = [canonical_model_source]
|
|
1109
|
+
|
|
1110
|
+
if frozen_model_source is not None:
|
|
1111
|
+
# Publication retries retain a valid choice from the frozen source set.
|
|
1112
|
+
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
|
|
1113
|
+
if not isinstance(model, dict):
|
|
1114
|
+
raise ContractError("Frozen stopped-scan model source is unavailable.")
|
|
1115
|
+
model = recovered_model(
|
|
1116
|
+
frozen_model_source, drafts_by_path[frozen_model_source], paths[frozen_model_source]
|
|
1117
|
+
)
|
|
1118
|
+
if model is not None:
|
|
1119
|
+
manifest["scan"]["threatModel"] = model
|
|
1120
|
+
if selected_model_source is not None:
|
|
1121
|
+
selected_model_source[:] = [] if model is None else [frozen_model_source]
|
|
1122
|
+
coverage = (
|
|
1123
|
+
copy.deepcopy(parent["coverage"])
|
|
1124
|
+
if parent and parent["coverage"]
|
|
1125
|
+
else {
|
|
1126
|
+
"mode": binding["coverageMode"],
|
|
1127
|
+
"inventoryStrategy": "diff"
|
|
1128
|
+
if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
|
|
1129
|
+
else "scoped_path"
|
|
1130
|
+
if binding["coverageMode"] == "scoped_path"
|
|
1131
|
+
else "repository",
|
|
1132
|
+
**binding["scope"],
|
|
1133
|
+
"surfaces": [],
|
|
1134
|
+
"explicitExclusions": [],
|
|
1135
|
+
"deferred": [],
|
|
1136
|
+
}
|
|
1137
|
+
)
|
|
1138
|
+
if isinstance(coverage.get("openQuestions"), list):
|
|
1139
|
+
coverage["openQuestions"] = [
|
|
1140
|
+
{"question": item.strip()} if isinstance(item, str) else item
|
|
1141
|
+
for item in coverage["openQuestions"]
|
|
1142
|
+
]
|
|
1143
|
+
# Closure reconciliation can reuse original parent rows instead of their copies.
|
|
1144
|
+
canonical_rows = (
|
|
1145
|
+
{
|
|
1146
|
+
id(item)
|
|
1147
|
+
for parent_coverage in (
|
|
1148
|
+
(coverage, parent["coverage"]) if preserve_canonical_parent else (coverage,)
|
|
1149
|
+
)
|
|
1150
|
+
for field in ("surfaces", "explicitExclusions", "deferred")
|
|
1151
|
+
for item in _coverage_rows(parent_coverage, field)
|
|
1152
|
+
}
|
|
1153
|
+
if parent_is_canonical
|
|
1154
|
+
else set()
|
|
1155
|
+
)
|
|
1156
|
+
# Links use source-local IDs until all retained surfaces receive their final IDs.
|
|
1157
|
+
coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] = {}
|
|
1158
|
+
source_owners = {relative: owner for relative, _, owner in all_sources}
|
|
1159
|
+
retained_surface_rows: set[int] = set()
|
|
1160
|
+
for relative, draft, _ in all_sources:
|
|
1161
|
+
for field in ("surfaces", "deferred"):
|
|
1162
|
+
rows = draft["coverage"].get(field)
|
|
1163
|
+
for row in rows if isinstance(rows, list) else []:
|
|
1164
|
+
if isinstance(row, dict):
|
|
1165
|
+
paths_for_row = coverage_sources.setdefault(id(row), (row, []))[1]
|
|
1166
|
+
if relative not in paths_for_row:
|
|
1167
|
+
paths_for_row.append(relative)
|
|
1168
|
+
if parent:
|
|
1169
|
+
for field in ("surfaces", "deferred"):
|
|
1170
|
+
rows, originals = coverage.get(field), parent["coverage"].get(field)
|
|
1171
|
+
if isinstance(rows, list) and isinstance(originals, list):
|
|
1172
|
+
for row, original in zip(rows, originals):
|
|
1173
|
+
_inherit_coverage_sources(row, [original], coverage_sources)
|
|
1174
|
+
if field == "surfaces":
|
|
1175
|
+
retained_surface_rows.update(id(original) for original in originals)
|
|
1176
|
+
|
|
1177
|
+
findings: list[dict[str, Any]] = []
|
|
1178
|
+
finding_positions: dict[str, tuple[int, dict[str, Any] | None]] = {}
|
|
1179
|
+
canonical_finding_keys: set[str] = set()
|
|
1180
|
+
represented: dict[str, str | None] = {}
|
|
1181
|
+
represented_candidates: dict[tuple[str, str, Any, Any, Any], str | None] = {}
|
|
1182
|
+
represented_history: dict[str, set[str]] = {}
|
|
1183
|
+
represented_candidate_history: dict[tuple[str, str, Any, Any, Any], set[str]] = {}
|
|
1184
|
+
rejected_history: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
1185
|
+
stopped_parent_seal = bool(
|
|
1186
|
+
stopped and parent_manifest and parent_manifest["scan"].get("sealedAt")
|
|
1187
|
+
)
|
|
1188
|
+
|
|
1189
|
+
# A damaged canonical row must not replace a checkpoint's established identity.
|
|
1190
|
+
legacy_identities: dict[str, dict[str, Any] | None] = {}
|
|
1191
|
+
legacy_identity_sources: set[str] = set()
|
|
1192
|
+
|
|
1193
|
+
def legacy_identity_key(finding: dict[str, Any], owner: str | None) -> str:
|
|
1194
|
+
content = _finding_content(finding)
|
|
1195
|
+
locations = content.get("locations")
|
|
1196
|
+
if isinstance(locations, list):
|
|
1197
|
+
content["locations"] = sorted(
|
|
1198
|
+
[
|
|
1199
|
+
{**row, "endLine": row.get("endLine", row.get("startLine"))}
|
|
1200
|
+
if isinstance(row, dict)
|
|
1201
|
+
else row
|
|
1202
|
+
for row in locations
|
|
1203
|
+
],
|
|
1204
|
+
key=_encoded,
|
|
1205
|
+
)
|
|
1206
|
+
return _digest([owner, _identity_candidate(finding), content])
|
|
1207
|
+
|
|
1208
|
+
def restore_legacy_identity(
|
|
1209
|
+
finding: Any, normalized: Any = None, *, allow_synthesis: bool = True
|
|
1210
|
+
) -> None:
|
|
1211
|
+
if isinstance(finding, dict) and "identity" not in finding:
|
|
1212
|
+
provenance = finding.get("provenance")
|
|
1213
|
+
owner = provenance.get("workerId") if isinstance(provenance, dict) else None
|
|
1214
|
+
key = legacy_identity_key(finding, owner)
|
|
1215
|
+
identity = legacy_identities.get(key)
|
|
1216
|
+
if identity is not None:
|
|
1217
|
+
finding["identity"] = copy.deepcopy(identity)
|
|
1218
|
+
allow_synthesis = allow_synthesis or key in legacy_identity_sources
|
|
1219
|
+
if not allow_synthesis:
|
|
1220
|
+
return
|
|
1221
|
+
if isinstance(finding, dict) and isinstance(normalized, dict):
|
|
1222
|
+
finding.setdefault("identity", normalized["identity"])
|
|
1223
|
+
else:
|
|
1224
|
+
_ensure_finding_identity(finding)
|
|
1225
|
+
|
|
1226
|
+
def finding_within_scope(finding: dict[str, Any]) -> bool:
|
|
1227
|
+
locations = finding.get("locations", [])
|
|
1228
|
+
return isinstance(locations, list) and any(
|
|
1229
|
+
isinstance(location, dict)
|
|
1230
|
+
and isinstance(location.get("path"), str)
|
|
1231
|
+
and any(
|
|
1232
|
+
PurePosixPath(location["path"]) == PurePosixPath(path)
|
|
1233
|
+
if is_file
|
|
1234
|
+
else path_within_scope(location["path"], path)
|
|
1235
|
+
for path, is_file in (
|
|
1236
|
+
finding_scope
|
|
1237
|
+
if finding_scope is not None
|
|
1238
|
+
else [(path, False) for path in binding["scope"]["includePaths"]]
|
|
1239
|
+
)
|
|
1240
|
+
)
|
|
1241
|
+
for location in locations
|
|
1242
|
+
)
|
|
1243
|
+
|
|
1244
|
+
valid_findings: dict[bytes, bool] = {}
|
|
1245
|
+
|
|
1246
|
+
def valid_finding(value: Any) -> bool:
|
|
1247
|
+
# Use the finalizer's own per-record recovery before a draft can suppress
|
|
1248
|
+
# an earlier checkpoint. Invalid latest records must not hide valid history.
|
|
1249
|
+
document = {"scanId": scan_id, "findings": [copy.deepcopy(value)]}
|
|
1250
|
+
if isinstance(document["findings"][0], dict):
|
|
1251
|
+
restore_legacy_identity(document["findings"][0])
|
|
1252
|
+
key = _encoded(document["findings"][0])
|
|
1253
|
+
if key in valid_findings:
|
|
1254
|
+
return valid_findings[key]
|
|
1255
|
+
_recover_unsealed_findings(
|
|
1256
|
+
{"scan": {"id": scan_id, "target": binding["target"]}},
|
|
1257
|
+
document,
|
|
1258
|
+
Path(__file__).resolve().parents[2] / "schemas",
|
|
1259
|
+
scan_dir,
|
|
1260
|
+
[],
|
|
1261
|
+
current_contract=current_contract,
|
|
1262
|
+
)
|
|
1263
|
+
valid_findings[key] = bool(document["findings"])
|
|
1264
|
+
return valid_findings[key]
|
|
1265
|
+
|
|
1266
|
+
for _, draft, owner in all_sources:
|
|
1267
|
+
for value in draft["findings"]:
|
|
1268
|
+
if not isinstance(value, dict):
|
|
1269
|
+
continue
|
|
1270
|
+
for saved in _retained_findings(value):
|
|
1271
|
+
identity = saved.get("identity")
|
|
1272
|
+
provenance = saved.get("provenance")
|
|
1273
|
+
saved_owner = (
|
|
1274
|
+
provenance.get("workerId", owner) if isinstance(provenance, dict) else owner
|
|
1275
|
+
)
|
|
1276
|
+
key = legacy_identity_key(saved, saved_owner)
|
|
1277
|
+
if "identity" in saved:
|
|
1278
|
+
legacy_identity_sources.add(key)
|
|
1279
|
+
if not isinstance(identity, dict) or not valid_finding(saved):
|
|
1280
|
+
continue
|
|
1281
|
+
if key not in legacy_identities:
|
|
1282
|
+
legacy_identities[key] = identity
|
|
1283
|
+
elif legacy_identities[key] != identity:
|
|
1284
|
+
legacy_identities[key] = None
|
|
1285
|
+
|
|
1286
|
+
source_order["parent"] = (0, parent_modified)
|
|
1287
|
+
|
|
1288
|
+
def projection_key(field: str, item: dict[str, Any]) -> tuple[str, str]:
|
|
1289
|
+
# Publication may rename canonical IDs and add retained finding history.
|
|
1290
|
+
ignored = {"id", "previousFindings"} if field == "surfaces" else {"id"}
|
|
1291
|
+
content = {
|
|
1292
|
+
key: value
|
|
1293
|
+
for key, value in canonical_coverage_record(field, item).items()
|
|
1294
|
+
if key not in ignored
|
|
1295
|
+
}
|
|
1296
|
+
return field, _digest(content)
|
|
1297
|
+
|
|
1298
|
+
projected_sources: dict[tuple[str, str], tuple[str, dict[str, Any], tuple[int, int]]] = {}
|
|
1299
|
+
accepted_pending_candidates: set[tuple[str, int, str]] = set()
|
|
1300
|
+
selected_reported_candidates: dict[tuple[str, str], tuple[int, int]] = {}
|
|
1301
|
+
for relative, draft, worker_id in sources:
|
|
1302
|
+
if relative not in selected_observations:
|
|
1303
|
+
continue
|
|
1304
|
+
for finding in draft["findings"]:
|
|
1305
|
+
if isinstance(finding, dict) and valid_finding(finding):
|
|
1306
|
+
candidate_id = finding_candidate_id(finding)
|
|
1307
|
+
if candidate_id:
|
|
1308
|
+
key = (worker_id, candidate_id)
|
|
1309
|
+
selected_reported_candidates[key] = max(
|
|
1310
|
+
selected_reported_candidates.get(key, source_order[relative]),
|
|
1311
|
+
source_order[relative],
|
|
1312
|
+
)
|
|
1313
|
+
for relative, draft, worker_id in sources:
|
|
1314
|
+
if relative not in accepted_sources:
|
|
1315
|
+
continue
|
|
1316
|
+
worker = workers_by_id[worker_id]
|
|
1317
|
+
reported = {
|
|
1318
|
+
finding_candidate_id(finding)
|
|
1319
|
+
for finding in draft["findings"]
|
|
1320
|
+
if isinstance(finding, dict) and valid_finding(finding)
|
|
1321
|
+
}
|
|
1322
|
+
for field in ("surfaces", "deferred", "explicitExclusions"):
|
|
1323
|
+
rows = draft["coverage"].get(field, [])
|
|
1324
|
+
for index, row in enumerate(rows if isinstance(rows, list) else [], 1):
|
|
1325
|
+
if not isinstance(row, dict):
|
|
1326
|
+
continue
|
|
1327
|
+
if (
|
|
1328
|
+
field == "deferred"
|
|
1329
|
+
and isinstance(candidate_id := row.get("candidateId"), str)
|
|
1330
|
+
and candidate_id not in reported
|
|
1331
|
+
and selected_reported_candidates.get((worker_id, candidate_id), (-1, -1))
|
|
1332
|
+
< source_order[relative]
|
|
1333
|
+
):
|
|
1334
|
+
accepted_pending_candidates.add((worker_id, worker["attempt"], candidate_id))
|
|
1335
|
+
projected = project_missing_record(field, row, index, worker, draft["coverage"])
|
|
1336
|
+
if field == "surfaces":
|
|
1337
|
+
projected["receiptRefs"] = coverage_receipts(projected, worker, relative)
|
|
1338
|
+
key = projection_key(field, projected)
|
|
1339
|
+
projected_sources[key] = (worker_id, row, source_order[relative])
|
|
1340
|
+
|
|
1341
|
+
def record_source(
|
|
1342
|
+
field: str, owner: str | None, item: dict[str, Any]
|
|
1343
|
+
) -> tuple[str | None, dict[str, Any]]:
|
|
1344
|
+
# Parent-authored provenance is data, not authority over another worker.
|
|
1345
|
+
if (source := projected_sources.get(projection_key(field, item))) and (
|
|
1346
|
+
owner is None or owner == source[0]
|
|
1347
|
+
):
|
|
1348
|
+
return source[0], source[1]
|
|
1349
|
+
return owner, item
|
|
1350
|
+
|
|
1351
|
+
def record_key(field: str, owner: str | None, item: dict[str, Any]) -> tuple[str | None, Any]:
|
|
1352
|
+
source_owner, original = record_source(field, owner, item)
|
|
1353
|
+
identity = original.get("id")
|
|
1354
|
+
return source_owner, identity if isinstance(identity, str) else None
|
|
1355
|
+
|
|
1356
|
+
def coverage_candidate(
|
|
1357
|
+
owner: str | None, item: dict[str, Any]
|
|
1358
|
+
) -> tuple[str | None, str] | None:
|
|
1359
|
+
for field in ("surfaces", "deferred", "explicitExclusions"):
|
|
1360
|
+
source_owner, original = record_source(field, owner, item)
|
|
1361
|
+
if original is not item:
|
|
1362
|
+
owner, item = source_owner, original
|
|
1363
|
+
break
|
|
1364
|
+
else:
|
|
1365
|
+
# A later parent can complete a known worker proof gap, but its
|
|
1366
|
+
# claimed provenance cannot overturn an accepted worker finding.
|
|
1367
|
+
provenance = item.get("provenance")
|
|
1368
|
+
if owner is None and isinstance(provenance, dict):
|
|
1369
|
+
source_owner = provenance.get("workerId")
|
|
1370
|
+
attempt = provenance.get("attempt")
|
|
1371
|
+
candidate_id = provenance.get("candidateId")
|
|
1372
|
+
if (
|
|
1373
|
+
isinstance(source_owner, str)
|
|
1374
|
+
and isinstance(attempt, int)
|
|
1375
|
+
and isinstance(candidate_id, str)
|
|
1376
|
+
and (source_owner, attempt, candidate_id) in accepted_pending_candidates
|
|
1377
|
+
):
|
|
1378
|
+
return source_owner, candidate_id
|
|
1379
|
+
candidate_id = (
|
|
1380
|
+
_deferred_candidate_id(item, owner, ambiguous_deferred)
|
|
1381
|
+
if any(key in item for key in ("candidate", "finding"))
|
|
1382
|
+
else item.get("candidateId")
|
|
1383
|
+
)
|
|
1384
|
+
return (owner, candidate_id) if isinstance(candidate_id, str) else None
|
|
1385
|
+
|
|
1386
|
+
def candidate_order(
|
|
1387
|
+
relative: str, owner: str | None, item: dict[str, Any] | None = None
|
|
1388
|
+
) -> tuple[int, int]:
|
|
1389
|
+
order = source_order[relative]
|
|
1390
|
+
if source_owners[relative] is None and owner is not None:
|
|
1391
|
+
if item is not None:
|
|
1392
|
+
for field in ("surfaces", "deferred", "explicitExclusions"):
|
|
1393
|
+
key = projection_key(field, item)
|
|
1394
|
+
if (source := projected_sources.get(key)) and source[0] == owner:
|
|
1395
|
+
return source[2]
|
|
1396
|
+
attempt = int(workers_by_id[owner]["attempt"] or 0) if owner in headed_workers else 0
|
|
1397
|
+
return attempt, order[1]
|
|
1398
|
+
return order
|
|
1399
|
+
|
|
1400
|
+
deferred_rows = {
|
|
1401
|
+
relative: _coverage_rows(draft["coverage"], "deferred")
|
|
1402
|
+
for relative, draft, _ in all_sources
|
|
1403
|
+
}
|
|
1404
|
+
# Accepted heads can replace equal tasks before they reach output coalescing.
|
|
1405
|
+
equivalent_tasks: dict[str | None, list[dict[str, Any]]] = {}
|
|
1406
|
+
for relative, _, owner in all_sources:
|
|
1407
|
+
known = equivalent_tasks.setdefault(owner, [])
|
|
1408
|
+
for row in deferred_rows[relative]:
|
|
1409
|
+
if not isinstance(row, dict):
|
|
1410
|
+
continue
|
|
1411
|
+
for previous in known:
|
|
1412
|
+
if previous == row:
|
|
1413
|
+
_inherit_coverage_sources(previous, [row], coverage_sources)
|
|
1414
|
+
coverage_sources[id(row)] = (row, coverage_sources[id(previous)][1])
|
|
1415
|
+
break
|
|
1416
|
+
else:
|
|
1417
|
+
known.append(row)
|
|
1418
|
+
# A legacy source can contain independent tasks with the same explicit ID.
|
|
1419
|
+
# Later rewrites cannot make an ID-only closure identify one of those tasks.
|
|
1420
|
+
ambiguous_deferred: set[tuple[str | None, str]] = set()
|
|
1421
|
+
for relative, _, owner in all_sources:
|
|
1422
|
+
by_id: dict[tuple[str | None, Any], dict[str, Any]] = {}
|
|
1423
|
+
candidate_aliases = {
|
|
1424
|
+
(source_owner, identity)
|
|
1425
|
+
for row in deferred_rows[relative]
|
|
1426
|
+
if isinstance(row, dict) and _has_candidate_metadata(row)
|
|
1427
|
+
for source_owner, original in [record_source("deferred", owner, row)]
|
|
1428
|
+
for identity in (original.get("id"), original.get("candidateId"))
|
|
1429
|
+
if isinstance(identity, str)
|
|
1430
|
+
}
|
|
1431
|
+
for row in deferred_rows[relative]:
|
|
1432
|
+
if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
|
|
1433
|
+
continue
|
|
1434
|
+
if _has_candidate_metadata(row):
|
|
1435
|
+
continue
|
|
1436
|
+
key = record_key("deferred", owner, row)
|
|
1437
|
+
if key in candidate_aliases or (key in by_id and row != by_id[key]):
|
|
1438
|
+
ambiguous_deferred.add(key)
|
|
1439
|
+
by_id[key] = row
|
|
1440
|
+
current_drafts = ([("parent", parent, None)] if parent else []) + [
|
|
1441
|
+
source for source in sources if source[0] in current_results | selected_observations.keys()
|
|
1442
|
+
]
|
|
1443
|
+
|
|
1444
|
+
pending_surface_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1445
|
+
for relative, draft, owner in all_sources:
|
|
1446
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1447
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1448
|
+
if (
|
|
1449
|
+
isinstance(surface, dict)
|
|
1450
|
+
and surface.get("disposition") == "needs_follow_up"
|
|
1451
|
+
and (candidate := coverage_candidate(owner, surface)) is not None
|
|
1452
|
+
):
|
|
1453
|
+
order = candidate_order(relative, candidate[0], surface)
|
|
1454
|
+
pending_surface_orders[candidate] = max(
|
|
1455
|
+
pending_surface_orders.get(candidate, order), order
|
|
1456
|
+
)
|
|
1457
|
+
# Generic closures belong to one logical scan or worker, just like candidates.
|
|
1458
|
+
# Keep them when recovering a terminal checkpoint without its canonical write.
|
|
1459
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1460
|
+
|
|
1461
|
+
candidate_ids: set[tuple[str | None, str]] = set()
|
|
1462
|
+
accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1463
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1464
|
+
coverage_schema = _read_json(
|
|
1465
|
+
Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"
|
|
1466
|
+
)["properties"]
|
|
1467
|
+
|
|
1468
|
+
projected_closure_sources = _projected_worker_closure_sources(
|
|
1469
|
+
sources, source_order, worker_attempts, coverage_schema["resolvedDeferred"]
|
|
1470
|
+
)
|
|
1471
|
+
|
|
1472
|
+
for relative, draft, owner in all_sources:
|
|
1473
|
+
for item in deferred_rows[relative]:
|
|
1474
|
+
if not isinstance(item, dict):
|
|
1475
|
+
continue
|
|
1476
|
+
source_owner, original = record_source("deferred", owner, item)
|
|
1477
|
+
order = candidate_order(relative, source_owner, item)
|
|
1478
|
+
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
|
|
1479
|
+
candidate_ids.update(
|
|
1480
|
+
(source_owner, identity)
|
|
1481
|
+
for identity in (original.get("id"), original.get("candidateId"))
|
|
1482
|
+
if isinstance(identity, str)
|
|
1483
|
+
)
|
|
1484
|
+
identity = original.get("id") or original.get("candidateId")
|
|
1485
|
+
if isinstance(identity, str):
|
|
1486
|
+
key = (source_owner, identity)
|
|
1487
|
+
previous = active_deferred.get(key)
|
|
1488
|
+
if previous is None or order > previous[0]:
|
|
1489
|
+
active_deferred[key] = (order, original, relative)
|
|
1490
|
+
if (
|
|
1491
|
+
relative == "parent"
|
|
1492
|
+
or relative in current_results
|
|
1493
|
+
or relative in selected_observations
|
|
1494
|
+
):
|
|
1495
|
+
try:
|
|
1496
|
+
# A malformed update cannot discard valid saved evidence.
|
|
1497
|
+
_validate_schema_node(
|
|
1498
|
+
{"id": identity, **item},
|
|
1499
|
+
coverage_schema["deferred"]["items"],
|
|
1500
|
+
"coverage.deferred",
|
|
1501
|
+
)
|
|
1502
|
+
except ContractError:
|
|
1503
|
+
continue
|
|
1504
|
+
accepted_deferred_orders[key] = max(
|
|
1505
|
+
accepted_deferred_orders.get(key, order), order
|
|
1506
|
+
)
|
|
1507
|
+
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
|
|
1508
|
+
closure_owner, closure_relative = owner, relative
|
|
1509
|
+
if owner is None and (
|
|
1510
|
+
source := projected_closure_sources.get((closure["id"], closure["reason"]))
|
|
1511
|
+
):
|
|
1512
|
+
# A copied parent closeout keeps its worker's identity and observation
|
|
1513
|
+
# time, so a later parent write cannot hide reopened worker work.
|
|
1514
|
+
closure_owner, closure, closure_relative = source
|
|
1515
|
+
key = (closure_owner, closure["id"])
|
|
1516
|
+
order = candidate_order(closure_relative, closure_owner)
|
|
1517
|
+
previous = closed_deferred.get(key)
|
|
1518
|
+
if previous is None or order > previous[0]:
|
|
1519
|
+
# The parent comes first, preserving its reason on equal timestamps.
|
|
1520
|
+
closed_deferred[key] = (order, closure, closure_relative)
|
|
1521
|
+
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
|
|
1522
|
+
reopened_generic: set[tuple[str | None, str]] = set()
|
|
1523
|
+
for key, (order, _, _) in list(closed_deferred.items()):
|
|
1524
|
+
# Equal timestamps cannot distinguish closure from reopened work.
|
|
1525
|
+
reopened = [
|
|
1526
|
+
active
|
|
1527
|
+
for (owner, identity), active in active_deferred.items()
|
|
1528
|
+
if owner == key[0]
|
|
1529
|
+
and (identity == key[1] or active[1].get("candidateId") == key[1])
|
|
1530
|
+
and active[0] >= order
|
|
1531
|
+
]
|
|
1532
|
+
if key in candidate_ids or key in ambiguous_deferred or reopened:
|
|
1533
|
+
del closed_deferred[key]
|
|
1534
|
+
for _, item, _ in reopened:
|
|
1535
|
+
reopened_rows.append((key[0], item))
|
|
1536
|
+
if key not in candidate_ids:
|
|
1537
|
+
reopened_generic.add(key)
|
|
1538
|
+
# Retain every distinct ambiguous task even when a terminal result supersedes
|
|
1539
|
+
# its source. Existing output ID normalization keeps each row distinct.
|
|
1540
|
+
for relative, _, owner in all_sources:
|
|
1541
|
+
for row in deferred_rows[relative]:
|
|
1542
|
+
if (
|
|
1543
|
+
isinstance(row, dict)
|
|
1544
|
+
and isinstance(identity := row.get("id"), str)
|
|
1545
|
+
and record_key("deferred", owner, row) in ambiguous_deferred
|
|
1546
|
+
and not _has_candidate_metadata(row)
|
|
1547
|
+
and (owner, row) not in reopened_rows
|
|
1548
|
+
):
|
|
1549
|
+
reopened_rows.append((owner, row))
|
|
1550
|
+
retained_closures = [
|
|
1551
|
+
closure
|
|
1552
|
+
if owner is None
|
|
1553
|
+
else _project_worker_closure(relative, owner, closure, worker_attempts, source_order)
|
|
1554
|
+
for (owner, _), (_, closure, relative) in closed_deferred.items()
|
|
1555
|
+
]
|
|
1556
|
+
coverage.pop("resolvedDeferred", None)
|
|
1557
|
+
if retained_closures:
|
|
1558
|
+
coverage["resolvedDeferred"] = copy.deepcopy(retained_closures)
|
|
1559
|
+
if parent:
|
|
1560
|
+
coverage["deferred"] = [
|
|
1561
|
+
row
|
|
1562
|
+
for row in deferred_rows["parent"]
|
|
1563
|
+
if not isinstance(row, dict)
|
|
1564
|
+
or not isinstance(row.get("id"), str)
|
|
1565
|
+
or (None, row["id"]) not in closed_deferred
|
|
1566
|
+
]
|
|
1567
|
+
|
|
1568
|
+
ordered_candidates = {
|
|
1569
|
+
(owner, identity)
|
|
1570
|
+
for owner, row in reopened_rows
|
|
1571
|
+
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
|
|
1572
|
+
and (owner, identity) in candidate_ids
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
outcomes, pending_candidate_orders = _saved_candidate_outcomes(
|
|
1576
|
+
scan_dir,
|
|
1577
|
+
current_drafts,
|
|
1578
|
+
valid_finding,
|
|
1579
|
+
coverage_candidate,
|
|
1580
|
+
candidate_order,
|
|
1581
|
+
workers_by_id,
|
|
1582
|
+
coverage_schema,
|
|
1583
|
+
)
|
|
1584
|
+
ordered_candidates.update(pending_candidate_orders)
|
|
1585
|
+
|
|
1586
|
+
def select_outcomes(unusable_receipts):
|
|
1587
|
+
permitted = [entry for entry in outcomes if not unusable_receipts.intersection(entry[-1])]
|
|
1588
|
+
resolved, ordered_outcomes = {}, {}
|
|
1589
|
+
candidates = ordered_candidates.union(
|
|
1590
|
+
(owner, candidate_id)
|
|
1591
|
+
for relative, owner, candidate_id, _, _, _ in permitted
|
|
1592
|
+
if relative in selected_observations
|
|
1593
|
+
)
|
|
1594
|
+
selected_head_candidates = {
|
|
1595
|
+
(Path(relative).parent, owner, candidate_id)
|
|
1596
|
+
for relative, owner, candidate_id, _, _, _ in permitted
|
|
1597
|
+
if relative in authoritative_heads
|
|
1598
|
+
}
|
|
1599
|
+
other_root_order = _unselected_root_order(saved_heads, checkpoint_heads, source_order)
|
|
1600
|
+
for relative, owner, candidate_id, disposition, order, _ in permitted:
|
|
1601
|
+
if (
|
|
1602
|
+
relative in selected_observations
|
|
1603
|
+
and relative not in authoritative_heads
|
|
1604
|
+
and relative not in current_results
|
|
1605
|
+
and (Path(relative).parent, owner, candidate_id) in selected_head_candidates
|
|
1606
|
+
):
|
|
1607
|
+
continue
|
|
1608
|
+
key = (owner, candidate_id)
|
|
1609
|
+
if pending_surface_orders.get(key, (-1, -1)) >= order:
|
|
1610
|
+
continue
|
|
1611
|
+
if key not in candidates:
|
|
1612
|
+
if relative == "parent" or relative in current_results:
|
|
1613
|
+
resolved.setdefault(key, disposition)
|
|
1614
|
+
continue
|
|
1615
|
+
root_terminal = (
|
|
1616
|
+
owner is None
|
|
1617
|
+
and (relative == "parent" or relative in authoritative_heads)
|
|
1618
|
+
and (parent if relative == "parent" else drafts_by_path[relative]).get("complete")
|
|
1619
|
+
is not False
|
|
1620
|
+
and other_root_order < order
|
|
1621
|
+
)
|
|
1622
|
+
pending_order = pending_candidate_orders.get(key, (-1, -1))
|
|
1623
|
+
if (pending_order > order or (pending_order == order and not root_terminal)) or any(
|
|
1624
|
+
(
|
|
1625
|
+
coverage_candidate(saved_owner, row)
|
|
1626
|
+
or (saved_owner, _deferred_candidate_id(row, saved_owner, ambiguous_deferred))
|
|
1627
|
+
)
|
|
1628
|
+
== key
|
|
1629
|
+
and (
|
|
1630
|
+
saved_order > order
|
|
1631
|
+
or saved_order == order
|
|
1632
|
+
and (
|
|
1633
|
+
not root_terminal
|
|
1634
|
+
or accepted_deferred_orders.get((saved_owner, identity)) == saved_order
|
|
1635
|
+
)
|
|
1636
|
+
)
|
|
1637
|
+
for (saved_owner, identity), (saved_order, row, _) in active_deferred.items()
|
|
1638
|
+
):
|
|
1639
|
+
continue
|
|
1640
|
+
previous = ordered_outcomes.get(key)
|
|
1641
|
+
if (
|
|
1642
|
+
previous is None
|
|
1643
|
+
or order > previous[0]
|
|
1644
|
+
or (
|
|
1645
|
+
order == previous[0]
|
|
1646
|
+
and previous[1] != relative
|
|
1647
|
+
and previous[1] in current_results
|
|
1648
|
+
and (
|
|
1649
|
+
relative in checkpoint_heads.values()
|
|
1650
|
+
or relative in authoritative_observations
|
|
1651
|
+
)
|
|
1652
|
+
)
|
|
1653
|
+
):
|
|
1654
|
+
resolved[key] = disposition
|
|
1655
|
+
ordered_outcomes[key] = (order, relative)
|
|
1656
|
+
return resolved, ordered_outcomes
|
|
1657
|
+
|
|
1658
|
+
resolved, ordered_outcomes = select_outcomes(set())
|
|
1659
|
+
# Only the current parent may claim that another worker finding was absorbed.
|
|
1660
|
+
# A superseded checkpoint must not suppress a newer independent result.
|
|
1661
|
+
if parent:
|
|
1662
|
+
for finding in parent["findings"]:
|
|
1663
|
+
if valid_finding(finding):
|
|
1664
|
+
canonical = copy.deepcopy(finding)
|
|
1665
|
+
restore_legacy_identity(canonical)
|
|
1666
|
+
canonical_key = _finding_key(canonical)
|
|
1667
|
+
for retained in _retained_findings(finding):
|
|
1668
|
+
retained_key = _finding_key(retained)
|
|
1669
|
+
if retained is not finding:
|
|
1670
|
+
represented_history.setdefault(retained_key, set()).add(
|
|
1671
|
+
_digest(_finding_content(retained))
|
|
1672
|
+
)
|
|
1673
|
+
previous_key = represented.get(retained_key)
|
|
1674
|
+
if retained_key not in represented:
|
|
1675
|
+
represented[retained_key] = canonical_key
|
|
1676
|
+
elif previous_key != canonical_key:
|
|
1677
|
+
# Ambiguous history cannot suppress an independent source.
|
|
1678
|
+
represented[retained_key] = None
|
|
1679
|
+
originals = finding["provenance"].get("sourceFindings", [])
|
|
1680
|
+
for original in originals if isinstance(originals, list) else []:
|
|
1681
|
+
if isinstance(original, dict) and isinstance(original.get("finding"), dict):
|
|
1682
|
+
source_id = original.get("id")
|
|
1683
|
+
candidate_id = finding_candidate_id(original["finding"])
|
|
1684
|
+
if isinstance(source_id, str) and ":" in source_id and candidate_id:
|
|
1685
|
+
candidate_key = _worker_candidate_key(
|
|
1686
|
+
source_id.rsplit(":", 1)[0],
|
|
1687
|
+
candidate_id,
|
|
1688
|
+
original["finding"],
|
|
1689
|
+
)
|
|
1690
|
+
previous_key = represented_candidates.get(candidate_key)
|
|
1691
|
+
if candidate_key not in represented_candidates:
|
|
1692
|
+
represented_candidates[candidate_key] = canonical_key
|
|
1693
|
+
elif previous_key != canonical_key:
|
|
1694
|
+
# Candidate ids are only authoritative within one
|
|
1695
|
+
# logical worker. Multiple canonical owners make
|
|
1696
|
+
# that worker-local identity ambiguous.
|
|
1697
|
+
represented_candidates[candidate_key] = None
|
|
1698
|
+
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
1699
|
+
_digest(_finding_content(original["finding"]))
|
|
1700
|
+
)
|
|
1701
|
+
selected_terminal_orders: dict[str | None, tuple[int, int]] = {}
|
|
1702
|
+
for relative, draft, worker_id in sources:
|
|
1703
|
+
if (
|
|
1704
|
+
relative in authoritative_heads
|
|
1705
|
+
and draft.get("complete") is not False
|
|
1706
|
+
and relative not in discarded_by_source
|
|
1707
|
+
and relative not in discarded_coverage_sources
|
|
1708
|
+
):
|
|
1709
|
+
order = source_order[relative]
|
|
1710
|
+
selected_terminal_orders[worker_id] = max(
|
|
1711
|
+
selected_terminal_orders.get(worker_id, order), order
|
|
1712
|
+
)
|
|
1713
|
+
terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
|
|
1714
|
+
terminal_coverage_orders: dict[str | None, tuple[int, int]] = {}
|
|
1715
|
+
for relative, draft, worker_id in sources:
|
|
1716
|
+
if (
|
|
1717
|
+
relative in current_results
|
|
1718
|
+
and draft.get("complete") is not False
|
|
1719
|
+
and relative not in discarded_by_source
|
|
1720
|
+
):
|
|
1721
|
+
order = source_order[relative]
|
|
1722
|
+
terminal_worker_orders[worker_id] = max(
|
|
1723
|
+
terminal_worker_orders.get(worker_id, order), order
|
|
1724
|
+
)
|
|
1725
|
+
if relative not in reducer_paths and relative not in discarded_coverage_sources:
|
|
1726
|
+
terminal_coverage_orders[worker_id] = max(
|
|
1727
|
+
terminal_coverage_orders.get(worker_id, order), order
|
|
1728
|
+
)
|
|
1729
|
+
prior_terminal_orders: dict[str | None, tuple[int, int]] = {}
|
|
1730
|
+
partial_head_orders = {
|
|
1731
|
+
Path(path).parent: selected_observations[path]
|
|
1732
|
+
for path in authoritative_heads
|
|
1733
|
+
if drafts_by_path[path].get("complete") is False
|
|
1734
|
+
}
|
|
1735
|
+
for head, checkpoint in saved_heads.items():
|
|
1736
|
+
directory = _checkpoint_head_directory(head)
|
|
1737
|
+
relative = (directory / "checkpoints" / checkpoint).as_posix()
|
|
1738
|
+
partial_order = partial_head_orders.get(Path(relative).parent)
|
|
1739
|
+
if partial_order is None:
|
|
1740
|
+
continue
|
|
1741
|
+
order = (partial_order[0], source_order[head][1])
|
|
1742
|
+
if (
|
|
1743
|
+
drafts_by_path[relative].get("complete") is not False
|
|
1744
|
+
and relative not in discarded_by_source
|
|
1745
|
+
and relative not in discarded_coverage_sources
|
|
1746
|
+
and order <= partial_order
|
|
1747
|
+
):
|
|
1748
|
+
# A partial head adds coverage after the preceding terminal replacement.
|
|
1749
|
+
worker_id = paths[relative]
|
|
1750
|
+
prior_terminal_orders[worker_id] = max(
|
|
1751
|
+
prior_terminal_orders.get(worker_id, order), order
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
# Reconcile raw records before generating context-dependent publication IDs.
|
|
1755
|
+
logical_rows: dict[tuple[Any, ...], dict[str, dict[str, Any]]] = {}
|
|
1756
|
+
row_groups: dict[tuple[str, int], dict[str, Any]] = {}
|
|
1757
|
+
published_groups: list[dict[str, Any]] = []
|
|
1758
|
+
|
|
1759
|
+
def same_owner(left: str | None, right: str | None) -> bool:
|
|
1760
|
+
# Canonical recovery needs identity or recorded lineage to absorb worker evidence.
|
|
1761
|
+
return left == right or (not preserve_canonical_parent and (left is None or right is None))
|
|
1762
|
+
|
|
1763
|
+
def finding_source_order(source, within_worker: bool = False):
|
|
1764
|
+
relative, draft, _ = source
|
|
1765
|
+
attempt, modified = source_order[relative]
|
|
1766
|
+
return (
|
|
1767
|
+
draft.get("complete") is not False,
|
|
1768
|
+
attempt if within_worker else 0,
|
|
1769
|
+
modified,
|
|
1770
|
+
relative in selected_observations,
|
|
1771
|
+
relative in current_results,
|
|
1772
|
+
relative == "parent",
|
|
1773
|
+
)
|
|
1774
|
+
|
|
1775
|
+
terminal_findings: dict[str, list[dict[str, Any]]] = {}
|
|
1776
|
+
# A reducer replaces only its claimed batch, not the whole parent scan.
|
|
1777
|
+
parent_sources = {
|
|
1778
|
+
relative
|
|
1779
|
+
for relative, _, _ in all_sources
|
|
1780
|
+
if Path(relative).parent == Path("checkpoints") or (relative == "parent" and parent_is_root)
|
|
1781
|
+
}
|
|
1782
|
+
projected_paths = {
|
|
1783
|
+
relative
|
|
1784
|
+
for relative, draft, _ in all_sources
|
|
1785
|
+
if relative in parent_sources
|
|
1786
|
+
and (
|
|
1787
|
+
relative == "parent"
|
|
1788
|
+
or relative == parent_context.get("canonical")
|
|
1789
|
+
# The semantic tool schema forbids this host-owned projection field.
|
|
1790
|
+
or draft["coverage"].get("mode") == binding["coverageMode"]
|
|
1791
|
+
)
|
|
1792
|
+
}
|
|
1793
|
+
projected_parents = [draft for relative, draft, _ in all_sources if relative in projected_paths]
|
|
1794
|
+
|
|
1795
|
+
def same_published_finding(left: dict[str, Any], right: dict[str, Any]) -> bool:
|
|
1796
|
+
if not valid_finding(left) or not valid_finding(right):
|
|
1797
|
+
return False
|
|
1798
|
+
return (
|
|
1799
|
+
same_raw_value(_finding_content(left), _finding_content(right))
|
|
1800
|
+
and (
|
|
1801
|
+
"identity" not in left
|
|
1802
|
+
or same_raw_value(_finding_identity(left), _finding_identity(right))
|
|
1803
|
+
)
|
|
1804
|
+
and same_raw_value(
|
|
1805
|
+
{
|
|
1806
|
+
key: value
|
|
1807
|
+
for key, value in left.get("provenance", {}).items()
|
|
1808
|
+
if key not in ("previousFindings", "preservedIdentity")
|
|
1809
|
+
},
|
|
1810
|
+
{
|
|
1811
|
+
key: value
|
|
1812
|
+
for key, value in right.get("provenance", {}).items()
|
|
1813
|
+
if key not in ("previousFindings", "preservedIdentity")
|
|
1814
|
+
},
|
|
1815
|
+
)
|
|
1816
|
+
)
|
|
1817
|
+
|
|
1818
|
+
def represented_by_parent(values: list[dict[str, Any]], draft: dict[str, Any]) -> bool:
|
|
1819
|
+
remaining = [row for row in draft["findings"] if isinstance(row, dict)]
|
|
1820
|
+
for value in values:
|
|
1821
|
+
match = next(
|
|
1822
|
+
(
|
|
1823
|
+
index
|
|
1824
|
+
for index, row in enumerate(remaining)
|
|
1825
|
+
if same_published_finding(value, row)
|
|
1826
|
+
),
|
|
1827
|
+
None,
|
|
1828
|
+
)
|
|
1829
|
+
if match is None:
|
|
1830
|
+
return False
|
|
1831
|
+
remaining.pop(match)
|
|
1832
|
+
return True
|
|
1833
|
+
|
|
1834
|
+
terminal_sources: set[str] = set()
|
|
1835
|
+
for relative, draft, _ in all_sources:
|
|
1836
|
+
if (
|
|
1837
|
+
scan_mode == "deep"
|
|
1838
|
+
and not allow_frozen_legacy_parent
|
|
1839
|
+
and relative in parent_sources
|
|
1840
|
+
and draft.get("complete") is not False
|
|
1841
|
+
and all(valid_finding(value) for value in draft["findings"])
|
|
1842
|
+
):
|
|
1843
|
+
terminal_sources.add(relative)
|
|
1844
|
+
if relative not in projected_paths:
|
|
1845
|
+
identified = copy.deepcopy(draft["findings"])
|
|
1846
|
+
_ensure_finding_identities(identified)
|
|
1847
|
+
terminal_findings[relative] = identified
|
|
1848
|
+
# Legacy projections alone do not record the submitted replacement batch.
|
|
1849
|
+
if not terminal_findings:
|
|
1850
|
+
terminal_sources.clear()
|
|
1851
|
+
head_path = (
|
|
1852
|
+
f"checkpoints/{saved_heads[parent_context['head']]}"
|
|
1853
|
+
if "head" in parent_context and not allow_frozen_legacy_parent
|
|
1854
|
+
else None
|
|
1855
|
+
)
|
|
1856
|
+
head_input = saved_inputs.get(parent_context.get("head"))
|
|
1857
|
+
ancestors = _parent_publication_ancestors(head_input, drafts_by_path)
|
|
1858
|
+
committed_parent = parent
|
|
1859
|
+
committed_path = "parent" if parent_is_root else None
|
|
1860
|
+
if head_path and (
|
|
1861
|
+
not parent_is_canonical or any(parent == drafts_by_path.get(path) for path in ancestors)
|
|
1862
|
+
):
|
|
1863
|
+
committed_path = head_path
|
|
1864
|
+
committed_parent = drafts_by_path[head_path]
|
|
1865
|
+
committed_time = source_order.get(committed_path, (0, 0))[1]
|
|
1866
|
+
pending_terminals = set()
|
|
1867
|
+
superseded_terminals = set()
|
|
1868
|
+
unordered_terminals = set()
|
|
1869
|
+
for relative, values in terminal_findings.items():
|
|
1870
|
+
publication = drafts_by_path[relative].get("publication")
|
|
1871
|
+
if isinstance(publication, dict):
|
|
1872
|
+
previous = publication.get("previousCheckpoint")
|
|
1873
|
+
if Path(relative).name == head_input:
|
|
1874
|
+
continue
|
|
1875
|
+
if (
|
|
1876
|
+
f"checkpoints/{previous}" if previous is not None else None
|
|
1877
|
+
) != head_path or publication.get("previousInputCheckpoint") != head_input:
|
|
1878
|
+
superseded_terminals.add(relative)
|
|
1879
|
+
continue
|
|
1880
|
+
pending_terminals.add(relative)
|
|
1881
|
+
elif not any(represented_by_parent(values, draft) for draft in projected_parents):
|
|
1882
|
+
# Legacy inputs have no causal link to an accepted publication. A
|
|
1883
|
+
# backdated observation cannot prove that accepted findings were removed.
|
|
1884
|
+
if source_order[relative][1] >= committed_time:
|
|
1885
|
+
pending_terminals.add(relative)
|
|
1886
|
+
else:
|
|
1887
|
+
unordered_terminals.add(relative)
|
|
1888
|
+
terminal_cutoffs = (
|
|
1889
|
+
{committed_path}
|
|
1890
|
+
if committed_path in terminal_sources
|
|
1891
|
+
else terminal_sources - superseded_terminals
|
|
1892
|
+
)
|
|
1893
|
+
terminal_parent_time = max(
|
|
1894
|
+
(source_order[relative][1] for relative in pending_terminals or terminal_cutoffs),
|
|
1895
|
+
default=None,
|
|
1896
|
+
)
|
|
1897
|
+
pending_terminals = {
|
|
1898
|
+
relative
|
|
1899
|
+
for relative in pending_terminals
|
|
1900
|
+
if source_order[relative][1] == terminal_parent_time
|
|
1901
|
+
}
|
|
1902
|
+
for relative in pending_terminals | unordered_terminals:
|
|
1903
|
+
identified = copy.deepcopy(drafts_by_path[relative]["findings"])
|
|
1904
|
+
_ensure_finding_identities(identified, deep=True)
|
|
1905
|
+
terminal_findings[relative] = identified
|
|
1906
|
+
current_findings = [
|
|
1907
|
+
previous
|
|
1908
|
+
for value in (committed_parent["findings"] if committed_parent else [])
|
|
1909
|
+
if isinstance(value, dict)
|
|
1910
|
+
for previous in _retained_findings(value)
|
|
1911
|
+
]
|
|
1912
|
+
replaced_findings = [
|
|
1913
|
+
value
|
|
1914
|
+
for draft in projected_parents
|
|
1915
|
+
for value in draft["findings"]
|
|
1916
|
+
if isinstance(value, dict)
|
|
1917
|
+
and (
|
|
1918
|
+
pending_terminals
|
|
1919
|
+
or (
|
|
1920
|
+
terminal_sources
|
|
1921
|
+
and not any(
|
|
1922
|
+
same_published_finding(value, previous) for previous in current_findings
|
|
1923
|
+
)
|
|
1924
|
+
)
|
|
1925
|
+
)
|
|
1926
|
+
]
|
|
1927
|
+
|
|
1928
|
+
def replaced_parent_finding(relative: str, owner: str | None, value: Any) -> bool:
|
|
1929
|
+
if (
|
|
1930
|
+
owner is not None
|
|
1931
|
+
or relative in pending_terminals | unordered_terminals
|
|
1932
|
+
or not isinstance(value, dict)
|
|
1933
|
+
):
|
|
1934
|
+
return False
|
|
1935
|
+
return any(same_published_finding(value, previous) for previous in replaced_findings)
|
|
1936
|
+
|
|
1937
|
+
terminal_worker_attempts: dict[str, int] = {}
|
|
1938
|
+
if terminal_parent_time is not None:
|
|
1939
|
+
for relative, _, owner in all_sources:
|
|
1940
|
+
attempt, modified = source_order[relative]
|
|
1941
|
+
if owner is not None and modified <= terminal_parent_time:
|
|
1942
|
+
terminal_worker_attempts[owner] = max(
|
|
1943
|
+
terminal_worker_attempts.get(owner, attempt), attempt
|
|
1944
|
+
)
|
|
1945
|
+
|
|
1946
|
+
def precedes_terminal_parent(relative: str, owner: str | None) -> bool:
|
|
1947
|
+
attempt, modified = source_order[relative]
|
|
1948
|
+
return relative in superseded_terminals or (
|
|
1949
|
+
relative not in unordered_terminals
|
|
1950
|
+
and terminal_parent_time is not None
|
|
1951
|
+
and (
|
|
1952
|
+
modified < terminal_parent_time
|
|
1953
|
+
or attempt < terminal_worker_attempts.get(owner, attempt)
|
|
1954
|
+
)
|
|
1955
|
+
)
|
|
1956
|
+
|
|
1957
|
+
finding_sources = [
|
|
1958
|
+
source for source in all_sources if not precedes_terminal_parent(source[0], source[2])
|
|
1959
|
+
]
|
|
1960
|
+
# Publication chronology spans workers; attempt numbers only order one worker's revisions.
|
|
1961
|
+
for source in sorted(finding_sources, key=lambda item: finding_source_order(item)[2:]):
|
|
1962
|
+
relative, draft, source_owner = source
|
|
1963
|
+
explicit_identity_counts = Counter(
|
|
1964
|
+
_semantic_identity_key(sibling, _finding_identity(sibling))
|
|
1965
|
+
for sibling in draft["findings"]
|
|
1966
|
+
if isinstance(sibling, dict)
|
|
1967
|
+
and isinstance(sibling.get("identity"), dict)
|
|
1968
|
+
and ((relative == "parent" and parent_is_canonical) or finding_within_scope(sibling))
|
|
1969
|
+
)
|
|
1970
|
+
pending = []
|
|
1971
|
+
for index, value in enumerate(draft["findings"]):
|
|
1972
|
+
if (
|
|
1973
|
+
not isinstance(value, dict)
|
|
1974
|
+
or replaced_parent_finding(relative, source_owner, value)
|
|
1975
|
+
or not valid_finding(value)
|
|
1976
|
+
or (
|
|
1977
|
+
not (relative == "parent" and parent_is_canonical)
|
|
1978
|
+
and not finding_within_scope(value)
|
|
1979
|
+
)
|
|
1980
|
+
):
|
|
1981
|
+
continue
|
|
1982
|
+
owner = raw_owner(value, source_owner)
|
|
1983
|
+
scope = raw_scope(value)
|
|
1984
|
+
candidate = finding_candidate_id(value)
|
|
1985
|
+
scopes = [scope, (scope[0], candidate)] if candidate else [scope]
|
|
1986
|
+
publication_identity = (
|
|
1987
|
+
_finding_identity(
|
|
1988
|
+
value if "identity" in value else terminal_findings[relative][index]
|
|
1989
|
+
)
|
|
1990
|
+
if relative in terminal_findings
|
|
1991
|
+
else None
|
|
1992
|
+
)
|
|
1993
|
+
matches = [
|
|
1994
|
+
group
|
|
1995
|
+
for group in {
|
|
1996
|
+
key: group
|
|
1997
|
+
for saved_scope in scopes
|
|
1998
|
+
for key, group in logical_rows.get(saved_scope, {}).items()
|
|
1999
|
+
}.values()
|
|
2000
|
+
if same_owner(owner, group["owner"])
|
|
2001
|
+
and (
|
|
2002
|
+
publication_identity is None
|
|
2003
|
+
or group["identity"] is None
|
|
2004
|
+
or _semantic_identity_key(value, group["identity"])
|
|
2005
|
+
== _semantic_identity_key(value, publication_identity)
|
|
2006
|
+
)
|
|
2007
|
+
and (
|
|
2008
|
+
(relative == "parent" and parent_is_canonical)
|
|
2009
|
+
or group["identity"] is None
|
|
2010
|
+
or "identity" not in value
|
|
2011
|
+
or _semantic_identity_key(value, group["identity"])
|
|
2012
|
+
== _semantic_identity_key(value, _finding_identity(value))
|
|
2013
|
+
)
|
|
2014
|
+
and all(same_raw_finding(value, previous) for previous in group["rows"])
|
|
2015
|
+
]
|
|
2016
|
+
ranked = []
|
|
2017
|
+
for group in matches:
|
|
2018
|
+
latest = group["latest"]
|
|
2019
|
+
if (
|
|
2020
|
+
isinstance(value.get("identity"), dict)
|
|
2021
|
+
and explicit_identity_counts[
|
|
2022
|
+
_semantic_identity_key(value, _finding_identity(value))
|
|
2023
|
+
]
|
|
2024
|
+
== 1
|
|
2025
|
+
and group["identity"] is not None
|
|
2026
|
+
and _semantic_identity_key(value, _finding_identity(value))
|
|
2027
|
+
== _semantic_identity_key(value, group["identity"])
|
|
2028
|
+
):
|
|
2029
|
+
rank = 4
|
|
2030
|
+
elif (
|
|
2031
|
+
owner == group["owner"]
|
|
2032
|
+
and same_raw_content(value, latest)
|
|
2033
|
+
and _identity_candidate(value) == _identity_candidate(latest)
|
|
2034
|
+
and identifiers(value) == identifiers(latest)
|
|
2035
|
+
):
|
|
2036
|
+
rank = 3
|
|
2037
|
+
elif any(same_raw_content(value, previous) for previous in group["rows"]):
|
|
2038
|
+
rank = 2
|
|
2039
|
+
elif (
|
|
2040
|
+
latest.get("title") == value.get("title")
|
|
2041
|
+
and raw_scope(latest) == scope
|
|
2042
|
+
and all(
|
|
2043
|
+
previous is None or current == previous
|
|
2044
|
+
for current, previous in zip(
|
|
2045
|
+
identifiers(value), identifiers(latest), strict=True
|
|
2046
|
+
)
|
|
2047
|
+
)
|
|
2048
|
+
):
|
|
2049
|
+
rank = 1
|
|
2050
|
+
else:
|
|
2051
|
+
rank = 0
|
|
2052
|
+
ranked.append((group, rank))
|
|
2053
|
+
pending.append((index, value, owner, scopes, ranked))
|
|
2054
|
+
assigned = {}
|
|
2055
|
+
claimed = set()
|
|
2056
|
+
for index, value, owner, _, matches in pending:
|
|
2057
|
+
for group, _ in matches:
|
|
2058
|
+
if (
|
|
2059
|
+
group["key"] not in claimed
|
|
2060
|
+
and owner == group["owner"]
|
|
2061
|
+
and same_raw_value(value, group["latest"])
|
|
2062
|
+
):
|
|
2063
|
+
assigned[index] = group
|
|
2064
|
+
claimed.add(group["key"])
|
|
2065
|
+
break
|
|
2066
|
+
for rank in (4, 3, 2, 1, 0):
|
|
2067
|
+
identity_claims = {
|
|
2068
|
+
group["key"]
|
|
2069
|
+
for index, _, _, _, matches in pending
|
|
2070
|
+
for group, score in matches
|
|
2071
|
+
if score == 4 and assigned.get(index) is group
|
|
2072
|
+
}
|
|
2073
|
+
options = {
|
|
2074
|
+
index: [
|
|
2075
|
+
group
|
|
2076
|
+
for group, score in matches
|
|
2077
|
+
if score >= rank
|
|
2078
|
+
and (
|
|
2079
|
+
group["key"] not in claimed
|
|
2080
|
+
or (rank == 0 and group["key"] not in identity_claims)
|
|
2081
|
+
)
|
|
2082
|
+
]
|
|
2083
|
+
for index, _, _, _, matches in pending
|
|
2084
|
+
if index not in assigned
|
|
2085
|
+
}
|
|
2086
|
+
counts = {}
|
|
2087
|
+
for groups in options.values():
|
|
2088
|
+
for group in groups:
|
|
2089
|
+
counts[group["key"]] = counts.get(group["key"], 0) + 1
|
|
2090
|
+
chosen = {
|
|
2091
|
+
index: groups[0]
|
|
2092
|
+
for index, groups in options.items()
|
|
2093
|
+
if len(groups) == 1
|
|
2094
|
+
and groups[0]["key"] not in claimed
|
|
2095
|
+
and counts[groups[0]["key"]] == 1
|
|
2096
|
+
}
|
|
2097
|
+
assigned.update(chosen)
|
|
2098
|
+
claimed.update(group["key"] for group in chosen.values())
|
|
2099
|
+
current_groups: list[tuple[int, dict[str, Any]]] = []
|
|
2100
|
+
for index, value, owner, scopes, _ in pending:
|
|
2101
|
+
match = assigned.get(index)
|
|
2102
|
+
if match is None:
|
|
2103
|
+
match = {
|
|
2104
|
+
"owner": owner,
|
|
2105
|
+
"rows": [],
|
|
2106
|
+
"identity": None,
|
|
2107
|
+
"key": f"raw:{len(row_groups)}",
|
|
2108
|
+
}
|
|
2109
|
+
for saved_scope in scopes:
|
|
2110
|
+
logical_rows.setdefault(saved_scope, {})[match["key"]] = match
|
|
2111
|
+
if match["owner"] is None:
|
|
2112
|
+
match["owner"] = owner
|
|
2113
|
+
if not any(same_raw_value(value, previous) for previous in match["rows"]):
|
|
2114
|
+
match["rows"].append(value)
|
|
2115
|
+
observations = match.setdefault("observations", {})
|
|
2116
|
+
previous = observations.get(source_owner)
|
|
2117
|
+
if previous is None or (
|
|
2118
|
+
finding_source_order(source, True),
|
|
2119
|
+
_finding_strength(value),
|
|
2120
|
+
) > (
|
|
2121
|
+
finding_source_order(previous[0], True),
|
|
2122
|
+
_finding_strength(previous[1]),
|
|
2123
|
+
):
|
|
2124
|
+
observations[source_owner] = (source, value)
|
|
2125
|
+
match["latest"] = max(
|
|
2126
|
+
observations.values(),
|
|
2127
|
+
key=lambda observation: (
|
|
2128
|
+
finding_source_order(observation[0]),
|
|
2129
|
+
_finding_strength(observation[1]),
|
|
2130
|
+
),
|
|
2131
|
+
)[1]
|
|
2132
|
+
if relative in terminal_findings:
|
|
2133
|
+
match["identity"] = copy.deepcopy(
|
|
2134
|
+
_finding_identity(
|
|
2135
|
+
value if "identity" in value else terminal_findings[relative][index]
|
|
2136
|
+
)
|
|
2137
|
+
)
|
|
2138
|
+
elif "identity" in value and (match["identity"] is None or relative == "parent"):
|
|
2139
|
+
match["identity"] = copy.deepcopy(_finding_identity(value))
|
|
2140
|
+
row_groups[(relative, index)] = match
|
|
2141
|
+
current_groups.append((index, match))
|
|
2142
|
+
if source_owner is None:
|
|
2143
|
+
current_ordered = [
|
|
2144
|
+
group for _, group in sorted(current_groups, key=lambda item: item[0])
|
|
2145
|
+
]
|
|
2146
|
+
ordered = current_ordered + [
|
|
2147
|
+
group for group in published_groups if group not in current_ordered
|
|
2148
|
+
]
|
|
2149
|
+
normalized = []
|
|
2150
|
+
for group in ordered:
|
|
2151
|
+
value = copy.deepcopy(group["latest"])
|
|
2152
|
+
if group["identity"] is not None:
|
|
2153
|
+
value["identity"] = copy.deepcopy(group["identity"])
|
|
2154
|
+
normalized.append(value)
|
|
2155
|
+
_ensure_finding_identities(normalized)
|
|
2156
|
+
for group, value in zip(ordered, normalized, strict=True):
|
|
2157
|
+
group["identity"] = value["identity"]
|
|
2158
|
+
published_groups = ordered
|
|
2159
|
+
|
|
2160
|
+
# A later worker draft can collapse raw duplicates that it fully contains.
|
|
2161
|
+
# Keep established identities and any evidence absent from the newer report.
|
|
2162
|
+
contained_groups = set()
|
|
2163
|
+
for group in {group["key"]: group for group in row_groups.values()}.values():
|
|
2164
|
+
owner = group["owner"]
|
|
2165
|
+
previous = group["observations"].get(owner)
|
|
2166
|
+
if owner is None or group["identity"] is not None or previous is None:
|
|
2167
|
+
continue
|
|
2168
|
+
for current in logical_rows.get(raw_scope(group["latest"]), {}).values():
|
|
2169
|
+
observed = current["observations"].get(owner)
|
|
2170
|
+
if (
|
|
2171
|
+
current["owner"] == owner
|
|
2172
|
+
and observed is not None
|
|
2173
|
+
and finding_source_order(observed[0], True)
|
|
2174
|
+
> finding_source_order(previous[0], True)
|
|
2175
|
+
and all(contains_raw_value(current["latest"], row) for row in group["rows"])
|
|
2176
|
+
):
|
|
2177
|
+
contained_groups.add(group["key"])
|
|
2178
|
+
break
|
|
2179
|
+
|
|
2180
|
+
# An absorbed source represents every reconciled version of that worker report.
|
|
2181
|
+
for groups in logical_rows.values():
|
|
2182
|
+
for group in groups.values():
|
|
2183
|
+
destinations = set()
|
|
2184
|
+
for value in group["rows"]:
|
|
2185
|
+
history_key = _finding_key(value)
|
|
2186
|
+
if _digest(_finding_content(value)) in represented_history.get(history_key, set()):
|
|
2187
|
+
destinations.add(represented.get(history_key))
|
|
2188
|
+
elif group["owner"] and (candidate_id := finding_candidate_id(value)):
|
|
2189
|
+
candidate_key = _worker_candidate_key(group["owner"], candidate_id, value)
|
|
2190
|
+
if _digest(_finding_content(value)) in represented_candidate_history.get(
|
|
2191
|
+
candidate_key, set()
|
|
2192
|
+
):
|
|
2193
|
+
destinations.add(represented_candidates[candidate_key])
|
|
2194
|
+
if len(destinations) == 1 and None not in destinations:
|
|
2195
|
+
group["parent_key"] = destinations.pop()
|
|
2196
|
+
|
|
2197
|
+
generated_worker_findings: list[dict[str, Any]] = []
|
|
2198
|
+
|
|
2199
|
+
def source_coverage_selection(
|
|
2200
|
+
relative: str,
|
|
2201
|
+
worker_id: str | None,
|
|
2202
|
+
) -> tuple[bool, set[str], bool, bool, bool]:
|
|
2203
|
+
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
2204
|
+
selected_terminal_order = selected_terminal_orders.get(worker_id)
|
|
2205
|
+
terminal_coverage_order = max(
|
|
2206
|
+
(
|
|
2207
|
+
order
|
|
2208
|
+
for order in (
|
|
2209
|
+
terminal_coverage_orders.get(worker_id),
|
|
2210
|
+
selected_terminal_order,
|
|
2211
|
+
prior_terminal_orders.get(worker_id),
|
|
2212
|
+
)
|
|
2213
|
+
if order is not None
|
|
2214
|
+
),
|
|
2215
|
+
default=None,
|
|
2216
|
+
)
|
|
2217
|
+
selected_coverage_superseded = (
|
|
2218
|
+
terminal_coverage_order is not None and source_order[relative] < terminal_coverage_order
|
|
2219
|
+
) or (
|
|
2220
|
+
selected_terminal_order is not None
|
|
2221
|
+
and source_order[relative][0] == selected_terminal_order[0]
|
|
2222
|
+
and (
|
|
2223
|
+
relative in selected_observations
|
|
2224
|
+
and relative not in authoritative_heads
|
|
2225
|
+
and relative not in current_results
|
|
2226
|
+
or (
|
|
2227
|
+
relative in current_results
|
|
2228
|
+
and relative not in authoritative_heads
|
|
2229
|
+
and source_order[relative] <= selected_terminal_order
|
|
2230
|
+
)
|
|
2231
|
+
)
|
|
2232
|
+
)
|
|
2233
|
+
selected_candidates = {
|
|
2234
|
+
candidate_id
|
|
2235
|
+
for (owner, candidate_id), (_, source) in ordered_outcomes.items()
|
|
2236
|
+
if owner == worker_id and source == relative
|
|
2237
|
+
}
|
|
2238
|
+
superseded = (
|
|
2239
|
+
worker_id is None
|
|
2240
|
+
and saved_parent is not None
|
|
2241
|
+
and saved_parent.get("complete") is not False
|
|
2242
|
+
and relative != "parent"
|
|
2243
|
+
and source_order[relative] <= (0, parent_modified)
|
|
2244
|
+
and (not stopped_parent_seal or relative in parent_preserved_sources)
|
|
2245
|
+
and (
|
|
2246
|
+
# Explicit recovery can admit checkpoints written after this parent.
|
|
2247
|
+
frozen_parent_path is None
|
|
2248
|
+
or relative == frozen_parent_path
|
|
2249
|
+
or relative in saved_parent.get("previousParentCheckpoints", [])
|
|
2250
|
+
)
|
|
2251
|
+
) or (
|
|
2252
|
+
worker_id is not None
|
|
2253
|
+
and relative not in current_results
|
|
2254
|
+
and worker_result_order is not None
|
|
2255
|
+
and (
|
|
2256
|
+
relative not in selected_observations
|
|
2257
|
+
or source_order[relative] < worker_result_order
|
|
2258
|
+
)
|
|
2259
|
+
)
|
|
2260
|
+
worker_coverage_order = terminal_coverage_orders.get(worker_id)
|
|
2261
|
+
coverage_superseded = (
|
|
2262
|
+
superseded
|
|
2263
|
+
if worker_id is None
|
|
2264
|
+
else (
|
|
2265
|
+
relative not in current_results
|
|
2266
|
+
and worker_coverage_order is not None
|
|
2267
|
+
and (
|
|
2268
|
+
relative not in selected_observations
|
|
2269
|
+
or source_order[relative] < worker_coverage_order
|
|
2270
|
+
)
|
|
2271
|
+
)
|
|
2272
|
+
)
|
|
2273
|
+
# A failed result write can leave pending work outside the accepted result.
|
|
2274
|
+
accepted_order = (
|
|
2275
|
+
(0, parent_modified)
|
|
2276
|
+
if worker_id is None and parent is not None
|
|
2277
|
+
else worker_coverage_order
|
|
2278
|
+
)
|
|
2279
|
+
retain_pending = (
|
|
2280
|
+
coverage_superseded
|
|
2281
|
+
and (worker_id in headed_workers or (worker_id is None and parent_heads))
|
|
2282
|
+
and accepted_order is not None
|
|
2283
|
+
and source_order[relative] >= accepted_order
|
|
2284
|
+
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
2285
|
+
)
|
|
2286
|
+
return (
|
|
2287
|
+
selected_coverage_superseded,
|
|
2288
|
+
selected_candidates,
|
|
2289
|
+
superseded,
|
|
2290
|
+
retain_pending,
|
|
2291
|
+
coverage_superseded,
|
|
2292
|
+
)
|
|
2293
|
+
|
|
2294
|
+
def coverage_row_is_retained(
|
|
2295
|
+
field: str,
|
|
2296
|
+
item: Any,
|
|
2297
|
+
worker_id: str | None,
|
|
2298
|
+
selection: tuple[bool, set[str], bool, bool, bool],
|
|
2299
|
+
) -> bool:
|
|
2300
|
+
(
|
|
2301
|
+
selected_coverage_superseded,
|
|
2302
|
+
selected_candidates,
|
|
2303
|
+
superseded,
|
|
2304
|
+
retain_pending,
|
|
2305
|
+
coverage_superseded,
|
|
2306
|
+
) = selection
|
|
2307
|
+
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
2308
|
+
# candidate evidence and pending tasks retain their own reconciliation.
|
|
2309
|
+
if (
|
|
2310
|
+
selected_coverage_superseded
|
|
2311
|
+
and field != "deferred"
|
|
2312
|
+
and not (isinstance(item, dict) and _has_candidate_metadata(item))
|
|
2313
|
+
):
|
|
2314
|
+
return False
|
|
2315
|
+
# A selected outcome must retain its evidence even if its result write failed.
|
|
2316
|
+
if coverage_superseded and not (
|
|
2317
|
+
isinstance(item, dict)
|
|
2318
|
+
and (
|
|
2319
|
+
(field == "deferred" and retain_pending)
|
|
2320
|
+
or (
|
|
2321
|
+
isinstance(item.get("candidateId"), str)
|
|
2322
|
+
and item["candidateId"] in selected_candidates
|
|
2323
|
+
and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
|
|
2324
|
+
)
|
|
2325
|
+
)
|
|
2326
|
+
):
|
|
2327
|
+
return False
|
|
2328
|
+
if (
|
|
2329
|
+
isinstance(item, dict)
|
|
2330
|
+
and isinstance(
|
|
2331
|
+
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
2332
|
+
if field == "deferred"
|
|
2333
|
+
else item.get("candidateId"),
|
|
2334
|
+
str,
|
|
2335
|
+
)
|
|
2336
|
+
and (coverage_candidate(worker_id, item) or (worker_id, identity)) in resolved
|
|
2337
|
+
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
2338
|
+
):
|
|
2339
|
+
return False
|
|
2340
|
+
return True
|
|
2341
|
+
|
|
2342
|
+
def source_surface_receipts(relative: str, row: dict[str, Any]) -> Any:
|
|
2343
|
+
worker = workers_by_id.get(source_owners.get(relative))
|
|
2344
|
+
return coverage_receipts(row, worker, relative) if worker else row.get("receiptRefs", [])
|
|
2345
|
+
|
|
2346
|
+
replaced_deferred = {
|
|
2347
|
+
id(row)
|
|
2348
|
+
for relative, _, owner in all_sources
|
|
2349
|
+
for row in deferred_rows[relative]
|
|
2350
|
+
if isinstance(row, dict)
|
|
2351
|
+
and isinstance(row.get("id"), str)
|
|
2352
|
+
and (key := record_key("deferred", owner, row)) not in candidate_ids
|
|
2353
|
+
and key not in ambiguous_deferred
|
|
2354
|
+
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
2355
|
+
and updated > candidate_order(relative, key[0], row)
|
|
2356
|
+
}
|
|
2357
|
+
ambiguous_surface_ids = {
|
|
2358
|
+
(owner, identity)
|
|
2359
|
+
for owner, row in reopened_rows
|
|
2360
|
+
if isinstance(row.get("id"), str)
|
|
2361
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
2362
|
+
and not _has_candidate_metadata(row)
|
|
2363
|
+
for identity in [
|
|
2364
|
+
row["id"],
|
|
2365
|
+
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
2366
|
+
]
|
|
2367
|
+
if isinstance(identity, str)
|
|
2368
|
+
}
|
|
2369
|
+
for _, draft, owner in all_sources:
|
|
2370
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
2371
|
+
for surface in _coverage_rows(draft["coverage"], "surfaces"):
|
|
2372
|
+
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
2373
|
+
continue
|
|
2374
|
+
if identity in by_id and surface != by_id[identity]:
|
|
2375
|
+
ambiguous_surface_ids.add((owner, identity))
|
|
2376
|
+
by_id[identity] = surface
|
|
2377
|
+
|
|
2378
|
+
def ambiguous_pending_surface(row: dict[str, Any], owner: str | None) -> bool:
|
|
2379
|
+
return (
|
|
2380
|
+
isinstance(row.get("id"), str)
|
|
2381
|
+
and (owner, row["id"]) in ambiguous_surface_ids
|
|
2382
|
+
and row.get("disposition") == "needs_follow_up"
|
|
2383
|
+
and not _has_candidate_metadata(row)
|
|
2384
|
+
)
|
|
2385
|
+
|
|
2386
|
+
def retain_findings(relative, draft, worker_id, selected_candidates, skip_superseded_findings):
|
|
2387
|
+
normalized_findings = copy.deepcopy(draft["findings"])
|
|
2388
|
+
_ensure_finding_identities(
|
|
2389
|
+
[finding for finding in normalized_findings if valid_finding(finding)]
|
|
2390
|
+
)
|
|
2391
|
+
for index, (value, normalized) in enumerate(
|
|
2392
|
+
zip(draft["findings"], normalized_findings, strict=True)
|
|
2393
|
+
):
|
|
2394
|
+
if precedes_terminal_parent(relative, worker_id) or replaced_parent_finding(
|
|
2395
|
+
relative, worker_id, value
|
|
2396
|
+
):
|
|
2397
|
+
continue
|
|
2398
|
+
_ensure_finding_identity(normalized)
|
|
2399
|
+
group = row_groups.get((relative, index))
|
|
2400
|
+
if group is not None and group["key"] in contained_groups:
|
|
2401
|
+
continue
|
|
2402
|
+
if skip_superseded_findings and not (
|
|
2403
|
+
isinstance(value, dict)
|
|
2404
|
+
and (candidate_id := finding_candidate_id(value)) in selected_candidates
|
|
2405
|
+
and resolved.get((worker_id, candidate_id)) == "reported"
|
|
2406
|
+
):
|
|
2407
|
+
continue
|
|
2408
|
+
if relative == "parent" and parent_is_canonical:
|
|
2409
|
+
finding = copy.deepcopy(value)
|
|
2410
|
+
group = row_groups.get((relative, index))
|
|
2411
|
+
restore_legacy_identity(
|
|
2412
|
+
finding,
|
|
2413
|
+
group or normalized,
|
|
2414
|
+
allow_synthesis=stopped
|
|
2415
|
+
or (isinstance(finding, dict) and _identity_candidate(finding) is not None)
|
|
2416
|
+
or (group is not None and any("identity" in row for row in group["rows"])),
|
|
2417
|
+
)
|
|
2418
|
+
provenance = finding.get("provenance") if isinstance(finding, dict) else None
|
|
2419
|
+
owner = provenance.get("workerId") if isinstance(provenance, dict) else None
|
|
2420
|
+
candidate_id = finding_candidate_id(finding) if isinstance(finding, dict) else None
|
|
2421
|
+
if (
|
|
2422
|
+
stopped_parent_seal
|
|
2423
|
+
and isinstance(owner, str)
|
|
2424
|
+
and candidate_id
|
|
2425
|
+
and resolved.get((owner, candidate_id)) in ("rejected", "not_applicable")
|
|
2426
|
+
):
|
|
2427
|
+
rejected_history.setdefault((owner, candidate_id), []).append(finding)
|
|
2428
|
+
continue
|
|
2429
|
+
if valid_finding(finding):
|
|
2430
|
+
key = _finding_key(finding)
|
|
2431
|
+
if group is not None:
|
|
2432
|
+
key = group.setdefault("merge_key", key)
|
|
2433
|
+
finding_positions.setdefault(key, (len(findings), group))
|
|
2434
|
+
if preserve_canonical_parent:
|
|
2435
|
+
canonical_finding_keys.add(key)
|
|
2436
|
+
findings.append(finding)
|
|
2437
|
+
continue
|
|
2438
|
+
if (
|
|
2439
|
+
relative != "parent"
|
|
2440
|
+
and parent
|
|
2441
|
+
and value in parent["findings"]
|
|
2442
|
+
and not precedes_terminal_parent("parent", None)
|
|
2443
|
+
and not replaced_parent_finding("parent", None, value)
|
|
2444
|
+
):
|
|
2445
|
+
continue
|
|
2446
|
+
if not isinstance(value, dict):
|
|
2447
|
+
warnings.append(f"Retained malformed finding evidence in {relative}.")
|
|
2448
|
+
continue
|
|
2449
|
+
finding = copy.deepcopy(value)
|
|
2450
|
+
candidate_id = finding_candidate_id(finding)
|
|
2451
|
+
if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
|
|
2452
|
+
"rejected",
|
|
2453
|
+
"not_applicable",
|
|
2454
|
+
}:
|
|
2455
|
+
if preserve_canonical_parent:
|
|
2456
|
+
rejected_history.setdefault((worker_id, candidate_id), []).append(finding)
|
|
2457
|
+
for item in _coverage_rows(coverage, "surfaces"):
|
|
2458
|
+
if isinstance(item, dict) and (
|
|
2459
|
+
coverage_candidate(None, item) == (worker_id, candidate_id)
|
|
2460
|
+
or (
|
|
2461
|
+
preserve_canonical_parent
|
|
2462
|
+
and item.get("candidateId") == candidate_id
|
|
2463
|
+
and any(
|
|
2464
|
+
source_owners[source] == worker_id
|
|
2465
|
+
for source in coverage_sources.get(id(item), (item, []))[1]
|
|
2466
|
+
)
|
|
2467
|
+
)
|
|
2468
|
+
or (
|
|
2469
|
+
not preserve_canonical_parent
|
|
2470
|
+
and coverage_candidate(None, item) == (None, candidate_id)
|
|
2471
|
+
)
|
|
2472
|
+
):
|
|
2473
|
+
_append_previous_finding(item, finding)
|
|
2474
|
+
continue
|
|
2475
|
+
for key in ("findingId", "occurrenceId", "fingerprints"):
|
|
2476
|
+
finding.pop(key, None)
|
|
2477
|
+
if not finding_within_scope(finding):
|
|
2478
|
+
warnings.append(f"Skipped out-of-scope finding from {relative}.")
|
|
2479
|
+
continue
|
|
2480
|
+
provenance = finding.setdefault("provenance", {"source": "local_plugin"})
|
|
2481
|
+
if not isinstance(provenance, dict):
|
|
2482
|
+
findings.append(finding)
|
|
2483
|
+
continue
|
|
2484
|
+
if worker_id:
|
|
2485
|
+
provenance.setdefault("workerId", worker_id)
|
|
2486
|
+
group = row_groups.get((relative, index))
|
|
2487
|
+
if relative in terminal_findings:
|
|
2488
|
+
terminal = terminal_findings[relative][index]
|
|
2489
|
+
finding["identity"] = copy.deepcopy(terminal["identity"])
|
|
2490
|
+
if "preservedIdentity" in terminal.get("provenance", {}):
|
|
2491
|
+
provenance.setdefault(
|
|
2492
|
+
"preservedIdentity",
|
|
2493
|
+
copy.deepcopy(terminal["provenance"]["preservedIdentity"]),
|
|
2494
|
+
)
|
|
2495
|
+
elif group is not None and group["identity"] is not None:
|
|
2496
|
+
finding.setdefault("identity", copy.deepcopy(group["identity"]))
|
|
2497
|
+
if worker_id is not None and "identity" not in finding:
|
|
2498
|
+
generated_worker_findings.append(finding)
|
|
2499
|
+
finding.setdefault("identity", normalized["identity"])
|
|
2500
|
+
if not valid_finding(finding):
|
|
2501
|
+
findings.append(finding)
|
|
2502
|
+
continue
|
|
2503
|
+
# Terminal siblings remain separate even when their preserved authored
|
|
2504
|
+
# identities and locations coincide. Worker revisions reuse the group.
|
|
2505
|
+
key = (
|
|
2506
|
+
group["key"]
|
|
2507
|
+
if group is not None
|
|
2508
|
+
and (group["identity"] is None or relative in terminal_findings)
|
|
2509
|
+
else _finding_key(finding)
|
|
2510
|
+
)
|
|
2511
|
+
if group is not None:
|
|
2512
|
+
key = group.setdefault("merge_key", key)
|
|
2513
|
+
# Distinct reconciled reports may share an anchor and source location.
|
|
2514
|
+
if key in finding_positions and distinct_reports(group, finding_positions[key][1]):
|
|
2515
|
+
key = group["merge_key"] = group["key"]
|
|
2516
|
+
key = group.get("parent_key", key)
|
|
2517
|
+
represented_by_parent = False
|
|
2518
|
+
if relative != "parent":
|
|
2519
|
+
history_key = _finding_key(value)
|
|
2520
|
+
# A singleton key is not evidence that a new batch sibling was
|
|
2521
|
+
# absorbed. Only recorded content may reuse that historical edge.
|
|
2522
|
+
if _digest(_finding_content(value)) not in represented_history.get(
|
|
2523
|
+
history_key, set()
|
|
2524
|
+
):
|
|
2525
|
+
history_key = key
|
|
2526
|
+
if history_key in represented:
|
|
2527
|
+
mapped_key = represented[history_key]
|
|
2528
|
+
historical_contents = represented_history.get(history_key, set())
|
|
2529
|
+
elif worker_id and candidate_id:
|
|
2530
|
+
candidate_key = _worker_candidate_key(worker_id, candidate_id, finding)
|
|
2531
|
+
mapped_key = represented_candidates.get(candidate_key)
|
|
2532
|
+
historical_contents = represented_candidate_history.get(candidate_key, set())
|
|
2533
|
+
else:
|
|
2534
|
+
mapped_key = None
|
|
2535
|
+
historical_contents = set()
|
|
2536
|
+
if mapped_key is not None:
|
|
2537
|
+
key = mapped_key
|
|
2538
|
+
represented_by_parent = _digest(_finding_content(value)) in historical_contents
|
|
2539
|
+
if key in finding_positions:
|
|
2540
|
+
position, retained_group = finding_positions[key]
|
|
2541
|
+
retained = findings[position]
|
|
2542
|
+
for field in ("anchor", "instance"):
|
|
2543
|
+
if field in retained["identity"]:
|
|
2544
|
+
finding["identity"][field] = retained["identity"][field]
|
|
2545
|
+
else:
|
|
2546
|
+
finding["identity"].pop(field, None)
|
|
2547
|
+
if finding != retained:
|
|
2548
|
+
if (
|
|
2549
|
+
key not in canonical_finding_keys
|
|
2550
|
+
and not represented_by_parent
|
|
2551
|
+
and (
|
|
2552
|
+
value is group["latest"]
|
|
2553
|
+
if group is not None and group is retained_group
|
|
2554
|
+
else _finding_strength(finding) > _finding_strength(retained)
|
|
2555
|
+
)
|
|
2556
|
+
):
|
|
2557
|
+
previous = copy.deepcopy(retained)
|
|
2558
|
+
previous_history = previous["provenance"].pop("previousFindings", [])
|
|
2559
|
+
if "preservedIdentity" in retained["provenance"]:
|
|
2560
|
+
finding["provenance"]["preservedIdentity"] = copy.deepcopy(
|
|
2561
|
+
retained["provenance"]["preservedIdentity"]
|
|
2562
|
+
)
|
|
2563
|
+
retained = finding
|
|
2564
|
+
findings[position] = retained
|
|
2565
|
+
finding_positions[key] = (position, group)
|
|
2566
|
+
else:
|
|
2567
|
+
previous = copy.deepcopy(value)
|
|
2568
|
+
previous_history = previous.get("provenance", {}).pop(
|
|
2569
|
+
"previousFindings", []
|
|
2570
|
+
)
|
|
2571
|
+
retained_provenance = retained["provenance"]
|
|
2572
|
+
for field in ("sourceFindingIds", "sourceFindings", "originalCandidates"):
|
|
2573
|
+
values = retained_provenance.get(field, [])
|
|
2574
|
+
values = list(values) if isinstance(values, list) else []
|
|
2575
|
+
previous_values = previous.get("provenance", {}).get(field, [])
|
|
2576
|
+
for item in previous_values if isinstance(previous_values, list) else []:
|
|
2577
|
+
if item not in values:
|
|
2578
|
+
values.append(copy.deepcopy(item))
|
|
2579
|
+
if values:
|
|
2580
|
+
retained_provenance[field] = values
|
|
2581
|
+
retained_history = retained_provenance.get("previousFindings")
|
|
2582
|
+
history = (
|
|
2583
|
+
[item for item in retained_history if isinstance(item, dict)]
|
|
2584
|
+
if isinstance(retained_history, list)
|
|
2585
|
+
else []
|
|
2586
|
+
)
|
|
2587
|
+
for original in [
|
|
2588
|
+
*(previous_history if isinstance(previous_history, list) else []),
|
|
2589
|
+
previous,
|
|
2590
|
+
]:
|
|
2591
|
+
if not isinstance(original, dict):
|
|
2592
|
+
continue
|
|
2593
|
+
owner = retained_provenance.get("workerId", worker_id)
|
|
2594
|
+
source_key = _finding_key(original, owner)
|
|
2595
|
+
source_content = _finding_content(original)
|
|
2596
|
+
already_retained = any(
|
|
2597
|
+
source_key == _finding_key(historical, owner)
|
|
2598
|
+
and source_content == _finding_content(historical)
|
|
2599
|
+
for historical in _retained_findings(retained)
|
|
2600
|
+
)
|
|
2601
|
+
if (
|
|
2602
|
+
not already_retained
|
|
2603
|
+
and original not in history
|
|
2604
|
+
and original != retained
|
|
2605
|
+
):
|
|
2606
|
+
history.append(original)
|
|
2607
|
+
if history or retained_history is not None:
|
|
2608
|
+
retained_provenance["previousFindings"] = history
|
|
2609
|
+
continue
|
|
2610
|
+
finding_positions[key] = (len(findings), group)
|
|
2611
|
+
findings.append(finding)
|
|
2612
|
+
|
|
2613
|
+
competing_receipts = _competing_rejection_receipts(
|
|
2614
|
+
outcomes, all_sources, source_surface_receipts
|
|
2615
|
+
)
|
|
2616
|
+
check_outcome_receipts = bool(competing_receipts)
|
|
2617
|
+
dry_run = check_outcome_receipts
|
|
2618
|
+
unavailable = set()
|
|
2619
|
+
if dry_run:
|
|
2620
|
+
initial_coverage = coverage
|
|
2621
|
+
initial_canonical_rows = canonical_rows
|
|
2622
|
+
initial_sources = coverage_sources
|
|
2623
|
+
initial_surface_rows = retained_surface_rows
|
|
2624
|
+
initial_ids = surface_ids_by_source
|
|
2625
|
+
initial_projection = projection
|
|
2626
|
+
output_warnings = warnings
|
|
2627
|
+
|
|
2628
|
+
def retain_surface_links(owner, row):
|
|
2629
|
+
if isinstance(row, dict):
|
|
2630
|
+
links = row.get("surfaceIds", [])
|
|
2631
|
+
for identity in [row.get("id"), *(links if isinstance(links, list) else [])]:
|
|
2632
|
+
if isinstance(identity, str):
|
|
2633
|
+
pending_surface_ids.add((owner, identity))
|
|
2634
|
+
|
|
2635
|
+
def surface_is_retained(relative: str, owner: str | None, row: Any) -> bool:
|
|
2636
|
+
selection = source_coverage_selection(relative, owner)
|
|
2637
|
+
if (
|
|
2638
|
+
isinstance(row, dict)
|
|
2639
|
+
and isinstance(row.get("id"), str)
|
|
2640
|
+
and (owner, row["id"]) in pending_surface_ids
|
|
2641
|
+
):
|
|
2642
|
+
selection = (False, *selection[1:])
|
|
2643
|
+
return coverage_row_is_retained("surfaces", row, owner, selection)
|
|
2644
|
+
|
|
2645
|
+
while True:
|
|
2646
|
+
if check_outcome_receipts:
|
|
2647
|
+
warnings = [] if dry_run else output_warnings
|
|
2648
|
+
coverage = copy.deepcopy(initial_coverage)
|
|
2649
|
+
canonical_rows = initial_canonical_rows.copy()
|
|
2650
|
+
coverage_sources = {
|
|
2651
|
+
identity: (row, paths.copy()) for identity, (row, paths) in initial_sources.items()
|
|
2652
|
+
}
|
|
2653
|
+
for field in ("surfaces", "deferred", "explicitExclusions"):
|
|
2654
|
+
for initial, current in zip(
|
|
2655
|
+
_coverage_rows(initial_coverage, field),
|
|
2656
|
+
_coverage_rows(coverage, field),
|
|
2657
|
+
strict=True,
|
|
2658
|
+
):
|
|
2659
|
+
if id(initial) in initial_canonical_rows:
|
|
2660
|
+
canonical_rows.add(id(current))
|
|
2661
|
+
_inherit_coverage_sources(current, [initial], coverage_sources)
|
|
2662
|
+
retained_surface_rows = initial_surface_rows.copy()
|
|
2663
|
+
surface_ids_by_source = {key: ids.copy() for key, ids in initial_ids.items()}
|
|
2664
|
+
reconciled_surface_ids.clear()
|
|
2665
|
+
projection = type(initial_projection)(
|
|
2666
|
+
initial_projection.project_worker_coverage,
|
|
2667
|
+
initial_projection.legacy_surface_links,
|
|
2668
|
+
sources,
|
|
2669
|
+
)
|
|
2670
|
+
# Apply reconciled outcomes to coverage already copied from the parent too.
|
|
2671
|
+
for field in ("surfaces", "deferred"):
|
|
2672
|
+
rows = coverage.get(field)
|
|
2673
|
+
if isinstance(rows, list):
|
|
2674
|
+
coverage[field] = [
|
|
2675
|
+
row
|
|
2676
|
+
for row in rows
|
|
2677
|
+
if not (
|
|
2678
|
+
isinstance(row, dict)
|
|
2679
|
+
and (
|
|
2680
|
+
(
|
|
2681
|
+
field == "deferred"
|
|
2682
|
+
and record_key(field, None, row) in closed_deferred
|
|
2683
|
+
)
|
|
2684
|
+
or (
|
|
2685
|
+
coverage_candidate(None, row) in resolved
|
|
2686
|
+
and (
|
|
2687
|
+
field == "deferred"
|
|
2688
|
+
or row.get("disposition") == "needs_follow_up"
|
|
2689
|
+
)
|
|
2690
|
+
)
|
|
2691
|
+
)
|
|
2692
|
+
)
|
|
2693
|
+
]
|
|
2694
|
+
|
|
2695
|
+
# Reserve receipts for supporting surfaces before repairing historical refs.
|
|
2696
|
+
prospective_tasks = []
|
|
2697
|
+
for relative, _, owner in all_sources:
|
|
2698
|
+
selection = source_coverage_selection(relative, owner)
|
|
2699
|
+
for row in deferred_rows[relative]:
|
|
2700
|
+
if (
|
|
2701
|
+
coverage_row_is_retained("deferred", row, owner, selection)
|
|
2702
|
+
and id(row) not in replaced_deferred
|
|
2703
|
+
and not (
|
|
2704
|
+
isinstance(row, dict)
|
|
2705
|
+
and isinstance(row.get("id"), str)
|
|
2706
|
+
and record_key("deferred", owner, row) in closed_deferred
|
|
2707
|
+
)
|
|
2708
|
+
):
|
|
2709
|
+
prospective_tasks.append((owner, row))
|
|
2710
|
+
prospective_tasks.extend(
|
|
2711
|
+
(owner, row)
|
|
2712
|
+
for owner, row in reopened_rows
|
|
2713
|
+
if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
|
|
2714
|
+
)
|
|
2715
|
+
pending_surface_ids = set()
|
|
2716
|
+
|
|
2717
|
+
for owner, row in prospective_tasks:
|
|
2718
|
+
retain_surface_links(owner, row)
|
|
2719
|
+
if parent and isinstance(coverage.get("deferred"), list):
|
|
2720
|
+
previous = parent["coverage"].get("deferred", [])
|
|
2721
|
+
removed_parent = (
|
|
2722
|
+
[row for row in previous if id(row) in replaced_deferred]
|
|
2723
|
+
if isinstance(previous, list)
|
|
2724
|
+
else []
|
|
2725
|
+
)
|
|
2726
|
+
for row in coverage["deferred"]:
|
|
2727
|
+
if row not in removed_parent:
|
|
2728
|
+
retain_surface_links(None, row)
|
|
2729
|
+
|
|
2730
|
+
fallback_surfaces = projection.fallback_surfaces(
|
|
2731
|
+
coverage,
|
|
2732
|
+
all_sources,
|
|
2733
|
+
prospective_tasks,
|
|
2734
|
+
surface_is_retained=surface_is_retained,
|
|
2735
|
+
ambiguous_surface_ids=ambiguous_surface_ids,
|
|
2736
|
+
)
|
|
2737
|
+
|
|
2738
|
+
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
2739
|
+
all_sources,
|
|
2740
|
+
closed_deferred,
|
|
2741
|
+
active_deferred,
|
|
2742
|
+
resolved,
|
|
2743
|
+
reopened_generic,
|
|
2744
|
+
coverage_schema["surfaces"]["items"],
|
|
2745
|
+
deferred_rows,
|
|
2746
|
+
ambiguous_deferred,
|
|
2747
|
+
record_source,
|
|
2748
|
+
candidate_order,
|
|
2749
|
+
source_surface_receipts,
|
|
2750
|
+
canonical_rows=canonical_rows,
|
|
2751
|
+
coverage_sources=coverage_sources,
|
|
2752
|
+
receipt_root=scan_dir if project_worker_coverage else None,
|
|
2753
|
+
receipt_warnings=warnings,
|
|
2754
|
+
fallback_surfaces=fallback_surfaces,
|
|
2755
|
+
surface_is_retained=surface_is_retained,
|
|
2756
|
+
eligible_receipt_sources={
|
|
2757
|
+
relative
|
|
2758
|
+
for relative, draft, _ in all_sources
|
|
2759
|
+
if draft.get("complete") is not False or relative in selected_observations
|
|
2760
|
+
},
|
|
2761
|
+
)
|
|
2762
|
+
replaced_rows = {
|
|
2763
|
+
"surfaces": replaced_surfaces,
|
|
2764
|
+
"deferred": {
|
|
2765
|
+
id(row)
|
|
2766
|
+
for relative, _, owner in all_sources
|
|
2767
|
+
for row in deferred_rows[relative]
|
|
2768
|
+
if isinstance(row, dict)
|
|
2769
|
+
and isinstance(identity := row.get("id"), str)
|
|
2770
|
+
and (key := record_key("deferred", owner, row)) not in candidate_ids
|
|
2771
|
+
and key not in ambiguous_deferred
|
|
2772
|
+
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
2773
|
+
and updated > candidate_order(relative, key[0], row)
|
|
2774
|
+
},
|
|
2775
|
+
}
|
|
2776
|
+
for field, replaced in replaced_rows.items():
|
|
2777
|
+
if parent and isinstance(coverage.get(field), list):
|
|
2778
|
+
previous = parent["coverage"].get(field, [])
|
|
2779
|
+
if isinstance(previous, list):
|
|
2780
|
+
removed_parent = [row for row in previous if id(row) in replaced]
|
|
2781
|
+
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
2782
|
+
if isinstance(coverage.get("surfaces"), list):
|
|
2783
|
+
for relative, owner, source_surface, surface in surface_updates:
|
|
2784
|
+
original_surface = surface
|
|
2785
|
+
# Generic updates already resolve each receipt at its saved source.
|
|
2786
|
+
surface = projection.repair_reviewed_surface(
|
|
2787
|
+
coverage,
|
|
2788
|
+
projected_coverage if parent_is_canonical else {},
|
|
2789
|
+
workers_by_id.get(owner),
|
|
2790
|
+
source_surface,
|
|
2791
|
+
surface,
|
|
2792
|
+
) or project_record("surfaces", surface, source_surface, receipts_resolved=True)
|
|
2793
|
+
reconciled_surface_ids[record_key("surfaces", owner, source_surface)] = surface[
|
|
2794
|
+
"id"
|
|
2795
|
+
]
|
|
2796
|
+
if surface not in coverage["surfaces"] and (
|
|
2797
|
+
surface is original_surface
|
|
2798
|
+
or retained_coverage_record("surfaces", surface) is None
|
|
2799
|
+
):
|
|
2800
|
+
retain_coverage_row(
|
|
2801
|
+
coverage["surfaces"],
|
|
2802
|
+
surface,
|
|
2803
|
+
owner,
|
|
2804
|
+
"surfaces",
|
|
2805
|
+
source_surface,
|
|
2806
|
+
surface is not original_surface or relative == "parent",
|
|
2807
|
+
)
|
|
2808
|
+
for surface in coverage["surfaces"]:
|
|
2809
|
+
if isinstance(surface, dict):
|
|
2810
|
+
surface.setdefault("receiptRefs", [])
|
|
2811
|
+
|
|
2812
|
+
# Reopened work survives a superseded checkpoint, but current candidate
|
|
2813
|
+
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
2814
|
+
for owner, item in reopened_rows:
|
|
2815
|
+
source_owner, original = record_source("deferred", owner, item)
|
|
2816
|
+
if (
|
|
2817
|
+
identity := _deferred_candidate_id(original, source_owner, ambiguous_deferred)
|
|
2818
|
+
) is not None and (
|
|
2819
|
+
source_owner,
|
|
2820
|
+
identity,
|
|
2821
|
+
) in resolved:
|
|
2822
|
+
continue
|
|
2823
|
+
original_item = item
|
|
2824
|
+
item = project_record("deferred", item)
|
|
2825
|
+
pending = coverage.setdefault("deferred", [])
|
|
2826
|
+
if (
|
|
2827
|
+
isinstance(pending, list)
|
|
2828
|
+
and item not in pending
|
|
2829
|
+
and (item is original_item or retained_coverage_record("deferred", item) is None)
|
|
2830
|
+
):
|
|
2831
|
+
retain_coverage_row(
|
|
2832
|
+
pending, item, owner, "deferred", original_item, item is not original_item
|
|
2833
|
+
)
|
|
2834
|
+
for _, draft, owner in all_sources:
|
|
2835
|
+
for surface in _coverage_rows(draft["coverage"], "surfaces"):
|
|
2836
|
+
if (
|
|
2837
|
+
isinstance(surface, dict)
|
|
2838
|
+
and ambiguous_pending_surface(surface, owner)
|
|
2839
|
+
and isinstance(coverage.get("surfaces"), list)
|
|
2840
|
+
):
|
|
2841
|
+
original_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
|
|
2842
|
+
retained_surface = project_record("surfaces", original_surface, surface)
|
|
2843
|
+
if retained_surface not in coverage["surfaces"] and (
|
|
2844
|
+
retained_surface is original_surface
|
|
2845
|
+
or retained_coverage_record("surfaces", retained_surface) is None
|
|
2846
|
+
):
|
|
2847
|
+
retain_coverage_row(
|
|
2848
|
+
coverage["surfaces"],
|
|
2849
|
+
retained_surface,
|
|
2850
|
+
owner,
|
|
2851
|
+
"surfaces",
|
|
2852
|
+
surface,
|
|
2853
|
+
retained_surface is not original_surface,
|
|
2854
|
+
)
|
|
2855
|
+
for relative, draft, worker_id in all_sources:
|
|
2856
|
+
selection = source_coverage_selection(relative, worker_id)
|
|
2857
|
+
(
|
|
2858
|
+
selected_coverage_superseded,
|
|
2859
|
+
selected_candidates,
|
|
2860
|
+
superseded,
|
|
2861
|
+
retain_pending,
|
|
2862
|
+
coverage_superseded,
|
|
2863
|
+
) = selection
|
|
2864
|
+
skip_superseded_findings = (
|
|
2865
|
+
superseded
|
|
2866
|
+
and not stopped
|
|
2867
|
+
and all(
|
|
2868
|
+
valid_finding(finding) for finding in (parent["findings"] if parent else [])
|
|
2869
|
+
)
|
|
2870
|
+
)
|
|
2871
|
+
if (
|
|
2872
|
+
skip_superseded_findings
|
|
2873
|
+
and coverage_superseded
|
|
2874
|
+
and not selected_candidates
|
|
2875
|
+
and not retain_pending
|
|
2876
|
+
):
|
|
2877
|
+
continue
|
|
2878
|
+
if (
|
|
2879
|
+
not dry_run
|
|
2880
|
+
and not skip_superseded_findings
|
|
2881
|
+
and "threatModel" not in manifest["scan"]
|
|
2882
|
+
and isinstance(draft.get("threatModel"), dict)
|
|
2883
|
+
):
|
|
2884
|
+
model = recovered_model(relative, draft, worker_id)
|
|
2885
|
+
model_path = relative
|
|
2886
|
+
checkpoint_dir = Path(relative).parent
|
|
2887
|
+
root_checkpoint = checkpoint_dir == Path("checkpoints")
|
|
2888
|
+
prefer_checkpoint_head = root_checkpoint or (
|
|
2889
|
+
worker_id is not None
|
|
2890
|
+
and (checkpoint_dir.name == "checkpoints" or draft.get("complete") is False)
|
|
2891
|
+
)
|
|
2892
|
+
if worker_id is not None or root_checkpoint:
|
|
2893
|
+
if checkpoint_dir.name != "checkpoints":
|
|
2894
|
+
checkpoint_dir /= "checkpoints"
|
|
2895
|
+
selected_models = [
|
|
2896
|
+
path
|
|
2897
|
+
for path in drafts_by_path
|
|
2898
|
+
if (
|
|
2899
|
+
path in selected_observations
|
|
2900
|
+
or (
|
|
2901
|
+
path in current_results
|
|
2902
|
+
and drafts_by_path[path].get("complete") is not False
|
|
2903
|
+
)
|
|
2904
|
+
)
|
|
2905
|
+
and (
|
|
2906
|
+
Path(path).parent == checkpoint_dir
|
|
2907
|
+
or (
|
|
2908
|
+
path in current_results
|
|
2909
|
+
and Path(path).parent / "checkpoints" == checkpoint_dir
|
|
2910
|
+
)
|
|
2911
|
+
)
|
|
2912
|
+
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
2913
|
+
and recovered_model(path, drafts_by_path[path], worker_id) is not None
|
|
2914
|
+
]
|
|
2915
|
+
if selected_models:
|
|
2916
|
+
authoritative_models = [
|
|
2917
|
+
path
|
|
2918
|
+
for path in selected_models
|
|
2919
|
+
if path in authoritative_heads or path in current_results
|
|
2920
|
+
]
|
|
2921
|
+
if not any(path in authoritative_heads for path in selected_models):
|
|
2922
|
+
authoritative_models = selected_models
|
|
2923
|
+
head_path = max(
|
|
2924
|
+
authoritative_models or selected_models,
|
|
2925
|
+
key=lambda path: (source_order[path], path in authoritative_heads),
|
|
2926
|
+
)
|
|
2927
|
+
current = drafts_by_path[head_path]
|
|
2928
|
+
# A terminal checkpoint is committed before result.json is replaced.
|
|
2929
|
+
# Use the admitted observation, including its frozen ordering on retries.
|
|
2930
|
+
if not prefer_checkpoint_head and current.get("complete") is not False:
|
|
2931
|
+
prefer_checkpoint_head = (
|
|
2932
|
+
source_order[head_path] >= source_order[relative]
|
|
2933
|
+
)
|
|
2934
|
+
if prefer_checkpoint_head:
|
|
2935
|
+
model = recovered_model(head_path, current, worker_id)
|
|
2936
|
+
model_path = head_path
|
|
2937
|
+
if model is not None:
|
|
2938
|
+
manifest["scan"]["threatModel"] = model
|
|
2939
|
+
if selected_model_source is not None and model_path in source_digests:
|
|
2940
|
+
selected_model_source[:] = [model_path]
|
|
2941
|
+
if not dry_run:
|
|
2942
|
+
retain_findings(
|
|
2943
|
+
relative, draft, worker_id, selected_candidates, skip_superseded_findings
|
|
2944
|
+
)
|
|
2945
|
+
worker = workers_by_id.get(worker_id)
|
|
2946
|
+
# Only the accepted current output belongs to this worker attempt. An
|
|
2947
|
+
# unsuperseded archive can remain evidence after that output is lost.
|
|
2948
|
+
project_coverage = (
|
|
2949
|
+
worker is not None
|
|
2950
|
+
and worker["status"] == "succeeded"
|
|
2951
|
+
and worker["result_manifest_path"] == str(scan_dir / relative)
|
|
2952
|
+
and draft.get("complete") is not False
|
|
2953
|
+
)
|
|
2954
|
+
if coverage_superseded and not selected_candidates and not retain_pending:
|
|
2955
|
+
continue
|
|
2956
|
+
if project_coverage:
|
|
2957
|
+
reviewed_attempts.add((worker_id, worker["attempt"]))
|
|
2958
|
+
reviews = coverage.setdefault("reviews", [])
|
|
2959
|
+
if isinstance(reviews, list) and not any(
|
|
2960
|
+
isinstance(review, dict)
|
|
2961
|
+
and review.get("workerId") == worker_id
|
|
2962
|
+
and review.get("attempt") == worker["attempt"]
|
|
2963
|
+
for review in reviews
|
|
2964
|
+
):
|
|
2965
|
+
reviews.append(
|
|
2966
|
+
{
|
|
2967
|
+
"workerId": worker_id,
|
|
2968
|
+
"attempt": worker["attempt"],
|
|
2969
|
+
"completeness": draft["coverage"].get("completeness", "partial"),
|
|
2970
|
+
}
|
|
2971
|
+
)
|
|
2972
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions", "reviews"):
|
|
2973
|
+
if field == "reviews" and (worker_id is not None or field not in draft["coverage"]):
|
|
2974
|
+
continue
|
|
2975
|
+
if coverage_superseded and field not in {
|
|
2976
|
+
"surfaces",
|
|
2977
|
+
"explicitExclusions",
|
|
2978
|
+
"deferred",
|
|
2979
|
+
}:
|
|
2980
|
+
continue
|
|
2981
|
+
items = draft["coverage"].get(field, [])
|
|
2982
|
+
if field == "deferred":
|
|
2983
|
+
items = deferred_rows[relative]
|
|
2984
|
+
if not isinstance(items, list):
|
|
2985
|
+
continue
|
|
2986
|
+
output = coverage.setdefault(field, [])
|
|
2987
|
+
if not isinstance(output, list):
|
|
2988
|
+
# Keep malformed canonical collections for the existing finalizer's
|
|
2989
|
+
# recovery and warnings rather than silently changing its contract.
|
|
2990
|
+
continue
|
|
2991
|
+
for index, item in enumerate(items, 1):
|
|
2992
|
+
source_item = item
|
|
2993
|
+
if not (
|
|
2994
|
+
surface_is_retained(relative, worker_id, item)
|
|
2995
|
+
if field == "surfaces"
|
|
2996
|
+
else coverage_row_is_retained(field, item, worker_id, selection)
|
|
2997
|
+
):
|
|
2998
|
+
continue
|
|
2999
|
+
if id(item) in replaced_rows.get(field, ()):
|
|
3000
|
+
continue
|
|
3001
|
+
if (
|
|
3002
|
+
field == "deferred"
|
|
3003
|
+
and isinstance(item, dict)
|
|
3004
|
+
and isinstance(item.get("id"), str)
|
|
3005
|
+
and record_key("deferred", worker_id, item) in closed_deferred
|
|
3006
|
+
):
|
|
3007
|
+
continue
|
|
3008
|
+
if field == "openQuestions" and isinstance(item, str):
|
|
3009
|
+
item = {"question": item.strip()}
|
|
3010
|
+
if (
|
|
3011
|
+
field == "surfaces"
|
|
3012
|
+
and isinstance(item, dict)
|
|
3013
|
+
and item.get("disposition") in ("rejected", "not_applicable")
|
|
3014
|
+
and isinstance(item.get("candidateId"), str)
|
|
3015
|
+
and (
|
|
3016
|
+
history_findings := rejected_history.get(
|
|
3017
|
+
(worker_id, item["candidateId"])
|
|
3018
|
+
)
|
|
3019
|
+
)
|
|
3020
|
+
):
|
|
3021
|
+
item = copy.deepcopy(item)
|
|
3022
|
+
for finding in history_findings:
|
|
3023
|
+
_append_previous_finding(item, finding, clone=True)
|
|
3024
|
+
projected_row = False
|
|
3025
|
+
if isinstance(item, dict):
|
|
3026
|
+
original_item = item
|
|
3027
|
+
item = (
|
|
3028
|
+
project_missing_record(field, item, index, worker, draft["coverage"])
|
|
3029
|
+
if project_coverage
|
|
3030
|
+
else project_record(field, item, source_item)
|
|
3031
|
+
)
|
|
3032
|
+
projected_row = item is not original_item
|
|
3033
|
+
if field == "surfaces" and isinstance(item, dict):
|
|
3034
|
+
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
3035
|
+
if (
|
|
3036
|
+
field == "surfaces"
|
|
3037
|
+
and worker is not None
|
|
3038
|
+
and isinstance(item, dict)
|
|
3039
|
+
and (project_coverage or not projected_row)
|
|
3040
|
+
):
|
|
3041
|
+
item = copy.deepcopy(item)
|
|
3042
|
+
item["receiptRefs"] = coverage_receipts(item, worker, relative)
|
|
3043
|
+
if (
|
|
3044
|
+
projected_row
|
|
3045
|
+
and isinstance(item, dict)
|
|
3046
|
+
and retained_coverage_record(field, item) is not None
|
|
3047
|
+
):
|
|
3048
|
+
continue
|
|
3049
|
+
if isinstance(item, dict) and "id" not in item:
|
|
3050
|
+
semantic_item = dict(item)
|
|
3051
|
+
existing = next(
|
|
3052
|
+
(
|
|
3053
|
+
existing
|
|
3054
|
+
for existing in output
|
|
3055
|
+
if isinstance(existing, dict)
|
|
3056
|
+
and {key: value for key, value in existing.items() if key != "id"}
|
|
3057
|
+
== semantic_item
|
|
3058
|
+
),
|
|
3059
|
+
None,
|
|
3060
|
+
)
|
|
3061
|
+
if existing is not None:
|
|
3062
|
+
_inherit_coverage_sources(existing, [source_item], coverage_sources)
|
|
3063
|
+
if field == "surfaces":
|
|
3064
|
+
retained_surface_rows.add(id(source_item))
|
|
3065
|
+
continue
|
|
3066
|
+
retain_coverage_row(
|
|
3067
|
+
output,
|
|
3068
|
+
item,
|
|
3069
|
+
worker_id,
|
|
3070
|
+
field,
|
|
3071
|
+
source_item,
|
|
3072
|
+
projected_row or project_coverage,
|
|
3073
|
+
)
|
|
3074
|
+
|
|
3075
|
+
projection.finalize(
|
|
3076
|
+
coverage,
|
|
3077
|
+
canonical_rows,
|
|
3078
|
+
_digest,
|
|
3079
|
+
preserve_canonical_parent=preserve_canonical_parent,
|
|
3080
|
+
all_sources=all_sources,
|
|
3081
|
+
source_order=source_order,
|
|
3082
|
+
coverage_sources=coverage_sources,
|
|
3083
|
+
surface_ids_by_source=surface_ids_by_source,
|
|
3084
|
+
retained_surface_rows=retained_surface_rows,
|
|
3085
|
+
resolved=resolved,
|
|
3086
|
+
)
|
|
3087
|
+
if not dry_run:
|
|
3088
|
+
break
|
|
3089
|
+
invalid = _conflicting_outcome_receipts(
|
|
3090
|
+
coverage,
|
|
3091
|
+
scan_dir,
|
|
3092
|
+
competing_receipts - unavailable,
|
|
3093
|
+
)
|
|
3094
|
+
if invalid:
|
|
3095
|
+
unavailable.update(invalid)
|
|
3096
|
+
resolved, ordered_outcomes = select_outcomes(unavailable)
|
|
3097
|
+
else:
|
|
3098
|
+
dry_run = False
|
|
3099
|
+
|
|
3100
|
+
# Generate new worker identities from the retained collection, as a normal
|
|
3101
|
+
# Deep parent does. Authored, restored and canonical identities stay stable.
|
|
3102
|
+
generated_worker_ids = {id(finding) for finding in generated_worker_findings}
|
|
3103
|
+
for finding in generated_worker_findings:
|
|
3104
|
+
finding.pop("identity", None)
|
|
3105
|
+
|
|
3106
|
+
def publication_order(item):
|
|
3107
|
+
_, group = item
|
|
3108
|
+
source, value = group["observations"][group["owner"]]
|
|
3109
|
+
index = next(index for index, row in enumerate(source[1]["findings"]) if row is value)
|
|
3110
|
+
return finding_source_order(source, True), -index
|
|
3111
|
+
|
|
3112
|
+
identity_order = list(findings)
|
|
3113
|
+
for owner in {group["owner"] for _, group in finding_positions.values() if group is not None}:
|
|
3114
|
+
positions = [
|
|
3115
|
+
(position, group)
|
|
3116
|
+
for position, group in finding_positions.values()
|
|
3117
|
+
if group is not None and group["owner"] == owner and group["identity"] is None
|
|
3118
|
+
]
|
|
3119
|
+
|
|
3120
|
+
for (position, _), (published_position, _) in zip(
|
|
3121
|
+
positions, sorted(positions, key=publication_order, reverse=True), strict=True
|
|
3122
|
+
):
|
|
3123
|
+
identity_order[position] = findings[published_position]
|
|
3124
|
+
identity_order = [
|
|
3125
|
+
finding
|
|
3126
|
+
for finding in identity_order
|
|
3127
|
+
if valid_finding(finding) and ("identity" in finding or id(finding) in generated_worker_ids)
|
|
3128
|
+
]
|
|
3129
|
+
_ensure_finding_identities(identity_order)
|
|
3130
|
+
|
|
3131
|
+
identities: dict[bytes, list[tuple[str, dict[str, Any] | None, str | None]]] = {}
|
|
3132
|
+
retained_groups = {
|
|
3133
|
+
position: group for position, group in finding_positions.values() if group is not None
|
|
3134
|
+
}
|
|
3135
|
+
reserved_identities = {_semantic_identity_key(finding) for finding in identity_order}
|
|
3136
|
+
for position, finding in enumerate(findings):
|
|
3137
|
+
if not valid_finding(finding):
|
|
3138
|
+
continue
|
|
3139
|
+
identity = finding.get("identity")
|
|
3140
|
+
if not isinstance(identity, dict):
|
|
3141
|
+
continue
|
|
3142
|
+
key = _semantic_identity_key(finding)
|
|
3143
|
+
variant = _finding_key(finding)
|
|
3144
|
+
group = retained_groups.get(position)
|
|
3145
|
+
previous = next(
|
|
3146
|
+
(
|
|
3147
|
+
entry
|
|
3148
|
+
for entry in identities.get(key, [])
|
|
3149
|
+
if entry[0] == variant and not distinct_reports(group, entry[1])
|
|
3150
|
+
),
|
|
3151
|
+
None,
|
|
3152
|
+
)
|
|
3153
|
+
if previous is not None and previous[2] is not None:
|
|
3154
|
+
if identity.get("instance") != previous[2]:
|
|
3155
|
+
finding.setdefault("provenance", {}).setdefault(
|
|
3156
|
+
"preservedIdentity", copy.deepcopy(identity)
|
|
3157
|
+
)
|
|
3158
|
+
identity["instance"] = previous[2]
|
|
3159
|
+
elif previous is None and key in identities:
|
|
3160
|
+
finding.setdefault("provenance", {}).setdefault(
|
|
3161
|
+
"preservedIdentity", copy.deepcopy(identity)
|
|
3162
|
+
)
|
|
3163
|
+
base_instance = identity.get("instance", "saved")
|
|
3164
|
+
suffix = 2
|
|
3165
|
+
while True:
|
|
3166
|
+
identity["instance"] = f"{base_instance}-{suffix}"
|
|
3167
|
+
distinct = _semantic_identity_key(finding)
|
|
3168
|
+
if distinct not in reserved_identities:
|
|
3169
|
+
reserved_identities.add(distinct)
|
|
3170
|
+
break
|
|
3171
|
+
suffix += 1
|
|
3172
|
+
identities.setdefault(key, []).append((variant, group, identity.get("instance")))
|
|
3173
|
+
coverage.pop("completeness", None)
|
|
3174
|
+
if stopped:
|
|
3175
|
+
if not isinstance(coverage.get("deferred"), list):
|
|
3176
|
+
coverage["deferred"] = []
|
|
3177
|
+
item = {"id": "scan-stopped", "reason": reason}
|
|
3178
|
+
if item not in coverage["deferred"]:
|
|
3179
|
+
coverage["deferred"].append(item)
|
|
3180
|
+
surfaces = coverage.get("surfaces", [])
|
|
3181
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
3182
|
+
if isinstance(surface, dict) and isinstance(surface.get("provenance"), dict):
|
|
3183
|
+
surface["provenance"].pop("scanReceiptRefs", None)
|
|
3184
|
+
if (
|
|
3185
|
+
frozen_source_digests is None
|
|
3186
|
+
or heads_snapshotted
|
|
3187
|
+
or snapshot_error is not None
|
|
3188
|
+
or any(_is_source_order_snapshot(path) for path in source_digests)
|
|
3189
|
+
or allow_frozen_legacy_parent
|
|
3190
|
+
):
|
|
3191
|
+
try:
|
|
3192
|
+
if snapshot_error is not None:
|
|
3193
|
+
raise snapshot_error
|
|
3194
|
+
_freeze_source_times(
|
|
3195
|
+
scan_dir,
|
|
3196
|
+
scan_id,
|
|
3197
|
+
source_digests,
|
|
3198
|
+
source_times,
|
|
3199
|
+
parent_context,
|
|
3200
|
+
created_paths=created_paths,
|
|
3201
|
+
)
|
|
3202
|
+
except (ContractError, OSError) as exc:
|
|
3203
|
+
failure = _CheckpointSnapshotError(
|
|
3204
|
+
str(exc),
|
|
3205
|
+
{
|
|
3206
|
+
path: (digest, source_times[path])
|
|
3207
|
+
for path, digest in source_digests.items()
|
|
3208
|
+
if not _is_source_order_snapshot(path)
|
|
3209
|
+
},
|
|
3210
|
+
)
|
|
3211
|
+
failure.parent_context = parent_context
|
|
3212
|
+
failure.heads = checkpoint_heads
|
|
3213
|
+
failure.documents = (manifest, {"findings": findings}, coverage)
|
|
3214
|
+
if isinstance(manifest["scan"].get("threatModel"), dict):
|
|
3215
|
+
failure.model_source = list(selected_model_source or [])
|
|
3216
|
+
raise failure from exc
|
|
3217
|
+
|
|
3218
|
+
return manifest, {"findings": findings}, coverage
|