@openai/codex-security 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -9
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/core-scan.md +2 -0
- package/_bundled_plugin/references/final-report.md +5 -1
- package/_bundled_plugin/references/investigation-time-budget.md +11 -0
- package/_bundled_plugin/references/sarif-adapter.md +1 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -6
- package/_bundled_plugin/references/scan-runtime.md +15 -0
- package/_bundled_plugin/references/threat-model.md +2 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +6 -2
- package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +92 -44
- package/_bundled_plugin/scripts/finding_preview.py +5 -3
- package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
- package/_bundled_plugin/scripts/generate_rank_input.py +13 -6
- package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
- package/_bundled_plugin/scripts/report_projection.py +2 -1
- package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
- package/_bundled_plugin/scripts/workbench_cli.py +5 -10
- package/_bundled_plugin/scripts/workbench_constants.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +68 -39
- package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
- package/_bundled_plugin/scripts/workbench_progress.py +0 -50
- package/_bundled_plugin/scripts/workbench_publication.py +4 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
- package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
- package/_bundled_plugin/scripts/workbench_scan_history.py +82 -80
- package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
- package/_bundled_plugin/scripts/workbench_scan_usage.py +246 -38
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
- package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
- package/_bundled_plugin/scripts/workbench_target.py +28 -12
- package/_bundled_plugin/scripts/workbench_validation.py +52 -13
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
- package/_bundled_plugin/shared/workbench-migrations.json +15 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/dist/api.d.ts +4 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +215 -93
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +2 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +38 -23
- package/dist/artifact-export.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +16 -18
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity-records.d.ts +76 -0
- package/dist/classify-severity-records.d.ts.map +1 -0
- package/dist/classify-severity-records.js +141 -0
- package/dist/classify-severity-records.js.map +1 -0
- package/dist/classify-severity.d.ts +1 -23
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -24
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-daybreak.d.ts +5 -0
- package/dist/cli-daybreak.d.ts.map +1 -0
- package/dist/cli-daybreak.js +89 -0
- package/dist/cli-daybreak.js.map +1 -0
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +2 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +957 -738
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +1 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.js +2 -2
- package/dist/codeowners.js.map +1 -1
- package/dist/codex-sdk-environment.js +1 -1
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +3 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.js +2 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -5
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +5 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +65 -17
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +7 -2
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +2 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +17 -8
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +2 -5
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +8 -6
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.js +1 -1
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/retry.d.ts.map +1 -1
- package/dist/deduplication/retry.js +5 -0
- package/dist/deduplication/retry.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +15 -10
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +4 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +21 -1
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan.d.ts +6 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +8 -6
- package/dist/deep-scan.js.map +1 -1
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +26 -6
- package/dist/findings-import.js.map +1 -1
- package/dist/git-remote.d.ts +10 -0
- package/dist/git-remote.d.ts.map +1 -0
- package/dist/git-remote.js +24 -0
- package/dist/git-remote.js.map +1 -0
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +5 -7
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -0
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +15 -0
- package/dist/host-records-protocol.d.ts.map +1 -0
- package/dist/host-records-protocol.js +15 -0
- package/dist/host-records-protocol.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +5 -0
- package/dist/hosted-remediation-protocol.d.ts.map +1 -0
- package/dist/hosted-remediation-protocol.js +26 -0
- package/dist/hosted-remediation-protocol.js.map +1 -0
- package/dist/hosted-remediation.d.ts +89 -0
- package/dist/hosted-remediation.d.ts.map +1 -0
- package/dist/hosted-remediation.js +122 -0
- package/dist/hosted-remediation.js.map +1 -0
- package/dist/hosted-scan.d.ts +1 -6
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +2 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.d.ts +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +5 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +87 -7
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +65 -29
- package/dist/multiscan.js.map +1 -1
- package/dist/patch-publication.d.ts +26 -0
- package/dist/patch-publication.d.ts.map +1 -0
- package/dist/patch-publication.js +105 -0
- package/dist/patch-publication.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +4 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +29 -15
- package/dist/project-config.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +135 -19
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +48 -25
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +63 -0
- package/dist/remediation.d.ts.map +1 -0
- package/dist/remediation.js +110 -0
- package/dist/remediation.js.map +1 -0
- package/dist/request-metadata.d.ts +17 -7
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +43 -8
- package/dist/request-metadata.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +104 -36
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +16 -4
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.js +8 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +15 -9
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +12 -6
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +5 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +9 -8
- package/dist/scan-executor.js.map +1 -1
- package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
- package/dist/scan-progress.d.ts.map +1 -0
- package/dist/scan-progress.js +138 -0
- package/dist/scan-progress.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +7 -1
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-time-budget.d.ts +9 -0
- package/dist/scan-time-budget.d.ts.map +1 -0
- package/dist/scan-time-budget.js +16 -0
- package/dist/scan-time-budget.js.map +1 -0
- package/dist/security-policy.js +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +4 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +14 -2
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-review.d.ts +29 -0
- package/dist/severity-review.d.ts.map +1 -0
- package/dist/severity-review.js +47 -0
- package/dist/severity-review.js.map +1 -0
- package/dist/severity-store.js +4 -1
- package/dist/severity-store.js.map +1 -1
- package/dist/timestamps.d.ts +3 -0
- package/dist/timestamps.d.ts.map +1 -0
- package/dist/timestamps.js +14 -0
- package/dist/timestamps.js.map +1 -0
- package/dist/validation-records.d.ts +38 -0
- package/dist/validation-records.d.ts.map +1 -0
- package/dist/validation-records.js +76 -0
- package/dist/validation-records.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +17 -4
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts.map +1 -1
- package/dist/workbench-arguments.js +9 -3
- package/dist/workbench-arguments.js.map +1 -1
- package/dist/worker-progress.d.ts +13 -7
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +12 -44
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +89 -0
- package/docs/cli.md +352 -45
- package/docs/findings-service.md +7 -2
- package/docs/hosted-remediation.md +117 -0
- package/docs/hosted-scans.md +8 -3
- package/docs/validate-records.md +69 -0
- package/package.json +13 -10
- package/schemas/project-config.schema.json +7 -1
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
- package/_bundled_plugin/references/config-preflight.md +0 -93
- package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
- package/_bundled_plugin/scripts/config_preflight.py +0 -911
- package/dist/deep-progress.d.ts.map +0 -1
- package/dist/deep-progress.js +0 -92
- package/dist/deep-progress.js.map +0 -1
|
@@ -12,7 +12,7 @@ import re
|
|
|
12
12
|
import sqlite3
|
|
13
13
|
import stat
|
|
14
14
|
import sys
|
|
15
|
-
from collections.abc import Iterator
|
|
15
|
+
from collections.abc import Callable, Iterator
|
|
16
16
|
from contextlib import contextmanager
|
|
17
17
|
from pathlib import Path
|
|
18
18
|
from typing import Any
|
|
@@ -31,6 +31,8 @@ from finalize_scan_contract import (
|
|
|
31
31
|
_read_scan_local_json_with_metadata,
|
|
32
32
|
_recover_unsealed_findings,
|
|
33
33
|
_remove_scan_local_file_if_exists,
|
|
34
|
+
_require_portable_relative_path,
|
|
35
|
+
_require_str,
|
|
34
36
|
_validate_completion_binding,
|
|
35
37
|
_validate_resolved_deferred,
|
|
36
38
|
_validate_schema_node,
|
|
@@ -41,6 +43,11 @@ from finalize_scan_contract import (
|
|
|
41
43
|
write_scan_local_bytes,
|
|
42
44
|
write_threat_model_projection_if_possible,
|
|
43
45
|
)
|
|
46
|
+
from workbench.saved_coverage import (
|
|
47
|
+
_prepare_worker_coverage_recovery,
|
|
48
|
+
_project_worker_coverage_row,
|
|
49
|
+
_saved_coverage_id,
|
|
50
|
+
)
|
|
44
51
|
from workbench_constants import PHASES
|
|
45
52
|
from workbench_target import committed_diff_snapshot_digest
|
|
46
53
|
from workbench_validation import path_within_scope
|
|
@@ -714,10 +721,6 @@ def _merge_tied_parent_observations(
|
|
|
714
721
|
return merged
|
|
715
722
|
|
|
716
723
|
|
|
717
|
-
def _saved_coverage_id(item: dict[str, Any]) -> str:
|
|
718
|
-
return item.get("candidateId") or f"saved-{_digest(item)[:16]}"
|
|
719
|
-
|
|
720
|
-
|
|
721
724
|
def _deferred_candidate_id(
|
|
722
725
|
row: dict[str, Any],
|
|
723
726
|
owner: str | None,
|
|
@@ -743,10 +746,28 @@ def _generic_surface_updates(
|
|
|
743
746
|
surface_schema: dict[str, Any],
|
|
744
747
|
deferred_rows: dict[str, list[Any]],
|
|
745
748
|
ambiguous_deferred: set[tuple[str | None, str]],
|
|
746
|
-
|
|
747
|
-
|
|
749
|
+
*,
|
|
750
|
+
surface_is_retained: Callable[[str, str | None, dict[str, Any]], bool],
|
|
751
|
+
eligible_receipt_sources: set[str],
|
|
752
|
+
receipt_root: Path | None = None,
|
|
753
|
+
receipt_warnings: list[str] | None = None,
|
|
754
|
+
fallback_surfaces: list[dict[str, Any]] | None = None,
|
|
755
|
+
) -> tuple[set[int], list[tuple[str | None, dict[str, Any], set[int]]]]:
|
|
756
|
+
if not closed_deferred and not reopened_generic and receipt_root is None:
|
|
748
757
|
return set(), []
|
|
749
758
|
|
|
759
|
+
def receipt_error(ref: Any) -> ContractError | None:
|
|
760
|
+
if receipt_root is None or not isinstance(ref, str):
|
|
761
|
+
return ContractError("Saved coverage receipt is unavailable.")
|
|
762
|
+
try:
|
|
763
|
+
descriptor = open_scan_local_file_descriptor(
|
|
764
|
+
receipt_root, ref, "Saved coverage receipt"
|
|
765
|
+
)
|
|
766
|
+
os.close(descriptor)
|
|
767
|
+
except ContractError as exc:
|
|
768
|
+
return exc
|
|
769
|
+
return None
|
|
770
|
+
|
|
750
771
|
def linked(row: dict[str, Any], identity: str | None) -> bool:
|
|
751
772
|
surface_ids = row.get("surfaceIds", [])
|
|
752
773
|
return identity is not None and (
|
|
@@ -760,7 +781,51 @@ def _generic_surface_updates(
|
|
|
760
781
|
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
761
782
|
saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
|
|
762
783
|
replaced: set[int] = set()
|
|
763
|
-
updates: list[dict[str, Any]] = []
|
|
784
|
+
updates: list[tuple[str | None, dict[str, Any], set[int]]] = []
|
|
785
|
+
# Current selections across all surfaces take precedence over historical refs.
|
|
786
|
+
receipt_paths: dict[str, str] = {}
|
|
787
|
+
retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
|
|
788
|
+
if receipt_root is not None:
|
|
789
|
+
receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
|
|
790
|
+
for key, matches in saved_surfaces.items():
|
|
791
|
+
retained = [
|
|
792
|
+
match for match in matches if surface_is_retained(match[0], key[0], match[1])
|
|
793
|
+
]
|
|
794
|
+
if not retained:
|
|
795
|
+
continue
|
|
796
|
+
latest_source, latest_surface = max(retained, key=lambda match: source_order[match[0]])
|
|
797
|
+
retained_surfaces[key] = latest_surface
|
|
798
|
+
receipt_surfaces.extend(
|
|
799
|
+
(key[0], row)
|
|
800
|
+
for relative, row in retained
|
|
801
|
+
if source_order[relative] == source_order[latest_source]
|
|
802
|
+
)
|
|
803
|
+
for relative, draft, owner in sources:
|
|
804
|
+
rows = draft["coverage"].get("surfaces", [])
|
|
805
|
+
for row in rows if isinstance(rows, list) else []:
|
|
806
|
+
if (
|
|
807
|
+
isinstance(row, dict)
|
|
808
|
+
and "id" not in row
|
|
809
|
+
and surface_is_retained(relative, owner, row)
|
|
810
|
+
):
|
|
811
|
+
receipt_surfaces.append((owner, row))
|
|
812
|
+
receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
|
|
813
|
+
for owner, latest_surface in receipt_surfaces:
|
|
814
|
+
# Recovery repairs disposition and receipts before validating row
|
|
815
|
+
# metadata. A row discarded there cannot reserve a receipt spelling.
|
|
816
|
+
metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
|
|
817
|
+
metadata.setdefault("id", _saved_coverage_id(metadata))
|
|
818
|
+
metadata.update(disposition="needs_follow_up", receiptRefs=[])
|
|
819
|
+
try:
|
|
820
|
+
_require_str(metadata, "id", "coverage.surfaces")
|
|
821
|
+
_validate_schema_node(metadata, surface_schema, "coverage.surfaces")
|
|
822
|
+
except ContractError:
|
|
823
|
+
continue
|
|
824
|
+
refs = latest_surface.get("receiptRefs", [])
|
|
825
|
+
for ref in refs if isinstance(refs, list) else []:
|
|
826
|
+
if receipt_error(ref) is None:
|
|
827
|
+
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
828
|
+
receipt_paths.setdefault(normalized.lower(), normalized)
|
|
764
829
|
for relative, draft, owner in sources:
|
|
765
830
|
closed_ids = {
|
|
766
831
|
identity
|
|
@@ -772,7 +837,7 @@ def _generic_surface_updates(
|
|
|
772
837
|
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
773
838
|
if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
|
|
774
839
|
}
|
|
775
|
-
if not closed_ids and not reopened_ids:
|
|
840
|
+
if not closed_ids and not reopened_ids and receipt_root is None:
|
|
776
841
|
continue
|
|
777
842
|
current = draft["coverage"].get("surfaces", [])
|
|
778
843
|
for surface in current if isinstance(current, list) else []:
|
|
@@ -780,12 +845,18 @@ def _generic_surface_updates(
|
|
|
780
845
|
continue
|
|
781
846
|
reopening = surface.get("disposition") == "needs_follow_up"
|
|
782
847
|
work_ids = reopened_ids if reopening else closed_ids
|
|
783
|
-
|
|
848
|
+
repair_only = not work_ids
|
|
849
|
+
if repair_only and receipt_root is None:
|
|
784
850
|
continue
|
|
785
851
|
identity = surface.get("id")
|
|
786
852
|
if not isinstance(identity, str):
|
|
787
853
|
continue
|
|
788
854
|
matches = saved_surfaces[(owner, identity)]
|
|
855
|
+
# Receipt repair alone cannot close or reopen a saved surface.
|
|
856
|
+
if repair_only and any(
|
|
857
|
+
row.get("disposition") != surface.get("disposition") for _, row in matches
|
|
858
|
+
):
|
|
859
|
+
continue
|
|
789
860
|
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
790
861
|
# A closure cannot identify which of those observations it replaces.
|
|
791
862
|
by_source = dict(matches)
|
|
@@ -832,27 +903,93 @@ def _generic_surface_updates(
|
|
|
832
903
|
continue
|
|
833
904
|
# An accepted checkpoint can update a saved surface by ID without
|
|
834
905
|
# optional surfaceIds links on its generic task.
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
906
|
+
receipt_matches = [
|
|
907
|
+
(saved_path, row)
|
|
908
|
+
for saved_path, row in matches
|
|
909
|
+
if receipt_root is None
|
|
910
|
+
or saved_path in eligible_receipt_sources
|
|
911
|
+
or surface_is_retained(saved_path, owner, row)
|
|
912
|
+
]
|
|
913
|
+
if not receipt_matches:
|
|
914
|
+
continue
|
|
915
|
+
latest_surface = max(receipt_matches, key=lambda match: source_order[match[0]])[1]
|
|
916
|
+
target = retained_surfaces.get((owner, identity)) if repair_only else latest_surface
|
|
917
|
+
if target is None:
|
|
918
|
+
continue
|
|
919
|
+
update = copy.deepcopy(target)
|
|
920
|
+
# Historical evidence can repair a retained row without promoting
|
|
921
|
+
# an unselected observation's label, notes, or other metadata.
|
|
922
|
+
refs = copy.deepcopy(latest_surface.get("receiptRefs", []))
|
|
923
|
+
update["receiptRefs"] = refs
|
|
924
|
+
replaced_receipt = False
|
|
925
|
+
replacement_paths = dict(receipt_paths)
|
|
838
926
|
if isinstance(refs, list):
|
|
839
|
-
|
|
927
|
+
repaired = (
|
|
928
|
+
not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
|
|
929
|
+
)
|
|
930
|
+
if repaired:
|
|
931
|
+
for ref in refs:
|
|
932
|
+
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
933
|
+
key = normalized.lower()
|
|
934
|
+
if key in replacement_paths and replacement_paths[key] != normalized:
|
|
935
|
+
repaired = False
|
|
936
|
+
break
|
|
937
|
+
replacement_paths[key] = normalized
|
|
938
|
+
for _, row in receipt_matches:
|
|
840
939
|
previous_refs = row.get("receiptRefs", [])
|
|
841
940
|
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
842
941
|
if ref not in refs:
|
|
942
|
+
failure = receipt_error(ref) if repaired else None
|
|
943
|
+
replaceable = failure is not None and isinstance(
|
|
944
|
+
failure.__cause__, FileNotFoundError
|
|
945
|
+
)
|
|
946
|
+
if repaired and isinstance(ref, str):
|
|
947
|
+
try:
|
|
948
|
+
normalized = _require_portable_relative_path(
|
|
949
|
+
ref, "Saved coverage receipt"
|
|
950
|
+
)
|
|
951
|
+
except ContractError as error:
|
|
952
|
+
failure = error
|
|
953
|
+
replaceable = True
|
|
954
|
+
else:
|
|
955
|
+
key = normalized.lower()
|
|
956
|
+
previous = replacement_paths.get(key)
|
|
957
|
+
if previous is not None and previous != normalized:
|
|
958
|
+
failure = ContractError(
|
|
959
|
+
f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
|
|
960
|
+
)
|
|
961
|
+
replaceable = True
|
|
962
|
+
elif not replaceable:
|
|
963
|
+
replacement_paths[key] = normalized
|
|
964
|
+
if replaceable:
|
|
965
|
+
replaced_receipt = True
|
|
966
|
+
if receipt_warnings is not None:
|
|
967
|
+
receipt_warnings.append(
|
|
968
|
+
f"Replaced invalid saved coverage receipt {ref}: {failure}"
|
|
969
|
+
)
|
|
970
|
+
continue
|
|
843
971
|
refs.append(ref)
|
|
972
|
+
if repair_only and not replaced_receipt:
|
|
973
|
+
continue
|
|
844
974
|
try:
|
|
845
975
|
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
846
976
|
except ContractError:
|
|
847
977
|
continue
|
|
848
|
-
|
|
978
|
+
replaced_sources = {
|
|
849
979
|
id(row)
|
|
850
980
|
for _, row in matches
|
|
851
981
|
if reopening
|
|
852
982
|
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
983
|
+
}
|
|
984
|
+
replaced.update(replaced_sources)
|
|
985
|
+
existing = next(
|
|
986
|
+
(entry for entry in updates if entry[0] == owner and entry[1] == update), None
|
|
853
987
|
)
|
|
854
|
-
if
|
|
855
|
-
|
|
988
|
+
if existing is None:
|
|
989
|
+
receipt_paths.update(replacement_paths)
|
|
990
|
+
updates.append((owner, update, replaced_sources))
|
|
991
|
+
else:
|
|
992
|
+
existing[2].update(replaced_sources)
|
|
856
993
|
return replaced, updates
|
|
857
994
|
|
|
858
995
|
|
|
@@ -908,6 +1045,7 @@ def merge_saved_results(
|
|
|
908
1045
|
allow_frozen_legacy_parent: bool = False,
|
|
909
1046
|
frozen_model_source: str | None = None,
|
|
910
1047
|
selected_model_source: list[str] | None = None,
|
|
1048
|
+
project_worker_coverage: bool = False,
|
|
911
1049
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
912
1050
|
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
913
1051
|
initial_warnings = set(warnings)
|
|
@@ -1173,6 +1311,70 @@ def merge_saved_results(
|
|
|
1173
1311
|
if parent is None and latest_reducer is not None:
|
|
1174
1312
|
parent = drafts_by_path[latest_reducer]
|
|
1175
1313
|
|
|
1314
|
+
project_worker_coverage, surface_links = _prepare_worker_coverage_recovery(
|
|
1315
|
+
sources, parent, source_order, enabled=project_worker_coverage
|
|
1316
|
+
)
|
|
1317
|
+
# Keep the actual retained row even when recovery adds previous findings.
|
|
1318
|
+
linked_surface_sources = {id(surface) for _, _, surface, _, _ in surface_links}
|
|
1319
|
+
linked_surface_outputs: dict[int, dict[str, Any]] = {}
|
|
1320
|
+
worker_row_owners: dict[int, str] = {}
|
|
1321
|
+
worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
|
|
1322
|
+
|
|
1323
|
+
def record_coverage_owner(retained: Any, item: Any, owner: str | None, field: str) -> None:
|
|
1324
|
+
# Parent aggregates can carry worker metadata without originating in
|
|
1325
|
+
# that worker's retained source. Track the actual source of output rows.
|
|
1326
|
+
if project_worker_coverage and owner is not None and isinstance(retained, dict):
|
|
1327
|
+
worker_row_owners[id(retained)] = owner
|
|
1328
|
+
if field == "surfaces" and isinstance(item.get("id"), str):
|
|
1329
|
+
worker_surface_outputs.setdefault((owner, item["id"]), {})[id(retained)] = retained
|
|
1330
|
+
|
|
1331
|
+
def append_coverage_row(output: list[Any], item: Any, owner: str | None, field: str) -> Any:
|
|
1332
|
+
retained = _project_worker_coverage_row(
|
|
1333
|
+
item, owner if project_worker_coverage else None, field
|
|
1334
|
+
)
|
|
1335
|
+
if retained not in output:
|
|
1336
|
+
output.append(retained)
|
|
1337
|
+
retained = output[output.index(retained)]
|
|
1338
|
+
record_coverage_owner(retained, item, owner, field)
|
|
1339
|
+
return retained
|
|
1340
|
+
|
|
1341
|
+
def linked_tasks(
|
|
1342
|
+
task: dict[str, Any],
|
|
1343
|
+
published_id: str,
|
|
1344
|
+
source_task: dict[str, Any] | None,
|
|
1345
|
+
candidates: list[tuple[str | None, Any]],
|
|
1346
|
+
) -> list[Any]:
|
|
1347
|
+
expected = (
|
|
1348
|
+
_project_worker_coverage_row(source_task, source_task["workerId"], "deferred")
|
|
1349
|
+
if source_task is not None
|
|
1350
|
+
else None
|
|
1351
|
+
)
|
|
1352
|
+
return [
|
|
1353
|
+
retained
|
|
1354
|
+
for owner, retained in candidates
|
|
1355
|
+
if retained == task
|
|
1356
|
+
or (
|
|
1357
|
+
expected is not None
|
|
1358
|
+
and owner == expected["workerId"]
|
|
1359
|
+
and isinstance(retained, dict)
|
|
1360
|
+
and retained.get("id") == expected.get("id")
|
|
1361
|
+
and isinstance(retained.get("surfaceIds"), list)
|
|
1362
|
+
and published_id in retained["surfaceIds"]
|
|
1363
|
+
)
|
|
1364
|
+
]
|
|
1365
|
+
|
|
1366
|
+
def linked_surface(surface, outputs, named_outputs):
|
|
1367
|
+
retained = outputs.get(id(surface))
|
|
1368
|
+
if (
|
|
1369
|
+
retained is None
|
|
1370
|
+
and isinstance(owner := surface.get("workerId"), str)
|
|
1371
|
+
and isinstance(identity := surface.get("id"), str)
|
|
1372
|
+
):
|
|
1373
|
+
matches = named_outputs.get((owner, identity), {})
|
|
1374
|
+
if len(matches) == 1:
|
|
1375
|
+
retained = next(iter(matches.values()))
|
|
1376
|
+
return retained
|
|
1377
|
+
|
|
1176
1378
|
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
1177
1379
|
# Older checkpoints can omit IDs already assigned in their published output.
|
|
1178
1380
|
for field in ("deferred", "surfaces"):
|
|
@@ -1544,91 +1746,6 @@ def merge_saved_results(
|
|
|
1544
1746
|
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
1545
1747
|
_digest(_finding_content(original["finding"]))
|
|
1546
1748
|
)
|
|
1547
|
-
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1548
|
-
all_sources,
|
|
1549
|
-
source_order,
|
|
1550
|
-
closed_deferred,
|
|
1551
|
-
active_deferred,
|
|
1552
|
-
resolved,
|
|
1553
|
-
reopened_generic,
|
|
1554
|
-
coverage_schema["surfaces"]["items"],
|
|
1555
|
-
deferred_rows,
|
|
1556
|
-
ambiguous_deferred,
|
|
1557
|
-
)
|
|
1558
|
-
replaced_rows = {
|
|
1559
|
-
"surfaces": replaced_surfaces,
|
|
1560
|
-
"deferred": {
|
|
1561
|
-
id(row)
|
|
1562
|
-
for relative, _, owner in all_sources
|
|
1563
|
-
for row in deferred_rows[relative]
|
|
1564
|
-
if isinstance(row, dict)
|
|
1565
|
-
and isinstance(identity := row.get("id"), str)
|
|
1566
|
-
and (key := (owner, identity)) not in candidate_ids
|
|
1567
|
-
and key not in ambiguous_deferred
|
|
1568
|
-
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1569
|
-
and updated > source_order[relative]
|
|
1570
|
-
},
|
|
1571
|
-
}
|
|
1572
|
-
for field, replaced in replaced_rows.items():
|
|
1573
|
-
if parent and isinstance(coverage.get(field), list):
|
|
1574
|
-
previous = parent["coverage"].get(field, [])
|
|
1575
|
-
if isinstance(previous, list):
|
|
1576
|
-
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1577
|
-
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1578
|
-
if isinstance(coverage.get("surfaces"), list):
|
|
1579
|
-
for surface in surface_updates:
|
|
1580
|
-
if surface not in coverage["surfaces"]:
|
|
1581
|
-
coverage["surfaces"].append(surface)
|
|
1582
|
-
for surface in coverage["surfaces"]:
|
|
1583
|
-
if isinstance(surface, dict):
|
|
1584
|
-
surface.setdefault("receiptRefs", [])
|
|
1585
|
-
|
|
1586
|
-
# Reopened work survives a superseded checkpoint, but current candidate
|
|
1587
|
-
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
1588
|
-
for owner, item in reopened_rows:
|
|
1589
|
-
if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
|
|
1590
|
-
owner,
|
|
1591
|
-
identity,
|
|
1592
|
-
) in resolved:
|
|
1593
|
-
continue
|
|
1594
|
-
pending = coverage.setdefault("deferred", [])
|
|
1595
|
-
if isinstance(pending, list) and item not in pending:
|
|
1596
|
-
pending.append(copy.deepcopy(item))
|
|
1597
|
-
ambiguous_surface_ids = {
|
|
1598
|
-
(owner, identity)
|
|
1599
|
-
for owner, row in reopened_rows
|
|
1600
|
-
if isinstance(row.get("id"), str)
|
|
1601
|
-
and (owner, row["id"]) in ambiguous_deferred
|
|
1602
|
-
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1603
|
-
for identity in [
|
|
1604
|
-
row["id"],
|
|
1605
|
-
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1606
|
-
]
|
|
1607
|
-
if isinstance(identity, str)
|
|
1608
|
-
}
|
|
1609
|
-
for _, draft, owner in all_sources:
|
|
1610
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
1611
|
-
by_id: dict[str, dict[str, Any]] = {}
|
|
1612
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1613
|
-
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1614
|
-
continue
|
|
1615
|
-
if identity in by_id and surface != by_id[identity]:
|
|
1616
|
-
ambiguous_surface_ids.add((owner, identity))
|
|
1617
|
-
by_id[identity] = surface
|
|
1618
|
-
for _, draft, owner in all_sources:
|
|
1619
|
-
surfaces = draft["coverage"].get("surfaces", [])
|
|
1620
|
-
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1621
|
-
if (
|
|
1622
|
-
isinstance(surface, dict)
|
|
1623
|
-
and isinstance(surface.get("id"), str)
|
|
1624
|
-
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
1625
|
-
and surface.get("disposition") == "needs_follow_up"
|
|
1626
|
-
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
1627
|
-
and isinstance(coverage.get("surfaces"), list)
|
|
1628
|
-
):
|
|
1629
|
-
retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
|
|
1630
|
-
if retained_surface not in coverage["surfaces"]:
|
|
1631
|
-
coverage["surfaces"].append(copy.deepcopy(retained_surface))
|
|
1632
1749
|
selected_terminal_orders: dict[str, tuple[int, int]] = {}
|
|
1633
1750
|
for relative, draft, worker_id in sources:
|
|
1634
1751
|
if (
|
|
@@ -1647,7 +1764,11 @@ def merge_saved_results(
|
|
|
1647
1764
|
terminal_worker_orders[worker_id] = max(
|
|
1648
1765
|
terminal_worker_orders.get(worker_id, order), order
|
|
1649
1766
|
)
|
|
1650
|
-
|
|
1767
|
+
|
|
1768
|
+
def source_coverage_selection(
|
|
1769
|
+
relative: str,
|
|
1770
|
+
worker_id: str | None,
|
|
1771
|
+
) -> tuple[bool, set[str], bool, bool]:
|
|
1651
1772
|
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
1652
1773
|
selected_coverage_superseded = worker_id in selected_terminal_orders and (
|
|
1653
1774
|
source_order[relative] < selected_terminal_orders[worker_id]
|
|
@@ -1689,6 +1810,221 @@ def merge_saved_results(
|
|
|
1689
1810
|
and source_order[relative] >= accepted_order
|
|
1690
1811
|
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
1691
1812
|
)
|
|
1813
|
+
return selected_coverage_superseded, selected_candidates, superseded, retain_pending
|
|
1814
|
+
|
|
1815
|
+
def coverage_row_is_retained(
|
|
1816
|
+
field: str,
|
|
1817
|
+
item: Any,
|
|
1818
|
+
worker_id: str | None,
|
|
1819
|
+
selection: tuple[bool, set[str], bool, bool],
|
|
1820
|
+
) -> bool:
|
|
1821
|
+
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
1822
|
+
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
1823
|
+
# candidate evidence and pending tasks retain their own reconciliation.
|
|
1824
|
+
if (
|
|
1825
|
+
selected_coverage_superseded
|
|
1826
|
+
and field != "deferred"
|
|
1827
|
+
and not (
|
|
1828
|
+
isinstance(item, dict)
|
|
1829
|
+
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1830
|
+
)
|
|
1831
|
+
):
|
|
1832
|
+
return False
|
|
1833
|
+
# A selected outcome must retain its evidence even if its result write failed.
|
|
1834
|
+
if superseded and not (
|
|
1835
|
+
isinstance(item, dict)
|
|
1836
|
+
and (
|
|
1837
|
+
(field == "deferred" and retain_pending)
|
|
1838
|
+
or (
|
|
1839
|
+
isinstance(item.get("candidateId"), str)
|
|
1840
|
+
and item["candidateId"] in selected_candidates
|
|
1841
|
+
and item.get("disposition") == resolved.get((worker_id, item["candidateId"]))
|
|
1842
|
+
)
|
|
1843
|
+
)
|
|
1844
|
+
):
|
|
1845
|
+
return False
|
|
1846
|
+
if (
|
|
1847
|
+
isinstance(item, dict)
|
|
1848
|
+
and isinstance(
|
|
1849
|
+
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1850
|
+
if field == "deferred"
|
|
1851
|
+
else item.get("candidateId"),
|
|
1852
|
+
str,
|
|
1853
|
+
)
|
|
1854
|
+
and (worker_id, identity) in resolved
|
|
1855
|
+
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1856
|
+
):
|
|
1857
|
+
return False
|
|
1858
|
+
return True
|
|
1859
|
+
|
|
1860
|
+
ambiguous_surface_ids = {
|
|
1861
|
+
(owner, identity)
|
|
1862
|
+
for owner, row in reopened_rows
|
|
1863
|
+
if isinstance(row.get("id"), str)
|
|
1864
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
1865
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1866
|
+
for identity in [
|
|
1867
|
+
row["id"],
|
|
1868
|
+
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1869
|
+
]
|
|
1870
|
+
if isinstance(identity, str)
|
|
1871
|
+
}
|
|
1872
|
+
for _, draft, owner in all_sources:
|
|
1873
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1874
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1875
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1876
|
+
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1877
|
+
continue
|
|
1878
|
+
if identity in by_id and surface != by_id[identity]:
|
|
1879
|
+
ambiguous_surface_ids.add((owner, identity))
|
|
1880
|
+
by_id[identity] = surface
|
|
1881
|
+
|
|
1882
|
+
replaced_deferred = {
|
|
1883
|
+
id(row)
|
|
1884
|
+
for relative, _, owner in all_sources
|
|
1885
|
+
for row in deferred_rows[relative]
|
|
1886
|
+
if isinstance(row, dict)
|
|
1887
|
+
and isinstance(identity := row.get("id"), str)
|
|
1888
|
+
and (key := (owner, identity)) not in candidate_ids
|
|
1889
|
+
and key not in ambiguous_deferred
|
|
1890
|
+
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1891
|
+
and updated > source_order[relative]
|
|
1892
|
+
}
|
|
1893
|
+
fallback_surfaces = []
|
|
1894
|
+
if surface_links and isinstance(coverage.get("surfaces"), list):
|
|
1895
|
+
# Plan the surviving associations before historical receipts can claim
|
|
1896
|
+
# a spelling needed by a surface that must be restored for its task.
|
|
1897
|
+
prospective_tasks = []
|
|
1898
|
+
prospective_surfaces = {}
|
|
1899
|
+
prospective_named = {}
|
|
1900
|
+
for relative, draft, owner in all_sources:
|
|
1901
|
+
selection = source_coverage_selection(relative, owner)
|
|
1902
|
+
for row in deferred_rows[relative]:
|
|
1903
|
+
if (
|
|
1904
|
+
coverage_row_is_retained("deferred", row, owner, selection)
|
|
1905
|
+
and id(row) not in replaced_deferred
|
|
1906
|
+
and not (
|
|
1907
|
+
isinstance(row, dict)
|
|
1908
|
+
and isinstance(row.get("id"), str)
|
|
1909
|
+
and (owner, row["id"]) in closed_deferred
|
|
1910
|
+
)
|
|
1911
|
+
):
|
|
1912
|
+
prospective_tasks.append(
|
|
1913
|
+
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1914
|
+
)
|
|
1915
|
+
rows = draft["coverage"].get("surfaces", [])
|
|
1916
|
+
for row in rows if isinstance(rows, list) else []:
|
|
1917
|
+
if owner is None or not isinstance(row, dict):
|
|
1918
|
+
continue
|
|
1919
|
+
emitted = coverage_row_is_retained("surfaces", row, owner, selection)
|
|
1920
|
+
ambiguous = (
|
|
1921
|
+
isinstance(row.get("id"), str)
|
|
1922
|
+
and (owner, row["id"]) in ambiguous_surface_ids
|
|
1923
|
+
and row.get("disposition") == "needs_follow_up"
|
|
1924
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1925
|
+
)
|
|
1926
|
+
if not emitted and not ambiguous:
|
|
1927
|
+
continue
|
|
1928
|
+
projected = _project_worker_coverage_row(row, owner, "surfaces")
|
|
1929
|
+
if isinstance(row.get("id"), str):
|
|
1930
|
+
named = prospective_named.setdefault((owner, row["id"]), {})
|
|
1931
|
+
projected = next(
|
|
1932
|
+
(existing for existing in named.values() if existing == projected),
|
|
1933
|
+
projected,
|
|
1934
|
+
)
|
|
1935
|
+
named[id(projected)] = projected
|
|
1936
|
+
if emitted:
|
|
1937
|
+
prospective_surfaces[id(row)] = projected
|
|
1938
|
+
prospective_tasks.extend(
|
|
1939
|
+
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
1940
|
+
for owner, row in reopened_rows
|
|
1941
|
+
if (owner, _deferred_candidate_id(row, owner, ambiguous_deferred)) not in resolved
|
|
1942
|
+
)
|
|
1943
|
+
bound = {
|
|
1944
|
+
(id(retained), published_id)
|
|
1945
|
+
for task, published_id, surface, _, task_source in surface_links
|
|
1946
|
+
if linked_surface(surface, prospective_surfaces, prospective_named) is not None
|
|
1947
|
+
for retained in linked_tasks(task, published_id, task_source, prospective_tasks)
|
|
1948
|
+
}
|
|
1949
|
+
for task, published_id, _, parent_surface, task_source in surface_links:
|
|
1950
|
+
for retained in linked_tasks(task, published_id, task_source, prospective_tasks):
|
|
1951
|
+
key = (id(retained), published_id)
|
|
1952
|
+
if key not in bound:
|
|
1953
|
+
fallback_surfaces.append(parent_surface)
|
|
1954
|
+
bound.add(key)
|
|
1955
|
+
|
|
1956
|
+
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1957
|
+
all_sources,
|
|
1958
|
+
source_order,
|
|
1959
|
+
closed_deferred,
|
|
1960
|
+
active_deferred,
|
|
1961
|
+
resolved,
|
|
1962
|
+
reopened_generic,
|
|
1963
|
+
coverage_schema["surfaces"]["items"],
|
|
1964
|
+
deferred_rows,
|
|
1965
|
+
ambiguous_deferred,
|
|
1966
|
+
receipt_root=scan_dir if project_worker_coverage else None,
|
|
1967
|
+
receipt_warnings=warnings,
|
|
1968
|
+
fallback_surfaces=fallback_surfaces,
|
|
1969
|
+
surface_is_retained=lambda relative, owner, row: coverage_row_is_retained(
|
|
1970
|
+
"surfaces", row, owner, source_coverage_selection(relative, owner)
|
|
1971
|
+
),
|
|
1972
|
+
eligible_receipt_sources={
|
|
1973
|
+
relative
|
|
1974
|
+
for relative, draft, _ in all_sources
|
|
1975
|
+
if draft.get("complete") is not False or relative in selected_observations
|
|
1976
|
+
},
|
|
1977
|
+
)
|
|
1978
|
+
replaced_rows = {
|
|
1979
|
+
"surfaces": replaced_surfaces,
|
|
1980
|
+
"deferred": replaced_deferred,
|
|
1981
|
+
}
|
|
1982
|
+
for field, replaced in replaced_rows.items():
|
|
1983
|
+
if parent and isinstance(coverage.get(field), list):
|
|
1984
|
+
previous = parent["coverage"].get(field, [])
|
|
1985
|
+
if isinstance(previous, list):
|
|
1986
|
+
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1987
|
+
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1988
|
+
if isinstance(coverage.get("surfaces"), list):
|
|
1989
|
+
for owner, surface, replaced_sources in surface_updates:
|
|
1990
|
+
retained = append_coverage_row(coverage["surfaces"], surface, owner, "surfaces")
|
|
1991
|
+
for source_id in replaced_sources & linked_surface_sources:
|
|
1992
|
+
linked_surface_outputs[source_id] = retained
|
|
1993
|
+
for surface in coverage["surfaces"]:
|
|
1994
|
+
if isinstance(surface, dict):
|
|
1995
|
+
surface.setdefault("receiptRefs", [])
|
|
1996
|
+
|
|
1997
|
+
# Reopened work survives a superseded checkpoint, but current candidate
|
|
1998
|
+
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
1999
|
+
for owner, item in reopened_rows:
|
|
2000
|
+
if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
|
|
2001
|
+
owner,
|
|
2002
|
+
identity,
|
|
2003
|
+
) in resolved:
|
|
2004
|
+
continue
|
|
2005
|
+
pending = coverage.setdefault("deferred", [])
|
|
2006
|
+
if isinstance(pending, list):
|
|
2007
|
+
append_coverage_row(pending, item, owner, "deferred")
|
|
2008
|
+
for _, draft, owner in all_sources:
|
|
2009
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
2010
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2011
|
+
if (
|
|
2012
|
+
isinstance(surface, dict)
|
|
2013
|
+
and isinstance(surface.get("id"), str)
|
|
2014
|
+
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
2015
|
+
and surface.get("disposition") == "needs_follow_up"
|
|
2016
|
+
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
2017
|
+
and isinstance(coverage.get("surfaces"), list)
|
|
2018
|
+
):
|
|
2019
|
+
append_coverage_row(
|
|
2020
|
+
coverage["surfaces"],
|
|
2021
|
+
{**surface, "receiptRefs": surface.get("receiptRefs", [])},
|
|
2022
|
+
owner,
|
|
2023
|
+
"surfaces",
|
|
2024
|
+
)
|
|
2025
|
+
for relative, draft, worker_id in all_sources:
|
|
2026
|
+
selection = source_coverage_selection(relative, worker_id)
|
|
2027
|
+
selected_coverage_superseded, selected_candidates, superseded, retain_pending = selection
|
|
1692
2028
|
skip_superseded_findings = (
|
|
1693
2029
|
superseded
|
|
1694
2030
|
and not stopped
|
|
@@ -1884,30 +2220,8 @@ def merge_saved_results(
|
|
|
1884
2220
|
# recovery and warnings rather than silently changing its contract.
|
|
1885
2221
|
continue
|
|
1886
2222
|
for item in items:
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
if (
|
|
1890
|
-
selected_coverage_superseded
|
|
1891
|
-
and field != "deferred"
|
|
1892
|
-
and not (
|
|
1893
|
-
isinstance(item, dict)
|
|
1894
|
-
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1895
|
-
)
|
|
1896
|
-
):
|
|
1897
|
-
continue
|
|
1898
|
-
# A selected outcome must retain its evidence even if its result write failed.
|
|
1899
|
-
if superseded and not (
|
|
1900
|
-
isinstance(item, dict)
|
|
1901
|
-
and (
|
|
1902
|
-
(field == "deferred" and retain_pending)
|
|
1903
|
-
or (
|
|
1904
|
-
isinstance(item.get("candidateId"), str)
|
|
1905
|
-
and item["candidateId"] in selected_candidates
|
|
1906
|
-
and item.get("disposition")
|
|
1907
|
-
== resolved.get((worker_id, item["candidateId"]))
|
|
1908
|
-
)
|
|
1909
|
-
)
|
|
1910
|
-
):
|
|
2223
|
+
source_item = item
|
|
2224
|
+
if not coverage_row_is_retained(field, item, worker_id, selection):
|
|
1911
2225
|
continue
|
|
1912
2226
|
if id(item) in replaced_rows.get(field, ()):
|
|
1913
2227
|
continue
|
|
@@ -1939,31 +2253,29 @@ def merge_saved_results(
|
|
|
1939
2253
|
for previous in history
|
|
1940
2254
|
):
|
|
1941
2255
|
history.append(copy.deepcopy(finding))
|
|
1942
|
-
if (
|
|
1943
|
-
isinstance(item, dict)
|
|
1944
|
-
and isinstance(
|
|
1945
|
-
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1946
|
-
if field == "deferred"
|
|
1947
|
-
else item.get("candidateId"),
|
|
1948
|
-
str,
|
|
1949
|
-
)
|
|
1950
|
-
and (worker_id, identity) in resolved
|
|
1951
|
-
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1952
|
-
):
|
|
1953
|
-
continue
|
|
1954
2256
|
if field == "surfaces" and isinstance(item, dict):
|
|
1955
2257
|
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
1956
2258
|
if isinstance(item, dict) and "id" not in item:
|
|
1957
2259
|
semantic_item = dict(item)
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
2260
|
+
existing = next(
|
|
2261
|
+
(
|
|
2262
|
+
row
|
|
2263
|
+
for row in output
|
|
2264
|
+
if isinstance(row, dict)
|
|
2265
|
+
and {key: value for key, value in row.items() if key != "id"}
|
|
2266
|
+
== semantic_item
|
|
2267
|
+
),
|
|
2268
|
+
None,
|
|
2269
|
+
)
|
|
2270
|
+
if existing is not None:
|
|
2271
|
+
if project_worker_coverage and worker_id is not None:
|
|
2272
|
+
worker_row_owners[id(existing)] = worker_id
|
|
2273
|
+
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2274
|
+
linked_surface_outputs[id(source_item)] = existing
|
|
1964
2275
|
continue
|
|
1965
|
-
|
|
1966
|
-
|
|
2276
|
+
retained = append_coverage_row(output, item, worker_id, field)
|
|
2277
|
+
if field == "surfaces" and id(source_item) in linked_surface_sources:
|
|
2278
|
+
linked_surface_outputs[id(source_item)] = retained
|
|
1967
2279
|
|
|
1968
2280
|
identities: dict[str, str] = {}
|
|
1969
2281
|
for finding in findings:
|
|
@@ -1978,9 +2290,77 @@ def merge_saved_results(
|
|
|
1978
2290
|
finding.setdefault("provenance", {})["preservedIdentity"] = copy.deepcopy(identity)
|
|
1979
2291
|
identity["instance"] = f"{identity.get('instance', 'saved')}-{variant[:16]}"
|
|
1980
2292
|
identities[key] = variant
|
|
2293
|
+
# Bind links to retained output objects before their IDs become unique.
|
|
2294
|
+
# Candidate metadata can give an anonymous row the same preliminary ID as
|
|
2295
|
+
# another surface, so only the final surface ID can be used for its links.
|
|
2296
|
+
retained_surface_links: dict[int, tuple[dict[str, Any], dict[str, dict[str, Any]]]] = {}
|
|
2297
|
+
output_tasks = coverage.get("deferred", [])
|
|
2298
|
+
task_candidates = [
|
|
2299
|
+
(worker_row_owners.get(id(row)), row)
|
|
2300
|
+
for row in (output_tasks if isinstance(output_tasks, list) else [])
|
|
2301
|
+
]
|
|
2302
|
+
for task, published_id, surface, _, task_source in surface_links:
|
|
2303
|
+
retained_surface = linked_surface(surface, linked_surface_outputs, worker_surface_outputs)
|
|
2304
|
+
if retained_surface is None or not isinstance(retained_surface.get("id"), str):
|
|
2305
|
+
continue
|
|
2306
|
+
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2307
|
+
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2308
|
+
links.setdefault(published_id, retained_surface)
|
|
2309
|
+
# A newer worker checkpoint may omit a surface still needed by a retained
|
|
2310
|
+
# task. Restore its parent copy only when no worker replacement survived.
|
|
2311
|
+
for task, published_id, surface, parent_surface, task_source in surface_links:
|
|
2312
|
+
for retained_task in linked_tasks(task, published_id, task_source, task_candidates):
|
|
2313
|
+
_, links = retained_surface_links.setdefault(id(retained_task), (retained_task, {}))
|
|
2314
|
+
if published_id not in links and isinstance(coverage.get("surfaces"), list):
|
|
2315
|
+
retained_surface = append_coverage_row(
|
|
2316
|
+
coverage["surfaces"], parent_surface, None, "surfaces"
|
|
2317
|
+
)
|
|
2318
|
+
record_coverage_owner(
|
|
2319
|
+
retained_surface, surface, surface.get("workerId"), "surfaces"
|
|
2320
|
+
)
|
|
2321
|
+
links[published_id] = retained_surface
|
|
2322
|
+
if project_worker_coverage:
|
|
2323
|
+
worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
|
|
2324
|
+
named_surface_outputs = {
|
|
2325
|
+
identity for outputs in worker_surface_outputs.values() for identity in outputs
|
|
2326
|
+
}
|
|
2327
|
+
surfaces = coverage.get("surfaces", [])
|
|
2328
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
2329
|
+
if not isinstance(surface, dict):
|
|
2330
|
+
continue
|
|
2331
|
+
owner, identity = worker_row_owners.get(id(surface)), surface.get("id")
|
|
2332
|
+
if isinstance(owner, str) and isinstance(identity, str):
|
|
2333
|
+
key = (owner, identity)
|
|
2334
|
+
existing = worker_surfaces.get(key)
|
|
2335
|
+
# An anonymous candidate ID can collide with an explicit surface ID.
|
|
2336
|
+
# Bind a named worker link to the explicitly named source.
|
|
2337
|
+
if existing is None or (
|
|
2338
|
+
id(surface) in named_surface_outputs
|
|
2339
|
+
and id(existing) not in named_surface_outputs
|
|
2340
|
+
):
|
|
2341
|
+
worker_surfaces[key] = surface
|
|
2342
|
+
for task in output_tasks if isinstance(output_tasks, list) else []:
|
|
2343
|
+
if (
|
|
2344
|
+
not isinstance(task, dict)
|
|
2345
|
+
or (owner := worker_row_owners.get(id(task))) is None
|
|
2346
|
+
or not isinstance(task.get("surfaceIds"), list)
|
|
2347
|
+
):
|
|
2348
|
+
continue
|
|
2349
|
+
for identity in task["surfaceIds"]:
|
|
2350
|
+
if (
|
|
2351
|
+
isinstance(identity, str)
|
|
2352
|
+
and (surface := worker_surfaces.get((owner, identity))) is not None
|
|
2353
|
+
):
|
|
2354
|
+
_, links = retained_surface_links.setdefault(id(task), (task, {}))
|
|
2355
|
+
links.setdefault(identity, surface)
|
|
1981
2356
|
for field in ("surfaces", "explicitExclusions", "deferred"):
|
|
1982
2357
|
used: set[str] = set()
|
|
1983
2358
|
items = coverage.setdefault(field, [])
|
|
2359
|
+
reserved = {
|
|
2360
|
+
row["id"]
|
|
2361
|
+
for row in (items if isinstance(items, list) else [])
|
|
2362
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
2363
|
+
}
|
|
1984
2364
|
for item in items if isinstance(items, list) else []:
|
|
1985
2365
|
if not isinstance(item, dict):
|
|
1986
2366
|
continue
|
|
@@ -1993,10 +2373,22 @@ def merge_saved_results(
|
|
|
1993
2373
|
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
1994
2374
|
continue
|
|
1995
2375
|
if item["id"] in used:
|
|
1996
|
-
|
|
2376
|
+
if project_worker_coverage:
|
|
2377
|
+
base = item["id"]
|
|
2378
|
+
suffix = 2
|
|
2379
|
+
while item["id"] in used or item["id"] in reserved:
|
|
2380
|
+
item["id"] = f"{base}-{suffix}"
|
|
2381
|
+
suffix += 1
|
|
2382
|
+
else:
|
|
2383
|
+
item["id"] = f"{item['id']}-{_digest(item)[:16]}"
|
|
1997
2384
|
used.add(item["id"])
|
|
1998
2385
|
if field == "surfaces":
|
|
1999
2386
|
item.setdefault("receiptRefs", [])
|
|
2387
|
+
for task, links in retained_surface_links.values():
|
|
2388
|
+
task["surfaceIds"] = [
|
|
2389
|
+
links[identity]["id"] if isinstance(identity, str) and identity in links else identity
|
|
2390
|
+
for identity in task["surfaceIds"]
|
|
2391
|
+
]
|
|
2000
2392
|
coverage.pop("completeness", None)
|
|
2001
2393
|
if stopped:
|
|
2002
2394
|
if not isinstance(coverage.get("deferred"), list):
|
|
@@ -2085,6 +2477,7 @@ def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterato
|
|
|
2085
2477
|
"The configured discovery time limit elapsed before any source review completed. "
|
|
2086
2478
|
"Saved findings and pending review were preserved without a completed reduction."
|
|
2087
2479
|
),
|
|
2480
|
+
project_worker_coverage=True,
|
|
2088
2481
|
)
|
|
2089
2482
|
if documents is None:
|
|
2090
2483
|
raise ContractError("Deep Scan deadline has no saved parent draft.")
|
|
@@ -2271,6 +2664,7 @@ def preserve_scan_results_locked(
|
|
|
2271
2664
|
frozen_source_digests=frozen_source_digests,
|
|
2272
2665
|
frozen_model_source=model_source[0] if model_source else None,
|
|
2273
2666
|
selected_model_source=model_source,
|
|
2667
|
+
project_worker_coverage=scan["mode"] == "deep",
|
|
2274
2668
|
allow_frozen_legacy_parent=(
|
|
2275
2669
|
include_parent_with_recovery
|
|
2276
2670
|
or (
|