@openai/codex-security 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -36
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +45 -189
- package/_bundled_plugin/references/scan-contract.md +2 -0
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
- package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
- package/_bundled_plugin/scripts/report_projection.py +44 -15
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +51 -22
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +53 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
- package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +44 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -30
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +134 -0
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +8 -4
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +8 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +414 -72
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/classify-severity-records.d.ts +5 -2
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +40 -34
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +5 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +20 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +1 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +802 -149
- package/dist/cli.js.map +1 -1
- package/dist/cloud-endpoint.d.ts +6 -0
- package/dist/cloud-endpoint.d.ts.map +1 -0
- package/dist/cloud-endpoint.js +51 -0
- package/dist/cloud-endpoint.js.map +1 -0
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +7 -5
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +67 -33
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +30 -4
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +7 -5
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +3 -0
- package/dist/default-severity-rubric.d.ts.map +1 -0
- package/dist/default-severity-rubric.js +67 -0
- package/dist/default-severity-rubric.js.map +1 -0
- package/dist/endor-findings.d.ts +8 -0
- package/dist/endor-findings.d.ts.map +1 -0
- package/dist/endor-findings.js +238 -0
- package/dist/endor-findings.js.map +1 -0
- package/dist/external-findings-import.d.ts +5 -0
- package/dist/external-findings-import.d.ts.map +1 -0
- package/dist/external-findings-import.js +46 -0
- package/dist/external-findings-import.js.map +1 -0
- package/dist/external-findings-local.d.ts +63 -0
- package/dist/external-findings-local.d.ts.map +1 -0
- package/dist/external-findings-local.js +344 -0
- package/dist/external-findings-local.js.map +1 -0
- package/dist/external-findings-publish.d.ts +73 -0
- package/dist/external-findings-publish.d.ts.map +1 -0
- package/dist/external-findings-publish.js +721 -0
- package/dist/external-findings-publish.js.map +1 -0
- package/dist/external-findings-read.d.ts +16 -0
- package/dist/external-findings-read.d.ts.map +1 -0
- package/dist/external-findings-read.js +97 -0
- package/dist/external-findings-read.js.map +1 -0
- package/dist/external-findings.d.ts +21 -0
- package/dist/external-findings.d.ts.map +1 -0
- package/dist/external-findings.js +2 -0
- package/dist/external-findings.js.map +1 -0
- package/dist/external-import-contract.d.ts +15 -0
- package/dist/external-import-contract.d.ts.map +1 -0
- package/dist/external-import-contract.js +262 -0
- package/dist/external-import-contract.js.map +1 -0
- package/dist/external-import-models.d.ts +362 -0
- package/dist/external-import-models.d.ts.map +1 -0
- package/dist/external-import-models.js +3 -0
- package/dist/external-import-models.js.map +1 -0
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +23 -6
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +23 -10
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts +386 -0
- package/dist/github-advisories.d.ts.map +1 -0
- package/dist/github-advisories.js +133 -0
- package/dist/github-advisories.js.map +1 -0
- package/dist/github-findings.d.ts +11 -0
- package/dist/github-findings.d.ts.map +1 -0
- package/dist/github-findings.js +258 -0
- package/dist/github-findings.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +2 -1
- package/dist/hosted-remediation-protocol.d.ts.map +1 -1
- package/dist/hosted-remediation-protocol.js +17 -11
- package/dist/hosted-remediation-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts +4 -0
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +22 -3
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +16 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +9 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +1 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +17 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/models.d.ts +1 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -0
- package/dist/project-config.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +7 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +18 -28
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +10 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +10 -8
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +8 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +8 -0
- package/dist/scan-owner-suggestions.d.ts.map +1 -0
- package/dist/scan-owner-suggestions.js +33 -0
- package/dist/scan-owner-suggestions.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +20 -0
- package/dist/scanner-import.d.ts.map +1 -0
- package/dist/scanner-import.js +208 -0
- package/dist/scanner-import.js.map +1 -0
- package/dist/security-policy-diff.d.ts +2 -0
- package/dist/security-policy-diff.d.ts.map +1 -0
- package/dist/security-policy-diff.js +197 -0
- package/dist/security-policy-diff.js.map +1 -0
- package/dist/security-policy.d.ts +7 -3
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +11 -76
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +10 -0
- package/dist/semgrep-findings.d.ts.map +1 -0
- package/dist/semgrep-findings.js +293 -0
- package/dist/semgrep-findings.js.map +1 -0
- package/dist/severity-review.d.ts +3 -0
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +19 -0
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-store.js +7 -2
- package/dist/severity-store.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +10 -0
- package/dist/snyk-findings.d.ts.map +1 -0
- package/dist/snyk-findings.js +350 -0
- package/dist/snyk-findings.js.map +1 -0
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +2 -2
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +16 -3
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +9 -0
- package/dist/wiz-csv.d.ts.map +1 -0
- package/dist/wiz-csv.js +116 -0
- package/dist/wiz-csv.js.map +1 -0
- package/dist/wiz-findings.d.ts +5 -0
- package/dist/wiz-findings.d.ts.map +1 -0
- package/dist/wiz-findings.js +464 -0
- package/dist/wiz-findings.js.map +1 -0
- package/docs/classify-severity-records.md +30 -13
- package/docs/cli.md +835 -47
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +43 -24
- package/docs/hosted-scans.md +38 -0
- package/package.json +11 -9
- package/schemas/external-findings.schema.json +1421 -0
- package/schemas/project-config.schema.json +4 -0
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -11,10 +11,40 @@ from typing import Any
|
|
|
11
11
|
from finalize_scan_contract import (
|
|
12
12
|
ContractError,
|
|
13
13
|
_normalize_unsealed_open_questions,
|
|
14
|
+
_read_json,
|
|
14
15
|
_require_portable_relative_path,
|
|
16
|
+
_validate_resolved_deferred,
|
|
17
|
+
_validate_schema_node,
|
|
15
18
|
)
|
|
16
19
|
|
|
17
20
|
|
|
21
|
+
def _has_candidate_metadata(row: dict[str, Any]) -> bool:
|
|
22
|
+
return any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _coverage_rows(coverage: dict[str, Any], field: str) -> list[Any]:
|
|
26
|
+
rows = coverage.get(field, [])
|
|
27
|
+
return rows if isinstance(rows, list) else []
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _inherit_coverage_sources(
|
|
31
|
+
row: Any,
|
|
32
|
+
originals: list[Any],
|
|
33
|
+
sources: dict[int, tuple[dict[str, Any], list[str]]],
|
|
34
|
+
) -> None:
|
|
35
|
+
if not isinstance(row, dict):
|
|
36
|
+
return
|
|
37
|
+
for original in originals:
|
|
38
|
+
saved = sources.get(id(original))
|
|
39
|
+
if saved is None:
|
|
40
|
+
continue
|
|
41
|
+
# Retain the object as well as its ID while copies are reconciled.
|
|
42
|
+
paths = sources.setdefault(id(row), (row, []))[1]
|
|
43
|
+
for relative in saved[1]:
|
|
44
|
+
if relative not in paths:
|
|
45
|
+
paths.append(relative)
|
|
46
|
+
|
|
47
|
+
|
|
18
48
|
def _saved_coverage_id(item: dict[str, Any]) -> str:
|
|
19
49
|
if candidate_id := item.get("candidateId"):
|
|
20
50
|
return candidate_id
|
|
@@ -98,6 +128,7 @@ def _prepare_worker_coverage_recovery(
|
|
|
98
128
|
source_order: dict[str, tuple[int, int]],
|
|
99
129
|
*,
|
|
100
130
|
enabled: bool,
|
|
131
|
+
receipt_resolver=None,
|
|
101
132
|
) -> tuple[
|
|
102
133
|
bool, list[tuple[dict[str, Any], str, dict[str, Any], dict[str, Any], dict[str, Any] | None]]
|
|
103
134
|
]:
|
|
@@ -156,7 +187,9 @@ def _prepare_worker_coverage_recovery(
|
|
|
156
187
|
if not isinstance(row, dict):
|
|
157
188
|
continue
|
|
158
189
|
row["workerId"] = owner
|
|
159
|
-
if field == "surfaces" and
|
|
190
|
+
if field == "surfaces" and receipt_resolver is not None:
|
|
191
|
+
row["receiptRefs"] = receipt_resolver(relative, row, owner)
|
|
192
|
+
elif field == "surfaces" and isinstance(row.get("receiptRefs"), list):
|
|
160
193
|
refs = []
|
|
161
194
|
for ref in row["receiptRefs"]:
|
|
162
195
|
if isinstance(ref, str):
|
|
@@ -232,3 +265,613 @@ def _prepare_worker_coverage_recovery(
|
|
|
232
265
|
if _coverage_comparison_row(row, field) not in projected[field]
|
|
233
266
|
]
|
|
234
267
|
return True, surface_links
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class SavedCoverageProjection:
|
|
271
|
+
"""Keep worker source links stable while legacy and accepted rows converge."""
|
|
272
|
+
|
|
273
|
+
def __init__(self, project_worker_coverage, legacy_surface_links, sources):
|
|
274
|
+
self.project_worker_coverage = project_worker_coverage
|
|
275
|
+
self.legacy_surface_links = legacy_surface_links
|
|
276
|
+
self.legacy_source_outputs: dict[int, dict[str, Any]] = {}
|
|
277
|
+
self.worker_row_owners: dict[int, str] = {}
|
|
278
|
+
self.worker_surface_outputs: dict[tuple[str, str], dict[int, dict[str, Any]]] = {}
|
|
279
|
+
self.source_owner_by_id = {
|
|
280
|
+
id(row): owner
|
|
281
|
+
for _, draft, owner in sources
|
|
282
|
+
for field in ("surfaces", "deferred")
|
|
283
|
+
for rows in (draft["coverage"].get(field, []),)
|
|
284
|
+
if isinstance(rows, list)
|
|
285
|
+
for row in rows
|
|
286
|
+
if isinstance(row, dict)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
def record_coverage_owner(self, retained, original, owner, field):
|
|
290
|
+
if self.project_worker_coverage and owner is not None and isinstance(retained, dict):
|
|
291
|
+
self.worker_row_owners[id(retained)] = owner
|
|
292
|
+
if field == "surfaces" and isinstance(original.get("id"), str):
|
|
293
|
+
self.worker_surface_outputs.setdefault((owner, original["id"]), {})[
|
|
294
|
+
id(retained)
|
|
295
|
+
] = retained
|
|
296
|
+
|
|
297
|
+
def retain_coverage_row(self, output, item, owner, field, original=None, qualified=False):
|
|
298
|
+
retained = (
|
|
299
|
+
copy.deepcopy(item)
|
|
300
|
+
if qualified or not self.project_worker_coverage
|
|
301
|
+
else _project_worker_coverage_row(item, owner, field)
|
|
302
|
+
)
|
|
303
|
+
if retained not in output:
|
|
304
|
+
output.append(retained)
|
|
305
|
+
retained = output[output.index(retained)]
|
|
306
|
+
if isinstance(retained, dict) and original is not None:
|
|
307
|
+
self.legacy_source_outputs[id(original)] = retained
|
|
308
|
+
self.record_coverage_owner(
|
|
309
|
+
retained, original if original is not None else item, owner, field
|
|
310
|
+
)
|
|
311
|
+
return retained
|
|
312
|
+
|
|
313
|
+
def retained_record(self, coverage, field, item, canonical_record):
|
|
314
|
+
item = canonical_record(field, item)
|
|
315
|
+
if field == "surfaces":
|
|
316
|
+
# Canonical IDs can change while source content and ownership stay the same.
|
|
317
|
+
item = {key: value for key, value in item.items() if key != "id"}
|
|
318
|
+
records = coverage.get(field, [])
|
|
319
|
+
for record in records if isinstance(records, list) else []:
|
|
320
|
+
if not isinstance(record, dict):
|
|
321
|
+
continue
|
|
322
|
+
original = dict(canonical_record(field, record))
|
|
323
|
+
if "id" not in item:
|
|
324
|
+
original.pop("id", None) # Canonical publication can assign an ID.
|
|
325
|
+
if field == "openQuestions":
|
|
326
|
+
# Compare the same canonical form that publication writes.
|
|
327
|
+
previous_question = {"openQuestions": [original]}
|
|
328
|
+
source_question = {"openQuestions": [item]}
|
|
329
|
+
_normalize_unsealed_open_questions(previous_question)
|
|
330
|
+
_normalize_unsealed_open_questions(source_question)
|
|
331
|
+
if previous_question == source_question:
|
|
332
|
+
return record
|
|
333
|
+
elif original == item:
|
|
334
|
+
return record
|
|
335
|
+
return None
|
|
336
|
+
|
|
337
|
+
def linked_surface(self, surface, outputs, named_outputs):
|
|
338
|
+
retained = outputs.get(id(surface))
|
|
339
|
+
owner = self.source_owner_by_id.get(id(surface), surface.get("workerId"))
|
|
340
|
+
if retained is None and isinstance(owner, str) and isinstance(surface.get("id"), str):
|
|
341
|
+
matches = named_outputs.get((owner, surface["id"]), {})
|
|
342
|
+
if len(matches) == 1:
|
|
343
|
+
retained = next(iter(matches.values()))
|
|
344
|
+
return retained
|
|
345
|
+
|
|
346
|
+
def is_reviewed_worker(self, canonical, worker):
|
|
347
|
+
if (
|
|
348
|
+
worker is None
|
|
349
|
+
or not {"status", "merge_state", "attempt"}.issubset(worker.keys())
|
|
350
|
+
or worker["status"] != "succeeded"
|
|
351
|
+
or worker["merge_state"] != "merged"
|
|
352
|
+
):
|
|
353
|
+
return False
|
|
354
|
+
reviews = canonical.get("reviews", [])
|
|
355
|
+
return any(
|
|
356
|
+
isinstance(review, dict)
|
|
357
|
+
and review.get("workerId") == worker["id"]
|
|
358
|
+
and review.get("attempt") == worker["attempt"]
|
|
359
|
+
for review in (reviews if isinstance(reviews, list) else [])
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
def repair_reviewed_surface(self, coverage, canonical, worker, source, surface):
|
|
363
|
+
if not self.is_reviewed_worker(canonical, worker):
|
|
364
|
+
return None
|
|
365
|
+
published = next(
|
|
366
|
+
(
|
|
367
|
+
row
|
|
368
|
+
for row in canonical.get("surfaces", [])
|
|
369
|
+
if isinstance(row, dict)
|
|
370
|
+
and isinstance(provenance := row.get("provenance"), dict)
|
|
371
|
+
and provenance.get("workerId") == worker["id"]
|
|
372
|
+
and provenance.get("attempt") == worker["attempt"]
|
|
373
|
+
and provenance.get("sourceId") == surface.get("id")
|
|
374
|
+
and row.get("disposition") == surface.get("disposition")
|
|
375
|
+
),
|
|
376
|
+
None,
|
|
377
|
+
)
|
|
378
|
+
if published is None:
|
|
379
|
+
return None
|
|
380
|
+
retained = next(
|
|
381
|
+
(
|
|
382
|
+
row
|
|
383
|
+
for row in coverage["surfaces"]
|
|
384
|
+
if row.get("id") == published.get("id")
|
|
385
|
+
and row.get("provenance") == published.get("provenance")
|
|
386
|
+
),
|
|
387
|
+
None,
|
|
388
|
+
)
|
|
389
|
+
if retained is None:
|
|
390
|
+
retained = copy.deepcopy(published)
|
|
391
|
+
# Receipt repair can recover earlier evidence without replacing the
|
|
392
|
+
# accepted surface's identity, disposition, or host-owned provenance.
|
|
393
|
+
retained["receiptRefs"] = surface["receiptRefs"]
|
|
394
|
+
return self.retain_coverage_row(
|
|
395
|
+
coverage["surfaces"], retained, worker["id"], "surfaces", source, True
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
def linked_tasks(self, task, published_id, source_task, candidates):
|
|
399
|
+
expected = None
|
|
400
|
+
if source_task is not None:
|
|
401
|
+
expected = self.legacy_source_outputs.get(id(source_task))
|
|
402
|
+
if expected is None:
|
|
403
|
+
expected = _project_worker_coverage_row(
|
|
404
|
+
source_task, self.source_owner_by_id.get(id(source_task)), "deferred"
|
|
405
|
+
)
|
|
406
|
+
return [
|
|
407
|
+
retained
|
|
408
|
+
for owner, retained in candidates
|
|
409
|
+
if retained == task
|
|
410
|
+
or (
|
|
411
|
+
expected is not None
|
|
412
|
+
and (
|
|
413
|
+
retained is expected
|
|
414
|
+
or (
|
|
415
|
+
owner == self.source_owner_by_id.get(id(source_task))
|
|
416
|
+
and isinstance(retained, dict)
|
|
417
|
+
and retained.get("id") == expected.get("id")
|
|
418
|
+
and isinstance(retained.get("surfaceIds"), list)
|
|
419
|
+
and published_id in retained["surfaceIds"]
|
|
420
|
+
)
|
|
421
|
+
)
|
|
422
|
+
)
|
|
423
|
+
]
|
|
424
|
+
|
|
425
|
+
def finalize(
|
|
426
|
+
self,
|
|
427
|
+
coverage,
|
|
428
|
+
canonical_rows,
|
|
429
|
+
digest,
|
|
430
|
+
*,
|
|
431
|
+
preserve_canonical_parent,
|
|
432
|
+
all_sources,
|
|
433
|
+
source_order,
|
|
434
|
+
coverage_sources,
|
|
435
|
+
surface_ids_by_source,
|
|
436
|
+
retained_surface_rows,
|
|
437
|
+
resolved,
|
|
438
|
+
):
|
|
439
|
+
|
|
440
|
+
source_owners = {relative: owner for relative, _, owner in all_sources}
|
|
441
|
+
recovered_worker_ids = {
|
|
442
|
+
worker_id for _, _, worker_id in all_sources if worker_id is not None
|
|
443
|
+
}
|
|
444
|
+
surface_positions: dict[str, int] = {}
|
|
445
|
+
resolved_surface_targets: dict[tuple[str | None, str], list[str]] = {}
|
|
446
|
+
|
|
447
|
+
def remember_surface_id(item: dict[str, Any], local_id: str) -> None:
|
|
448
|
+
final_id = item["id"]
|
|
449
|
+
surface_positions.setdefault(final_id, len(surface_positions))
|
|
450
|
+
shadowed_alias = (
|
|
451
|
+
self.project_worker_coverage
|
|
452
|
+
and id(item) not in named_surface_outputs
|
|
453
|
+
and (named := worker_surfaces.get((self.worker_row_owners.get(id(item)), local_id)))
|
|
454
|
+
is not None
|
|
455
|
+
and id(named) in named_surface_outputs
|
|
456
|
+
)
|
|
457
|
+
for relative in coverage_sources.get(id(item), (item, []))[1]:
|
|
458
|
+
# A candidate-derived ID cannot claim a link explicitly naming another surface.
|
|
459
|
+
if not shadowed_alias:
|
|
460
|
+
targets = surface_ids_by_source.setdefault((relative, local_id), [])
|
|
461
|
+
if final_id not in targets:
|
|
462
|
+
targets.append(final_id)
|
|
463
|
+
candidate_id = item.get("candidateId")
|
|
464
|
+
if isinstance(candidate_id, str):
|
|
465
|
+
key = (source_owners[relative], candidate_id)
|
|
466
|
+
if key in resolved and resolved[key] == item.get("disposition"):
|
|
467
|
+
targets = resolved_surface_targets.setdefault(key, [])
|
|
468
|
+
if final_id not in targets:
|
|
469
|
+
targets.append(final_id)
|
|
470
|
+
|
|
471
|
+
retained_legacy_surface_links: dict[
|
|
472
|
+
int, tuple[dict[str, Any], dict[str, dict[str, Any]]]
|
|
473
|
+
] = {}
|
|
474
|
+
output_tasks = coverage.get("deferred", [])
|
|
475
|
+
task_candidates = [
|
|
476
|
+
(self.worker_row_owners.get(id(row)), row)
|
|
477
|
+
for row in (output_tasks if isinstance(output_tasks, list) else [])
|
|
478
|
+
]
|
|
479
|
+
for task, published_id, surface, _, task_source in self.legacy_surface_links:
|
|
480
|
+
retained_surface = self.linked_surface(
|
|
481
|
+
surface, self.legacy_source_outputs, self.worker_surface_outputs
|
|
482
|
+
)
|
|
483
|
+
if retained_surface is None or not isinstance(retained_surface.get("id"), str):
|
|
484
|
+
continue
|
|
485
|
+
for retained_task in self.linked_tasks(
|
|
486
|
+
task, published_id, task_source, task_candidates
|
|
487
|
+
):
|
|
488
|
+
_, links = retained_legacy_surface_links.setdefault(
|
|
489
|
+
id(retained_task), (retained_task, {})
|
|
490
|
+
)
|
|
491
|
+
links.setdefault(published_id, retained_surface)
|
|
492
|
+
# A newer worker checkpoint may omit a surface still needed by a retained
|
|
493
|
+
# task. Restore its parent copy only when no worker replacement survived.
|
|
494
|
+
for task, published_id, surface, parent_surface, task_source in self.legacy_surface_links:
|
|
495
|
+
for retained_task in self.linked_tasks(
|
|
496
|
+
task, published_id, task_source, task_candidates
|
|
497
|
+
):
|
|
498
|
+
_, links = retained_legacy_surface_links.setdefault(
|
|
499
|
+
id(retained_task), (retained_task, {})
|
|
500
|
+
)
|
|
501
|
+
if published_id not in links and isinstance(coverage.get("surfaces"), list):
|
|
502
|
+
retained_surface = self.retain_coverage_row(
|
|
503
|
+
coverage["surfaces"], parent_surface, None, "surfaces"
|
|
504
|
+
)
|
|
505
|
+
self.record_coverage_owner(
|
|
506
|
+
retained_surface,
|
|
507
|
+
surface,
|
|
508
|
+
self.source_owner_by_id.get(id(surface), surface.get("workerId")),
|
|
509
|
+
"surfaces",
|
|
510
|
+
)
|
|
511
|
+
_inherit_coverage_sources(retained_surface, [parent_surface], coverage_sources)
|
|
512
|
+
links[published_id] = retained_surface
|
|
513
|
+
if self.project_worker_coverage:
|
|
514
|
+
worker_surfaces: dict[tuple[str, str], dict[str, Any]] = {}
|
|
515
|
+
named_surface_outputs = {
|
|
516
|
+
identity for outputs in self.worker_surface_outputs.values() for identity in outputs
|
|
517
|
+
}
|
|
518
|
+
surfaces = coverage.get("surfaces", [])
|
|
519
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
520
|
+
if not isinstance(surface, dict):
|
|
521
|
+
continue
|
|
522
|
+
owner, identity = self.worker_row_owners.get(id(surface)), surface.get("id")
|
|
523
|
+
if isinstance(owner, str) and isinstance(identity, str):
|
|
524
|
+
key = (owner, identity)
|
|
525
|
+
existing = worker_surfaces.get(key)
|
|
526
|
+
# An anonymous candidate ID can collide with an explicit surface ID.
|
|
527
|
+
# Bind a named worker link to the explicitly named source.
|
|
528
|
+
if existing is None or (
|
|
529
|
+
id(surface) in named_surface_outputs
|
|
530
|
+
and id(existing) not in named_surface_outputs
|
|
531
|
+
):
|
|
532
|
+
worker_surfaces[key] = surface
|
|
533
|
+
for task in output_tasks if isinstance(output_tasks, list) else []:
|
|
534
|
+
if (
|
|
535
|
+
not isinstance(task, dict)
|
|
536
|
+
or (owner := self.worker_row_owners.get(id(task))) is None
|
|
537
|
+
or not isinstance(task.get("surfaceIds"), list)
|
|
538
|
+
):
|
|
539
|
+
continue
|
|
540
|
+
for identity in task["surfaceIds"]:
|
|
541
|
+
if (
|
|
542
|
+
isinstance(identity, str)
|
|
543
|
+
and (surface := worker_surfaces.get((owner, identity))) is not None
|
|
544
|
+
):
|
|
545
|
+
_, links = retained_legacy_surface_links.setdefault(id(task), (task, {}))
|
|
546
|
+
links.setdefault(identity, surface)
|
|
547
|
+
for field in ("surfaces", "explicitExclusions", "deferred"):
|
|
548
|
+
used: set[str] = set()
|
|
549
|
+
items = coverage.setdefault(field, [])
|
|
550
|
+
reserved = {
|
|
551
|
+
row["id"]
|
|
552
|
+
for row in (items if isinstance(items, list) else [])
|
|
553
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
554
|
+
}
|
|
555
|
+
normalized_items = []
|
|
556
|
+
for item in items if isinstance(items, list) else []:
|
|
557
|
+
normalized_items.append(item)
|
|
558
|
+
if not isinstance(item, dict):
|
|
559
|
+
continue
|
|
560
|
+
if id(item) in canonical_rows:
|
|
561
|
+
if isinstance(item.get("id"), str):
|
|
562
|
+
used.add(item["id"])
|
|
563
|
+
if field == "surfaces":
|
|
564
|
+
remember_surface_id(item, item["id"])
|
|
565
|
+
continue
|
|
566
|
+
item.setdefault("id", _saved_coverage_id(item))
|
|
567
|
+
if not isinstance(item["id"], str):
|
|
568
|
+
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
569
|
+
continue
|
|
570
|
+
local_id = item["id"]
|
|
571
|
+
scoped = (
|
|
572
|
+
preserve_canonical_parent
|
|
573
|
+
and isinstance(worker_id := item.get("workerId"), str)
|
|
574
|
+
and worker_id in recovered_worker_ids
|
|
575
|
+
)
|
|
576
|
+
if scoped:
|
|
577
|
+
# Resolve worker-local candidate/deferred history before scoping IDs.
|
|
578
|
+
item["id"] = f"{worker_id}/{item['id']}"
|
|
579
|
+
if isinstance(item.get("surfaceIds"), list):
|
|
580
|
+
links: list[Any] = []
|
|
581
|
+
for surface_id in item["surfaceIds"]:
|
|
582
|
+
targets: list[str] = []
|
|
583
|
+
declared = False
|
|
584
|
+
if field == "deferred" and isinstance(surface_id, str):
|
|
585
|
+
for relative in coverage_sources.get(id(item), (item, []))[1]:
|
|
586
|
+
source_key = (relative, surface_id)
|
|
587
|
+
history_keys = [source_key]
|
|
588
|
+
if source_key not in surface_ids_by_source:
|
|
589
|
+
# A deferred-only checkpoint can refer to an earlier
|
|
590
|
+
# observation from this logical worker, including retries.
|
|
591
|
+
# Reused IDs can denote several retained observations.
|
|
592
|
+
history_keys = [
|
|
593
|
+
(historical, surface_id)
|
|
594
|
+
for historical, _, owner in all_sources
|
|
595
|
+
if owner == source_owners[relative]
|
|
596
|
+
and source_order[historical] <= source_order[relative]
|
|
597
|
+
]
|
|
598
|
+
for key in history_keys:
|
|
599
|
+
declared |= key in surface_ids_by_source
|
|
600
|
+
for target in surface_ids_by_source.get(key, []):
|
|
601
|
+
if target not in targets:
|
|
602
|
+
targets.append(target)
|
|
603
|
+
if targets:
|
|
604
|
+
targets.sort(key=surface_positions.__getitem__)
|
|
605
|
+
for target in targets:
|
|
606
|
+
if target not in links:
|
|
607
|
+
links.append(target)
|
|
608
|
+
elif not declared:
|
|
609
|
+
links.append(
|
|
610
|
+
f"{worker_id}/{surface_id}"
|
|
611
|
+
if scoped and isinstance(surface_id, str)
|
|
612
|
+
else surface_id
|
|
613
|
+
)
|
|
614
|
+
item["surfaceIds"] = links
|
|
615
|
+
if scoped:
|
|
616
|
+
if preserve_canonical_parent and item in normalized_items[:-1]:
|
|
617
|
+
# A previous finish may have published files before its DB commit.
|
|
618
|
+
if field == "surfaces":
|
|
619
|
+
remember_surface_id(item, local_id)
|
|
620
|
+
normalized_items.pop()
|
|
621
|
+
continue
|
|
622
|
+
if item["id"] in used:
|
|
623
|
+
if self.project_worker_coverage:
|
|
624
|
+
base = item["id"]
|
|
625
|
+
suffix = 2
|
|
626
|
+
while item["id"] in used or item["id"] in reserved:
|
|
627
|
+
item["id"] = f"{base}-{suffix}"
|
|
628
|
+
suffix += 1
|
|
629
|
+
else:
|
|
630
|
+
item["id"] = f"{item['id']}-{digest(item)[:16]}"
|
|
631
|
+
if preserve_canonical_parent and item in normalized_items[:-1]:
|
|
632
|
+
if field == "surfaces":
|
|
633
|
+
remember_surface_id(item, local_id)
|
|
634
|
+
normalized_items.pop()
|
|
635
|
+
continue
|
|
636
|
+
used.add(item["id"])
|
|
637
|
+
if field == "surfaces":
|
|
638
|
+
item.setdefault("receiptRefs", [])
|
|
639
|
+
remember_surface_id(item, local_id)
|
|
640
|
+
if preserve_canonical_parent and isinstance(items, list):
|
|
641
|
+
coverage[field] = normalized_items
|
|
642
|
+
if field == "surfaces":
|
|
643
|
+
# Candidate resolution replaces pending surfaces independently of generic
|
|
644
|
+
# tasks. Preserve their original links, even when the outcome changes ID
|
|
645
|
+
# or a distinct retained surface shares the old source-local ID.
|
|
646
|
+
for relative, draft, owner in all_sources:
|
|
647
|
+
originals = draft["coverage"].get("surfaces", [])
|
|
648
|
+
for original in originals if isinstance(originals, list) else []:
|
|
649
|
+
if (
|
|
650
|
+
not isinstance(original, dict)
|
|
651
|
+
or id(original) in retained_surface_rows
|
|
652
|
+
or original.get("disposition") != "needs_follow_up"
|
|
653
|
+
or not isinstance(local_id := original.get("id"), str)
|
|
654
|
+
or not isinstance(candidate_id := original.get("candidateId"), str)
|
|
655
|
+
):
|
|
656
|
+
continue
|
|
657
|
+
targets = surface_ids_by_source.setdefault((relative, local_id), [])
|
|
658
|
+
for target in resolved_surface_targets.get((owner, candidate_id), []):
|
|
659
|
+
if target not in targets:
|
|
660
|
+
targets.append(target)
|
|
661
|
+
for task, links in retained_legacy_surface_links.values():
|
|
662
|
+
task["surfaceIds"] = [
|
|
663
|
+
links[identity]["id"]
|
|
664
|
+
if isinstance(identity, str) and identity in links
|
|
665
|
+
else identity
|
|
666
|
+
for identity in task["surfaceIds"]
|
|
667
|
+
]
|
|
668
|
+
|
|
669
|
+
def fallback_surfaces(
|
|
670
|
+
self,
|
|
671
|
+
coverage,
|
|
672
|
+
all_sources,
|
|
673
|
+
prospective_tasks,
|
|
674
|
+
*,
|
|
675
|
+
surface_is_retained,
|
|
676
|
+
ambiguous_surface_ids,
|
|
677
|
+
):
|
|
678
|
+
fallback_surfaces = []
|
|
679
|
+
if self.legacy_surface_links and isinstance(coverage.get("surfaces"), list):
|
|
680
|
+
# Plan the surviving associations before historical receipts can claim
|
|
681
|
+
# a spelling needed by a surface that must be restored for its task.
|
|
682
|
+
prospective_tasks = [
|
|
683
|
+
(owner, _project_worker_coverage_row(row, owner, "deferred"))
|
|
684
|
+
for owner, row in prospective_tasks
|
|
685
|
+
]
|
|
686
|
+
prospective_surfaces = {}
|
|
687
|
+
prospective_named = {}
|
|
688
|
+
for relative, draft, owner in all_sources:
|
|
689
|
+
rows = draft["coverage"].get("surfaces", [])
|
|
690
|
+
for row in rows if isinstance(rows, list) else []:
|
|
691
|
+
if owner is None or not isinstance(row, dict):
|
|
692
|
+
continue
|
|
693
|
+
emitted = surface_is_retained(relative, owner, row)
|
|
694
|
+
ambiguous = (
|
|
695
|
+
isinstance(row.get("id"), str)
|
|
696
|
+
and (owner, row["id"]) in ambiguous_surface_ids
|
|
697
|
+
and row.get("disposition") == "needs_follow_up"
|
|
698
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
699
|
+
)
|
|
700
|
+
if not emitted and not ambiguous:
|
|
701
|
+
continue
|
|
702
|
+
projected = _project_worker_coverage_row(row, owner, "surfaces")
|
|
703
|
+
if isinstance(row.get("id"), str):
|
|
704
|
+
named = prospective_named.setdefault((owner, row["id"]), {})
|
|
705
|
+
projected = next(
|
|
706
|
+
(existing for existing in named.values() if existing == projected),
|
|
707
|
+
projected,
|
|
708
|
+
)
|
|
709
|
+
named[id(projected)] = projected
|
|
710
|
+
if emitted:
|
|
711
|
+
prospective_surfaces[id(row)] = projected
|
|
712
|
+
bound = {
|
|
713
|
+
(id(retained), published_id)
|
|
714
|
+
for task, published_id, surface, _, task_source in self.legacy_surface_links
|
|
715
|
+
if self.linked_surface(surface, prospective_surfaces, prospective_named) is not None
|
|
716
|
+
for retained in self.linked_tasks(
|
|
717
|
+
task, published_id, task_source, prospective_tasks
|
|
718
|
+
)
|
|
719
|
+
}
|
|
720
|
+
for task, published_id, _, parent_surface, task_source in self.legacy_surface_links:
|
|
721
|
+
for retained in self.linked_tasks(
|
|
722
|
+
task, published_id, task_source, prospective_tasks
|
|
723
|
+
):
|
|
724
|
+
key = (id(retained), published_id)
|
|
725
|
+
if key not in bound:
|
|
726
|
+
fallback_surfaces.append(parent_surface)
|
|
727
|
+
bound.add(key)
|
|
728
|
+
return fallback_surfaces
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def prepare_saved_coverage_projection(
|
|
732
|
+
sources,
|
|
733
|
+
parent,
|
|
734
|
+
source_order,
|
|
735
|
+
*,
|
|
736
|
+
project_worker_coverage,
|
|
737
|
+
receipt_resolver,
|
|
738
|
+
preserve_canonical_parent=False,
|
|
739
|
+
):
|
|
740
|
+
if preserve_canonical_parent:
|
|
741
|
+
return SavedCoverageProjection(False, [], sources)
|
|
742
|
+
# Older parent publications use worker/id identities. Reconcile those copies
|
|
743
|
+
# against their bound sources before projecting accepted attempt provenance.
|
|
744
|
+
projected_metadata = {
|
|
745
|
+
id(row): ("workerId" in row, row.get("workerId"))
|
|
746
|
+
for relative, draft, _ in sources
|
|
747
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions")
|
|
748
|
+
for rows in (draft["coverage"].get(field, []),)
|
|
749
|
+
if isinstance(rows, list)
|
|
750
|
+
for row in rows
|
|
751
|
+
if isinstance(row, dict)
|
|
752
|
+
}
|
|
753
|
+
project_worker_coverage, legacy_surface_links = _prepare_worker_coverage_recovery(
|
|
754
|
+
sources,
|
|
755
|
+
parent,
|
|
756
|
+
source_order,
|
|
757
|
+
enabled=project_worker_coverage,
|
|
758
|
+
receipt_resolver=receipt_resolver,
|
|
759
|
+
)
|
|
760
|
+
for _, draft, _ in sources:
|
|
761
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
762
|
+
rows = draft["coverage"].get(field, [])
|
|
763
|
+
if not isinstance(rows, list):
|
|
764
|
+
continue
|
|
765
|
+
for row in rows:
|
|
766
|
+
if not isinstance(row, dict):
|
|
767
|
+
continue
|
|
768
|
+
present, value = projected_metadata.get(id(row), (False, None))
|
|
769
|
+
if present:
|
|
770
|
+
row["workerId"] = value
|
|
771
|
+
else:
|
|
772
|
+
row.pop("workerId", None)
|
|
773
|
+
return SavedCoverageProjection(project_worker_coverage, legacy_surface_links, sources)
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
|
|
777
|
+
rows = coverage.get("deferred", [])
|
|
778
|
+
return rows if isinstance(rows, list) else []
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
782
|
+
if draft.get("complete") is False:
|
|
783
|
+
return []
|
|
784
|
+
coverage = draft["coverage"]
|
|
785
|
+
rows = coverage.get("resolvedDeferred", [])
|
|
786
|
+
try:
|
|
787
|
+
# Invalid closure metadata cannot discard the evidence it names.
|
|
788
|
+
_validate_schema_node(rows, schema, "coverage.resolvedDeferred")
|
|
789
|
+
_validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
|
|
790
|
+
except ContractError:
|
|
791
|
+
return []
|
|
792
|
+
return rows
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _project_worker_closure(
|
|
796
|
+
relative: str,
|
|
797
|
+
owner: str,
|
|
798
|
+
closure: dict[str, Any],
|
|
799
|
+
worker_attempts: dict[str, tuple[str, int]],
|
|
800
|
+
source_order: dict[str, tuple[int, int]],
|
|
801
|
+
) -> dict[str, Any]:
|
|
802
|
+
directory = Path(relative).parent
|
|
803
|
+
if directory.name == "checkpoints":
|
|
804
|
+
directory = directory.parent
|
|
805
|
+
_, attempt = worker_attempts.get(directory.as_posix(), (owner, source_order[relative][0]))
|
|
806
|
+
return {
|
|
807
|
+
"id": f"{owner}-attempt-{attempt}-resolved-{closure['id']}",
|
|
808
|
+
"reason": closure["reason"],
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _projected_worker_closure_sources(
|
|
813
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
814
|
+
source_order: dict[str, tuple[int, int]],
|
|
815
|
+
worker_attempts: dict[str, tuple[str, int]],
|
|
816
|
+
schema: dict[str, Any],
|
|
817
|
+
) -> dict[tuple[str, str], tuple[str, dict[str, Any], str]]:
|
|
818
|
+
projected: dict[tuple[str, str], tuple[str, dict[str, Any], str]] = {}
|
|
819
|
+
for relative, draft, owner in sources:
|
|
820
|
+
if owner is None:
|
|
821
|
+
continue
|
|
822
|
+
for closure in _resolved_deferred_rows(draft, schema):
|
|
823
|
+
row = _project_worker_closure(relative, owner, closure, worker_attempts, source_order)
|
|
824
|
+
# Both fields identify the full strict closeout row, including its reason.
|
|
825
|
+
key = (row["id"], row["reason"])
|
|
826
|
+
previous = projected.get(key)
|
|
827
|
+
if previous is None or source_order[relative] > source_order[previous[2]]:
|
|
828
|
+
projected[key] = (owner, closure, relative)
|
|
829
|
+
return projected
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _parent_observation_content(parent: dict[str, Any]) -> dict[str, Any]:
|
|
833
|
+
# Frozen recovery ancestry does not change the observed scan result.
|
|
834
|
+
return {
|
|
835
|
+
key: value
|
|
836
|
+
for key, value in parent.items()
|
|
837
|
+
if key not in {"previousParentCheckpoints", "canonicalParent"}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _merge_tied_parent_observations(
|
|
842
|
+
current: dict[str, Any], previous: dict[str, Any]
|
|
843
|
+
) -> dict[str, Any]:
|
|
844
|
+
merged = copy.deepcopy(current)
|
|
845
|
+
for finding in previous["findings"]:
|
|
846
|
+
if finding not in merged["findings"]:
|
|
847
|
+
merged["findings"].append(copy.deepcopy(finding))
|
|
848
|
+
coverage = merged["coverage"]
|
|
849
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
850
|
+
rows = previous["coverage"].get(field, [])
|
|
851
|
+
output = coverage.setdefault(field, [])
|
|
852
|
+
if isinstance(rows, list) and isinstance(output, list):
|
|
853
|
+
for row in rows:
|
|
854
|
+
if row not in output:
|
|
855
|
+
output.append(copy.deepcopy(row))
|
|
856
|
+
pending_ids = {
|
|
857
|
+
identity
|
|
858
|
+
for row in _deferred_rows(coverage)
|
|
859
|
+
if isinstance(row, dict)
|
|
860
|
+
for identity in (row.get("id"), row.get("candidateId"))
|
|
861
|
+
if isinstance(identity, str)
|
|
862
|
+
}
|
|
863
|
+
closure_schema = _read_json(
|
|
864
|
+
Path(__file__).resolve().parents[2] / "schemas" / "coverage.schema.json"
|
|
865
|
+
)["properties"]["resolvedDeferred"]
|
|
866
|
+
closures = {}
|
|
867
|
+
for observed in (current, previous):
|
|
868
|
+
for row in _resolved_deferred_rows(observed, closure_schema):
|
|
869
|
+
if row["id"] not in pending_ids:
|
|
870
|
+
closures.setdefault(row["id"], copy.deepcopy(row))
|
|
871
|
+
coverage.pop("resolvedDeferred", None)
|
|
872
|
+
if closures:
|
|
873
|
+
coverage["resolvedDeferred"] = list(closures.values())
|
|
874
|
+
if current.get("complete") is False or previous.get("complete") is False:
|
|
875
|
+
merged["complete"] = False
|
|
876
|
+
coverage.pop("completeness", None)
|
|
877
|
+
return merged
|