@openai/codex-security 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -39
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +34 -34
- package/_bundled_plugin/examples/completed-scan/report.md +22 -31
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +27 -29
- package/_bundled_plugin/examples/{completed-scan → legacy-completed-scan}/coverage.json +1 -0
- package/_bundled_plugin/examples/legacy-completed-scan/findings.json +57 -0
- package/_bundled_plugin/examples/legacy-completed-scan/report.md +98 -0
- package/_bundled_plugin/examples/legacy-completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +50 -184
- package/_bundled_plugin/references/scan-artifacts.md +5 -5
- package/_bundled_plugin/references/scan-contract.md +12 -12
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +37 -11
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +5 -5
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +146 -86
- package/_bundled_plugin/scripts/finalize_scan_contract.py +249 -76
- package/_bundled_plugin/scripts/report_projection.py +49 -17
- package/_bundled_plugin/scripts/review_state.py +68 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3230 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +651 -8
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +730 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_constants.py +0 -1
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +55 -171
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +808 -2389
- package/_bundled_plugin/scripts/workbench_scan_history.py +77 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +28 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -24
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +29 -17
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +5 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +359 -53
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +696 -430
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +2 -23
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +5 -2
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +93 -45
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-publish.js +2 -2
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.js +5 -5
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +47 -19
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +2 -2
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -0
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +3 -0
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +21 -1
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +9 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/endor-findings.d.ts +4 -3
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +16 -11
- package/dist/endor-findings.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +13 -5
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +1 -0
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +14 -3
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +2 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +64 -28
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +12 -1
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +25 -6
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +2 -0
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +7 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +63 -31
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +24 -5
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +24 -9
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +3 -1
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-findings.d.ts +4 -3
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +16 -14
- package/dist/github-findings.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -1
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +19 -2
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +0 -6
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +17 -17
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +20 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/mock-scan.d.ts +1 -2
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +12 -17
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +10 -1
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +1 -6
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +30 -0
- package/dist/native-json-findings.d.ts.map +1 -0
- package/dist/native-json-findings.js +114 -0
- package/dist/native-json-findings.js.map +1 -0
- package/dist/opengrep-findings.d.ts +10 -0
- package/dist/opengrep-findings.d.ts.map +1 -0
- package/dist/opengrep-findings.js +198 -0
- package/dist/opengrep-findings.js.map +1 -0
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +2 -0
- package/dist/patch-tui.js.map +1 -1
- package/dist/publication.js +1 -2
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +1 -0
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +10 -0
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +1 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -24
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +5 -3
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +9 -6
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +7 -7
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +9 -3
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +1 -1
- package/dist/scan-owner-suggestions.d.ts.map +1 -1
- package/dist/scan-owner-suggestions.js +4 -4
- package/dist/scan-owner-suggestions.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +1 -1
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +7 -3
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +17 -22
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +4 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +22 -89
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +5 -4
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +18 -17
- package/dist/snyk-findings.js.map +1 -1
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +4 -1
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -1
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +3 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -1
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +27 -23
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +3 -4
- package/docs/cli.md +171 -41
- package/docs/dedupe-records.md +15 -1
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +19 -17
- package/docs/hosted-scans.md +37 -11
- package/package.json +9 -9
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -59,9 +59,11 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
59
59
|
|
|
60
60
|
inspect_setup = add_command("inspect-setup", "--target-path", "--scope")
|
|
61
61
|
inspect_setup.add_argument("--mode", choices=MODES, required=True)
|
|
62
|
+
inspect_setup.add_argument("--deep-scan-project-json")
|
|
62
63
|
|
|
63
64
|
save_workspace = add_command("save-workspace", "--workspace-id", "--target-path", "--scope")
|
|
64
65
|
save_workspace.add_argument("--mode", choices=MODES, required=True)
|
|
66
|
+
save_workspace.add_argument("--deep-scan-project-json")
|
|
65
67
|
save_workspace.add_argument("--target-summary")
|
|
66
68
|
add_user_context(save_workspace)
|
|
67
69
|
|
|
@@ -101,6 +103,7 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
101
103
|
|
|
102
104
|
get_scan = add_command("get-scan", "--scan-id")
|
|
103
105
|
get_scan.add_argument("--occurrence-id")
|
|
106
|
+
get_scan.add_argument("--check-target", action="store_true")
|
|
104
107
|
|
|
105
108
|
add_command("rename-scan", "--scan-id", "--name")
|
|
106
109
|
|
|
@@ -19,7 +19,6 @@ FINDING_CLOSE_REASONS = ("already_fixed", "wont_fix", "false_positive")
|
|
|
19
19
|
REMEDIATION_UPDATE_STATES = ("generated", "applied", "verifying", "verified", "failed")
|
|
20
20
|
EXPORT_FORMATS = ("csv", "json", "sarif")
|
|
21
21
|
ARTIFACTS = {
|
|
22
|
-
"coverage": "coverage.json",
|
|
23
22
|
"findings": "findings.json",
|
|
24
23
|
"manifest": "scan-manifest.json",
|
|
25
24
|
"markdownReport": "report.md",
|
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
"""Resolve accepted worker receipt references without changing their ownership."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
from pathlib import Path, PurePosixPath
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from finalize_scan_contract import (
|
|
13
|
+
ContractError,
|
|
14
|
+
_require_portable_relative_path,
|
|
15
|
+
_require_str,
|
|
16
|
+
_validate_schema_node,
|
|
17
|
+
open_scan_local_file_descriptor,
|
|
18
|
+
path_within_scope,
|
|
19
|
+
)
|
|
20
|
+
from workbench.saved_coverage import _project_worker_coverage_row, _saved_coverage_id
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
24
|
+
path = Path(relative)
|
|
25
|
+
if path.name == "checkpoint-head.json":
|
|
26
|
+
return path.parent
|
|
27
|
+
if path.parent.name == "checkpoint-heads":
|
|
28
|
+
return path.parent.parent
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _is_source_order_snapshot(relative: str) -> bool:
|
|
33
|
+
path = Path(relative)
|
|
34
|
+
return path.parent == Path("source-order") and bool(
|
|
35
|
+
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def capture_receipt_checkpoints(
|
|
40
|
+
relative: str,
|
|
41
|
+
kind: str | None,
|
|
42
|
+
captured: dict[str, tuple[str, int]],
|
|
43
|
+
checkpoint_paths: Callable[[str], list[str]],
|
|
44
|
+
read_checkpoint: Callable[[str], tuple[dict[str, Any], str, int]],
|
|
45
|
+
) -> dict[str, tuple[str, int]]:
|
|
46
|
+
"""Bind a captured worker result to checkpoints visible after reading it."""
|
|
47
|
+
if kind == "discovery" and Path(relative).name == "result.json":
|
|
48
|
+
directory = (Path(relative).parent / "checkpoints").as_posix()
|
|
49
|
+
for path in checkpoint_paths(directory):
|
|
50
|
+
try:
|
|
51
|
+
_, digest, observed = read_checkpoint(path)
|
|
52
|
+
except (ContractError, OSError, ValueError):
|
|
53
|
+
continue
|
|
54
|
+
captured[path] = (digest, observed)
|
|
55
|
+
return captured
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def receipt_checkpoints(
|
|
59
|
+
relative: str,
|
|
60
|
+
worker_id: str | None,
|
|
61
|
+
known: dict[str, str | None],
|
|
62
|
+
checkpoint_paths: Callable[[str], list[str]],
|
|
63
|
+
) -> dict[str, str]:
|
|
64
|
+
"""Capture host checkpoints written after enumeration but before result.json."""
|
|
65
|
+
if worker_id is None or Path(relative).name != "result.json":
|
|
66
|
+
return {}
|
|
67
|
+
directory = (Path(relative).parent / "checkpoints").as_posix()
|
|
68
|
+
added = {path: worker_id for path in checkpoint_paths(directory) if path not in known}
|
|
69
|
+
known.update(added)
|
|
70
|
+
return added
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def retain_recorded_receipt_ownership(
|
|
74
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
75
|
+
digest: Callable[[Any], str],
|
|
76
|
+
include_paths: Sequence[str],
|
|
77
|
+
finding_scope: Sequence[tuple[str, bool]] | None = None,
|
|
78
|
+
) -> None:
|
|
79
|
+
# Checkpoints are written by the host tool; direct result files can be authored
|
|
80
|
+
# without it. Compare the entire payload within the same worker attempt.
|
|
81
|
+
def payload_digest(draft: dict[str, Any]) -> str:
|
|
82
|
+
return digest({key: value for key, value in draft.items() if key != "handoffClaimToken"})
|
|
83
|
+
|
|
84
|
+
scope = (
|
|
85
|
+
finding_scope if finding_scope is not None else [(path, False) for path in include_paths]
|
|
86
|
+
)
|
|
87
|
+
recorded: dict[Path, set[str]] = {}
|
|
88
|
+
for relative, draft, worker_id in sources:
|
|
89
|
+
directory = Path(relative).parent
|
|
90
|
+
if worker_id is not None and directory.name == "checkpoints":
|
|
91
|
+
payloads = recorded.setdefault(directory.parent, set())
|
|
92
|
+
payloads.add(payload_digest(draft))
|
|
93
|
+
if "." not in include_paths:
|
|
94
|
+
# Legacy writers saved the checkpoint before filtering inherited findings.
|
|
95
|
+
scoped = [
|
|
96
|
+
finding
|
|
97
|
+
for finding in draft["findings"]
|
|
98
|
+
if isinstance(finding, dict)
|
|
99
|
+
and isinstance(finding.get("locations"), list)
|
|
100
|
+
and any(
|
|
101
|
+
isinstance(location, dict)
|
|
102
|
+
and isinstance(location.get("path"), str)
|
|
103
|
+
and any(
|
|
104
|
+
PurePosixPath(location["path"]) == PurePosixPath(path)
|
|
105
|
+
if is_file
|
|
106
|
+
else path_within_scope(location["path"], path)
|
|
107
|
+
for path, is_file in scope
|
|
108
|
+
)
|
|
109
|
+
for location in finding["locations"]
|
|
110
|
+
)
|
|
111
|
+
]
|
|
112
|
+
payloads.add(payload_digest({**draft, "findings": scoped}))
|
|
113
|
+
for relative, draft, worker_id in sources:
|
|
114
|
+
directory = Path(relative).parent
|
|
115
|
+
if worker_id is None or directory.name == "checkpoints":
|
|
116
|
+
continue
|
|
117
|
+
if payload_digest(draft) in recorded.get(directory, set()):
|
|
118
|
+
continue
|
|
119
|
+
surfaces = draft["reviewState"].get("surfaces", [])
|
|
120
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
121
|
+
if isinstance(surface, dict) and isinstance(surface.get("provenance"), dict):
|
|
122
|
+
surface["provenance"].pop("scanReceiptRefs", None)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def resolve_coverage_receipts(
|
|
126
|
+
scan_dir: Path, item: dict[str, Any], worker: Any, relative: str
|
|
127
|
+
) -> Any:
|
|
128
|
+
refs = item.get("receiptRefs", [])
|
|
129
|
+
if not isinstance(refs, list):
|
|
130
|
+
return refs
|
|
131
|
+
directory = Path(relative).parent
|
|
132
|
+
if directory.name == "checkpoints":
|
|
133
|
+
directory = directory.parent
|
|
134
|
+
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
135
|
+
worker_prefix = (output.parent if output.name == "output" else output).as_posix() + "/"
|
|
136
|
+
active_prefix = output.as_posix() + "/"
|
|
137
|
+
if directory != output:
|
|
138
|
+
refs = [
|
|
139
|
+
f"{directory.as_posix()}/{Path(ref).as_posix()[len(active_prefix) :]}"
|
|
140
|
+
if isinstance(ref, str) and Path(ref).as_posix().startswith(active_prefix)
|
|
141
|
+
else ref
|
|
142
|
+
for ref in refs
|
|
143
|
+
]
|
|
144
|
+
|
|
145
|
+
def scan_receipt(ref: str) -> bool:
|
|
146
|
+
try:
|
|
147
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, ref, "coverage receipt")
|
|
148
|
+
except (ContractError, OSError):
|
|
149
|
+
return False
|
|
150
|
+
os.close(descriptor)
|
|
151
|
+
return True
|
|
152
|
+
|
|
153
|
+
def qualified_receipt(ref: Any) -> Any:
|
|
154
|
+
if not isinstance(ref, str):
|
|
155
|
+
return ref
|
|
156
|
+
try:
|
|
157
|
+
normalized = _require_portable_relative_path(ref, "Worker coverage receipt")
|
|
158
|
+
except ContractError:
|
|
159
|
+
return ref # Leave malformed references to per-record recovery.
|
|
160
|
+
if not normalized.startswith("artifacts/"):
|
|
161
|
+
# A worker prefix must not make result.json a valid coverage receipt.
|
|
162
|
+
return ref
|
|
163
|
+
ref = normalized
|
|
164
|
+
provenance = item.get("provenance")
|
|
165
|
+
inherited_scan_refs = (
|
|
166
|
+
provenance.get("scanReceiptRefs", []) if isinstance(provenance, dict) else []
|
|
167
|
+
)
|
|
168
|
+
if ref.startswith(worker_prefix) or (
|
|
169
|
+
isinstance(inherited_scan_refs, list) and ref in inherited_scan_refs
|
|
170
|
+
):
|
|
171
|
+
return ref
|
|
172
|
+
local_ref = Path(f"{directory}/{ref}").as_posix()
|
|
173
|
+
return local_ref if scan_receipt(local_ref) or not scan_receipt(ref) else ref
|
|
174
|
+
|
|
175
|
+
return [qualified_receipt(ref) for ref in refs]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _deferred_candidate_id(
|
|
179
|
+
row: dict[str, Any],
|
|
180
|
+
owner: str | None,
|
|
181
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
182
|
+
) -> str | None:
|
|
183
|
+
identity = row.get("candidateId") or row.get("id")
|
|
184
|
+
if not isinstance(identity, str):
|
|
185
|
+
return None
|
|
186
|
+
if (owner, identity) in ambiguous_deferred and not any(
|
|
187
|
+
key in row for key in ("candidateId", "candidate", "finding")
|
|
188
|
+
):
|
|
189
|
+
return None
|
|
190
|
+
return identity
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _generic_surface_updates(
|
|
194
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
195
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
196
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
197
|
+
resolved_candidates: dict[tuple[str | None, str], str],
|
|
198
|
+
reopened_generic: set[tuple[str | None, str]],
|
|
199
|
+
surface_schema: dict[str, Any],
|
|
200
|
+
deferred_rows: dict[str, list[Any]],
|
|
201
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
202
|
+
record_source: Any,
|
|
203
|
+
record_order: Any,
|
|
204
|
+
surface_receipts: Any,
|
|
205
|
+
*,
|
|
206
|
+
canonical_rows: set[int] | None = None,
|
|
207
|
+
coverage_sources: dict[int, tuple[dict[str, Any], list[str]]] | None = None,
|
|
208
|
+
surface_is_retained: Any = None,
|
|
209
|
+
eligible_receipt_sources: set[str] | None = None,
|
|
210
|
+
receipt_root: Path | None = None,
|
|
211
|
+
receipt_warnings: list[str] | None = None,
|
|
212
|
+
fallback_surfaces: list[dict[str, Any]] | None = None,
|
|
213
|
+
) -> tuple[set[int], list[tuple[str, str | None, dict[str, Any], dict[str, Any]]]]:
|
|
214
|
+
if not closed_deferred and not reopened_generic and receipt_root is None:
|
|
215
|
+
return set(), []
|
|
216
|
+
|
|
217
|
+
def receipt_error(ref: Any) -> ContractError | None:
|
|
218
|
+
if receipt_root is None or not isinstance(ref, str):
|
|
219
|
+
return ContractError("Saved coverage receipt is unavailable.")
|
|
220
|
+
try:
|
|
221
|
+
descriptor = open_scan_local_file_descriptor(
|
|
222
|
+
receipt_root, ref, "Saved coverage receipt"
|
|
223
|
+
)
|
|
224
|
+
os.close(descriptor)
|
|
225
|
+
except ContractError as exc:
|
|
226
|
+
return exc
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
def record_key(field: str, owner: str | None, row: dict[str, Any]) -> tuple[str | None, Any]:
|
|
230
|
+
source_owner, original = record_source(field, owner, row)
|
|
231
|
+
identity = original.get("id")
|
|
232
|
+
return source_owner, identity if isinstance(identity, str) else None
|
|
233
|
+
|
|
234
|
+
def linked(row: dict[str, Any], identity: str | None, owner: str | None) -> bool:
|
|
235
|
+
_, row = record_source("deferred", owner, row)
|
|
236
|
+
surface_ids = row.get("surfaceIds", [])
|
|
237
|
+
return identity is not None and (
|
|
238
|
+
row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
|
|
242
|
+
for relative, draft, owner in sources:
|
|
243
|
+
rows = draft["reviewState"].get("surfaces", [])
|
|
244
|
+
for row in rows if isinstance(rows, list) else []:
|
|
245
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
246
|
+
saved_surfaces.setdefault(record_key("surfaces", owner, row), []).append(
|
|
247
|
+
(relative, row)
|
|
248
|
+
)
|
|
249
|
+
replaced: set[int] = set()
|
|
250
|
+
updates: list[tuple[str, str | None, dict[str, Any], dict[str, Any]]] = []
|
|
251
|
+
# Current selections across all surfaces take precedence over historical refs.
|
|
252
|
+
receipt_paths: dict[str, str] = {}
|
|
253
|
+
retained_surfaces: dict[tuple[str | None, str], dict[str, Any]] = {}
|
|
254
|
+
if receipt_root is not None:
|
|
255
|
+
receipt_surfaces: list[tuple[str | None, dict[str, Any]]] = []
|
|
256
|
+
for key, matches in saved_surfaces.items():
|
|
257
|
+
retained = [
|
|
258
|
+
match for match in matches if surface_is_retained(match[0], key[0], match[1])
|
|
259
|
+
]
|
|
260
|
+
if not retained:
|
|
261
|
+
continue
|
|
262
|
+
latest_source, latest_surface = max(
|
|
263
|
+
retained, key=lambda match: record_order(match[0], key[0], match[1])
|
|
264
|
+
)
|
|
265
|
+
retained_surfaces[key] = latest_surface
|
|
266
|
+
receipt_surfaces.extend(
|
|
267
|
+
(key[0], row)
|
|
268
|
+
for relative, row in retained
|
|
269
|
+
if record_order(relative, key[0], row)
|
|
270
|
+
== record_order(latest_source, key[0], latest_surface)
|
|
271
|
+
)
|
|
272
|
+
for relative, draft, owner in sources:
|
|
273
|
+
rows = draft["reviewState"].get("surfaces", [])
|
|
274
|
+
for row in rows if isinstance(rows, list) else []:
|
|
275
|
+
if (
|
|
276
|
+
isinstance(row, dict)
|
|
277
|
+
and "id" not in row
|
|
278
|
+
and surface_is_retained(relative, owner, row)
|
|
279
|
+
):
|
|
280
|
+
receipt_surfaces.append((owner, row))
|
|
281
|
+
receipt_surfaces.extend((None, row) for row in fallback_surfaces or [])
|
|
282
|
+
for owner, latest_surface in receipt_surfaces:
|
|
283
|
+
# Recovery repairs disposition and receipts before validating row
|
|
284
|
+
# metadata. A row discarded there cannot reserve a receipt spelling.
|
|
285
|
+
metadata = _project_worker_coverage_row(latest_surface, owner, "surfaces")
|
|
286
|
+
metadata.setdefault("id", _saved_coverage_id(metadata))
|
|
287
|
+
metadata.update(disposition="needs_follow_up", receiptRefs=[])
|
|
288
|
+
try:
|
|
289
|
+
_require_str(metadata, "id", "coverage.surfaces")
|
|
290
|
+
_validate_schema_node(metadata, surface_schema, "coverage.surfaces")
|
|
291
|
+
except ContractError:
|
|
292
|
+
continue
|
|
293
|
+
refs = latest_surface.get("receiptRefs", [])
|
|
294
|
+
for ref in refs if isinstance(refs, list) else []:
|
|
295
|
+
if receipt_error(ref) is None:
|
|
296
|
+
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
297
|
+
receipt_paths.setdefault(normalized.lower(), normalized)
|
|
298
|
+
for relative, draft, owner in sources:
|
|
299
|
+
current = draft["reviewState"].get("surfaces", [])
|
|
300
|
+
for surface in current if isinstance(current, list) else []:
|
|
301
|
+
if not isinstance(surface, dict):
|
|
302
|
+
continue
|
|
303
|
+
surface_owner, identity = record_key("surfaces", owner, surface)
|
|
304
|
+
closed_ids = {
|
|
305
|
+
identity
|
|
306
|
+
for (saved_owner, identity), (_, _, source) in closed_deferred.items()
|
|
307
|
+
if saved_owner == surface_owner and source == relative
|
|
308
|
+
}
|
|
309
|
+
reopened_ids = {
|
|
310
|
+
identity
|
|
311
|
+
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
312
|
+
if saved_owner == surface_owner
|
|
313
|
+
and source == relative
|
|
314
|
+
and (owner, identity) in reopened_generic
|
|
315
|
+
}
|
|
316
|
+
reopening = surface.get("disposition") == "needs_follow_up"
|
|
317
|
+
work_ids = reopened_ids if reopening else closed_ids
|
|
318
|
+
repair_only = not work_ids
|
|
319
|
+
if repair_only and receipt_root is None:
|
|
320
|
+
continue
|
|
321
|
+
if not isinstance(identity, str):
|
|
322
|
+
continue
|
|
323
|
+
matches = saved_surfaces[(surface_owner, identity)]
|
|
324
|
+
# Receipt repair alone cannot close or reopen a saved surface.
|
|
325
|
+
if repair_only and any(
|
|
326
|
+
row.get("disposition") != surface.get("disposition") for _, row in matches
|
|
327
|
+
):
|
|
328
|
+
continue
|
|
329
|
+
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
330
|
+
# A closure cannot identify which of those observations it replaces.
|
|
331
|
+
by_source = dict(matches)
|
|
332
|
+
if any(row != by_source[saved_path] for saved_path, row in matches):
|
|
333
|
+
continue
|
|
334
|
+
if any(
|
|
335
|
+
"candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
|
|
336
|
+
):
|
|
337
|
+
continue
|
|
338
|
+
if any(
|
|
339
|
+
row is not surface
|
|
340
|
+
and record_order(saved_path, surface_owner, row)
|
|
341
|
+
>= record_order(relative, surface_owner, surface)
|
|
342
|
+
and row.get("disposition") != surface.get("disposition")
|
|
343
|
+
for saved_path, row in matches
|
|
344
|
+
):
|
|
345
|
+
continue
|
|
346
|
+
|
|
347
|
+
if not reopening and any(
|
|
348
|
+
saved_owner == surface_owner
|
|
349
|
+
and (
|
|
350
|
+
not isinstance(row.get("id") or row.get("candidateId"), str)
|
|
351
|
+
or (
|
|
352
|
+
isinstance(row.get("id"), str)
|
|
353
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
354
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
355
|
+
)
|
|
356
|
+
)
|
|
357
|
+
and linked(row, identity, saved_owner)
|
|
358
|
+
for saved_path, _, saved_owner in sources
|
|
359
|
+
for row in deferred_rows[saved_path]
|
|
360
|
+
if isinstance(row, dict)
|
|
361
|
+
):
|
|
362
|
+
continue
|
|
363
|
+
if not reopening and any(
|
|
364
|
+
saved_owner == surface_owner
|
|
365
|
+
and (owner, deferred_id) not in closed_deferred
|
|
366
|
+
and (
|
|
367
|
+
(candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
|
|
368
|
+
or (owner, candidate_id) not in resolved_candidates
|
|
369
|
+
)
|
|
370
|
+
and linked(row, identity, saved_owner)
|
|
371
|
+
for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
|
|
372
|
+
):
|
|
373
|
+
continue
|
|
374
|
+
# An accepted checkpoint can update a saved surface by ID without
|
|
375
|
+
# optional surfaceIds links on its generic task.
|
|
376
|
+
receipt_matches = [
|
|
377
|
+
(saved_path, row)
|
|
378
|
+
for saved_path, row in matches
|
|
379
|
+
if receipt_root is None
|
|
380
|
+
or saved_path in eligible_receipt_sources
|
|
381
|
+
or surface_is_retained(saved_path, owner, row)
|
|
382
|
+
]
|
|
383
|
+
if not receipt_matches:
|
|
384
|
+
continue
|
|
385
|
+
latest_relative, latest_surface = max(
|
|
386
|
+
receipt_matches, key=lambda match: record_order(match[0], surface_owner, match[1])
|
|
387
|
+
)
|
|
388
|
+
target = (
|
|
389
|
+
retained_surfaces.get((surface_owner, identity)) if repair_only else latest_surface
|
|
390
|
+
)
|
|
391
|
+
if target is None:
|
|
392
|
+
continue
|
|
393
|
+
update = copy.deepcopy(target)
|
|
394
|
+
# Historical evidence can repair a retained row without promoting
|
|
395
|
+
# an unselected observation's label, notes, or other metadata.
|
|
396
|
+
refs = copy.deepcopy(surface_receipts(latest_relative, latest_surface))
|
|
397
|
+
update["receiptRefs"] = refs
|
|
398
|
+
replaced_receipt = False
|
|
399
|
+
replacement_paths = dict(receipt_paths)
|
|
400
|
+
if isinstance(refs, list):
|
|
401
|
+
repaired = (
|
|
402
|
+
not reopening and bool(refs) and all(receipt_error(ref) is None for ref in refs)
|
|
403
|
+
)
|
|
404
|
+
if repaired:
|
|
405
|
+
for ref in refs:
|
|
406
|
+
normalized = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
407
|
+
key = normalized.lower()
|
|
408
|
+
if key in replacement_paths and replacement_paths[key] != normalized:
|
|
409
|
+
repaired = False
|
|
410
|
+
break
|
|
411
|
+
replacement_paths[key] = normalized
|
|
412
|
+
for saved_relative, row in receipt_matches:
|
|
413
|
+
previous_refs = surface_receipts(saved_relative, row)
|
|
414
|
+
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
415
|
+
if ref not in refs:
|
|
416
|
+
failure = receipt_error(ref) if repaired else None
|
|
417
|
+
replaceable = failure is not None and isinstance(
|
|
418
|
+
failure.__cause__, FileNotFoundError
|
|
419
|
+
)
|
|
420
|
+
if repaired and isinstance(ref, str):
|
|
421
|
+
try:
|
|
422
|
+
normalized = _require_portable_relative_path(
|
|
423
|
+
ref, "Saved coverage receipt"
|
|
424
|
+
)
|
|
425
|
+
except ContractError as error:
|
|
426
|
+
failure = error
|
|
427
|
+
replaceable = True
|
|
428
|
+
else:
|
|
429
|
+
key = normalized.lower()
|
|
430
|
+
previous = replacement_paths.get(key)
|
|
431
|
+
if previous is not None and previous != normalized:
|
|
432
|
+
failure = ContractError(
|
|
433
|
+
f"Saved coverage receipt: duplicate artifact path: {previous} and {normalized}"
|
|
434
|
+
)
|
|
435
|
+
replaceable = True
|
|
436
|
+
elif not replaceable:
|
|
437
|
+
replacement_paths[key] = normalized
|
|
438
|
+
if replaceable:
|
|
439
|
+
replaced_receipt = True
|
|
440
|
+
if receipt_warnings is not None:
|
|
441
|
+
receipt_warnings.append(
|
|
442
|
+
f"Replaced invalid saved coverage receipt {ref}: {failure}"
|
|
443
|
+
)
|
|
444
|
+
continue
|
|
445
|
+
refs.append(ref)
|
|
446
|
+
if repair_only and not replaced_receipt:
|
|
447
|
+
continue
|
|
448
|
+
try:
|
|
449
|
+
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
450
|
+
except ContractError:
|
|
451
|
+
continue
|
|
452
|
+
replaced_sources = {
|
|
453
|
+
id(row)
|
|
454
|
+
for _, row in matches
|
|
455
|
+
if reopening
|
|
456
|
+
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
457
|
+
}
|
|
458
|
+
replaced.update(replaced_sources)
|
|
459
|
+
entry = (latest_relative, surface_owner, latest_surface, update)
|
|
460
|
+
if entry not in updates:
|
|
461
|
+
receipt_paths.update(replacement_paths)
|
|
462
|
+
updates.append(entry)
|
|
463
|
+
retained = next(
|
|
464
|
+
entry[3]
|
|
465
|
+
for entry in updates
|
|
466
|
+
if entry[:3] == (latest_relative, surface_owner, latest_surface)
|
|
467
|
+
and entry[3] == update
|
|
468
|
+
)
|
|
469
|
+
if coverage_sources is not None:
|
|
470
|
+
from workbench.retained_coverage import _inherit_coverage_sources
|
|
471
|
+
|
|
472
|
+
_inherit_coverage_sources(
|
|
473
|
+
retained,
|
|
474
|
+
[latest_surface, *(row for _, row in matches if id(row) in replaced_sources)],
|
|
475
|
+
coverage_sources,
|
|
476
|
+
)
|
|
477
|
+
if canonical_rows is not None and id(latest_surface) in canonical_rows:
|
|
478
|
+
canonical_rows.add(id(retained))
|
|
479
|
+
return replaced, updates
|