@openai/codex-security 0.1.31 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +442 -2434
- package/_bundled_plugin/.app.json +5 -1
- package/_bundled_plugin/.codex-plugin/plugin.json +9 -5
- package/_bundled_plugin/.mcp.json +3 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
- package/_bundled_plugin/examples/completed-scan/report.md +28 -26
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
- package/_bundled_plugin/mcp/helpers.mjs +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +7 -214
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +563 -0
- package/_bundled_plugin/mcp/server.mjs +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
- package/_bundled_plugin/references/artifact-storage.md +5 -5
- package/_bundled_plugin/references/config-preflight.md +1 -5
- package/_bundled_plugin/references/core-scan.md +34 -10
- package/_bundled_plugin/references/final-report.md +13 -13
- package/_bundled_plugin/references/finding-detail-fields.md +90 -80
- package/_bundled_plugin/references/sarif-adapter.md +1 -1
- package/_bundled_plugin/references/scan-artifacts.md +3 -4
- package/_bundled_plugin/references/scan-contract.md +11 -6
- package/_bundled_plugin/references/scan-prologue.md +12 -0
- package/_bundled_plugin/references/security-guidance.md +2 -2
- package/_bundled_plugin/references/standalone-validation.md +11 -0
- package/_bundled_plugin/references/threat-model.md +2 -2
- package/_bundled_plugin/schemas/coverage.schema.json +23 -47
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
- package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
- package/_bundled_plugin/schemas/findings.schema.json +71 -192
- package/_bundled_plugin/schemas/scan-manifest.schema.json +83 -95
- package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
- package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
- package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
- package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +245 -356
- package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
- package/_bundled_plugin/scripts/codex_session.mjs +53 -0
- package/_bundled_plugin/scripts/config_preflight.py +29 -69
- package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
- package/_bundled_plugin/scripts/deep_scan_workbench.py +208 -363
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
- package/_bundled_plugin/scripts/finalize_scan_contract.py +733 -375
- package/_bundled_plugin/scripts/finding_preview.py +81 -91
- package/_bundled_plugin/scripts/generate_in_scope_files.py +48 -102
- package/_bundled_plugin/scripts/generate_rank_input.py +117 -888
- package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -6
- package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
- package/_bundled_plugin/scripts/rank_preview.py +43 -902
- package/_bundled_plugin/scripts/report_projection.py +143 -219
- package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
- package/_bundled_plugin/scripts/threat_model_projection.py +161 -0
- package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
- package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
- package/_bundled_plugin/scripts/windows_scan_local_files.py +59 -9
- package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
- package/_bundled_plugin/scripts/workbench/storage.py +21 -6
- package/_bundled_plugin/scripts/workbench_cli.py +99 -190
- package/_bundled_plugin/scripts/workbench_constants.py +6 -14
- package/_bundled_plugin/scripts/workbench_db.py +524 -672
- package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
- package/_bundled_plugin/scripts/workbench_finding_index.py +20 -7
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +28 -18
- package/_bundled_plugin/scripts/workbench_progress.py +26 -92
- package/_bundled_plugin/scripts/workbench_publication.py +77 -116
- package/_bundled_plugin/scripts/workbench_remediation.py +14 -20
- package/_bundled_plugin/scripts/workbench_saved_results.py +1402 -263
- package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
- package/_bundled_plugin/scripts/workbench_scan_start.py +109 -52
- package/_bundled_plugin/scripts/workbench_scan_usage.py +30 -49
- package/_bundled_plugin/scripts/workbench_schema.py +71 -1370
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +16 -7
- package/_bundled_plugin/scripts/workbench_target.py +526 -150
- package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
- package/_bundled_plugin/scripts/workbench_validation.py +19 -44
- package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
- package/_bundled_plugin/shared/workbench-migrations.json +397 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
- package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +6 -6
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +13 -1
- package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +46 -178
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
- package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
- package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
- package/bin/codex-security.mjs +9 -2
- package/dist/api.d.ts +63 -15
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +970 -553
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +70 -0
- package/dist/artifact-export.d.ts.map +1 -0
- package/dist/artifact-export.js +292 -0
- package/dist/artifact-export.js.map +1 -0
- package/dist/auth.d.ts +2 -6
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +27 -12
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +9 -9
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +2 -5
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity.d.ts +28 -17
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +16 -35
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts +3 -0
- package/dist/cli-help.d.ts.map +1 -0
- package/dist/cli-help.js +285 -0
- package/dist/cli-help.js.map +1 -0
- package/dist/cli-signals.d.ts +8 -0
- package/dist/cli-signals.d.ts.map +1 -0
- package/dist/cli-signals.js +28 -0
- package/dist/cli-signals.js.map +1 -0
- package/dist/cli.d.ts +22 -24
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1660 -1556
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +11 -17
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.d.ts +19 -0
- package/dist/codeowners.d.ts.map +1 -0
- package/dist/codeowners.js +138 -0
- package/dist/codeowners.js.map +1 -0
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +1 -2
- package/dist/codex-prompt.js.map +1 -1
- package/dist/codex-sdk-environment.d.ts +3 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -0
- package/dist/codex-sdk-environment.js +68 -0
- package/dist/codex-sdk-environment.js.map +1 -0
- package/dist/component-plan.d.ts +2 -5
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +38 -34
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts +7 -5
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +54 -44
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +93 -104
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +15 -2
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +124 -348
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts +13 -6
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +75 -34
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +10 -2
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +116 -134
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +7 -6
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +1 -2
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +23 -16
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts +4 -2
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +33 -20
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +9 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +129 -20
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +5 -2
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -4
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +26 -4
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +59 -47
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +59 -113
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/diagnostics.d.ts +17 -0
- package/dist/deduplication/diagnostics.d.ts.map +1 -0
- package/dist/deduplication/diagnostics.js +5 -0
- package/dist/deduplication/diagnostics.js.map +1 -0
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +7 -1
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts +31 -0
- package/dist/deduplication/local.d.ts.map +1 -0
- package/dist/deduplication/local.js +166 -0
- package/dist/deduplication/local.js.map +1 -0
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +12 -28
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.d.ts +188 -12
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +34 -24
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/review.d.ts +7 -0
- package/dist/deduplication/review.d.ts.map +1 -1
- package/dist/deduplication/review.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -3
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +159 -44
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +2 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +7 -23
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +2 -1
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +32 -47
- package/dist/deep-progress.js.map +1 -1
- package/dist/deep-scan.d.ts +20 -0
- package/dist/deep-scan.d.ts.map +1 -0
- package/dist/deep-scan.js +113 -0
- package/dist/deep-scan.js.map +1 -0
- package/dist/errors.d.ts +5 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -7
- package/dist/errors.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +53 -21
- package/dist/feedback.js.map +1 -1
- package/dist/finding-catalogue.d.ts.map +1 -1
- package/dist/finding-catalogue.js +26 -30
- package/dist/finding-catalogue.js.map +1 -1
- package/dist/finding-normalization.d.ts +5 -0
- package/dist/finding-normalization.d.ts.map +1 -0
- package/dist/finding-normalization.js +120 -0
- package/dist/finding-normalization.js.map +1 -0
- package/dist/finding-retrieval.d.ts +9 -0
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +13 -8
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +18 -13
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-client.d.ts +2 -1
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +36 -9
- package/dist/findings-client.js.map +1 -1
- package/dist/findings-errors.d.ts +16 -0
- package/dist/findings-errors.d.ts.map +1 -0
- package/dist/findings-errors.js +24 -0
- package/dist/findings-errors.js.map +1 -0
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +16 -44
- package/dist/findings-import.js.map +1 -1
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +9 -4
- package/dist/github.js.map +1 -1
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +49 -39
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +11 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +7 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +115 -47
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js +4 -10
- package/dist/linear.js.map +1 -1
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +0 -1
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +24 -2
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts +9 -3
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +225 -153
- package/dist/multiscan.js.map +1 -1
- package/dist/owner-evidence.d.ts +10 -2
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +69 -14
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +5 -6
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +10 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +5 -3
- package/dist/project-config.js.map +1 -1
- package/dist/prompt-files.d.ts +0 -1
- package/dist/prompt-files.d.ts.map +1 -1
- package/dist/prompt-files.js +2 -4
- package/dist/prompt-files.js.map +1 -1
- package/dist/provider-profile.d.ts +21 -0
- package/dist/provider-profile.d.ts.map +1 -0
- package/dist/provider-profile.js +133 -0
- package/dist/provider-profile.js.map +1 -0
- package/dist/publication-events.d.ts +6 -8
- package/dist/publication-events.d.ts.map +1 -1
- package/dist/publication-events.js +40 -79
- package/dist/publication-events.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +48 -76
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts +1 -2
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +7 -16
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +1 -5
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +138 -199
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +2 -0
- package/dist/record.d.ts.map +1 -0
- package/dist/record.js +4 -0
- package/dist/record.js.map +1 -0
- package/dist/request-metadata.d.ts +9 -0
- package/dist/request-metadata.d.ts.map +1 -0
- package/dist/request-metadata.js +13 -0
- package/dist/request-metadata.js.map +1 -0
- package/dist/result.d.ts +4 -1
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +7 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +25 -17
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +619 -255
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +14 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -0
- package/dist/saved-scan-bootstrap.js +261 -0
- package/dist/saved-scan-bootstrap.js.map +1 -0
- package/dist/saved-scan.d.ts +1 -1
- package/dist/saved-scan.d.ts.map +1 -1
- package/dist/saved-scan.js +16 -32
- package/dist/saved-scan.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +19 -30
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +31 -18
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +307 -347
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +351 -92
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +55 -15
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-inputs.d.ts +6 -0
- package/dist/scan-inputs.d.ts.map +1 -0
- package/dist/scan-inputs.js +71 -0
- package/dist/scan-inputs.js.map +1 -0
- package/dist/scan-logs.d.ts +2 -3
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +14 -12
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-sessions.d.ts +4 -0
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +18 -3
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +5 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-skill-paths.d.ts +8 -0
- package/dist/scan-skill-paths.d.ts.map +1 -0
- package/dist/scan-skill-paths.js +47 -0
- package/dist/scan-skill-paths.js.map +1 -0
- package/dist/security-policy-cli.d.ts.map +1 -1
- package/dist/security-policy-cli.js +15 -33
- package/dist/security-policy-cli.js.map +1 -1
- package/dist/security-policy.d.ts +17 -9
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +131 -98
- package/dist/security-policy.js.map +1 -1
- package/dist/server/api.d.ts +0 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js +0 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embeddings.d.ts +3 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +11 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/errors.d.ts +5 -2
- package/dist/server/errors.d.ts.map +1 -1
- package/dist/server/errors.js +7 -0
- package/dist/server/errors.js.map +1 -1
- package/dist/server/sqlite-store.d.ts +0 -2
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +55 -55
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/server/storage.d.ts +0 -2
- package/dist/server/storage.d.ts.map +1 -1
- package/dist/server/storage.js.map +1 -1
- package/dist/severity-store.js +20 -32
- package/dist/severity-store.js.map +1 -1
- package/dist/standalone-validation-prompt.d.ts +2 -0
- package/dist/standalone-validation-prompt.d.ts.map +1 -0
- package/dist/standalone-validation-prompt.js +10 -0
- package/dist/standalone-validation-prompt.js.map +1 -0
- package/dist/suggest-owners.d.ts +6 -5
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +15 -10
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +12 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +124 -62
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.d.ts +6 -1
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +28 -9
- package/dist/trusted-executable.js.map +1 -1
- package/dist/value.d.ts +2 -0
- package/dist/value.d.ts.map +1 -0
- package/dist/value.js +27 -0
- package/dist/value.js.map +1 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +3 -6
- package/dist/version.js.map +1 -1
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +1 -2
- package/dist/windows-path.js.map +1 -1
- package/dist/workbench-arguments.d.ts +3 -0
- package/dist/workbench-arguments.d.ts.map +1 -0
- package/dist/workbench-arguments.js +15 -0
- package/dist/workbench-arguments.js.map +1 -0
- package/dist/worker-progress.d.ts +2 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +40 -46
- package/dist/worker-progress.js.map +1 -1
- package/docs/cli.md +1778 -0
- package/docs/dedupe-records.md +24 -0
- package/docs/findings-service.md +402 -0
- package/package.json +23 -23
- package/schemas/project-config.schema.json +5 -0
- package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
- package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
- package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
- package/_bundled_plugin/scripts/workbench_findings.py +0 -221
- package/_bundled_plugin/scripts/workbench_severity.py +0 -119
- package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
- package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
- package/dist/server/dashboard/app.css +0 -2
- package/dist/server/dashboard/app.js +0 -66
- package/dist/server/dashboard/index.html +0 -18
- package/dist/server/dashboard-types.d.ts +0 -48
- package/dist/server/dashboard-types.d.ts.map +0 -1
- package/dist/server/dashboard-types.js +0 -2
- package/dist/server/dashboard-types.js.map +0 -1
- package/dist/server/dashboard.d.ts +0 -6
- package/dist/server/dashboard.d.ts.map +0 -1
- package/dist/server/dashboard.js +0 -59
- package/dist/server/dashboard.js.map +0 -1
- package/dist/server/findings-service.d.ts +0 -20
- package/dist/server/findings-service.d.ts.map +0 -1
- package/dist/server/findings-service.js +0 -31
- package/dist/server/findings-service.js.map +0 -1
- package/dist/server/index.d.ts +0 -2
- package/dist/server/index.d.ts.map +0 -1
- package/dist/server/index.js +0 -6
- package/dist/server/index.js.map +0 -1
- package/dist/server/routes.d.ts +0 -6
- package/dist/server/routes.d.ts.map +0 -1
- package/dist/server/routes.js +0 -86
- package/dist/server/routes.js.map +0 -1
- package/dist/server/serve.d.ts +0 -2
- package/dist/server/serve.d.ts.map +0 -1
- package/dist/server/serve.js +0 -28
- package/dist/server/serve.js.map +0 -1
- package/dist/server/server.d.ts +0 -10
- package/dist/server/server.d.ts.map +0 -1
- package/dist/server/server.js +0 -17
- package/dist/server/server.js.map +0 -1
- package/dist/server/validation.d.ts +0 -17
- package/dist/server/validation.d.ts.map +0 -1
- package/dist/server/validation.js +0 -53
- package/dist/server/validation.js.map +0 -1
|
@@ -12,10 +12,9 @@ import re
|
|
|
12
12
|
import sqlite3
|
|
13
13
|
import stat
|
|
14
14
|
import sys
|
|
15
|
-
from collections.abc import
|
|
16
|
-
from
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from contextlib import contextmanager
|
|
17
17
|
from pathlib import Path
|
|
18
|
-
from types import ModuleType
|
|
19
18
|
from typing import Any
|
|
20
19
|
|
|
21
20
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
@@ -25,18 +24,25 @@ from finalize_scan_contract import (
|
|
|
25
24
|
_populate_unsealed_artifact_envelope,
|
|
26
25
|
_populate_unsealed_manifest_envelope,
|
|
27
26
|
_prepare_scan_finalization,
|
|
27
|
+
_read_json,
|
|
28
|
+
_read_saved_threat_model,
|
|
28
29
|
_read_scan_local_json,
|
|
29
30
|
_read_scan_local_json_bytes,
|
|
31
|
+
_read_scan_local_json_with_metadata,
|
|
30
32
|
_recover_unsealed_findings,
|
|
31
33
|
_remove_scan_local_file_if_exists,
|
|
32
34
|
_validate_completion_binding,
|
|
35
|
+
_validate_resolved_deferred,
|
|
36
|
+
_validate_schema_node,
|
|
33
37
|
_write_prepared_scan_finalization,
|
|
34
38
|
finalize_scan,
|
|
35
39
|
finding_candidate_id,
|
|
36
40
|
open_scan_local_file_descriptor,
|
|
37
41
|
write_scan_local_bytes,
|
|
42
|
+
write_threat_model_projection_if_possible,
|
|
38
43
|
)
|
|
39
44
|
from workbench_constants import PHASES
|
|
45
|
+
from workbench_target import committed_diff_snapshot_digest
|
|
40
46
|
from workbench_validation import path_within_scope
|
|
41
47
|
|
|
42
48
|
_PUBLISHED_OUTPUTS = (
|
|
@@ -44,6 +50,7 @@ _PUBLISHED_OUTPUTS = (
|
|
|
44
50
|
"coverage.json",
|
|
45
51
|
"scan-manifest.json",
|
|
46
52
|
"report.md",
|
|
53
|
+
"threatmodel.md",
|
|
47
54
|
"report.html",
|
|
48
55
|
"exports/results.sarif",
|
|
49
56
|
)
|
|
@@ -55,29 +62,59 @@ _RESERVED_ARTIFACT_PATHS = json.loads(
|
|
|
55
62
|
)
|
|
56
63
|
|
|
57
64
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
65
|
+
def threat_model_fields(db: Any, scan: sqlite3.Row) -> dict[str, Any]:
|
|
66
|
+
scan_dir = Path(scan["scan_dir"])
|
|
67
|
+
fields: dict[str, Any] = {"threatModelAvailable": False}
|
|
68
|
+
try:
|
|
69
|
+
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
70
|
+
db.verify_manifest_binding(
|
|
71
|
+
scan, _read_scan_local_json(scan_dir, db.ARTIFACTS["manifest"], "scan manifest")
|
|
72
|
+
)
|
|
73
|
+
saved_model = _read_saved_threat_model(scan_dir)
|
|
74
|
+
if saved_model is not None:
|
|
75
|
+
description = saved_model[0]
|
|
76
|
+
fields.update(
|
|
77
|
+
threatModelAvailable=True,
|
|
78
|
+
threatModelProvenance=description["provenance"],
|
|
79
|
+
)
|
|
80
|
+
if description["path"] is not None:
|
|
81
|
+
fields["threatModelPath"] = description["path"]
|
|
82
|
+
except (ContractError, OSError, SystemExit):
|
|
83
|
+
# Unavailable optional model data must not prevent reading the saved scan.
|
|
84
|
+
pass
|
|
85
|
+
return fields
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def refresh_completed_scan(
|
|
89
|
+
db: Any,
|
|
90
|
+
connection: sqlite3.Connection,
|
|
91
|
+
scan: sqlite3.Row,
|
|
92
|
+
cost_json: str | None,
|
|
93
|
+
) -> dict[str, Any]:
|
|
94
|
+
warnings = json.loads(scan["completion_warnings_json"])
|
|
95
|
+
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
96
|
+
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
97
|
+
db.verify_manifest_binding(scan, db.read_json_object(scan_dir / db.ARTIFACTS["manifest"]))
|
|
98
|
+
try:
|
|
99
|
+
manifest, _, _ = finalize_scan(
|
|
100
|
+
scan_dir,
|
|
101
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
102
|
+
projection_warnings=warnings,
|
|
103
|
+
)
|
|
104
|
+
except ContractError as exc:
|
|
105
|
+
raise SystemExit(str(exc)) from exc
|
|
106
|
+
db.verify_manifest_binding(scan, manifest)
|
|
107
|
+
manifest_digest = db.published_manifest_digest(scan_dir, manifest)
|
|
108
|
+
db.pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
|
|
109
|
+
if cost_json is not None and scan["recipe_json"] is not None:
|
|
110
|
+
db.scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json)
|
|
111
|
+
if warnings != json.loads(scan["completion_warnings_json"]):
|
|
112
|
+
with connection:
|
|
113
|
+
connection.execute(
|
|
114
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
115
|
+
(json.dumps(warnings), scan["id"]),
|
|
116
|
+
)
|
|
117
|
+
return db.scan_context(connection, scan["id"])
|
|
81
118
|
|
|
82
119
|
|
|
83
120
|
def _encoded(value: Any) -> bytes:
|
|
@@ -118,35 +155,51 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
|
118
155
|
)
|
|
119
156
|
|
|
120
157
|
|
|
121
|
-
def
|
|
122
|
-
|
|
158
|
+
def _checkpoint_paths(scan_dir: Path, directory: str) -> list[str]:
|
|
159
|
+
return [
|
|
160
|
+
f"{directory}/{name}"
|
|
161
|
+
for name in _children(scan_dir, directory)
|
|
162
|
+
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
163
|
+
]
|
|
123
164
|
|
|
124
|
-
def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
|
|
125
|
-
for name in _children(scan_dir, directory):
|
|
126
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name):
|
|
127
|
-
yield f"{directory}/{name}", kind
|
|
128
165
|
|
|
129
|
-
|
|
166
|
+
def _worker_outputs(scan_dir: Path, worker: Any) -> list[tuple[str, int]]:
|
|
167
|
+
output = Path(worker["artifact_dir"]).relative_to(scan_dir)
|
|
168
|
+
attempts = (output.parent if output.name == "output" else output) / "attempts"
|
|
169
|
+
archived = [
|
|
170
|
+
((attempts / name).as_posix(), int(name.split("-")[1]))
|
|
171
|
+
for name in _children(scan_dir, attempts.as_posix())
|
|
172
|
+
if re.fullmatch(r"attempt-\d+", name)
|
|
173
|
+
]
|
|
174
|
+
attempt = int(worker["attempt"] or 0) if "attempt" in worker.keys() else 0
|
|
175
|
+
if worker["kind"] == "discovery" and not attempt:
|
|
176
|
+
attempt = max((attempt for _, attempt in archived), default=0) + 1
|
|
177
|
+
return [(output.as_posix(), attempt), *archived]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
181
|
+
latest_reducer = _latest_successful_reducer(workers)
|
|
182
|
+
yield "checkpoint-head.json", None
|
|
183
|
+
for directory in ("checkpoint-heads", "checkpoints"):
|
|
184
|
+
yield from ((path, None) for path in _checkpoint_paths(scan_dir, directory))
|
|
130
185
|
for worker in workers:
|
|
131
186
|
if worker["kind"] not in {"dedup", "discovery"}:
|
|
132
187
|
continue
|
|
133
188
|
try:
|
|
134
|
-
|
|
189
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
135
190
|
except (TypeError, ValueError):
|
|
136
191
|
continue
|
|
137
|
-
|
|
138
|
-
"
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
for directory in directories:
|
|
146
|
-
checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
|
|
192
|
+
for directory, _ in outputs:
|
|
193
|
+
if worker["kind"] == "discovery":
|
|
194
|
+
yield f"{directory}/checkpoint-head.json", worker["kind"]
|
|
195
|
+
yield from (
|
|
196
|
+
(path, worker["kind"])
|
|
197
|
+
for path in _checkpoint_paths(scan_dir, f"{directory}/checkpoint-heads")
|
|
198
|
+
)
|
|
199
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
147
200
|
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
148
201
|
yield f"{directory}/result.json", worker["kind"]
|
|
149
|
-
yield from checkpoint_paths
|
|
202
|
+
yield from ((path, worker["kind"]) for path in checkpoint_paths)
|
|
150
203
|
if worker["result_manifest_path"] and (
|
|
151
204
|
worker["kind"] == "discovery"
|
|
152
205
|
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
@@ -160,17 +213,143 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[st
|
|
|
160
213
|
continue
|
|
161
214
|
|
|
162
215
|
|
|
216
|
+
def _checkpoint_head_directory(relative: str) -> Path | None:
|
|
217
|
+
path = Path(relative)
|
|
218
|
+
if path.name == "checkpoint-head.json":
|
|
219
|
+
return path.parent
|
|
220
|
+
if path.parent.name == "checkpoint-heads":
|
|
221
|
+
return path.parent.parent
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _capture_saved_source(
|
|
226
|
+
scan_dir: Path,
|
|
227
|
+
relative: str,
|
|
228
|
+
scan_id: str,
|
|
229
|
+
*,
|
|
230
|
+
kind: str | None = None,
|
|
231
|
+
snapshot_head: bool = True,
|
|
232
|
+
write: bool = True,
|
|
233
|
+
) -> dict[str, tuple[str, int]]:
|
|
234
|
+
if not snapshot_head or Path(relative).name != "checkpoint-head.json":
|
|
235
|
+
_, digest, observed = _read_saved_result(scan_dir, relative, scan_id, kind=kind)
|
|
236
|
+
return {relative: (digest, observed)}
|
|
237
|
+
head, _, observed = _read_saved_result(scan_dir, relative, scan_id)
|
|
238
|
+
observation = {"checkpoint": head["checkpoint"], "observedAtNs": str(observed)}
|
|
239
|
+
directory = Path(relative).parent
|
|
240
|
+
selected = (directory / "checkpoints" / observation["checkpoint"]).as_posix()
|
|
241
|
+
_, selected_digest, selected_time = _read_saved_result(scan_dir, selected, scan_id)
|
|
242
|
+
digest = _digest(observation)
|
|
243
|
+
snapshot = (directory / "checkpoint-heads" / f"{digest}.json").as_posix()
|
|
244
|
+
# Capture the selected file even if the worker created it after directory enumeration.
|
|
245
|
+
if write and not (scan_dir / snapshot).exists():
|
|
246
|
+
write_scan_local_bytes(scan_dir, snapshot, _encoded(observation))
|
|
247
|
+
return {
|
|
248
|
+
snapshot: (digest, int(observation["observedAtNs"])),
|
|
249
|
+
selected: (selected_digest, selected_time),
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _is_source_order_snapshot(relative: str) -> bool:
|
|
254
|
+
path = Path(relative)
|
|
255
|
+
return path.parent == Path("source-order") and bool(
|
|
256
|
+
re.fullmatch(r"[0-9a-f]{64}\.json", path.name)
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
163
260
|
def _read_saved_result(
|
|
164
261
|
scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
|
|
165
|
-
) -> tuple[dict[str, Any], str]:
|
|
166
|
-
draft
|
|
262
|
+
) -> tuple[dict[str, Any], str, int]:
|
|
263
|
+
draft, _, metadata = _read_scan_local_json_with_metadata(
|
|
264
|
+
scan_dir, relative, "Saved scan checkpoint"
|
|
265
|
+
)
|
|
266
|
+
directory = _checkpoint_head_directory(relative)
|
|
267
|
+
if directory is not None:
|
|
268
|
+
checkpoint = draft.get("checkpoint")
|
|
269
|
+
if not isinstance(checkpoint, str) or not re.fullmatch(r"[0-9a-f]{64}\.json", checkpoint):
|
|
270
|
+
raise ContractError("checkpoint head does not name a saved checkpoint")
|
|
271
|
+
_read_saved_result(scan_dir, (directory / "checkpoints" / checkpoint).as_posix(), scan_id)
|
|
272
|
+
if Path(relative).name == "checkpoint-head.json":
|
|
273
|
+
return draft, _digest([draft, metadata.st_mtime_ns]), metadata.st_mtime_ns
|
|
274
|
+
observed = draft.get("observedAtNs")
|
|
275
|
+
if not isinstance(observed, str) or not re.fullmatch(r"-?[0-9]+", observed):
|
|
276
|
+
raise ContractError("checkpoint head has no observation time")
|
|
277
|
+
return draft, _digest(draft), int(observed)
|
|
167
278
|
if draft.get("scanId") != scan_id:
|
|
168
279
|
raise ContractError("checkpoint belongs to a different scan")
|
|
169
|
-
if not
|
|
170
|
-
draft.get("
|
|
280
|
+
if not _is_source_order_snapshot(relative) and (
|
|
281
|
+
not isinstance(draft.get("findings"), list)
|
|
282
|
+
or not isinstance(draft.get("coverage", {} if kind == "dedup" else None), dict)
|
|
171
283
|
):
|
|
172
284
|
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
173
|
-
return draft, _digest(draft)
|
|
285
|
+
return draft, _digest(draft), metadata.st_mtime_ns
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _frozen_source_times(scan_dir: Path, scan_id: str, sources: dict[str, str]) -> dict[str, int]:
|
|
289
|
+
times: dict[str, int] = {}
|
|
290
|
+
snapshots = [path for path in sources if _is_source_order_snapshot(path)]
|
|
291
|
+
for path in snapshots:
|
|
292
|
+
record, digest, _ = _read_saved_result(scan_dir, path, scan_id)
|
|
293
|
+
if digest != sources[path] or not isinstance(record.get("sources"), dict):
|
|
294
|
+
raise ContractError("saved source ordering changed after the scan stopped")
|
|
295
|
+
for relative, observation in record["sources"].items():
|
|
296
|
+
if (
|
|
297
|
+
not isinstance(observation, dict)
|
|
298
|
+
or relative not in sources
|
|
299
|
+
or observation.get("digest") != sources[relative]
|
|
300
|
+
or not isinstance(observation.get("observedAtNs"), str)
|
|
301
|
+
or not re.fullmatch(r"-?[0-9]+", observation["observedAtNs"])
|
|
302
|
+
):
|
|
303
|
+
raise ContractError("saved source ordering does not match its frozen sources")
|
|
304
|
+
observed = int(observation["observedAtNs"])
|
|
305
|
+
if relative in times and times[relative] != observed:
|
|
306
|
+
raise ContractError("saved source ordering has conflicting observations")
|
|
307
|
+
times[relative] = observed
|
|
308
|
+
if snapshots and sources.keys() - set(snapshots) - times.keys():
|
|
309
|
+
raise ContractError("saved source ordering is incomplete")
|
|
310
|
+
return times
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _freeze_source_times(
|
|
314
|
+
scan_dir: Path, scan_id: str, sources: dict[str, str], times: dict[str, int]
|
|
315
|
+
) -> None:
|
|
316
|
+
# Identical result rewrites must not change the order of frozen review evidence.
|
|
317
|
+
observations = {
|
|
318
|
+
path: {"digest": digest, "observedAtNs": str(times[path])}
|
|
319
|
+
for path, digest in sources.items()
|
|
320
|
+
if not _is_source_order_snapshot(path)
|
|
321
|
+
}
|
|
322
|
+
if not observations:
|
|
323
|
+
return
|
|
324
|
+
record = {"scanId": scan_id, "sources": observations}
|
|
325
|
+
digest = _digest(record)
|
|
326
|
+
path = f"source-order/{digest}.json"
|
|
327
|
+
if not (scan_dir / path).exists():
|
|
328
|
+
write_scan_local_bytes(scan_dir, path, _encoded(record))
|
|
329
|
+
if _read_saved_result(scan_dir, path, scan_id)[1] != digest:
|
|
330
|
+
raise ContractError("saved source ordering does not match its digest")
|
|
331
|
+
sources[path] = digest
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _parent_scan_draft(
|
|
335
|
+
scan_id: str,
|
|
336
|
+
parent_scan: dict[str, Any],
|
|
337
|
+
findings: dict[str, Any],
|
|
338
|
+
coverage: dict[str, Any],
|
|
339
|
+
) -> dict[str, Any]:
|
|
340
|
+
parent = {
|
|
341
|
+
"scanId": scan_id,
|
|
342
|
+
"findings": findings.get("findings"),
|
|
343
|
+
"coverage": coverage,
|
|
344
|
+
**{
|
|
345
|
+
key: parent_scan[key]
|
|
346
|
+
for key in ("scope", "threatModel", "complete")
|
|
347
|
+
if key in parent_scan
|
|
348
|
+
},
|
|
349
|
+
}
|
|
350
|
+
if not isinstance(parent["findings"], list):
|
|
351
|
+
raise ContractError("Saved parent draft has no findings array")
|
|
352
|
+
return parent
|
|
174
353
|
|
|
175
354
|
|
|
176
355
|
def _read_saved_parent_result(
|
|
@@ -188,29 +367,34 @@ def _read_saved_parent_result(
|
|
|
188
367
|
or coverage.get("scanId", scan_id) != scan_id
|
|
189
368
|
):
|
|
190
369
|
raise ContractError("Saved parent documents belong to a different scan")
|
|
191
|
-
|
|
192
|
-
"scanId": scan_id,
|
|
193
|
-
"findings": findings.get("findings"),
|
|
194
|
-
"coverage": coverage,
|
|
195
|
-
**{
|
|
196
|
-
key: parent_scan[key]
|
|
197
|
-
for key in ("scope", "threatModel", "complete")
|
|
198
|
-
if key in parent_scan
|
|
199
|
-
},
|
|
200
|
-
}
|
|
201
|
-
if not isinstance(parent["findings"], list):
|
|
202
|
-
raise ContractError("Saved parent draft has no findings array")
|
|
203
|
-
return manifest, parent
|
|
370
|
+
return manifest, _parent_scan_draft(scan_id, parent_scan, findings, coverage)
|
|
204
371
|
|
|
205
372
|
|
|
206
|
-
def _source_digests(value: Any,
|
|
373
|
+
def _source_digests(value: Any, error: str) -> dict[str, str]:
|
|
207
374
|
if not isinstance(value, dict) or not all(
|
|
208
375
|
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
209
376
|
):
|
|
210
|
-
raise ContractError(
|
|
377
|
+
raise ContractError(error)
|
|
211
378
|
return value
|
|
212
379
|
|
|
213
380
|
|
|
381
|
+
def _retained_source_state(value: Any) -> tuple[dict[str, str], str | None]:
|
|
382
|
+
if isinstance(value, dict) and isinstance(value.get("sources"), dict):
|
|
383
|
+
sources = _source_digests(
|
|
384
|
+
value["sources"], "Saved stopped-scan source digests are malformed."
|
|
385
|
+
)
|
|
386
|
+
model_source = value.get("threatModelSource")
|
|
387
|
+
if not isinstance(model_source, str) or model_source not in sources:
|
|
388
|
+
raise ContractError("Saved stopped-scan model source is outside its checkpoint set.")
|
|
389
|
+
return sources, model_source
|
|
390
|
+
return _source_digests(value, "Saved stopped-scan source digests are malformed."), None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _encode_retained_sources(sources: dict[str, str], model_source: list[str]) -> str:
|
|
394
|
+
state = {"sources": sources, "threatModelSource": model_source[0]} if model_source else sources
|
|
395
|
+
return json.dumps(state, sort_keys=True)
|
|
396
|
+
|
|
397
|
+
|
|
214
398
|
def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
215
399
|
try:
|
|
216
400
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
@@ -234,7 +418,7 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
234
418
|
|
|
235
419
|
if manifest_path is None:
|
|
236
420
|
if frozen_sources is not None:
|
|
237
|
-
return bool(
|
|
421
|
+
return bool(_retained_source_state(json.loads(frozen_sources))[0])
|
|
238
422
|
return has_saved_source()
|
|
239
423
|
if scan["seal_manifest_digest"] is None:
|
|
240
424
|
try:
|
|
@@ -252,14 +436,22 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
252
436
|
if not isinstance(manifest_scan, dict):
|
|
253
437
|
return True
|
|
254
438
|
published_sources = _source_digests(
|
|
255
|
-
manifest_scan.get("preservedSources", {}),
|
|
439
|
+
manifest_scan.get("preservedSources", {}),
|
|
440
|
+
"Published scan source digests are malformed.",
|
|
256
441
|
)
|
|
257
442
|
current_sources = dict(published_sources)
|
|
443
|
+
paths.update({path: None for path in published_sources if _is_source_order_snapshot(path)})
|
|
258
444
|
for path in paths:
|
|
259
445
|
try:
|
|
260
|
-
|
|
261
|
-
scan_dir,
|
|
446
|
+
captured = _capture_saved_source(
|
|
447
|
+
scan_dir,
|
|
448
|
+
path,
|
|
449
|
+
scan["id"],
|
|
450
|
+
kind=paths[path],
|
|
451
|
+
snapshot_head=path not in published_sources,
|
|
452
|
+
write=False,
|
|
262
453
|
)
|
|
454
|
+
current_sources.update({path: value[0] for path, value in captured.items()})
|
|
263
455
|
except (ContractError, OSError, ValueError):
|
|
264
456
|
continue
|
|
265
457
|
return current_sources != published_sources
|
|
@@ -273,7 +465,7 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
273
465
|
include_parent = True
|
|
274
466
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
275
467
|
if raw_frozen_sources is not None:
|
|
276
|
-
frozen_sources =
|
|
468
|
+
frozen_sources, _ = _retained_source_state(json.loads(raw_frozen_sources))
|
|
277
469
|
include_parent = False
|
|
278
470
|
|
|
279
471
|
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
@@ -291,7 +483,8 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
291
483
|
):
|
|
292
484
|
if "preservedSources" in manifest_scan:
|
|
293
485
|
published_sources = _source_digests(
|
|
294
|
-
manifest_scan["preservedSources"],
|
|
486
|
+
manifest_scan["preservedSources"],
|
|
487
|
+
"Published scan source digests are malformed.",
|
|
295
488
|
)
|
|
296
489
|
include_parent = not published_sources
|
|
297
490
|
if published_sources:
|
|
@@ -312,21 +505,30 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
312
505
|
).fetchall()
|
|
313
506
|
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
314
507
|
recovery_sources = dict(frozen_sources or {})
|
|
508
|
+
source_times = _frozen_source_times(scan_dir, scan["id"], recovery_sources)
|
|
315
509
|
for relative, expected_digest in recovery_sources.items():
|
|
316
510
|
try:
|
|
317
|
-
_, digest = _read_saved_result(
|
|
511
|
+
_, digest, observed = _read_saved_result(
|
|
512
|
+
scan_dir, relative, scan["id"], kind=paths.get(relative)
|
|
513
|
+
)
|
|
318
514
|
except (ContractError, OSError, ValueError) as exc:
|
|
319
515
|
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
320
516
|
if digest != expected_digest:
|
|
321
517
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
518
|
+
if not _is_source_order_snapshot(relative):
|
|
519
|
+
source_times.setdefault(relative, observed)
|
|
322
520
|
|
|
323
521
|
for relative in paths.keys() - recovery_sources.keys():
|
|
324
522
|
try:
|
|
325
|
-
|
|
326
|
-
scan_dir, relative, scan["id"], kind=paths[relative]
|
|
327
|
-
)
|
|
523
|
+
captured = _capture_saved_source(scan_dir, relative, scan["id"], kind=paths[relative])
|
|
328
524
|
except (ContractError, OSError, ValueError):
|
|
329
525
|
continue
|
|
526
|
+
for path, (digest, observed) in captured.items():
|
|
527
|
+
if path in recovery_sources and recovery_sources[path] != digest:
|
|
528
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
529
|
+
recovery_sources[path] = digest
|
|
530
|
+
source_times.setdefault(path, observed)
|
|
531
|
+
_freeze_source_times(scan_dir, scan["id"], recovery_sources, source_times)
|
|
330
532
|
return recovery_sources, include_parent
|
|
331
533
|
|
|
332
534
|
|
|
@@ -358,15 +560,10 @@ def _finding_key(finding: dict[str, Any]) -> str:
|
|
|
358
560
|
else finding.get("identity")
|
|
359
561
|
)
|
|
360
562
|
if not isinstance(identity, dict):
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
or "finding"
|
|
366
|
-
)
|
|
367
|
-
identity = {
|
|
368
|
-
"anchor": re.sub(r"[^a-z0-9._/-]+", "-", source.lower()).strip("._/-") or "finding"
|
|
369
|
-
}
|
|
563
|
+
normalized = dict(finding)
|
|
564
|
+
normalized.pop("identity", None)
|
|
565
|
+
_ensure_finding_identity(normalized)
|
|
566
|
+
identity = normalized["identity"]
|
|
370
567
|
locations = finding.get("locations", [])
|
|
371
568
|
if not isinstance(locations, list):
|
|
372
569
|
locations = []
|
|
@@ -459,6 +656,245 @@ def _retained_findings(finding: dict[str, Any]) -> Iterator[dict[str, Any]]:
|
|
|
459
656
|
)
|
|
460
657
|
|
|
461
658
|
|
|
659
|
+
def _deferred_rows(coverage: dict[str, Any]) -> list[Any]:
|
|
660
|
+
rows = coverage.get("deferred", [])
|
|
661
|
+
return rows if isinstance(rows, list) else []
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _resolved_deferred_rows(draft: dict[str, Any], schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
665
|
+
if draft.get("complete") is False:
|
|
666
|
+
return []
|
|
667
|
+
coverage = draft["coverage"]
|
|
668
|
+
rows = coverage.get("resolvedDeferred", [])
|
|
669
|
+
try:
|
|
670
|
+
# Invalid closure metadata cannot discard the evidence it names.
|
|
671
|
+
_validate_schema_node(rows, schema, "coverage.resolvedDeferred")
|
|
672
|
+
_validate_resolved_deferred({**coverage, "deferred": _deferred_rows(coverage)})
|
|
673
|
+
except ContractError:
|
|
674
|
+
return []
|
|
675
|
+
return rows
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _merge_tied_parent_observations(
|
|
679
|
+
current: dict[str, Any], previous: dict[str, Any]
|
|
680
|
+
) -> dict[str, Any]:
|
|
681
|
+
merged = copy.deepcopy(current)
|
|
682
|
+
for finding in previous["findings"]:
|
|
683
|
+
if finding not in merged["findings"]:
|
|
684
|
+
merged["findings"].append(copy.deepcopy(finding))
|
|
685
|
+
coverage = merged["coverage"]
|
|
686
|
+
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
687
|
+
rows = previous["coverage"].get(field, [])
|
|
688
|
+
output = coverage.setdefault(field, [])
|
|
689
|
+
if isinstance(rows, list) and isinstance(output, list):
|
|
690
|
+
for row in rows:
|
|
691
|
+
if row not in output:
|
|
692
|
+
output.append(copy.deepcopy(row))
|
|
693
|
+
pending_ids = {
|
|
694
|
+
identity
|
|
695
|
+
for row in _deferred_rows(coverage)
|
|
696
|
+
if isinstance(row, dict)
|
|
697
|
+
for identity in (row.get("id"), row.get("candidateId"))
|
|
698
|
+
if isinstance(identity, str)
|
|
699
|
+
}
|
|
700
|
+
closure_schema = _read_json(
|
|
701
|
+
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
702
|
+
)["properties"]["resolvedDeferred"]
|
|
703
|
+
closures = {}
|
|
704
|
+
for observed in (current, previous):
|
|
705
|
+
for row in _resolved_deferred_rows(observed, closure_schema):
|
|
706
|
+
if row["id"] not in pending_ids:
|
|
707
|
+
closures.setdefault(row["id"], copy.deepcopy(row))
|
|
708
|
+
coverage.pop("resolvedDeferred", None)
|
|
709
|
+
if closures:
|
|
710
|
+
coverage["resolvedDeferred"] = list(closures.values())
|
|
711
|
+
if current.get("complete") is False or previous.get("complete") is False:
|
|
712
|
+
merged["complete"] = False
|
|
713
|
+
coverage.pop("completeness", None)
|
|
714
|
+
return merged
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def _saved_coverage_id(item: dict[str, Any]) -> str:
|
|
718
|
+
return item.get("candidateId") or f"saved-{_digest(item)[:16]}"
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _deferred_candidate_id(
|
|
722
|
+
row: dict[str, Any],
|
|
723
|
+
owner: str | None,
|
|
724
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
725
|
+
) -> str | None:
|
|
726
|
+
identity = row.get("candidateId") or row.get("id")
|
|
727
|
+
if not isinstance(identity, str):
|
|
728
|
+
return None
|
|
729
|
+
if (owner, identity) in ambiguous_deferred and not any(
|
|
730
|
+
key in row for key in ("candidateId", "candidate", "finding")
|
|
731
|
+
):
|
|
732
|
+
return None
|
|
733
|
+
return identity
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _generic_surface_updates(
|
|
737
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
738
|
+
source_order: dict[str, tuple[int, int]],
|
|
739
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
740
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]],
|
|
741
|
+
resolved_candidates: dict[tuple[str | None, str], str],
|
|
742
|
+
reopened_generic: set[tuple[str | None, str]],
|
|
743
|
+
surface_schema: dict[str, Any],
|
|
744
|
+
deferred_rows: dict[str, list[Any]],
|
|
745
|
+
ambiguous_deferred: set[tuple[str | None, str]],
|
|
746
|
+
) -> tuple[set[int], list[dict[str, Any]]]:
|
|
747
|
+
if not closed_deferred and not reopened_generic:
|
|
748
|
+
return set(), []
|
|
749
|
+
|
|
750
|
+
def linked(row: dict[str, Any], identity: str | None) -> bool:
|
|
751
|
+
surface_ids = row.get("surfaceIds", [])
|
|
752
|
+
return identity is not None and (
|
|
753
|
+
row.get("id") == identity or (isinstance(surface_ids, list) and identity in surface_ids)
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
saved_surfaces: dict[tuple[str | None, str], list[tuple[str, dict[str, Any]]]] = {}
|
|
757
|
+
for relative, draft, owner in sources:
|
|
758
|
+
rows = draft["coverage"].get("surfaces", [])
|
|
759
|
+
for row in rows if isinstance(rows, list) else []:
|
|
760
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str):
|
|
761
|
+
saved_surfaces.setdefault((owner, row["id"]), []).append((relative, row))
|
|
762
|
+
replaced: set[int] = set()
|
|
763
|
+
updates: list[dict[str, Any]] = []
|
|
764
|
+
for relative, draft, owner in sources:
|
|
765
|
+
closed_ids = {
|
|
766
|
+
identity
|
|
767
|
+
for (saved_owner, identity), (_, _, source) in closed_deferred.items()
|
|
768
|
+
if saved_owner == owner and source == relative
|
|
769
|
+
}
|
|
770
|
+
reopened_ids = {
|
|
771
|
+
identity
|
|
772
|
+
for (saved_owner, identity), (_, _, source) in active_deferred.items()
|
|
773
|
+
if saved_owner == owner and source == relative and (owner, identity) in reopened_generic
|
|
774
|
+
}
|
|
775
|
+
if not closed_ids and not reopened_ids:
|
|
776
|
+
continue
|
|
777
|
+
current = draft["coverage"].get("surfaces", [])
|
|
778
|
+
for surface in current if isinstance(current, list) else []:
|
|
779
|
+
if not isinstance(surface, dict):
|
|
780
|
+
continue
|
|
781
|
+
reopening = surface.get("disposition") == "needs_follow_up"
|
|
782
|
+
work_ids = reopened_ids if reopening else closed_ids
|
|
783
|
+
if not work_ids:
|
|
784
|
+
continue
|
|
785
|
+
identity = surface.get("id")
|
|
786
|
+
if not isinstance(identity, str):
|
|
787
|
+
continue
|
|
788
|
+
matches = saved_surfaces[(owner, identity)]
|
|
789
|
+
# Older writers could assign one ID to distinct surfaces in a draft.
|
|
790
|
+
# A closure cannot identify which of those observations it replaces.
|
|
791
|
+
by_source = dict(matches)
|
|
792
|
+
if any(row != by_source[saved_path] for saved_path, row in matches):
|
|
793
|
+
continue
|
|
794
|
+
if any(
|
|
795
|
+
"candidateId" in row or "candidate" in row or "finding" in row for _, row in matches
|
|
796
|
+
):
|
|
797
|
+
continue
|
|
798
|
+
if any(
|
|
799
|
+
row is not surface
|
|
800
|
+
and source_order[saved_path] >= source_order[relative]
|
|
801
|
+
and row.get("disposition") != surface.get("disposition")
|
|
802
|
+
for saved_path, row in matches
|
|
803
|
+
):
|
|
804
|
+
continue
|
|
805
|
+
|
|
806
|
+
if not reopening and any(
|
|
807
|
+
saved_owner == owner
|
|
808
|
+
and (
|
|
809
|
+
not isinstance(row.get("id") or row.get("candidateId"), str)
|
|
810
|
+
or (
|
|
811
|
+
isinstance(row.get("id"), str)
|
|
812
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
813
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
814
|
+
)
|
|
815
|
+
)
|
|
816
|
+
and linked(row, identity)
|
|
817
|
+
for saved_path, _, saved_owner in sources
|
|
818
|
+
for row in deferred_rows[saved_path]
|
|
819
|
+
if isinstance(row, dict)
|
|
820
|
+
):
|
|
821
|
+
continue
|
|
822
|
+
if not reopening and any(
|
|
823
|
+
saved_owner == owner
|
|
824
|
+
and (owner, deferred_id) not in closed_deferred
|
|
825
|
+
and (
|
|
826
|
+
(candidate_id := _deferred_candidate_id(row, owner, ambiguous_deferred)) is None
|
|
827
|
+
or (owner, candidate_id) not in resolved_candidates
|
|
828
|
+
)
|
|
829
|
+
and linked(row, identity)
|
|
830
|
+
for (saved_owner, deferred_id), (_, row, _) in active_deferred.items()
|
|
831
|
+
):
|
|
832
|
+
continue
|
|
833
|
+
# An accepted checkpoint can update a saved surface by ID without
|
|
834
|
+
# optional surfaceIds links on its generic task.
|
|
835
|
+
latest_surface = max(matches, key=lambda match: source_order[match[0]])[1]
|
|
836
|
+
update = copy.deepcopy(latest_surface)
|
|
837
|
+
refs = update.setdefault("receiptRefs", [])
|
|
838
|
+
if isinstance(refs, list):
|
|
839
|
+
for _, row in matches:
|
|
840
|
+
previous_refs = row.get("receiptRefs", [])
|
|
841
|
+
for ref in previous_refs if isinstance(previous_refs, list) else []:
|
|
842
|
+
if ref not in refs:
|
|
843
|
+
refs.append(ref)
|
|
844
|
+
try:
|
|
845
|
+
_validate_schema_node(update, surface_schema, "coverage.surfaces")
|
|
846
|
+
except ContractError:
|
|
847
|
+
continue
|
|
848
|
+
replaced.update(
|
|
849
|
+
id(row)
|
|
850
|
+
for _, row in matches
|
|
851
|
+
if reopening
|
|
852
|
+
or row.get("disposition") in {"needs_follow_up", surface.get("disposition")}
|
|
853
|
+
)
|
|
854
|
+
if update not in updates:
|
|
855
|
+
updates.append(update)
|
|
856
|
+
return replaced, updates
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
@contextmanager
|
|
860
|
+
def preserve_parent_head_on_error(scan_dir: Path) -> Iterator[None]:
|
|
861
|
+
"""Keep rejected completion attempts from becoming accepted parent observations."""
|
|
862
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
863
|
+
previous = None
|
|
864
|
+
try:
|
|
865
|
+
head_path.lstat()
|
|
866
|
+
except FileNotFoundError:
|
|
867
|
+
pass
|
|
868
|
+
else:
|
|
869
|
+
descriptor = open_scan_local_file_descriptor(
|
|
870
|
+
scan_dir, "checkpoint-head.json", "Saved parent checkpoint head"
|
|
871
|
+
)
|
|
872
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
873
|
+
metadata = os.fstat(handle.fileno())
|
|
874
|
+
previous = (handle.read(), metadata)
|
|
875
|
+
directories = ("checkpoints", "checkpoint-heads")
|
|
876
|
+
previous_files = {
|
|
877
|
+
relative for directory in directories for relative in _checkpoint_paths(scan_dir, directory)
|
|
878
|
+
}
|
|
879
|
+
try:
|
|
880
|
+
yield
|
|
881
|
+
except BaseException:
|
|
882
|
+
if previous is None:
|
|
883
|
+
_remove_scan_local_file_if_exists(scan_dir, "checkpoint-head.json")
|
|
884
|
+
else:
|
|
885
|
+
payload, metadata = previous
|
|
886
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", payload)
|
|
887
|
+
os.utime(head_path, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
888
|
+
current_files = {
|
|
889
|
+
relative
|
|
890
|
+
for directory in directories
|
|
891
|
+
for relative in _checkpoint_paths(scan_dir, directory)
|
|
892
|
+
}
|
|
893
|
+
for relative in current_files - previous_files:
|
|
894
|
+
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
895
|
+
raise
|
|
896
|
+
|
|
897
|
+
|
|
462
898
|
def merge_saved_results(
|
|
463
899
|
scan_dir: Path,
|
|
464
900
|
scan_id: str,
|
|
@@ -470,14 +906,25 @@ def merge_saved_results(
|
|
|
470
906
|
reason: str,
|
|
471
907
|
frozen_source_digests: dict[str, str] | None = None,
|
|
472
908
|
allow_frozen_legacy_parent: bool = False,
|
|
909
|
+
frozen_model_source: str | None = None,
|
|
910
|
+
selected_model_source: list[str] | None = None,
|
|
473
911
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]] | None:
|
|
474
912
|
"""Read only bound parent/worker files; return an unsealed loss-preserving union."""
|
|
475
913
|
initial_warnings = set(warnings)
|
|
914
|
+
try:
|
|
915
|
+
source_times = _frozen_source_times(scan_dir, scan_id, frozen_source_digests or {})
|
|
916
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
917
|
+
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
476
918
|
parent: dict[str, Any] | None = None
|
|
477
919
|
parent_manifest: dict[str, Any] | None = None
|
|
920
|
+
parent_is_canonical = False
|
|
921
|
+
parent_modified = 0
|
|
478
922
|
if frozen_source_digests is None or allow_frozen_legacy_parent:
|
|
479
923
|
try:
|
|
480
924
|
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
925
|
+
# Without an accepted head, file-authored coverage is a full replacement.
|
|
926
|
+
parent_modified = (scan_dir / "coverage.json").lstat().st_mtime_ns
|
|
927
|
+
parent_is_canonical = True
|
|
481
928
|
except (ContractError, OSError, ValueError) as exc:
|
|
482
929
|
if not stopped:
|
|
483
930
|
raise
|
|
@@ -487,25 +934,66 @@ def merge_saved_results(
|
|
|
487
934
|
parent = None
|
|
488
935
|
if parent_manifest is not None and parent is not None:
|
|
489
936
|
parent_scan = parent_manifest["scan"]
|
|
937
|
+
try:
|
|
938
|
+
previous_head, _, head_modified = _read_saved_result(
|
|
939
|
+
scan_dir, "checkpoint-head.json", scan_id
|
|
940
|
+
)
|
|
941
|
+
except (ContractError, OSError, ValueError):
|
|
942
|
+
head_modified = None
|
|
943
|
+
if head_modified is not None:
|
|
944
|
+
# A partial tool publication must not outrank its accepted head.
|
|
945
|
+
parent_modified = min(
|
|
946
|
+
(scan_dir / name).lstat().st_mtime_ns
|
|
947
|
+
for name in ("findings.json", "coverage.json", "scan-manifest.json")
|
|
948
|
+
)
|
|
490
949
|
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
950
|
+
head_path = scan_dir / "checkpoint-head.json"
|
|
951
|
+
tied_observations = False
|
|
952
|
+
if head_modified == parent_modified:
|
|
953
|
+
previous_parent, _, _ = _read_saved_result(
|
|
954
|
+
scan_dir, f"checkpoints/{previous_head['checkpoint']}", scan_id
|
|
955
|
+
)
|
|
956
|
+
if previous_parent != parent:
|
|
957
|
+
# Tied observations cannot decide which pending work came last.
|
|
958
|
+
parent = _merge_tied_parent_observations(parent, previous_parent)
|
|
959
|
+
parent_is_canonical = False
|
|
960
|
+
tied_observations = True
|
|
491
961
|
payload = _encoded(parent)
|
|
492
962
|
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
493
963
|
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
494
|
-
|
|
964
|
+
checkpoint_path = scan_dir / parent_checkpoint
|
|
965
|
+
if not checkpoint_path.exists():
|
|
966
|
+
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
967
|
+
# A recovery copy must not appear newer than the review it copies.
|
|
968
|
+
os.utime(checkpoint_path, ns=(parent_modified, parent_modified))
|
|
969
|
+
if head_modified is None or head_modified < parent_modified or tied_observations:
|
|
970
|
+
write_scan_local_bytes(
|
|
971
|
+
scan_dir,
|
|
972
|
+
"checkpoint-head.json",
|
|
973
|
+
_encoded({"checkpoint": checkpoint_path.name}),
|
|
974
|
+
)
|
|
975
|
+
os.utime(head_path, ns=(parent_modified, parent_modified))
|
|
495
976
|
if frozen_source_digests is not None:
|
|
977
|
+
captured = _capture_saved_source(scan_dir, "checkpoint-head.json", scan_id)
|
|
496
978
|
frozen_source_digests = {
|
|
497
979
|
**frozen_source_digests,
|
|
498
980
|
parent_checkpoint: parent_digest,
|
|
981
|
+
**{path: value[0] for path, value in captured.items()},
|
|
499
982
|
}
|
|
500
983
|
|
|
501
984
|
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
502
985
|
parent_preserved_sources: dict[str, str] = {}
|
|
503
986
|
source_digests: dict[str, str] = {}
|
|
987
|
+
source_order: dict[str, tuple[int, int]] = {}
|
|
988
|
+
worker_attempts: dict[str, tuple[str, int]] = {}
|
|
989
|
+
saved_heads: dict[str, str] = {}
|
|
504
990
|
if parent_manifest:
|
|
505
991
|
recorded = parent_manifest["scan"].get("preservedSources", {})
|
|
506
992
|
if isinstance(recorded, dict):
|
|
507
993
|
parent_preserved_sources = recorded
|
|
508
994
|
source_digests.update(parent_preserved_sources)
|
|
995
|
+
if frozen_source_digests is None:
|
|
996
|
+
source_times.update(_frozen_source_times(scan_dir, scan_id, recorded))
|
|
509
997
|
paths: dict[str, str | None] = {}
|
|
510
998
|
reducer_paths: set[str] = set()
|
|
511
999
|
current_results: set[str] = set()
|
|
@@ -520,57 +1008,43 @@ def merge_saved_results(
|
|
|
520
1008
|
except ValueError:
|
|
521
1009
|
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
522
1010
|
|
|
523
|
-
def checkpoints(directory: str, worker_id: str | None) -> None:
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
1011
|
+
def checkpoints(directory: str, worker_id: str | None, attempt: int = 0) -> None:
|
|
1012
|
+
if worker_id is not None:
|
|
1013
|
+
root = Path(directory).parent.as_posix()
|
|
1014
|
+
worker_attempts[root] = (worker_id, attempt)
|
|
1015
|
+
paths[f"{root}/checkpoint-head.json"] = worker_id
|
|
1016
|
+
head_snapshots = (Path(directory).parent / "checkpoint-heads").as_posix()
|
|
1017
|
+
for saved_directory in (directory, head_snapshots):
|
|
1018
|
+
paths.update(dict.fromkeys(_checkpoint_paths(scan_dir, saved_directory), worker_id))
|
|
527
1019
|
|
|
1020
|
+
paths["checkpoint-head.json"] = None
|
|
528
1021
|
checkpoints("checkpoints", None)
|
|
529
1022
|
for worker in workers:
|
|
530
1023
|
try:
|
|
531
|
-
|
|
1024
|
+
outputs = _worker_outputs(scan_dir, worker)
|
|
532
1025
|
except (TypeError, ValueError):
|
|
533
1026
|
warnings.append("Skipped a worker checkpoint outside the scan directory.")
|
|
534
1027
|
continue
|
|
535
1028
|
if worker["kind"] == "dedup":
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
result_path = f"{directory}/result.json"
|
|
539
|
-
checkpoint_paths = [
|
|
540
|
-
f"{directory}/checkpoints/{name}"
|
|
541
|
-
for name in _children(scan_dir, f"{directory}/checkpoints")
|
|
542
|
-
if re.fullmatch(r"[0-9a-f]{64}\.json", name)
|
|
543
|
-
]
|
|
1029
|
+
for directory, attempt in outputs:
|
|
1030
|
+
checkpoint_paths = _checkpoint_paths(scan_dir, f"{directory}/checkpoints")
|
|
544
1031
|
if not checkpoint_paths:
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
reducer_paths.update(
|
|
550
|
-
reducer_outputs.append((
|
|
551
|
-
|
|
552
|
-
reducer_output(output, int(worker["attempt"] or 0), worker)
|
|
553
|
-
attempts = (
|
|
554
|
-
Path(output).parent if Path(output).name == "output" else Path(output)
|
|
555
|
-
) / "attempts"
|
|
556
|
-
for name in _children(scan_dir, attempts.as_posix()):
|
|
557
|
-
match = re.fullmatch(r"attempt-(\d+)", name)
|
|
558
|
-
if match:
|
|
559
|
-
reducer_output((attempts / name).as_posix(), int(match.group(1)), worker)
|
|
1032
|
+
continue
|
|
1033
|
+
result_path = f"{directory}/result.json"
|
|
1034
|
+
retained_paths = [result_path, *checkpoint_paths]
|
|
1035
|
+
paths.update(dict.fromkeys(retained_paths))
|
|
1036
|
+
reducer_paths.update(retained_paths)
|
|
1037
|
+
reducer_outputs.append((worker, result_path, checkpoint_paths, attempt))
|
|
560
1038
|
continue
|
|
561
1039
|
if worker["kind"] != "discovery":
|
|
562
1040
|
continue
|
|
1041
|
+
output, attempt = outputs[0]
|
|
563
1042
|
paths[f"{output}/result.json"] = worker["id"]
|
|
564
1043
|
current_results.add(f"{output}/result.json")
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
for name in _children(scan_dir, attempts.as_posix()):
|
|
570
|
-
if re.fullmatch(r"attempt-\d+", name):
|
|
571
|
-
archived = (attempts / name).as_posix()
|
|
572
|
-
paths[f"{archived}/result.json"] = worker["id"]
|
|
573
|
-
checkpoints(f"{archived}/checkpoints", worker["id"])
|
|
1044
|
+
for archived, archived_attempt in outputs[1:]:
|
|
1045
|
+
paths[f"{archived}/result.json"] = worker["id"]
|
|
1046
|
+
checkpoints(f"{archived}/checkpoints", worker["id"], archived_attempt)
|
|
1047
|
+
checkpoints(f"{output}/checkpoints", worker["id"], attempt)
|
|
574
1048
|
if worker["result_manifest_path"]:
|
|
575
1049
|
try:
|
|
576
1050
|
current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
@@ -579,7 +1053,26 @@ def merge_saved_results(
|
|
|
579
1053
|
except ValueError:
|
|
580
1054
|
warnings.append("Skipped a worker result outside the scan directory.")
|
|
581
1055
|
|
|
1056
|
+
if frozen_source_digests is None:
|
|
1057
|
+
for relative, worker_id in list(paths.items()):
|
|
1058
|
+
if Path(relative).name != "checkpoint-head.json":
|
|
1059
|
+
continue
|
|
1060
|
+
del paths[relative]
|
|
1061
|
+
try:
|
|
1062
|
+
captured = _capture_saved_source(scan_dir, relative, scan_id)
|
|
1063
|
+
paths.update({path: worker_id for path in captured})
|
|
1064
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
1065
|
+
if (scan_dir / relative).exists():
|
|
1066
|
+
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
1067
|
+
|
|
582
1068
|
if frozen_source_digests is not None:
|
|
1069
|
+
source_digests.update(
|
|
1070
|
+
{
|
|
1071
|
+
path: digest
|
|
1072
|
+
for path, digest in frozen_source_digests.items()
|
|
1073
|
+
if _is_source_order_snapshot(path)
|
|
1074
|
+
}
|
|
1075
|
+
)
|
|
583
1076
|
paths = {
|
|
584
1077
|
relative: worker_id
|
|
585
1078
|
for relative, worker_id in paths.items()
|
|
@@ -591,12 +1084,17 @@ def merge_saved_results(
|
|
|
591
1084
|
|
|
592
1085
|
for relative, worker_id in paths.items():
|
|
593
1086
|
try:
|
|
594
|
-
draft, digest = _read_saved_result(
|
|
1087
|
+
draft, digest, observed = _read_saved_result(
|
|
595
1088
|
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
596
1089
|
)
|
|
597
1090
|
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
598
1091
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
599
1092
|
source_digests[relative] = digest
|
|
1093
|
+
source_times.setdefault(relative, observed)
|
|
1094
|
+
source_order[relative] = (0, source_times[relative])
|
|
1095
|
+
if _checkpoint_head_directory(relative) is not None:
|
|
1096
|
+
saved_heads[relative] = draft["checkpoint"]
|
|
1097
|
+
continue
|
|
600
1098
|
# Recovery expects coverage, but reducer results only contain findings
|
|
601
1099
|
# and context. Add an empty value after hashing the original result.
|
|
602
1100
|
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
@@ -607,6 +1105,37 @@ def merge_saved_results(
|
|
|
607
1105
|
if frozen_source_digests.keys() - source_digests.keys():
|
|
608
1106
|
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.")
|
|
609
1107
|
|
|
1108
|
+
# Frozen observations retain checkpoint selection even if a worker moves its head.
|
|
1109
|
+
headed_workers = {
|
|
1110
|
+
worker_attempts[_checkpoint_head_directory(head).as_posix()][0]
|
|
1111
|
+
for head in saved_heads
|
|
1112
|
+
if _checkpoint_head_directory(head) != Path(".")
|
|
1113
|
+
}
|
|
1114
|
+
for relative, _, worker_id in sources:
|
|
1115
|
+
if worker_id not in headed_workers:
|
|
1116
|
+
continue
|
|
1117
|
+
directory = Path(relative).parent
|
|
1118
|
+
if directory.name == "checkpoints":
|
|
1119
|
+
directory = directory.parent
|
|
1120
|
+
_, attempt = worker_attempts.get(directory.as_posix(), (worker_id, 0))
|
|
1121
|
+
source_order[relative] = (attempt, source_order[relative][1])
|
|
1122
|
+
selected_observations: dict[str, tuple[int, int]] = {}
|
|
1123
|
+
parent_heads: list[tuple[int, str]] = []
|
|
1124
|
+
for head, checkpoint in saved_heads.items():
|
|
1125
|
+
directory = _checkpoint_head_directory(head)
|
|
1126
|
+
selected = (directory / "checkpoints" / checkpoint).as_posix()
|
|
1127
|
+
if selected not in source_order:
|
|
1128
|
+
raise ContractError("Checkpoint head is outside the saved source set.")
|
|
1129
|
+
observed = source_order[head][1]
|
|
1130
|
+
if directory == Path("."):
|
|
1131
|
+
order = (0, max(source_order[selected][1], observed))
|
|
1132
|
+
parent_heads.append((observed, selected))
|
|
1133
|
+
else:
|
|
1134
|
+
_, attempt = worker_attempts[directory.as_posix()]
|
|
1135
|
+
order = (attempt, observed)
|
|
1136
|
+
selected_observations[selected] = max(selected_observations.get(selected, order), order)
|
|
1137
|
+
source_order.update(selected_observations)
|
|
1138
|
+
|
|
610
1139
|
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
611
1140
|
latest_reducer_key = (
|
|
612
1141
|
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
@@ -627,8 +1156,67 @@ def merge_saved_results(
|
|
|
627
1156
|
latest_reducer_key = candidate_key
|
|
628
1157
|
latest_reducer = result_path
|
|
629
1158
|
|
|
1159
|
+
if parent_heads:
|
|
1160
|
+
latest_observation = max(observed for observed, _ in parent_heads)
|
|
1161
|
+
for observed, parent_path in parent_heads:
|
|
1162
|
+
if observed != latest_observation:
|
|
1163
|
+
continue
|
|
1164
|
+
draft = drafts_by_path[parent_path]
|
|
1165
|
+
modified = source_order[parent_path][1]
|
|
1166
|
+
if parent is None or modified > parent_modified:
|
|
1167
|
+
parent = draft
|
|
1168
|
+
parent_modified = modified
|
|
1169
|
+
parent_is_canonical = False
|
|
1170
|
+
elif modified == parent_modified and draft != parent:
|
|
1171
|
+
parent = _merge_tied_parent_observations(parent, draft)
|
|
1172
|
+
parent_is_canonical = False
|
|
630
1173
|
if parent is None and latest_reducer is not None:
|
|
631
|
-
parent =
|
|
1174
|
+
parent = drafts_by_path[latest_reducer]
|
|
1175
|
+
|
|
1176
|
+
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
1177
|
+
# Older checkpoints can omit IDs already assigned in their published output.
|
|
1178
|
+
for field in ("deferred", "surfaces"):
|
|
1179
|
+
named_rows: dict[str | None, list[dict[str, Any]]] = {}
|
|
1180
|
+
for _, draft, owner in all_sources:
|
|
1181
|
+
rows = draft["coverage"].get(field, [])
|
|
1182
|
+
named_rows.setdefault(owner, []).extend(
|
|
1183
|
+
row
|
|
1184
|
+
for row in (rows if isinstance(rows, list) else [])
|
|
1185
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1186
|
+
)
|
|
1187
|
+
for _, draft, owner in all_sources:
|
|
1188
|
+
rows = draft["coverage"].get(field, [])
|
|
1189
|
+
if not isinstance(rows, list):
|
|
1190
|
+
continue
|
|
1191
|
+
reserved = {
|
|
1192
|
+
row["id"]
|
|
1193
|
+
for row in rows
|
|
1194
|
+
if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
1195
|
+
}
|
|
1196
|
+
for row in rows:
|
|
1197
|
+
if not isinstance(row, dict) or "id" in row:
|
|
1198
|
+
continue
|
|
1199
|
+
if field == "deferred" and any(
|
|
1200
|
+
key in row for key in ("candidateId", "candidate", "finding")
|
|
1201
|
+
):
|
|
1202
|
+
continue
|
|
1203
|
+
content = {"receiptRefs": [], **row} if field == "surfaces" else row
|
|
1204
|
+
identity = next(
|
|
1205
|
+
(
|
|
1206
|
+
named["id"]
|
|
1207
|
+
for named in named_rows[owner]
|
|
1208
|
+
if named["id"] not in reserved
|
|
1209
|
+
and {
|
|
1210
|
+
**({"receiptRefs": []} if field == "surfaces" else {}),
|
|
1211
|
+
**{key: value for key, value in named.items() if key != "id"},
|
|
1212
|
+
}
|
|
1213
|
+
== content
|
|
1214
|
+
),
|
|
1215
|
+
None,
|
|
1216
|
+
)
|
|
1217
|
+
if identity is not None:
|
|
1218
|
+
row["id"] = identity
|
|
1219
|
+
reserved.add(identity)
|
|
632
1220
|
|
|
633
1221
|
if parent is None and not sources:
|
|
634
1222
|
return None
|
|
@@ -641,6 +1229,13 @@ def merge_saved_results(
|
|
|
641
1229
|
):
|
|
642
1230
|
return None
|
|
643
1231
|
|
|
1232
|
+
if (
|
|
1233
|
+
frozen_source_digests is None
|
|
1234
|
+
or any(_is_source_order_snapshot(path) for path in source_digests)
|
|
1235
|
+
or allow_frozen_legacy_parent
|
|
1236
|
+
):
|
|
1237
|
+
_freeze_source_times(scan_dir, scan_id, source_digests, source_times)
|
|
1238
|
+
|
|
644
1239
|
target_kind = binding["allowedTargetKinds"][0]
|
|
645
1240
|
if (
|
|
646
1241
|
target_kind == "git_worktree"
|
|
@@ -651,15 +1246,9 @@ def merge_saved_results(
|
|
|
651
1246
|
target = {"kind": target_kind, **binding["target"]}
|
|
652
1247
|
if target["kind"] == "git_diff" and "snapshotDigest" not in target:
|
|
653
1248
|
diff_kind = {"commit": "commit", "branch_diff": "range"}[binding["coverageMode"]]
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
+ b"\0"
|
|
658
|
-
+ target["baseRevision"].encode()
|
|
659
|
-
+ b"\0"
|
|
660
|
-
+ target["headRevision"].encode()
|
|
661
|
-
).hexdigest()
|
|
662
|
-
target["snapshotDigest"] = f"codex-security-snapshot/v1:sha256:{digest}"
|
|
1249
|
+
target["snapshotDigest"] = committed_diff_snapshot_digest(
|
|
1250
|
+
diff_kind, target["baseRevision"], target["headRevision"]
|
|
1251
|
+
)
|
|
663
1252
|
manifest = (
|
|
664
1253
|
copy.deepcopy(parent_manifest)
|
|
665
1254
|
if parent_manifest
|
|
@@ -668,11 +1257,31 @@ def merge_saved_results(
|
|
|
668
1257
|
for key in ("sealedAt", "artifacts"):
|
|
669
1258
|
manifest["scan"].pop(key, None)
|
|
670
1259
|
manifest["scan"]["preservedSources"] = source_digests
|
|
1260
|
+
# Completion follows the selected parent, including legacy terminal drafts
|
|
1261
|
+
# that omit the optional marker, rather than an older canonical manifest.
|
|
1262
|
+
if parent is not None:
|
|
1263
|
+
manifest["scan"].pop("complete", None)
|
|
1264
|
+
if "complete" in parent:
|
|
1265
|
+
manifest["scan"]["complete"] = parent["complete"]
|
|
1266
|
+
for key in ("scope", "threatModel"):
|
|
1267
|
+
if isinstance(parent.get(key), dict):
|
|
1268
|
+
manifest["scan"][key] = copy.deepcopy(parent[key])
|
|
1269
|
+
if isinstance(manifest["scan"].get("scope"), dict):
|
|
1270
|
+
manifest["scan"]["scope"].update(copy.deepcopy(binding["scope"]))
|
|
1271
|
+
if frozen_model_source is not None:
|
|
1272
|
+
# Publication retries retain the choice made with the frozen source set.
|
|
1273
|
+
model = drafts_by_path.get(frozen_model_source, {}).get("threatModel")
|
|
1274
|
+
if not isinstance(model, dict):
|
|
1275
|
+
raise ContractError("Frozen stopped-scan model source is unavailable.")
|
|
1276
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1277
|
+
if paths[frozen_model_source] is not None:
|
|
1278
|
+
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1279
|
+
if selected_model_source is not None:
|
|
1280
|
+
selected_model_source[:] = [frozen_model_source]
|
|
671
1281
|
coverage = (
|
|
672
1282
|
copy.deepcopy(parent["coverage"])
|
|
673
1283
|
if parent and parent["coverage"]
|
|
674
1284
|
else {
|
|
675
|
-
"completeness": "partial",
|
|
676
1285
|
"mode": binding["coverageMode"],
|
|
677
1286
|
"inventoryStrategy": "diff"
|
|
678
1287
|
if binding["coverageMode"] in {"commit", "branch_diff", "working_tree"}
|
|
@@ -696,7 +1305,7 @@ def merge_saved_results(
|
|
|
696
1305
|
for field in ("surfaces", "explicitExclusions", "deferred")
|
|
697
1306
|
for item in (coverage.get(field) if isinstance(coverage.get(field), list) else [])
|
|
698
1307
|
}
|
|
699
|
-
if
|
|
1308
|
+
if parent_is_canonical
|
|
700
1309
|
else set()
|
|
701
1310
|
)
|
|
702
1311
|
findings: list[dict[str, Any]] = []
|
|
@@ -725,32 +1334,180 @@ def merge_saved_results(
|
|
|
725
1334
|
)
|
|
726
1335
|
return bool(document["findings"])
|
|
727
1336
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
(
|
|
1337
|
+
source_order["parent"] = (0, parent_modified)
|
|
1338
|
+
deferred_rows = {
|
|
1339
|
+
relative: _deferred_rows(draft["coverage"]) for relative, draft, _ in all_sources
|
|
1340
|
+
}
|
|
1341
|
+
# A legacy source can contain independent tasks with the same explicit ID.
|
|
1342
|
+
# Later rewrites cannot make an ID-only closure identify one of those tasks.
|
|
1343
|
+
ambiguous_deferred: set[tuple[str | None, str]] = set()
|
|
1344
|
+
for relative, _, owner in all_sources:
|
|
1345
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1346
|
+
candidate_aliases = {
|
|
1347
|
+
identity
|
|
1348
|
+
for row in deferred_rows[relative]
|
|
1349
|
+
if isinstance(row, dict)
|
|
1350
|
+
and any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1351
|
+
for identity in (row.get("id"), row.get("candidateId"))
|
|
1352
|
+
if isinstance(identity, str)
|
|
1353
|
+
}
|
|
1354
|
+
for row in deferred_rows[relative]:
|
|
1355
|
+
if not isinstance(row, dict) or not isinstance(identity := row.get("id"), str):
|
|
1356
|
+
continue
|
|
1357
|
+
if any(key in row for key in ("candidateId", "candidate", "finding")):
|
|
1358
|
+
continue
|
|
1359
|
+
if identity in candidate_aliases or (identity in by_id and row != by_id[identity]):
|
|
1360
|
+
ambiguous_deferred.add((owner, identity))
|
|
1361
|
+
by_id[identity] = row
|
|
1362
|
+
current_drafts = ([("parent", parent, None)] if parent else []) + [
|
|
1363
|
+
source for source in sources if source[0] in current_results | selected_observations.keys()
|
|
731
1364
|
]
|
|
1365
|
+
# Generic closures belong to one logical scan or worker, just like candidates.
|
|
1366
|
+
# Keep them when recovering a terminal checkpoint without its canonical write.
|
|
1367
|
+
closed_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1368
|
+
|
|
1369
|
+
candidate_ids: set[tuple[str | None, str]] = set()
|
|
1370
|
+
accepted_deferred_orders: dict[tuple[str | None, str], tuple[int, int]] = {}
|
|
1371
|
+
active_deferred: dict[tuple[str | None, str], tuple[tuple[int, int], dict[str, Any], str]] = {}
|
|
1372
|
+
coverage_schema = _read_json(
|
|
1373
|
+
Path(__file__).resolve().parent.parent / "schemas" / "coverage.schema.json"
|
|
1374
|
+
)["properties"]
|
|
1375
|
+
for relative, draft, owner in all_sources:
|
|
1376
|
+
order = source_order[relative]
|
|
1377
|
+
for item in deferred_rows[relative]:
|
|
1378
|
+
if not isinstance(item, dict):
|
|
1379
|
+
continue
|
|
1380
|
+
if isinstance(item.get("candidateId"), str) or "candidate" in item or "finding" in item:
|
|
1381
|
+
candidate_ids.update(
|
|
1382
|
+
(owner, identity)
|
|
1383
|
+
for identity in (item.get("id"), item.get("candidateId"))
|
|
1384
|
+
if isinstance(identity, str)
|
|
1385
|
+
)
|
|
1386
|
+
identity = item.get("id") or item.get("candidateId")
|
|
1387
|
+
if isinstance(identity, str):
|
|
1388
|
+
key = (owner, identity)
|
|
1389
|
+
previous = active_deferred.get(key)
|
|
1390
|
+
if previous is None or order > previous[0]:
|
|
1391
|
+
active_deferred[key] = (order, item, relative)
|
|
1392
|
+
if (
|
|
1393
|
+
relative == "parent"
|
|
1394
|
+
or relative in current_results
|
|
1395
|
+
or relative in selected_observations
|
|
1396
|
+
):
|
|
1397
|
+
try:
|
|
1398
|
+
# A malformed update cannot discard valid saved evidence.
|
|
1399
|
+
_validate_schema_node(
|
|
1400
|
+
item, coverage_schema["deferred"]["items"], "coverage.deferred"
|
|
1401
|
+
)
|
|
1402
|
+
except ContractError:
|
|
1403
|
+
continue
|
|
1404
|
+
accepted_deferred_orders[key] = max(
|
|
1405
|
+
accepted_deferred_orders.get(key, order), order
|
|
1406
|
+
)
|
|
1407
|
+
for closure in _resolved_deferred_rows(draft, coverage_schema["resolvedDeferred"]):
|
|
1408
|
+
key = (owner, closure["id"])
|
|
1409
|
+
previous = closed_deferred.get(key)
|
|
1410
|
+
if previous is None or order > previous[0]:
|
|
1411
|
+
# The parent comes first, preserving its reason on equal timestamps.
|
|
1412
|
+
closed_deferred[key] = (order, closure, relative)
|
|
1413
|
+
reopened_rows: list[tuple[str | None, dict[str, Any]]] = []
|
|
1414
|
+
reopened_generic: set[tuple[str | None, str]] = set()
|
|
1415
|
+
for key, (order, _, _) in list(closed_deferred.items()):
|
|
1416
|
+
# Equal timestamps cannot distinguish closure from reopened work.
|
|
1417
|
+
reopened = [
|
|
1418
|
+
active
|
|
1419
|
+
for (owner, identity), active in active_deferred.items()
|
|
1420
|
+
if owner == key[0]
|
|
1421
|
+
and (identity == key[1] or active[1].get("candidateId") == key[1])
|
|
1422
|
+
and active[0] >= order
|
|
1423
|
+
]
|
|
1424
|
+
if key in candidate_ids or key in ambiguous_deferred or reopened:
|
|
1425
|
+
del closed_deferred[key]
|
|
1426
|
+
for _, item, _ in reopened:
|
|
1427
|
+
reopened_rows.append((key[0], item))
|
|
1428
|
+
if key not in candidate_ids:
|
|
1429
|
+
reopened_generic.add(key)
|
|
1430
|
+
# Retain every distinct ambiguous task even when a terminal result supersedes
|
|
1431
|
+
# its source. Existing output ID normalization keeps each row distinct.
|
|
1432
|
+
for relative, _, owner in all_sources:
|
|
1433
|
+
for row in deferred_rows[relative]:
|
|
1434
|
+
if (
|
|
1435
|
+
isinstance(row, dict)
|
|
1436
|
+
and isinstance(identity := row.get("id"), str)
|
|
1437
|
+
and (owner, identity) in ambiguous_deferred
|
|
1438
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1439
|
+
and (owner, row) not in reopened_rows
|
|
1440
|
+
):
|
|
1441
|
+
reopened_rows.append((owner, row))
|
|
1442
|
+
parent_closures = [
|
|
1443
|
+
closure for (owner, _), (_, closure, _) in closed_deferred.items() if owner is None
|
|
1444
|
+
]
|
|
1445
|
+
coverage.pop("resolvedDeferred", None)
|
|
1446
|
+
if parent_closures:
|
|
1447
|
+
coverage["resolvedDeferred"] = copy.deepcopy(parent_closures)
|
|
1448
|
+
if parent:
|
|
1449
|
+
coverage["deferred"] = [
|
|
1450
|
+
row
|
|
1451
|
+
for row in deferred_rows["parent"]
|
|
1452
|
+
if not isinstance(row, dict)
|
|
1453
|
+
or not isinstance(row.get("id"), str)
|
|
1454
|
+
or (None, row["id"]) not in closed_deferred
|
|
1455
|
+
]
|
|
732
1456
|
resolved: dict[tuple[str | None, str], str] = {}
|
|
733
|
-
|
|
1457
|
+
|
|
1458
|
+
ordered_candidates = {
|
|
1459
|
+
(owner, identity)
|
|
1460
|
+
for owner, row in reopened_rows
|
|
1461
|
+
if (identity := _deferred_candidate_id(row, owner, ambiguous_deferred)) is not None
|
|
1462
|
+
and (owner, identity) in candidate_ids
|
|
1463
|
+
}
|
|
1464
|
+
ordered_outcomes: dict[tuple[str | None, str], tuple[tuple[int, int], str]] = {}
|
|
1465
|
+
|
|
1466
|
+
outcomes: list[tuple[str, str | None, str, str]] = []
|
|
1467
|
+
for relative, draft, owner in current_drafts:
|
|
734
1468
|
for finding in draft["findings"]:
|
|
735
1469
|
if (
|
|
736
1470
|
isinstance(finding, dict)
|
|
737
1471
|
and valid_finding(finding)
|
|
738
1472
|
and (candidate_id := finding_candidate_id(finding))
|
|
739
1473
|
):
|
|
740
|
-
|
|
1474
|
+
outcomes.append((relative, owner, candidate_id, "reported"))
|
|
741
1475
|
for field in ("surfaces", "explicitExclusions"):
|
|
742
1476
|
items = draft["coverage"].get(field, [])
|
|
743
1477
|
for item in items if isinstance(items, list) else []:
|
|
744
1478
|
if (
|
|
745
1479
|
isinstance(item, dict)
|
|
746
1480
|
and isinstance(item.get("candidateId"), str)
|
|
747
|
-
and item.get("disposition") in {"
|
|
1481
|
+
and item.get("disposition") in {"rejected", "not_applicable"}
|
|
748
1482
|
):
|
|
749
|
-
|
|
1483
|
+
outcomes.append((relative, owner, item["candidateId"], item["disposition"]))
|
|
1484
|
+
ordered_candidates.update(
|
|
1485
|
+
(owner, candidate_id)
|
|
1486
|
+
for relative, owner, candidate_id, _ in outcomes
|
|
1487
|
+
if owner is not None and relative in selected_observations
|
|
1488
|
+
)
|
|
1489
|
+
# Reopened work and selected checkpoint outcomes follow the saved source order.
|
|
1490
|
+
for relative, owner, candidate_id, disposition in outcomes:
|
|
1491
|
+
key = (owner, candidate_id)
|
|
1492
|
+
if key not in ordered_candidates:
|
|
1493
|
+
if relative == "parent" or relative in current_results:
|
|
1494
|
+
resolved.setdefault(key, disposition)
|
|
1495
|
+
continue
|
|
1496
|
+
order = source_order[relative]
|
|
1497
|
+
if any(
|
|
1498
|
+
saved_owner == owner
|
|
1499
|
+
and _deferred_candidate_id(row, owner, ambiguous_deferred) == candidate_id
|
|
1500
|
+
and modified >= order
|
|
1501
|
+
for (saved_owner, _), (modified, row, _) in active_deferred.items()
|
|
1502
|
+
):
|
|
1503
|
+
continue
|
|
1504
|
+
if key not in ordered_outcomes or order > ordered_outcomes[key][0]:
|
|
1505
|
+
resolved[key] = disposition
|
|
1506
|
+
ordered_outcomes[key] = (order, relative)
|
|
750
1507
|
# Only the current parent may claim that another worker finding was absorbed.
|
|
751
1508
|
# A superseded checkpoint must not suppress a newer independent result.
|
|
752
|
-
|
|
753
|
-
for finding in
|
|
1509
|
+
if parent:
|
|
1510
|
+
for finding in parent["findings"]:
|
|
754
1511
|
if valid_finding(finding):
|
|
755
1512
|
canonical_key = _finding_key(finding)
|
|
756
1513
|
for retained in _retained_findings(finding):
|
|
@@ -787,43 +1544,201 @@ def merge_saved_results(
|
|
|
787
1544
|
represented_candidate_history.setdefault(candidate_key, set()).add(
|
|
788
1545
|
_digest(_finding_content(original["finding"]))
|
|
789
1546
|
)
|
|
790
|
-
|
|
1547
|
+
replaced_surfaces, surface_updates = _generic_surface_updates(
|
|
1548
|
+
all_sources,
|
|
1549
|
+
source_order,
|
|
1550
|
+
closed_deferred,
|
|
1551
|
+
active_deferred,
|
|
1552
|
+
resolved,
|
|
1553
|
+
reopened_generic,
|
|
1554
|
+
coverage_schema["surfaces"]["items"],
|
|
1555
|
+
deferred_rows,
|
|
1556
|
+
ambiguous_deferred,
|
|
1557
|
+
)
|
|
1558
|
+
replaced_rows = {
|
|
1559
|
+
"surfaces": replaced_surfaces,
|
|
1560
|
+
"deferred": {
|
|
1561
|
+
id(row)
|
|
1562
|
+
for relative, _, owner in all_sources
|
|
1563
|
+
for row in deferred_rows[relative]
|
|
1564
|
+
if isinstance(row, dict)
|
|
1565
|
+
and isinstance(identity := row.get("id"), str)
|
|
1566
|
+
and (key := (owner, identity)) not in candidate_ids
|
|
1567
|
+
and key not in ambiguous_deferred
|
|
1568
|
+
and (updated := accepted_deferred_orders.get(key)) is not None
|
|
1569
|
+
and updated > source_order[relative]
|
|
1570
|
+
},
|
|
1571
|
+
}
|
|
1572
|
+
for field, replaced in replaced_rows.items():
|
|
1573
|
+
if parent and isinstance(coverage.get(field), list):
|
|
1574
|
+
previous = parent["coverage"].get(field, [])
|
|
1575
|
+
if isinstance(previous, list):
|
|
1576
|
+
removed_parent = [row for row in previous if id(row) in replaced]
|
|
1577
|
+
coverage[field] = [row for row in coverage[field] if row not in removed_parent]
|
|
1578
|
+
if isinstance(coverage.get("surfaces"), list):
|
|
1579
|
+
for surface in surface_updates:
|
|
1580
|
+
if surface not in coverage["surfaces"]:
|
|
1581
|
+
coverage["surfaces"].append(surface)
|
|
1582
|
+
for surface in coverage["surfaces"]:
|
|
1583
|
+
if isinstance(surface, dict):
|
|
1584
|
+
surface.setdefault("receiptRefs", [])
|
|
1585
|
+
|
|
1586
|
+
# Reopened work survives a superseded checkpoint, but current candidate
|
|
1587
|
+
# outcomes still apply. Parent closures cannot remove another worker's row.
|
|
1588
|
+
for owner, item in reopened_rows:
|
|
1589
|
+
if (identity := _deferred_candidate_id(item, owner, ambiguous_deferred)) is not None and (
|
|
1590
|
+
owner,
|
|
1591
|
+
identity,
|
|
1592
|
+
) in resolved:
|
|
1593
|
+
continue
|
|
1594
|
+
pending = coverage.setdefault("deferred", [])
|
|
1595
|
+
if isinstance(pending, list) and item not in pending:
|
|
1596
|
+
pending.append(copy.deepcopy(item))
|
|
1597
|
+
ambiguous_surface_ids = {
|
|
1598
|
+
(owner, identity)
|
|
1599
|
+
for owner, row in reopened_rows
|
|
1600
|
+
if isinstance(row.get("id"), str)
|
|
1601
|
+
and (owner, row["id"]) in ambiguous_deferred
|
|
1602
|
+
and not any(key in row for key in ("candidateId", "candidate", "finding"))
|
|
1603
|
+
for identity in [
|
|
1604
|
+
row["id"],
|
|
1605
|
+
*(row.get("surfaceIds", []) if isinstance(row.get("surfaceIds", []), list) else []),
|
|
1606
|
+
]
|
|
1607
|
+
if isinstance(identity, str)
|
|
1608
|
+
}
|
|
1609
|
+
for _, draft, owner in all_sources:
|
|
1610
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1611
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
1612
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1613
|
+
if not isinstance(surface, dict) or not isinstance(identity := surface.get("id"), str):
|
|
1614
|
+
continue
|
|
1615
|
+
if identity in by_id and surface != by_id[identity]:
|
|
1616
|
+
ambiguous_surface_ids.add((owner, identity))
|
|
1617
|
+
by_id[identity] = surface
|
|
1618
|
+
for _, draft, owner in all_sources:
|
|
1619
|
+
surfaces = draft["coverage"].get("surfaces", [])
|
|
1620
|
+
for surface in surfaces if isinstance(surfaces, list) else []:
|
|
1621
|
+
if (
|
|
1622
|
+
isinstance(surface, dict)
|
|
1623
|
+
and isinstance(surface.get("id"), str)
|
|
1624
|
+
and (owner, surface["id"]) in ambiguous_surface_ids
|
|
1625
|
+
and surface.get("disposition") == "needs_follow_up"
|
|
1626
|
+
and not any(key in surface for key in ("candidateId", "candidate", "finding"))
|
|
1627
|
+
and isinstance(coverage.get("surfaces"), list)
|
|
1628
|
+
):
|
|
1629
|
+
retained_surface = {**surface, "receiptRefs": surface.get("receiptRefs", [])}
|
|
1630
|
+
if retained_surface not in coverage["surfaces"]:
|
|
1631
|
+
coverage["surfaces"].append(copy.deepcopy(retained_surface))
|
|
1632
|
+
selected_terminal_orders: dict[str, tuple[int, int]] = {}
|
|
1633
|
+
for relative, draft, worker_id in sources:
|
|
1634
|
+
if (
|
|
1635
|
+
worker_id is not None
|
|
1636
|
+
and relative in selected_observations
|
|
1637
|
+
and draft.get("complete") is not False
|
|
1638
|
+
):
|
|
1639
|
+
order = source_order[relative]
|
|
1640
|
+
selected_terminal_orders[worker_id] = max(
|
|
1641
|
+
selected_terminal_orders.get(worker_id, order), order
|
|
1642
|
+
)
|
|
1643
|
+
terminal_worker_orders: dict[str | None, tuple[int, int]] = {}
|
|
1644
|
+
for relative, draft, worker_id in sources:
|
|
1645
|
+
if relative in current_results and draft.get("complete") is not False:
|
|
1646
|
+
order = source_order[relative]
|
|
1647
|
+
terminal_worker_orders[worker_id] = max(
|
|
1648
|
+
terminal_worker_orders.get(worker_id, order), order
|
|
1649
|
+
)
|
|
791
1650
|
for relative, draft, worker_id in all_sources:
|
|
1651
|
+
worker_result_order = terminal_worker_orders.get(worker_id)
|
|
1652
|
+
selected_coverage_superseded = worker_id in selected_terminal_orders and (
|
|
1653
|
+
source_order[relative] < selected_terminal_orders[worker_id]
|
|
1654
|
+
or (
|
|
1655
|
+
relative in current_results
|
|
1656
|
+
and source_order[relative] <= selected_terminal_orders[worker_id]
|
|
1657
|
+
)
|
|
1658
|
+
)
|
|
1659
|
+
selected_candidates = {
|
|
1660
|
+
candidate_id
|
|
1661
|
+
for (owner, candidate_id), (_, source) in ordered_outcomes.items()
|
|
1662
|
+
if owner == worker_id and source == relative
|
|
1663
|
+
}
|
|
792
1664
|
superseded = (
|
|
793
1665
|
worker_id is None
|
|
794
1666
|
and parent is not None
|
|
795
1667
|
and parent.get("complete") is not False
|
|
796
1668
|
and relative != "parent"
|
|
1669
|
+
and source_order[relative] <= (0, parent_modified)
|
|
797
1670
|
and (not stopped_parent_seal or relative in parent_preserved_sources)
|
|
798
1671
|
) or (
|
|
799
1672
|
relative not in current_results
|
|
800
|
-
and
|
|
801
|
-
saved_worker == worker_id
|
|
802
|
-
and saved_path in current_results
|
|
803
|
-
and current.get("complete") is not False
|
|
804
|
-
for saved_path, current, saved_worker in sources
|
|
805
|
-
)
|
|
806
|
-
)
|
|
807
|
-
if (
|
|
808
|
-
(relative != "parent" or not parent_manifest)
|
|
809
|
-
and not superseded
|
|
1673
|
+
and worker_result_order is not None
|
|
810
1674
|
and (
|
|
811
|
-
|
|
812
|
-
or
|
|
1675
|
+
relative not in selected_observations
|
|
1676
|
+
or source_order[relative] < worker_result_order
|
|
813
1677
|
)
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
1678
|
+
)
|
|
1679
|
+
# A failed result write can leave pending work outside the accepted result.
|
|
1680
|
+
accepted_order = (
|
|
1681
|
+
(0, parent_modified)
|
|
1682
|
+
if worker_id is None and parent is not None
|
|
1683
|
+
else worker_result_order
|
|
1684
|
+
)
|
|
1685
|
+
retain_pending = (
|
|
1686
|
+
superseded
|
|
1687
|
+
and (worker_id in headed_workers or (worker_id is None and parent_heads))
|
|
1688
|
+
and accepted_order is not None
|
|
1689
|
+
and source_order[relative] >= accepted_order
|
|
1690
|
+
and any(isinstance(row, dict) for row in deferred_rows[relative])
|
|
1691
|
+
)
|
|
1692
|
+
skip_superseded_findings = (
|
|
818
1693
|
superseded
|
|
819
1694
|
and not stopped
|
|
820
1695
|
and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
|
|
821
|
-
)
|
|
1696
|
+
)
|
|
1697
|
+
if skip_superseded_findings and not selected_candidates and not retain_pending:
|
|
822
1698
|
continue
|
|
823
|
-
if
|
|
824
|
-
|
|
1699
|
+
if (
|
|
1700
|
+
not skip_superseded_findings
|
|
1701
|
+
and "threatModel" not in manifest["scan"]
|
|
1702
|
+
and isinstance(draft.get("threatModel"), dict)
|
|
1703
|
+
):
|
|
1704
|
+
model = draft["threatModel"]
|
|
1705
|
+
model_path = relative
|
|
1706
|
+
checkpoint_dir = Path(relative).parent
|
|
1707
|
+
prefer_worker_head = worker_id is not None and (
|
|
1708
|
+
checkpoint_dir.name == "checkpoints" or draft.get("complete") is False
|
|
1709
|
+
)
|
|
1710
|
+
if worker_id is not None:
|
|
1711
|
+
if checkpoint_dir.name != "checkpoints":
|
|
1712
|
+
checkpoint_dir /= "checkpoints"
|
|
1713
|
+
selected_models = [
|
|
1714
|
+
path
|
|
1715
|
+
for path in selected_observations
|
|
1716
|
+
if Path(path).parent == checkpoint_dir
|
|
1717
|
+
and isinstance(drafts_by_path[path].get("threatModel"), dict)
|
|
1718
|
+
]
|
|
1719
|
+
if selected_models:
|
|
1720
|
+
head_path = max(selected_models, key=source_order.__getitem__)
|
|
1721
|
+
current = drafts_by_path[head_path]
|
|
1722
|
+
# A terminal checkpoint is committed before result.json is replaced.
|
|
1723
|
+
# Use the admitted observation, including its frozen ordering on retries.
|
|
1724
|
+
if not prefer_worker_head and current.get("complete") is not False:
|
|
1725
|
+
prefer_worker_head = source_order[head_path] >= source_order[relative]
|
|
1726
|
+
if prefer_worker_head:
|
|
1727
|
+
model = current["threatModel"]
|
|
1728
|
+
model_path = head_path
|
|
1729
|
+
manifest["scan"]["threatModel"] = copy.deepcopy(model)
|
|
1730
|
+
if worker_id is not None:
|
|
1731
|
+
manifest["scan"]["threatModel"]["origin"] = "recovered"
|
|
1732
|
+
if selected_model_source is not None and worker_id is not None:
|
|
1733
|
+
selected_model_source[:] = [model_path]
|
|
825
1734
|
for value in draft["findings"]:
|
|
826
|
-
if
|
|
1735
|
+
if skip_superseded_findings and not (
|
|
1736
|
+
isinstance(value, dict)
|
|
1737
|
+
and (candidate_id := finding_candidate_id(value)) in selected_candidates
|
|
1738
|
+
and resolved.get((worker_id, candidate_id)) == "reported"
|
|
1739
|
+
):
|
|
1740
|
+
continue
|
|
1741
|
+
if relative == "parent" and parent_is_canonical:
|
|
827
1742
|
finding = copy.deepcopy(value)
|
|
828
1743
|
_ensure_finding_identity(finding, candidate_only=True)
|
|
829
1744
|
provenance = finding.get("provenance") if isinstance(finding, dict) else None
|
|
@@ -846,7 +1761,6 @@ def merge_saved_results(
|
|
|
846
1761
|
if not isinstance(value, dict):
|
|
847
1762
|
warnings.append(f"Retained malformed finding evidence in {relative}.")
|
|
848
1763
|
continue
|
|
849
|
-
source_value = copy.deepcopy(value)
|
|
850
1764
|
finding = copy.deepcopy(value)
|
|
851
1765
|
candidate_id = finding_candidate_id(finding)
|
|
852
1766
|
if relative != "parent" and resolved.get((worker_id, candidate_id)) in {
|
|
@@ -880,7 +1794,6 @@ def merge_saved_results(
|
|
|
880
1794
|
for location in locations
|
|
881
1795
|
):
|
|
882
1796
|
warnings.append(f"Skipped out-of-scope finding from {relative}.")
|
|
883
|
-
coverage["completeness"] = "partial"
|
|
884
1797
|
continue
|
|
885
1798
|
provenance = finding.setdefault("provenance", {"source": "local_plugin"})
|
|
886
1799
|
if not isinstance(provenance, dict):
|
|
@@ -909,24 +1822,19 @@ def merge_saved_results(
|
|
|
909
1822
|
historical_contents = set()
|
|
910
1823
|
if mapped_key is not None:
|
|
911
1824
|
key = mapped_key
|
|
912
|
-
represented_by_parent = (
|
|
913
|
-
_digest(_finding_content(source_value)) in historical_contents
|
|
914
|
-
)
|
|
1825
|
+
represented_by_parent = _digest(_finding_content(value)) in historical_contents
|
|
915
1826
|
if key in finding_positions:
|
|
916
1827
|
retained = findings[finding_positions[key]]
|
|
917
1828
|
if finding != retained:
|
|
918
|
-
if represented_by_parent
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
"previousFindings", []
|
|
922
|
-
)
|
|
923
|
-
elif _finding_strength(finding) > _finding_strength(retained):
|
|
1829
|
+
if not represented_by_parent and _finding_strength(finding) > _finding_strength(
|
|
1830
|
+
retained
|
|
1831
|
+
):
|
|
924
1832
|
previous = copy.deepcopy(retained)
|
|
925
1833
|
previous_history = previous["provenance"].pop("previousFindings", [])
|
|
926
1834
|
retained = finding
|
|
927
1835
|
findings[finding_positions[key]] = retained
|
|
928
1836
|
else:
|
|
929
|
-
previous = copy.deepcopy(
|
|
1837
|
+
previous = copy.deepcopy(value)
|
|
930
1838
|
previous_history = previous.get("provenance", {}).pop(
|
|
931
1839
|
"previousFindings", []
|
|
932
1840
|
)
|
|
@@ -960,10 +1868,14 @@ def merge_saved_results(
|
|
|
960
1868
|
continue
|
|
961
1869
|
finding_positions[key] = len(findings)
|
|
962
1870
|
findings.append(finding)
|
|
963
|
-
if superseded:
|
|
1871
|
+
if superseded and not selected_candidates and not retain_pending:
|
|
964
1872
|
continue
|
|
965
1873
|
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions"):
|
|
1874
|
+
if superseded and field not in {"surfaces", "explicitExclusions", "deferred"}:
|
|
1875
|
+
continue
|
|
966
1876
|
items = draft["coverage"].get(field, [])
|
|
1877
|
+
if field == "deferred":
|
|
1878
|
+
items = deferred_rows[relative]
|
|
967
1879
|
if not isinstance(items, list):
|
|
968
1880
|
continue
|
|
969
1881
|
output = coverage.setdefault(field, [])
|
|
@@ -972,6 +1884,40 @@ def merge_saved_results(
|
|
|
972
1884
|
# recovery and warnings rather than silently changing its contract.
|
|
973
1885
|
continue
|
|
974
1886
|
for item in items:
|
|
1887
|
+
# A selected terminal checkpoint replaces ordinary coverage, while
|
|
1888
|
+
# candidate evidence and pending tasks retain their own reconciliation.
|
|
1889
|
+
if (
|
|
1890
|
+
selected_coverage_superseded
|
|
1891
|
+
and field != "deferred"
|
|
1892
|
+
and not (
|
|
1893
|
+
isinstance(item, dict)
|
|
1894
|
+
and any(key in item for key in ("candidateId", "candidate", "finding"))
|
|
1895
|
+
)
|
|
1896
|
+
):
|
|
1897
|
+
continue
|
|
1898
|
+
# A selected outcome must retain its evidence even if its result write failed.
|
|
1899
|
+
if superseded and not (
|
|
1900
|
+
isinstance(item, dict)
|
|
1901
|
+
and (
|
|
1902
|
+
(field == "deferred" and retain_pending)
|
|
1903
|
+
or (
|
|
1904
|
+
isinstance(item.get("candidateId"), str)
|
|
1905
|
+
and item["candidateId"] in selected_candidates
|
|
1906
|
+
and item.get("disposition")
|
|
1907
|
+
== resolved.get((worker_id, item["candidateId"]))
|
|
1908
|
+
)
|
|
1909
|
+
)
|
|
1910
|
+
):
|
|
1911
|
+
continue
|
|
1912
|
+
if id(item) in replaced_rows.get(field, ()):
|
|
1913
|
+
continue
|
|
1914
|
+
if (
|
|
1915
|
+
field == "deferred"
|
|
1916
|
+
and isinstance(item, dict)
|
|
1917
|
+
and isinstance(item.get("id"), str)
|
|
1918
|
+
and (worker_id, item["id"]) in closed_deferred
|
|
1919
|
+
):
|
|
1920
|
+
continue
|
|
975
1921
|
if field == "openQuestions" and isinstance(item, str):
|
|
976
1922
|
item = {"question": item.strip()}
|
|
977
1923
|
if (
|
|
@@ -995,14 +1941,20 @@ def merge_saved_results(
|
|
|
995
1941
|
history.append(copy.deepcopy(finding))
|
|
996
1942
|
if (
|
|
997
1943
|
isinstance(item, dict)
|
|
998
|
-
and (
|
|
1944
|
+
and isinstance(
|
|
1945
|
+
identity := _deferred_candidate_id(item, worker_id, ambiguous_deferred)
|
|
1946
|
+
if field == "deferred"
|
|
1947
|
+
else item.get("candidateId"),
|
|
1948
|
+
str,
|
|
1949
|
+
)
|
|
1950
|
+
and (worker_id, identity) in resolved
|
|
999
1951
|
and (field == "deferred" or item.get("disposition") == "needs_follow_up")
|
|
1000
1952
|
):
|
|
1001
1953
|
continue
|
|
1954
|
+
if field == "surfaces" and isinstance(item, dict):
|
|
1955
|
+
item = {**item, "receiptRefs": item.get("receiptRefs", [])}
|
|
1002
1956
|
if isinstance(item, dict) and "id" not in item:
|
|
1003
1957
|
semantic_item = dict(item)
|
|
1004
|
-
if field == "surfaces":
|
|
1005
|
-
semantic_item.setdefault("receiptRefs", [])
|
|
1006
1958
|
if any(
|
|
1007
1959
|
isinstance(existing, dict)
|
|
1008
1960
|
and {key: value for key, value in existing.items() if key != "id"}
|
|
@@ -1036,14 +1988,16 @@ def merge_saved_results(
|
|
|
1036
1988
|
if isinstance(item.get("id"), str):
|
|
1037
1989
|
used.add(item["id"])
|
|
1038
1990
|
continue
|
|
1039
|
-
item.setdefault("id",
|
|
1991
|
+
item.setdefault("id", _saved_coverage_id(item))
|
|
1992
|
+
if not isinstance(item["id"], str):
|
|
1993
|
+
# Preserve malformed rows for per-record recovery, including frozen replay.
|
|
1994
|
+
continue
|
|
1040
1995
|
if item["id"] in used:
|
|
1041
1996
|
item["id"] = f"{item['id']}-{_digest(item)[:16]}"
|
|
1042
1997
|
used.add(item["id"])
|
|
1043
1998
|
if field == "surfaces":
|
|
1044
1999
|
item.setdefault("receiptRefs", [])
|
|
1045
|
-
|
|
1046
|
-
coverage["completeness"] = "partial"
|
|
2000
|
+
coverage.pop("completeness", None)
|
|
1047
2001
|
if stopped:
|
|
1048
2002
|
if not isinstance(coverage.get("deferred"), list):
|
|
1049
2003
|
coverage["deferred"] = []
|
|
@@ -1059,16 +2013,16 @@ def coverage_for_comparison(db: Any, scan: Any) -> dict[str, Any]:
|
|
|
1059
2013
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1060
2014
|
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
1061
2015
|
try:
|
|
1062
|
-
|
|
2016
|
+
prepared = _prepare_scan_finalization(scan_dir)
|
|
1063
2017
|
except ContractError as exc:
|
|
1064
2018
|
raise SystemExit(str(exc)) from exc
|
|
1065
|
-
if not was_sealed or manifest["scan"]["id"] != scan["id"]:
|
|
2019
|
+
if not prepared.was_sealed or prepared.manifest["scan"]["id"] != scan["id"]:
|
|
1066
2020
|
raise SystemExit("Only sealed scans can be compared.")
|
|
1067
|
-
return coverage
|
|
2021
|
+
return prepared.coverage
|
|
1068
2022
|
|
|
1069
2023
|
|
|
1070
|
-
def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
|
|
1071
|
-
snapshots: dict[str, bytes | None] = {}
|
|
2024
|
+
def _snapshot_published_outputs(scan_dir: Path) -> dict[str, tuple[bytes, os.stat_result] | None]:
|
|
2025
|
+
snapshots: dict[str, tuple[bytes, os.stat_result] | None] = {}
|
|
1072
2026
|
for relative in _PUBLISHED_OUTPUTS:
|
|
1073
2027
|
descriptor = -1
|
|
1074
2028
|
try:
|
|
@@ -1077,10 +2031,14 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
|
|
|
1077
2031
|
)
|
|
1078
2032
|
with os.fdopen(descriptor, "rb") as handle:
|
|
1079
2033
|
descriptor = -1
|
|
1080
|
-
|
|
2034
|
+
metadata = os.fstat(handle.fileno())
|
|
2035
|
+
snapshots[relative] = (handle.read(), metadata)
|
|
1081
2036
|
except ContractError:
|
|
1082
2037
|
path = scan_dir / relative
|
|
1083
2038
|
if path.exists() or path.is_symlink():
|
|
2039
|
+
if relative == "threatmodel.md":
|
|
2040
|
+
# This optional projection will not replace an unsafe destination.
|
|
2041
|
+
continue
|
|
1084
2042
|
raise
|
|
1085
2043
|
snapshots[relative] = None
|
|
1086
2044
|
finally:
|
|
@@ -1089,14 +2047,80 @@ def _snapshot_published_outputs(scan_dir: Path) -> dict[str, bytes | None]:
|
|
|
1089
2047
|
return snapshots
|
|
1090
2048
|
|
|
1091
2049
|
|
|
1092
|
-
def _restore_published_outputs(
|
|
1093
|
-
|
|
1094
|
-
|
|
2050
|
+
def _restore_published_outputs(
|
|
2051
|
+
scan_dir: Path, snapshots: dict[str, tuple[bytes, os.stat_result] | None]
|
|
2052
|
+
) -> None:
|
|
2053
|
+
for relative, snapshot in snapshots.items():
|
|
2054
|
+
if snapshot is None:
|
|
1095
2055
|
path = scan_dir / relative
|
|
1096
2056
|
if path.exists() or path.is_symlink():
|
|
1097
2057
|
_remove_scan_local_file_if_exists(scan_dir, relative)
|
|
1098
2058
|
else:
|
|
2059
|
+
contents, metadata = snapshot
|
|
1099
2060
|
write_scan_local_bytes(scan_dir, relative, contents)
|
|
2061
|
+
# Saved-result selection uses mtime; rollback must preserve its ordering.
|
|
2062
|
+
os.utime(scan_dir / relative, ns=(metadata.st_atime_ns, metadata.st_mtime_ns))
|
|
2063
|
+
|
|
2064
|
+
|
|
2065
|
+
@contextmanager
|
|
2066
|
+
def preserve_deep_deadline_draft(db: Any, connection: Any, scan: Any) -> Iterator[None]:
|
|
2067
|
+
"""Recover a settled, zero-accepted Deep deadline without sealing the parent."""
|
|
2068
|
+
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2069
|
+
snapshots = _snapshot_published_outputs(scan_dir)
|
|
2070
|
+
warnings = json.loads(scan["completion_warnings_json"])
|
|
2071
|
+
binding = db.workbench_completion_binding(scan, db.now())
|
|
2072
|
+
with preserve_parent_head_on_error(scan_dir):
|
|
2073
|
+
try:
|
|
2074
|
+
documents = merge_saved_results(
|
|
2075
|
+
scan_dir,
|
|
2076
|
+
scan["id"],
|
|
2077
|
+
binding,
|
|
2078
|
+
connection.execute(
|
|
2079
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ? ORDER BY created_at, id",
|
|
2080
|
+
(scan["id"],),
|
|
2081
|
+
).fetchall(),
|
|
2082
|
+
warnings,
|
|
2083
|
+
stopped=True,
|
|
2084
|
+
reason=(
|
|
2085
|
+
"The configured discovery time limit elapsed before any source review completed. "
|
|
2086
|
+
"Saved findings and pending review were preserved without a completed reduction."
|
|
2087
|
+
),
|
|
2088
|
+
)
|
|
2089
|
+
if documents is None:
|
|
2090
|
+
raise ContractError("Deep Scan deadline has no saved parent draft.")
|
|
2091
|
+
documents[0]["scan"].pop("complete", None)
|
|
2092
|
+
# Validate recovered evidence against the current finding contract,
|
|
2093
|
+
# scope, source lineage, and receipts before admitting this draft.
|
|
2094
|
+
prepared = _prepare_scan_finalization(
|
|
2095
|
+
scan_dir,
|
|
2096
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
2097
|
+
completion_binding=binding,
|
|
2098
|
+
recover_unsealed=True,
|
|
2099
|
+
draft_documents=documents,
|
|
2100
|
+
)
|
|
2101
|
+
warnings.extend(prepared.warnings)
|
|
2102
|
+
manifest, findings, coverage = prepared.manifest, prepared.findings, prepared.coverage
|
|
2103
|
+
manifest["scan"].pop("sealedAt", None)
|
|
2104
|
+
manifest["scan"].pop("artifacts", None)
|
|
2105
|
+
checkpoint = _parent_scan_draft(scan["id"], manifest["scan"], findings, coverage)
|
|
2106
|
+
contents = _encoded(checkpoint)
|
|
2107
|
+
name = f"{hashlib.sha256(contents).hexdigest()}.json"
|
|
2108
|
+
write_scan_local_bytes(scan_dir, f"checkpoints/{name}", contents)
|
|
2109
|
+
write_scan_local_bytes(scan_dir, "checkpoint-head.json", _encoded({"checkpoint": name}))
|
|
2110
|
+
for filename, document in (
|
|
2111
|
+
("findings.json", findings),
|
|
2112
|
+
("coverage.json", coverage),
|
|
2113
|
+
("scan-manifest.json", manifest),
|
|
2114
|
+
):
|
|
2115
|
+
write_scan_local_bytes(scan_dir, filename, _encoded(document))
|
|
2116
|
+
connection.execute(
|
|
2117
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ?",
|
|
2118
|
+
(json.dumps(list(dict.fromkeys(warnings))), scan["id"]),
|
|
2119
|
+
)
|
|
2120
|
+
yield
|
|
2121
|
+
except BaseException:
|
|
2122
|
+
_restore_published_outputs(scan_dir, snapshots)
|
|
2123
|
+
raise
|
|
1100
2124
|
|
|
1101
2125
|
|
|
1102
2126
|
def preserve_scan_results_locked(
|
|
@@ -1112,13 +2136,17 @@ def preserve_scan_results_locked(
|
|
|
1112
2136
|
if scan["status"] != "failed":
|
|
1113
2137
|
return False
|
|
1114
2138
|
frozen_source_digests: dict[str, str] | None = None
|
|
2139
|
+
model_source: list[str] = []
|
|
2140
|
+
saved_model_source: str | None = None
|
|
1115
2141
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
2142
|
+
if raw_frozen_sources is not None:
|
|
2143
|
+
frozen_source_digests, saved_model_source = _retained_source_state(
|
|
2144
|
+
json.loads(raw_frozen_sources)
|
|
2145
|
+
)
|
|
2146
|
+
if saved_model_source is not None:
|
|
2147
|
+
model_source.append(saved_model_source)
|
|
1116
2148
|
if recovery_source_digests is not None:
|
|
1117
2149
|
frozen_source_digests = recovery_source_digests
|
|
1118
|
-
elif raw_frozen_sources is not None:
|
|
1119
|
-
frozen_source_digests = _source_digests(
|
|
1120
|
-
json.loads(raw_frozen_sources), "Saved stopped-scan"
|
|
1121
|
-
)
|
|
1122
2150
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1123
2151
|
deep_run = connection.execute(
|
|
1124
2152
|
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
@@ -1141,12 +2169,10 @@ def preserve_scan_results_locked(
|
|
|
1141
2169
|
]
|
|
1142
2170
|
|
|
1143
2171
|
def record_publication(manifest: dict[str, Any], findings: dict[str, Any]) -> None:
|
|
1144
|
-
retained_sources =
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
):
|
|
1149
|
-
raise ContractError("Stopped scan source digests could not be frozen.")
|
|
2172
|
+
retained_sources = _source_digests(
|
|
2173
|
+
manifest.get("scan", {}).get("preservedSources"),
|
|
2174
|
+
"Stopped scan source digests could not be frozen.",
|
|
2175
|
+
)
|
|
1150
2176
|
digest = db.published_manifest_digest(scan_dir, manifest)
|
|
1151
2177
|
timestamp = db.now()
|
|
1152
2178
|
with connection:
|
|
@@ -1176,7 +2202,7 @@ def preserve_scan_results_locked(
|
|
|
1176
2202
|
"updated_at = ? WHERE id = ? AND status = 'failed'",
|
|
1177
2203
|
(
|
|
1178
2204
|
digest,
|
|
1179
|
-
|
|
2205
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
1180
2206
|
json.dumps(list(dict.fromkeys(warnings))),
|
|
1181
2207
|
timestamp,
|
|
1182
2208
|
scan_id,
|
|
@@ -1190,6 +2216,7 @@ def preserve_scan_results_locked(
|
|
|
1190
2216
|
|
|
1191
2217
|
existing_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
1192
2218
|
existing_scan = db.read_json_object(existing_path).get("scan", {}) if existing_path else {}
|
|
2219
|
+
existing = None
|
|
1193
2220
|
if scan["seal_manifest_digest"] is not None or (
|
|
1194
2221
|
isinstance(existing_scan, dict)
|
|
1195
2222
|
and (
|
|
@@ -1198,30 +2225,35 @@ def preserve_scan_results_locked(
|
|
|
1198
2225
|
):
|
|
1199
2226
|
db.require_recorded_manifest_digest(scan, scan_dir)
|
|
1200
2227
|
existing, existing_findings, _ = finalize_scan(
|
|
1201
|
-
scan_dir,
|
|
2228
|
+
scan_dir,
|
|
2229
|
+
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
2230
|
+
projection_warnings=warnings,
|
|
1202
2231
|
)
|
|
1203
2232
|
db.verify_manifest_binding(scan, existing)
|
|
1204
2233
|
if existing_scan.get("status") == outcome:
|
|
1205
2234
|
existing_sources = existing_scan.get("preservedSources")
|
|
1206
2235
|
if frozen_source_digests is None:
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
):
|
|
1211
|
-
raise ContractError("Stopped scan source digests could not be frozen.")
|
|
1212
|
-
frozen_source_digests = existing_sources
|
|
2236
|
+
frozen_source_digests = _source_digests(
|
|
2237
|
+
existing_sources, "Stopped scan source digests could not be frozen."
|
|
2238
|
+
)
|
|
1213
2239
|
if existing_sources == frozen_source_digests:
|
|
1214
2240
|
if (
|
|
1215
2241
|
raw_frozen_sources is not None
|
|
1216
2242
|
and scan["seal_manifest_digest"] is not None
|
|
1217
2243
|
and not publication_follow_up_warnings
|
|
2244
|
+
and warnings == stored_warnings
|
|
1218
2245
|
):
|
|
1219
2246
|
return True
|
|
1220
2247
|
record_publication(existing, existing_findings)
|
|
1221
2248
|
return True
|
|
1222
2249
|
if recovery_source_digests is None:
|
|
1223
2250
|
raise ContractError("Stopped scan sources changed after terminal publication.")
|
|
1224
|
-
binding = {
|
|
2251
|
+
binding = {
|
|
2252
|
+
**db.workbench_completion_binding(scan, scan["completed_at"], existing),
|
|
2253
|
+
"status": outcome,
|
|
2254
|
+
}
|
|
2255
|
+
if recovery_source_digests is not None:
|
|
2256
|
+
model_source.clear()
|
|
1225
2257
|
documents = merge_saved_results(
|
|
1226
2258
|
scan_dir,
|
|
1227
2259
|
scan_id,
|
|
@@ -1237,6 +2269,8 @@ def preserve_scan_results_locked(
|
|
|
1237
2269
|
f"{scan['failure_message'] or ''}"
|
|
1238
2270
|
).strip(),
|
|
1239
2271
|
frozen_source_digests=frozen_source_digests,
|
|
2272
|
+
frozen_model_source=model_source[0] if model_source else None,
|
|
2273
|
+
selected_model_source=model_source,
|
|
1240
2274
|
allow_frozen_legacy_parent=(
|
|
1241
2275
|
include_parent_with_recovery
|
|
1242
2276
|
or (
|
|
@@ -1258,29 +2292,37 @@ def preserve_scan_results_locked(
|
|
|
1258
2292
|
(json.dumps(unpublished_warnings), db.now(), scan_id),
|
|
1259
2293
|
)
|
|
1260
2294
|
return False
|
|
1261
|
-
if frozen_source_digests is None
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
2295
|
+
if frozen_source_digests is None or (
|
|
2296
|
+
recovery_source_digests is None and saved_model_source is None and model_source
|
|
2297
|
+
):
|
|
2298
|
+
retained_sources = _source_digests(
|
|
2299
|
+
documents[0].get("scan", {}).get("preservedSources"),
|
|
2300
|
+
"Stopped scan source digests could not be frozen.",
|
|
2301
|
+
)
|
|
1268
2302
|
with connection:
|
|
1269
2303
|
connection.execute(
|
|
1270
2304
|
"UPDATE scans SET retained_source_digests_json = ? "
|
|
1271
|
-
"WHERE id = ? AND retained_source_digests_json IS
|
|
1272
|
-
(
|
|
2305
|
+
"WHERE id = ? AND retained_source_digests_json IS ?",
|
|
2306
|
+
(
|
|
2307
|
+
_encode_retained_sources(retained_sources, model_source),
|
|
2308
|
+
scan_id,
|
|
2309
|
+
raw_frozen_sources,
|
|
2310
|
+
),
|
|
1273
2311
|
)
|
|
1274
2312
|
prepared = _prepare_scan_finalization(
|
|
1275
2313
|
scan_dir,
|
|
1276
2314
|
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
1277
2315
|
completion_binding=binding,
|
|
1278
|
-
|
|
2316
|
+
recover_unsealed=True,
|
|
1279
2317
|
draft_documents=documents,
|
|
2318
|
+
finding_scope=db.requested_finding_scope(scan) if scan["mode"] == "deep" else None,
|
|
1280
2319
|
)
|
|
2320
|
+
warnings.extend(prepared.warnings)
|
|
1281
2321
|
snapshots = _snapshot_published_outputs(scan_dir)
|
|
1282
2322
|
try:
|
|
1283
|
-
manifest, findings, _ = _write_prepared_scan_finalization(
|
|
2323
|
+
manifest, findings, _ = _write_prepared_scan_finalization(
|
|
2324
|
+
prepared, projection_warnings=warnings
|
|
2325
|
+
)
|
|
1284
2326
|
db.verify_manifest_binding(scan, manifest)
|
|
1285
2327
|
record_publication(manifest, findings)
|
|
1286
2328
|
except BaseException:
|
|
@@ -1387,6 +2429,66 @@ def save_scan_artifact(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1387
2429
|
return {"scanId": scan_id, "path": str(scan_dir / output)}
|
|
1388
2430
|
|
|
1389
2431
|
|
|
2432
|
+
def _require_current_deep_publication(
|
|
2433
|
+
db: Any, connection: Any, scan: Any, draft: dict[str, Any]
|
|
2434
|
+
) -> None:
|
|
2435
|
+
scan_id = scan["id"]
|
|
2436
|
+
run = connection.execute(
|
|
2437
|
+
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
2438
|
+
).fetchone()
|
|
2439
|
+
publication = draft.get("deepScanPublication")
|
|
2440
|
+
if (
|
|
2441
|
+
publication is None
|
|
2442
|
+
and draft["manifest"]["scan"].get("complete") is False
|
|
2443
|
+
and (run is None or run["status"] == "running")
|
|
2444
|
+
):
|
|
2445
|
+
return
|
|
2446
|
+
if run is None:
|
|
2447
|
+
run = db.deep_scan.require_deep_scan_run(connection, scan_id)
|
|
2448
|
+
# Finished discovery-only runs hand remaining publication back to the parent.
|
|
2449
|
+
if (
|
|
2450
|
+
publication is None
|
|
2451
|
+
and run["status"] != "running"
|
|
2452
|
+
and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
2453
|
+
):
|
|
2454
|
+
return
|
|
2455
|
+
db.deep_scan.require_current_coordinator(
|
|
2456
|
+
run,
|
|
2457
|
+
argparse.Namespace(
|
|
2458
|
+
coordinator_generation=publication.get("coordinatorGeneration") if publication else None
|
|
2459
|
+
),
|
|
2460
|
+
)
|
|
2461
|
+
# Once finish selects the canonical parent, only completion may replay it.
|
|
2462
|
+
if publication is None:
|
|
2463
|
+
if run["status"] != "running" and run["manifest_path"] == str(
|
|
2464
|
+
Path(scan["scan_dir"]) / "scan-manifest.json"
|
|
2465
|
+
):
|
|
2466
|
+
raise SystemExit("Deep Scan is terminal; drafts cannot replace its publication.")
|
|
2467
|
+
return
|
|
2468
|
+
|
|
2469
|
+
# Match the durable reducer sequence used by coordinator recovery.
|
|
2470
|
+
def reducer_order(worker: Any) -> tuple[int, str]:
|
|
2471
|
+
match = re.fullmatch(r"dedup-([0-9]+)", Path(worker["prompt_path"]).parent.name)
|
|
2472
|
+
return (int(match[1]) if match else 0, worker["id"])
|
|
2473
|
+
|
|
2474
|
+
reducer = max(
|
|
2475
|
+
(
|
|
2476
|
+
worker
|
|
2477
|
+
for worker in connection.execute(
|
|
2478
|
+
"SELECT * FROM deep_scan_workers WHERE scan_id = ?", (scan_id,)
|
|
2479
|
+
)
|
|
2480
|
+
if worker["kind"] == "dedup"
|
|
2481
|
+
and worker["status"] == "succeeded"
|
|
2482
|
+
and worker["result_manifest_path"]
|
|
2483
|
+
),
|
|
2484
|
+
key=reducer_order,
|
|
2485
|
+
default=None,
|
|
2486
|
+
)
|
|
2487
|
+
selected_result = reducer["result_manifest_path"] if reducer is not None else None
|
|
2488
|
+
if publication["resultPath"] != selected_result:
|
|
2489
|
+
raise SystemExit("Deep Scan aggregate belongs to a superseded publication selection.")
|
|
2490
|
+
|
|
2491
|
+
|
|
1390
2492
|
def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
1391
2493
|
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
1392
2494
|
with db.scan_completion_lock(scan_id):
|
|
@@ -1399,6 +2501,30 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1399
2501
|
"The scan stopped; its saved checkpoint was retained without replacing sealed results."
|
|
1400
2502
|
)
|
|
1401
2503
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2504
|
+
if (
|
|
2505
|
+
args.expected_draft_digest is not None
|
|
2506
|
+
and args.expected_draft_digest != _scan_draft_digest(scan_dir)
|
|
2507
|
+
):
|
|
2508
|
+
raise SystemExit(
|
|
2509
|
+
"scan_draft_conflict: canonical scan results changed; reconcile the saved checkpoint again."
|
|
2510
|
+
)
|
|
2511
|
+
try:
|
|
2512
|
+
relative = Path(args.draft_path).relative_to(scan_dir).as_posix()
|
|
2513
|
+
except ValueError as exc:
|
|
2514
|
+
raise SystemExit(
|
|
2515
|
+
"Scan draft must be inside the registered scan drafts directory."
|
|
2516
|
+
) from exc
|
|
2517
|
+
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
2518
|
+
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
2519
|
+
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
2520
|
+
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
|
|
2521
|
+
binding = db.workbench_completion_binding(scan, db.now())
|
|
2522
|
+
# Save scan IDs without sealing the draft.
|
|
2523
|
+
_populate_unsealed_manifest_envelope(manifest, manifest["scan"], binding)
|
|
2524
|
+
_populate_unsealed_artifact_envelope(manifest, findings, coverage, binding)
|
|
2525
|
+
_validate_completion_binding(manifest, findings, coverage, binding)
|
|
2526
|
+
checkpoint = manifest["scan"]
|
|
2527
|
+
checkpoint_contents = None
|
|
1402
2528
|
if args.checkpoint_path is not None:
|
|
1403
2529
|
try:
|
|
1404
2530
|
checkpoint_relative = Path(args.checkpoint_path).relative_to(scan_dir).as_posix()
|
|
@@ -1415,39 +2541,39 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1415
2541
|
)
|
|
1416
2542
|
if checkpoint.get("scanId") != scan_id:
|
|
1417
2543
|
raise SystemExit("Staged scan checkpoint belongs to another scan.")
|
|
2544
|
+
checkpoint_only = (
|
|
2545
|
+
scan["mode"] == "deep"
|
|
2546
|
+
and draft.get("deepScanPublication") is None
|
|
2547
|
+
and checkpoint.get("complete") is False
|
|
2548
|
+
and connection.execute(
|
|
2549
|
+
"SELECT 1 FROM deep_scan_runs WHERE scan_id = ? "
|
|
2550
|
+
"AND status = 'running' AND manifest_path = ?",
|
|
2551
|
+
(scan_id, str(scan_dir / "scan-manifest.json")),
|
|
2552
|
+
).fetchone()
|
|
2553
|
+
is not None
|
|
2554
|
+
)
|
|
2555
|
+
if scan["mode"] == "deep" and not checkpoint_only:
|
|
2556
|
+
_require_current_deep_publication(db, connection, scan, draft)
|
|
2557
|
+
if checkpoint_contents is not None:
|
|
1418
2558
|
checkpoint_digest = hashlib.sha256(checkpoint_contents).hexdigest()
|
|
1419
2559
|
write_scan_local_bytes(
|
|
1420
2560
|
scan_dir,
|
|
1421
2561
|
f"checkpoints/{checkpoint_digest}.json",
|
|
1422
2562
|
checkpoint_contents,
|
|
1423
2563
|
)
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
)
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
) from exc
|
|
1437
|
-
if not re.fullmatch(r"drafts/[0-9a-fA-F-]+\.json", relative):
|
|
1438
|
-
raise SystemExit("Scan draft must be inside the registered scan drafts directory.")
|
|
1439
|
-
draft = _read_scan_local_json(scan_dir, relative, "Staged scan draft")
|
|
1440
|
-
manifest, findings, coverage = draft["manifest"], draft["findings"], draft["coverage"]
|
|
1441
|
-
binding = db.workbench_completion_binding(scan, db.now())
|
|
1442
|
-
# Validate on copies: saved canonical documents remain ordinary unsealed drafts.
|
|
1443
|
-
copied_manifest = copy.deepcopy(manifest)
|
|
1444
|
-
copied_findings = copy.deepcopy(findings)
|
|
1445
|
-
copied_coverage = copy.deepcopy(coverage)
|
|
1446
|
-
_populate_unsealed_manifest_envelope(copied_manifest, copied_manifest["scan"], binding)
|
|
1447
|
-
_populate_unsealed_artifact_envelope(
|
|
1448
|
-
copied_manifest, copied_findings, copied_coverage, binding
|
|
2564
|
+
if checkpoint_only:
|
|
2565
|
+
return {"scanId": scan_id, "status": "draft_written"}
|
|
2566
|
+
checkpoint = _parent_scan_draft(scan_id, manifest["scan"], findings, coverage)
|
|
2567
|
+
checkpoint_contents = _encoded(checkpoint)
|
|
2568
|
+
checkpoint_name = f"{hashlib.sha256(checkpoint_contents).hexdigest()}.json"
|
|
2569
|
+
checkpoint_relative = f"checkpoints/{checkpoint_name}"
|
|
2570
|
+
if not (scan_dir / checkpoint_relative).exists():
|
|
2571
|
+
write_scan_local_bytes(scan_dir, checkpoint_relative, checkpoint_contents)
|
|
2572
|
+
if checkpoint_only:
|
|
2573
|
+
return {"scanId": scan_id, "status": "draft_written"}
|
|
2574
|
+
write_scan_local_bytes(
|
|
2575
|
+
scan_dir, "checkpoint-head.json", _encoded({"checkpoint": checkpoint_name})
|
|
1449
2576
|
)
|
|
1450
|
-
_validate_completion_binding(copied_manifest, copied_findings, copied_coverage, binding)
|
|
1451
2577
|
for filename, document in (
|
|
1452
2578
|
("findings.json", findings),
|
|
1453
2579
|
("coverage.json", coverage),
|
|
@@ -1458,9 +2584,25 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1458
2584
|
filename,
|
|
1459
2585
|
(json.dumps(document, allow_nan=False, indent=2) + "\n").encode(),
|
|
1460
2586
|
)
|
|
2587
|
+
if scan["mode"] == "deep" and manifest["scan"].get("complete") is not False:
|
|
2588
|
+
# Select the accepted final publication before releasing the write lock.
|
|
2589
|
+
with connection:
|
|
2590
|
+
connection.execute(
|
|
2591
|
+
"UPDATE deep_scan_runs SET manifest_path = ? "
|
|
2592
|
+
"WHERE scan_id = ? AND status = 'running'",
|
|
2593
|
+
(str(scan_dir / "scan-manifest.json"), scan_id),
|
|
2594
|
+
)
|
|
2595
|
+
model_warning = write_threat_model_projection_if_possible(scan_dir, manifest)
|
|
1461
2596
|
# Accepted Standard drafts are evidence of review or report assembly,
|
|
1462
2597
|
# even when the parent omitted its explicit progress call.
|
|
1463
|
-
|
|
2598
|
+
model_only_checkpoint = (
|
|
2599
|
+
manifest["scan"].get("complete") is False
|
|
2600
|
+
and isinstance(manifest["scan"].get("threatModel"), dict)
|
|
2601
|
+
and not findings.get("findings")
|
|
2602
|
+
and not coverage.get("surfaces")
|
|
2603
|
+
and not coverage.get("deferred")
|
|
2604
|
+
)
|
|
2605
|
+
if scan["mode"] == "standard" and not model_only_checkpoint:
|
|
1464
2606
|
phase = "discovery" if manifest["scan"].get("complete") is False else "reporting"
|
|
1465
2607
|
earlier = PHASES[: PHASES.index(phase)]
|
|
1466
2608
|
placeholders = ",".join("?" for _ in earlier)
|
|
@@ -1481,7 +2623,11 @@ def write_scan_draft(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1481
2623
|
)
|
|
1482
2624
|
except sqlite3.Error as exc:
|
|
1483
2625
|
print(f"Could not save scan progress: {exc}", file=sys.stderr)
|
|
1484
|
-
return {
|
|
2626
|
+
return {
|
|
2627
|
+
"scanId": scan_id,
|
|
2628
|
+
"status": "draft_written",
|
|
2629
|
+
**({"warnings": [model_warning]} if model_warning else {}),
|
|
2630
|
+
}
|
|
1485
2631
|
|
|
1486
2632
|
|
|
1487
2633
|
def _scan_draft_digest(scan_dir: Path) -> str:
|
|
@@ -1494,9 +2640,10 @@ def _scan_draft_digest(scan_dir: Path) -> str:
|
|
|
1494
2640
|
except FileNotFoundError:
|
|
1495
2641
|
digest.update(b"missing\0")
|
|
1496
2642
|
continue
|
|
1497
|
-
_, contents = _read_scan_local_json_bytes(scan_dir, filename, filename)
|
|
1498
2643
|
digest.update(b"present\0")
|
|
1499
|
-
|
|
2644
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, filename, filename)
|
|
2645
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
2646
|
+
digest.update(handle.read())
|
|
1500
2647
|
digest.update(b"\0")
|
|
1501
2648
|
return digest.hexdigest()
|
|
1502
2649
|
|
|
@@ -1510,7 +2657,7 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1510
2657
|
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
1511
2658
|
cost_json = db.parse_scan_cost(args.cost_json)
|
|
1512
2659
|
connection.execute("BEGIN IMMEDIATE")
|
|
1513
|
-
|
|
2660
|
+
with connection:
|
|
1514
2661
|
timestamp = db.now()
|
|
1515
2662
|
scan = db.require_scan(connection, scan_id)
|
|
1516
2663
|
if scan["status"] == "failed":
|
|
@@ -1542,10 +2689,6 @@ def fail_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1542
2689
|
)
|
|
1543
2690
|
if progress_updated.rowcount != 1:
|
|
1544
2691
|
raise SystemExit("Codex Security scan progress not found.")
|
|
1545
|
-
connection.commit()
|
|
1546
|
-
except BaseException:
|
|
1547
|
-
connection.rollback()
|
|
1548
|
-
raise
|
|
1549
2692
|
preserve_stopped_results_after_transition(db, connection, scan["id"])
|
|
1550
2693
|
return db.scan_context(connection, scan["id"])
|
|
1551
2694
|
|
|
@@ -1559,7 +2702,7 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1559
2702
|
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
1560
2703
|
thread_id = db.optional_text(args.thread_id, maximum=512)
|
|
1561
2704
|
connection.execute("BEGIN IMMEDIATE")
|
|
1562
|
-
|
|
2705
|
+
with connection:
|
|
1563
2706
|
timestamp = db.now()
|
|
1564
2707
|
scan = db.require_scan(connection, scan_id)
|
|
1565
2708
|
workspace = db.require_workspace(connection, scan["workspace_id"])
|
|
@@ -1588,10 +2731,6 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1588
2731
|
)
|
|
1589
2732
|
if progress_updated.rowcount != 1:
|
|
1590
2733
|
raise SystemExit("Codex Security scan progress not found.")
|
|
1591
|
-
connection.commit()
|
|
1592
|
-
except BaseException:
|
|
1593
|
-
connection.rollback()
|
|
1594
|
-
raise
|
|
1595
2734
|
preserve_stopped_results_after_transition(db, connection, scan["id"])
|
|
1596
2735
|
return db.workspace_state(connection, scan["workspace_id"])
|
|
1597
2736
|
|