@openai/codex-security 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +201 -178
- package/_bundled_plugin/.app.json +4 -0
- package/_bundled_plugin/.mcp.json +6 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
- package/_bundled_plugin/examples/completed-scan/report.md +11 -8
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
- package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
- package/_bundled_plugin/mcp/helpers.mjs +2 -11
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/package.json +1 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +51 -14
- package/_bundled_plugin/mcp/server.mjs +2 -11
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
- package/_bundled_plugin/references/artifact-storage.md +1 -1
- package/_bundled_plugin/references/core-scan.md +14 -5
- package/_bundled_plugin/references/default-severity-rubric.md +33 -0
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/finding-detail-fields.md +19 -18
- package/_bundled_plugin/references/sarif-adapter.md +8 -2
- package/_bundled_plugin/references/scan-artifacts.md +21 -12
- package/_bundled_plugin/references/scan-contract.md +27 -33
- package/_bundled_plugin/references/threat-model.md +1 -0
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
- package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
- package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +188 -454
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +317 -377
- package/_bundled_plugin/scripts/finding_preview.py +25 -29
- package/_bundled_plugin/scripts/launch_codex_security_mcp +1 -0
- package/_bundled_plugin/scripts/legacy_findings.py +148 -0
- package/_bundled_plugin/scripts/posix_scan_local_files.py +198 -0
- package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
- package/_bundled_plugin/scripts/report_projection.py +167 -178
- package/_bundled_plugin/scripts/review_state.py +8 -2
- package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +1 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
- package/_bundled_plugin/scripts/validate_scan_contract.py +9 -7
- package/_bundled_plugin/scripts/windows_scan_local_files.py +600 -72
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
- package/_bundled_plugin/scripts/workbench/handoff.py +47 -0
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +38 -75
- package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +152 -248
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
- package/_bundled_plugin/scripts/workbench/storage.py +71 -7
- package/_bundled_plugin/scripts/workbench_cli.py +121 -14
- package/_bundled_plugin/scripts/workbench_constants.py +19 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
- package/_bundled_plugin/scripts/workbench_db.py +550 -492
- package/_bundled_plugin/scripts/workbench_feedback.py +8 -7
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +90 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +29 -33
- package/_bundled_plugin/scripts/workbench_progress.py +10 -1
- package/_bundled_plugin/scripts/workbench_publication.py +14 -10
- package/_bundled_plugin/scripts/workbench_remediation.py +508 -11
- package/_bundled_plugin/scripts/workbench_saved_results.py +132 -77
- package/_bundled_plugin/scripts/workbench_scan_history.py +139 -80
- package/_bundled_plugin/scripts/workbench_scan_start.py +84 -25
- package/_bundled_plugin/scripts/workbench_scan_usage.py +666 -117
- package/_bundled_plugin/scripts/workbench_schema.py +44 -55
- package/_bundled_plugin/scripts/workbench_setup.py +251 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +33 -23
- package/_bundled_plugin/scripts/workbench_target.py +330 -66
- package/_bundled_plugin/scripts/workbench_target_state.py +3 -0
- package/_bundled_plugin/scripts/workbench_validation.py +44 -12
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -16
- package/_bundled_plugin/shared/workbench-migrations.json +53 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -1
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +4 -68
- package/_bundled_plugin/skills/check-skills/SKILL.md +43 -0
- package/_bundled_plugin/skills/check-skills/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +6 -4
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
- package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -2
- package/_bundled_plugin/skills/publish-findings/SKILL.md +81 -45
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +5 -24
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +464 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +36 -9
- package/_bundled_plugin/skills/track-findings/SKILL.md +5 -3
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +22 -68
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +12 -6
- package/dist/api.d.ts +45 -5
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1144 -411
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +10 -7
- package/dist/auth.js.map +1 -1
- package/dist/bedrock-oss-pricing.d.ts +6 -0
- package/dist/bedrock-oss-pricing.d.ts.map +1 -0
- package/dist/bedrock-oss-pricing.js +58 -0
- package/dist/bedrock-oss-pricing.js.map +1 -0
- package/dist/bedrock-pricing.d.ts +15 -0
- package/dist/bedrock-pricing.d.ts.map +1 -0
- package/dist/bedrock-pricing.js +117 -0
- package/dist/bedrock-pricing.js.map +1 -0
- package/dist/blackduck-findings.d.ts +8 -0
- package/dist/blackduck-findings.d.ts.map +1 -0
- package/dist/blackduck-findings.js +320 -0
- package/dist/blackduck-findings.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +51 -36
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/checkmarx-evidence.d.ts +19 -0
- package/dist/checkmarx-evidence.d.ts.map +1 -0
- package/dist/checkmarx-evidence.js +36 -0
- package/dist/checkmarx-evidence.js.map +1 -0
- package/dist/checkmarx-findings.d.ts +4 -0
- package/dist/checkmarx-findings.d.ts.map +1 -0
- package/dist/checkmarx-findings.js +27 -0
- package/dist/checkmarx-findings.js.map +1 -0
- package/dist/checkmarx-one.d.ts +4 -0
- package/dist/checkmarx-one.d.ts.map +1 -0
- package/dist/checkmarx-one.js +53 -0
- package/dist/checkmarx-one.js.map +1 -0
- package/dist/checkmarx-repository.d.ts +12 -0
- package/dist/checkmarx-repository.d.ts.map +1 -0
- package/dist/checkmarx-repository.js +181 -0
- package/dist/checkmarx-repository.js.map +1 -0
- package/dist/checkmarx-xml.d.ts +12 -0
- package/dist/checkmarx-xml.d.ts.map +1 -0
- package/dist/checkmarx-xml.js +196 -0
- package/dist/checkmarx-xml.js.map +1 -0
- package/dist/classify-severity-records.d.ts +6 -9
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +9 -21
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +4 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +30 -13
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli.d.ts +14 -5
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +2112 -1266
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +2 -2
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-prompt.d.ts +2 -0
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +6 -0
- package/dist/codex-prompt.js.map +1 -1
- package/dist/component-scan.d.ts +16 -3
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +32 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +2 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +221 -65
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts +7 -0
- package/dist/continuum-findings.d.ts.map +1 -0
- package/dist/continuum-findings.js +225 -0
- package/dist/continuum-findings.js.map +1 -0
- package/dist/contract-values.d.ts +30 -0
- package/dist/contract-values.d.ts.map +1 -0
- package/dist/contract-values.js +78 -0
- package/dist/contract-values.js.map +1 -0
- package/dist/contract.d.ts +2 -20
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +58 -669
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts +13 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +66 -115
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +61 -4
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +2560 -100
- package/dist/cost.js.map +1 -1
- package/dist/custom-publish.d.ts +4 -0
- package/dist/custom-publish.d.ts.map +1 -1
- package/dist/custom-publish.js +40 -3
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +5 -0
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +2 -0
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +4 -0
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -4
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +14 -11
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +11 -4
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/finding-schema.d.ts +19 -1
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +25 -0
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts +2 -2
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -10
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.d.ts +8 -2
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +36 -8
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +25 -5
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +3 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +1 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +0 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +22 -33
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan-defaults.d.ts +2 -0
- package/dist/deep-scan-defaults.d.ts.map +1 -1
- package/dist/deep-scan-defaults.js +28 -1
- package/dist/deep-scan-defaults.js.map +1 -1
- package/dist/deep-scan.d.ts +13 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +262 -7
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -2
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +2 -66
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts +3 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +108 -80
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +9 -2
- package/dist/errors.js.map +1 -1
- package/dist/external-findings-import.d.ts +5 -3
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +11 -16
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +15 -4
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +619 -225
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +20 -6
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +2356 -308
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +28 -2
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +174 -81
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +17 -3
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +153 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +33 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +335 -51
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +5 -4
- package/dist/feedback.js.map +1 -1
- package/dist/finding-prompt.d.ts +5 -0
- package/dist/finding-prompt.d.ts.map +1 -0
- package/dist/finding-prompt.js +32 -0
- package/dist/finding-prompt.js.map +1 -0
- package/dist/finding-retrieval.d.ts +3 -3
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +6 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +2 -2
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-client.d.ts +1 -0
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +27 -3
- package/dist/findings-client.js.map +1 -1
- package/dist/git-remote.d.ts +4 -0
- package/dist/git-remote.d.ts.map +1 -1
- package/dist/git-remote.js +20 -0
- package/dist/git-remote.js.map +1 -1
- package/dist/github-advisories.d.ts +3 -12
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +9 -31
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +22 -20
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts +4 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +392 -79
- package/dist/github-findings.js.map +1 -1
- package/dist/github.d.ts +17 -7
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +43 -30
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +2 -2
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +10 -11
- package/dist/host-records-protocol.d.ts.map +1 -1
- package/dist/host-records-protocol.js +69 -9
- package/dist/host-records-protocol.js.map +1 -1
- package/dist/hosted-git-remotes.d.ts +5 -0
- package/dist/hosted-git-remotes.d.ts.map +1 -0
- package/dist/hosted-git-remotes.js +285 -0
- package/dist/hosted-git-remotes.js.map +1 -0
- package/dist/hosted-git-repository.d.ts +28 -0
- package/dist/hosted-git-repository.d.ts.map +1 -0
- package/dist/hosted-git-repository.js +222 -0
- package/dist/hosted-git-repository.js.map +1 -0
- package/dist/hosted-scan-protocol.js +2 -2
- package/dist/hosted-scan-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts +15 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +71 -44
- package/dist/hosted-scan.js.map +1 -1
- package/dist/hosted-scanner-http.d.ts +21 -0
- package/dist/hosted-scanner-http.d.ts.map +1 -0
- package/dist/hosted-scanner-http.js +204 -0
- package/dist/hosted-scanner-http.js.map +1 -0
- package/dist/hosted-scanner-v1.d.ts +6 -0
- package/dist/hosted-scanner-v1.d.ts.map +1 -0
- package/dist/hosted-scanner-v1.js +60 -0
- package/dist/hosted-scanner-v1.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +3 -0
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +24 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -1
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +58 -81
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +86 -10
- package/dist/knowledge-base.js.map +1 -1
- package/dist/local-git-metadata.d.ts +5 -0
- package/dist/local-git-metadata.d.ts.map +1 -0
- package/dist/local-git-metadata.js +130 -0
- package/dist/local-git-metadata.js.map +1 -0
- package/dist/multiscan.d.ts +6 -4
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +160 -77
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +2 -1
- package/dist/native-json-findings.d.ts.map +1 -1
- package/dist/native-json-findings.js +12 -2
- package/dist/native-json-findings.js.map +1 -1
- package/dist/opengrep-findings.d.ts +1 -0
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +49 -31
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/operation-telemetry.d.ts +104 -0
- package/dist/operation-telemetry.d.ts.map +1 -0
- package/dist/operation-telemetry.js +293 -0
- package/dist/operation-telemetry.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +90 -35
- package/dist/patch-tui.js.map +1 -1
- package/dist/plugin-python-probe.d.ts +4 -0
- package/dist/plugin-python-probe.d.ts.map +1 -0
- package/dist/plugin-python-probe.js +10 -0
- package/dist/plugin-python-probe.js.map +1 -0
- package/dist/private-finding-import.d.ts +341 -0
- package/dist/private-finding-import.d.ts.map +1 -0
- package/dist/private-finding-import.js +411 -0
- package/dist/private-finding-import.js.map +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -2
- package/dist/project-config.js.map +1 -1
- package/dist/provider-profile.d.ts +11 -3
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +80 -11
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication-telemetry.d.ts +45 -0
- package/dist/publication-telemetry.d.ts.map +1 -0
- package/dist/publication-telemetry.js +50 -0
- package/dist/publication-telemetry.js.map +1 -0
- package/dist/publication.js +7 -1
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +4 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +42 -2
- package/dist/publish.js.map +1 -1
- package/dist/reader-selection.d.ts +15 -0
- package/dist/reader-selection.d.ts.map +1 -0
- package/dist/reader-selection.js +89 -0
- package/dist/reader-selection.js.map +1 -0
- package/dist/record.d.ts +6 -1
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +53 -2
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +3 -1
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +28 -2
- package/dist/remediation.js.map +1 -1
- package/dist/request-metadata.d.ts +3 -0
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +9 -0
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +6 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +3 -1
- package/dist/result.js.map +1 -1
- package/dist/retry-windows-rename.d.ts +8 -0
- package/dist/retry-windows-rename.d.ts.map +1 -0
- package/dist/retry-windows-rename.js +24 -0
- package/dist/retry-windows-rename.js.map +1 -0
- package/dist/runtime.d.ts +22 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +408 -162
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +18 -15
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +172 -64
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +10 -5
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +68 -25
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +7 -6
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +91 -79
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +12 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +22 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +31 -28
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +12 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +106 -77
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-measurement.d.ts +26 -0
- package/dist/scan-measurement.d.ts.map +1 -0
- package/dist/scan-measurement.js +77 -0
- package/dist/scan-measurement.js.map +1 -0
- package/dist/scan-progress.d.ts +1 -0
- package/dist/scan-progress.d.ts.map +1 -1
- package/dist/scan-progress.js +3 -0
- package/dist/scan-progress.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +4 -1
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -1
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +1 -24
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +64 -6
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +406 -48
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +17 -0
- package/dist/scanner-repository-url.d.ts.map +1 -0
- package/dist/scanner-repository-url.js +71 -0
- package/dist/scanner-repository-url.js.map +1 -0
- package/dist/security-policy.d.ts +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +6 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +124 -88
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-guardian-findings.d.ts +38 -0
- package/dist/semgrep-guardian-findings.d.ts.map +1 -0
- package/dist/semgrep-guardian-findings.js +165 -0
- package/dist/semgrep-guardian-findings.js.map +1 -0
- package/dist/semgrep-platform-collector.d.ts +26 -0
- package/dist/semgrep-platform-collector.d.ts.map +1 -0
- package/dist/semgrep-platform-collector.js +108 -0
- package/dist/semgrep-platform-collector.js.map +1 -0
- package/dist/semgrep-platform-findings.d.ts +213 -0
- package/dist/semgrep-platform-findings.d.ts.map +1 -0
- package/dist/semgrep-platform-findings.js +1078 -0
- package/dist/semgrep-platform-findings.js.map +1 -0
- package/dist/semgrep-platform-v1.d.ts +6 -0
- package/dist/semgrep-platform-v1.d.ts.map +1 -0
- package/dist/semgrep-platform-v1.js +83 -0
- package/dist/semgrep-platform-v1.js.map +1 -0
- package/dist/server/api.d.ts +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embedding-tokenizer.cdx.json +33 -0
- package/dist/server/embedding-tokenizer.d.ts +2 -0
- package/dist/server/embedding-tokenizer.d.ts.map +1 -0
- package/dist/server/embedding-tokenizer.js +322 -0
- package/dist/server/embedding-tokenizer.js.map +6 -0
- package/dist/server/embeddings.d.ts +16 -2
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +35 -5
- package/dist/server/embeddings.js.map +1 -1
- package/dist/session-lines.d.ts +5 -0
- package/dist/session-lines.d.ts.map +1 -0
- package/dist/session-lines.js +60 -0
- package/dist/session-lines.js.map +1 -0
- package/dist/severity-review.d.ts +3 -1
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +3 -2
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-rubric.d.ts +20 -0
- package/dist/severity-rubric.d.ts.map +1 -0
- package/dist/severity-rubric.js +73 -0
- package/dist/severity-rubric.js.map +1 -0
- package/dist/severity-store.js +15 -5
- package/dist/severity-store.js.map +1 -1
- package/dist/skill-authentication.d.ts +1 -0
- package/dist/skill-authentication.d.ts.map +1 -1
- package/dist/skill-authentication.js +1 -0
- package/dist/skill-authentication.js.map +1 -1
- package/dist/snyk-findings.d.ts +7 -2
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +108 -84
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +76 -0
- package/dist/snyk-rest-findings.d.ts.map +1 -0
- package/dist/snyk-rest-findings.js +1051 -0
- package/dist/snyk-rest-findings.js.map +1 -0
- package/dist/snyk-rest-v1.d.ts +28 -0
- package/dist/snyk-rest-v1.d.ts.map +1 -0
- package/dist/snyk-rest-v1.js +132 -0
- package/dist/snyk-rest-v1.js.map +1 -0
- package/dist/snyk-token-origin.d.ts +4 -0
- package/dist/snyk-token-origin.d.ts.map +1 -0
- package/dist/snyk-token-origin.js +82 -0
- package/dist/snyk-token-origin.js.map +1 -0
- package/dist/sonar-collector.d.ts +16 -0
- package/dist/sonar-collector.d.ts.map +1 -0
- package/dist/sonar-collector.js +380 -0
- package/dist/sonar-collector.js.map +1 -0
- package/dist/sonar-findings.d.ts +49 -0
- package/dist/sonar-findings.d.ts.map +1 -0
- package/dist/sonar-findings.js +288 -0
- package/dist/sonar-findings.js.map +1 -0
- package/dist/sonar-import.d.ts +111 -0
- package/dist/sonar-import.d.ts.map +1 -0
- package/dist/sonar-import.js +277 -0
- package/dist/sonar-import.js.map +1 -0
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +2 -1
- package/dist/suggest-owners.js.map +1 -1
- package/dist/target-drift.d.ts +9 -0
- package/dist/target-drift.d.ts.map +1 -0
- package/dist/target-drift.js +27 -0
- package/dist/target-drift.js.map +1 -0
- package/dist/targets.d.ts +5 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +73 -31
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +9 -0
- package/dist/tenable-findings.d.ts.map +1 -0
- package/dist/tenable-findings.js +122 -0
- package/dist/tenable-findings.js.map +1 -0
- package/dist/terminal-columns.d.ts +4 -0
- package/dist/terminal-columns.d.ts.map +1 -0
- package/dist/terminal-columns.js +19 -0
- package/dist/terminal-columns.js.map +1 -0
- package/dist/terminal-stream.d.ts +13 -0
- package/dist/terminal-stream.d.ts.map +1 -0
- package/dist/terminal-stream.js +35 -0
- package/dist/terminal-stream.js.map +1 -0
- package/dist/trivy-findings.d.ts +19 -0
- package/dist/trivy-findings.d.ts.map +1 -0
- package/dist/trivy-findings.js +435 -0
- package/dist/trivy-findings.js.map +1 -0
- package/dist/trivy-import.d.ts +19 -0
- package/dist/trivy-import.d.ts.map +1 -0
- package/dist/trivy-import.js +151 -0
- package/dist/trivy-import.js.map +1 -0
- package/dist/validation-records.d.ts +0 -3
- package/dist/validation-records.d.ts.map +1 -1
- package/dist/validation-records.js +1 -17
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +9 -0
- package/dist/veracode-findings.d.ts.map +1 -0
- package/dist/veracode-findings.js +409 -0
- package/dist/veracode-findings.js.map +1 -0
- package/dist/vertex-pricing.d.ts +8 -0
- package/dist/vertex-pricing.d.ts.map +1 -0
- package/dist/vertex-pricing.js +20 -0
- package/dist/vertex-pricing.js.map +1 -0
- package/dist/windows-path.d.ts +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +27 -1
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -2
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +1 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +14 -2
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +143 -50
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +28 -13
- package/docs/cli.md +623 -843
- package/docs/dedupe-records.md +54 -24
- package/docs/findings-service.md +87 -47
- package/docs/hosted-remediation.md +14 -7
- package/docs/hosted-scans.md +64 -41
- package/docs/telemetry.md +236 -0
- package/docs/validate-records.md +78 -44
- package/package.json +19 -17
- package/schemas/external-findings.schema.json +14 -1
- package/schemas/hosted-scan-v2.fixture.json +16 -1
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
- package/_bundled_plugin/scripts/validate_tracking_source.py +0 -82
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +0 -136
- package/dist/deduplication/records-protocol.d.ts +0 -4
- package/dist/deduplication/records-protocol.d.ts.map +0 -1
- package/dist/deduplication/records-protocol.js +0 -49
- package/dist/deduplication/records-protocol.js.map +0 -1
- package/dist/external-imports-feature.d.ts +0 -6
- package/dist/external-imports-feature.d.ts.map +0 -1
- package/dist/external-imports-feature.js +0 -51
- package/dist/external-imports-feature.js.map +0 -1
- package/dist/finding-normalization.d.ts +0 -5
- package/dist/finding-normalization.d.ts.map +0 -1
- package/dist/finding-normalization.js +0 -120
- package/dist/finding-normalization.js.map +0 -1
|
@@ -5,135 +5,33 @@ from __future__ import annotations
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
|
-
import shutil
|
|
9
8
|
import sqlite3
|
|
10
9
|
import sys
|
|
11
10
|
import uuid
|
|
12
11
|
from datetime import datetime, timedelta
|
|
13
12
|
from pathlib import Path
|
|
14
|
-
from typing import Any
|
|
13
|
+
from typing import Any
|
|
15
14
|
|
|
16
15
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
17
16
|
import workbench_saved_results as saved_results
|
|
18
17
|
from deep_scan_config import resolve_deep_scan_setup
|
|
19
|
-
from workbench.handoff import require_current_continuation
|
|
20
|
-
from workbench.storage import
|
|
18
|
+
from workbench.handoff import require_current_continuation, require_tokenless_scan_owner
|
|
19
|
+
from workbench.storage import (
|
|
20
|
+
archived_deep_path,
|
|
21
|
+
create_private_directory,
|
|
22
|
+
saved_deep_workers,
|
|
23
|
+
worker_usage_directory,
|
|
24
|
+
)
|
|
25
|
+
from workbench_constants import PHASES
|
|
21
26
|
from workbench_source_scopes import capture_source_scopes
|
|
22
27
|
from workbench_target import directory_snapshot_regular_file_count
|
|
23
28
|
from workbench_validation import optional_text, require_uuid, user_context_argument
|
|
24
29
|
|
|
25
|
-
DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup")
|
|
26
|
-
DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled")
|
|
27
|
-
DEEP_SCAN_REPLACEABLE_FAILURE_KINDS = (
|
|
28
|
-
"policy_refusal",
|
|
29
|
-
"transient_error",
|
|
30
|
-
"invalid_discovery_artifacts",
|
|
31
|
-
)
|
|
32
|
-
DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped")
|
|
33
|
-
DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1"
|
|
34
30
|
DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30
|
|
35
31
|
DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120
|
|
36
32
|
DEEP_SCAN_PUBLICATION_ERROR_SEPARATOR = "\nOriginal Deep Scan failure:\n"
|
|
37
33
|
|
|
38
34
|
|
|
39
|
-
def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) -> None:
|
|
40
|
-
begin_deep_scan = subparsers.add_parser("begin-deep-scan")
|
|
41
|
-
begin_deep_scan.add_argument("--thread-id", required=True)
|
|
42
|
-
begin_target = begin_deep_scan.add_mutually_exclusive_group(required=True)
|
|
43
|
-
begin_target.add_argument("--scan-id")
|
|
44
|
-
begin_target.add_argument("--target-path")
|
|
45
|
-
begin_deep_scan.add_argument("--scope", default=".")
|
|
46
|
-
begin_user_context = begin_deep_scan.add_mutually_exclusive_group()
|
|
47
|
-
begin_user_context.add_argument("--user-context")
|
|
48
|
-
begin_user_context.add_argument("--user-context-stdin", action="store_true")
|
|
49
|
-
begin_deep_scan.add_argument("--scan-root")
|
|
50
|
-
begin_deep_scan.add_argument("--claim-token")
|
|
51
|
-
begin_deep_scan.add_argument("--model")
|
|
52
|
-
begin_deep_scan.add_argument("--reasoning-effort")
|
|
53
|
-
begin_deep_scan.add_argument("--available-parallelism", type=positive_int)
|
|
54
|
-
begin_deep_scan.add_argument("--deep-scan-project-json")
|
|
55
|
-
begin_deep_scan.add_argument("--project-config-path")
|
|
56
|
-
begin_deep_scan.add_argument("--join-only", action="store_true")
|
|
57
|
-
begin_deep_scan.add_argument("--workflow-version", default=DEEP_SCAN_WORKFLOW_VERSION)
|
|
58
|
-
|
|
59
|
-
get_deep_scan = subparsers.add_parser("get-deep-scan")
|
|
60
|
-
get_deep_scan.add_argument("--scan-id", required=True)
|
|
61
|
-
get_deep_scan.add_argument("--thread-id", required=True)
|
|
62
|
-
|
|
63
|
-
acknowledge_result = subparsers.add_parser("acknowledge-deep-scan-result")
|
|
64
|
-
acknowledge_result.add_argument("--scan-id", required=True)
|
|
65
|
-
|
|
66
|
-
claim_coordinator = subparsers.add_parser("claim-deep-scan-coordinator")
|
|
67
|
-
claim_coordinator.add_argument("--scan-id", required=True)
|
|
68
|
-
claim_coordinator.add_argument("--thread-id", required=True)
|
|
69
|
-
claim_coordinator.add_argument("--claim-token")
|
|
70
|
-
claim_coordinator.add_argument("--coordinator-generation", type=positive_int)
|
|
71
|
-
|
|
72
|
-
upsert_deep_worker = subparsers.add_parser("upsert-deep-scan-worker")
|
|
73
|
-
upsert_deep_worker.add_argument("--scan-id", required=True)
|
|
74
|
-
upsert_deep_worker.add_argument("--worker-id", required=True)
|
|
75
|
-
upsert_deep_worker.add_argument("--kind", choices=DEEP_SCAN_WORKER_KINDS, required=True)
|
|
76
|
-
upsert_deep_worker.add_argument("--status", choices=DEEP_SCAN_WORKER_STATUSES, required=True)
|
|
77
|
-
upsert_deep_worker.add_argument("--prompt-path", required=True)
|
|
78
|
-
upsert_deep_worker.add_argument("--artifact-dir", required=True)
|
|
79
|
-
upsert_deep_worker.add_argument("--result-manifest-path")
|
|
80
|
-
upsert_deep_worker.add_argument("--attempt", type=non_negative_int)
|
|
81
|
-
upsert_deep_worker.add_argument("--sdk-thread-id")
|
|
82
|
-
upsert_deep_worker.add_argument("--error-message")
|
|
83
|
-
upsert_deep_worker.add_argument(
|
|
84
|
-
"--replaceable-failure-kind", choices=DEEP_SCAN_REPLACEABLE_FAILURE_KINDS
|
|
85
|
-
)
|
|
86
|
-
upsert_deep_worker.add_argument("--coordinator-generation", type=positive_int)
|
|
87
|
-
|
|
88
|
-
claim_deep_dedup = subparsers.add_parser("claim-deep-scan-dedup")
|
|
89
|
-
claim_deep_dedup.add_argument("--scan-id", required=True)
|
|
90
|
-
claim_deep_dedup.add_argument("--worker-id", required=True)
|
|
91
|
-
claim_deep_dedup.add_argument("--prompt-path", required=True)
|
|
92
|
-
claim_deep_dedup.add_argument("--artifact-dir", required=True)
|
|
93
|
-
claim_deep_dedup.add_argument("--input-worker-id", action="append", required=True)
|
|
94
|
-
claim_deep_dedup.add_argument("--coordinator-generation", type=positive_int)
|
|
95
|
-
|
|
96
|
-
commit_deep_dedup = subparsers.add_parser("commit-deep-scan-dedup")
|
|
97
|
-
commit_deep_dedup.add_argument("--scan-id", required=True)
|
|
98
|
-
commit_deep_dedup.add_argument("--worker-id", required=True)
|
|
99
|
-
commit_deep_dedup.add_argument("--result-manifest-path", required=True)
|
|
100
|
-
commit_deep_dedup.add_argument("--candidate-ledger-path")
|
|
101
|
-
commit_deep_dedup.add_argument("--new-findings-count", type=non_negative_int, required=True)
|
|
102
|
-
commit_deep_dedup.add_argument("--coordinator-generation", type=positive_int)
|
|
103
|
-
|
|
104
|
-
finish_deep_scan = subparsers.add_parser("finish-deep-scan")
|
|
105
|
-
finish_deep_scan.add_argument("--scan-id", required=True)
|
|
106
|
-
finish_deep_scan.add_argument(
|
|
107
|
-
"--terminal-reason", choices=DEEP_SCAN_TERMINAL_REASONS, required=True
|
|
108
|
-
)
|
|
109
|
-
finish_deep_scan.add_argument("--manifest-path", required=True)
|
|
110
|
-
finish_deep_scan.add_argument("--staged-manifest-path")
|
|
111
|
-
finish_deep_scan.add_argument("--omitted-worker-id", action="append", default=[])
|
|
112
|
-
finish_deep_scan.add_argument("--coordinator-generation", type=positive_int)
|
|
113
|
-
|
|
114
|
-
fail_deep_scan = subparsers.add_parser("fail-deep-scan")
|
|
115
|
-
fail_deep_scan.add_argument("--scan-id", required=True)
|
|
116
|
-
fail_deep_scan.add_argument("--message", required=True)
|
|
117
|
-
fail_deep_scan.add_argument("--manifest-path")
|
|
118
|
-
fail_deep_scan.add_argument("--staged-manifest-path")
|
|
119
|
-
fail_deep_scan.add_argument(
|
|
120
|
-
"--deep-status", choices=("failed", "interrupted"), default="failed"
|
|
121
|
-
)
|
|
122
|
-
fail_deep_scan.add_argument("--coordinator-generation", type=positive_int)
|
|
123
|
-
|
|
124
|
-
publication_failure = subparsers.add_parser("record-deep-scan-publication-failure")
|
|
125
|
-
publication_failure.add_argument("--scan-id", required=True)
|
|
126
|
-
publication_failure.add_argument("--message", required=True)
|
|
127
|
-
publication_failure.add_argument("--coordinator-generation", type=positive_int)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def non_negative_int(value: str) -> int:
|
|
131
|
-
parsed = int(value)
|
|
132
|
-
if parsed < 0:
|
|
133
|
-
raise argparse.ArgumentTypeError("expected a non-negative integer")
|
|
134
|
-
return parsed
|
|
135
|
-
|
|
136
|
-
|
|
137
35
|
_dependencies: Any = None
|
|
138
36
|
|
|
139
37
|
|
|
@@ -149,6 +47,7 @@ def dependencies() -> Any:
|
|
|
149
47
|
|
|
150
48
|
|
|
151
49
|
def deep_scan_error(run: sqlite3.Row) -> str | None:
|
|
50
|
+
"""Return recorded error messages, publication error first, or None if both are absent."""
|
|
152
51
|
original = run["error_message"]
|
|
153
52
|
publication = run["publication_error_message"]
|
|
154
53
|
if not isinstance(publication, str):
|
|
@@ -175,6 +74,7 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit
|
|
|
175
74
|
|
|
176
75
|
|
|
177
76
|
def deep_scan_deadline_reached(run: sqlite3.Row) -> bool:
|
|
77
|
+
"""Return whether the time since run creation has reached its configured limit in hours."""
|
|
178
78
|
elapsed = _parse_timestamp(dependencies().now()) - _parse_timestamp(str(run["created_at"]))
|
|
179
79
|
return elapsed.total_seconds() / 3600 >= run["max_time_hours"]
|
|
180
80
|
|
|
@@ -182,6 +82,11 @@ def deep_scan_deadline_reached(run: sqlite3.Row) -> bool:
|
|
|
182
82
|
def require_deep_scan_ready_for_parent_completion(
|
|
183
83
|
connection: sqlite3.Connection, scan: sqlite3.Row
|
|
184
84
|
) -> None:
|
|
85
|
+
"""Require a succeeded discovery run with a saved manifest before a Deep Scan completes.
|
|
86
|
+
|
|
87
|
+
For a capped run, also verify its workers have finished. Raise SystemExit
|
|
88
|
+
when a completion requirement fails. Other scan modes can complete directly.
|
|
89
|
+
"""
|
|
185
90
|
if scan["mode"] != "deep":
|
|
186
91
|
return
|
|
187
92
|
run = connection.execute(
|
|
@@ -276,6 +181,7 @@ def require_deep_scan_workers_finished(
|
|
|
276
181
|
def require_owned_scan(
|
|
277
182
|
connection: sqlite3.Connection, scan_id: str, thread_id: str
|
|
278
183
|
) -> tuple[sqlite3.Row, sqlite3.Row]:
|
|
184
|
+
"""Return the scan and workspace rows or raise SystemExit unless thread_id matches the recorded owner."""
|
|
279
185
|
scan = dependencies().require_scan(connection, scan_id)
|
|
280
186
|
workspace = dependencies().require_workspace(connection, scan["workspace_id"])
|
|
281
187
|
owner = optional_text(thread_id, maximum=512)
|
|
@@ -294,6 +200,7 @@ def deep_scan_path(
|
|
|
294
200
|
*,
|
|
295
201
|
kind: str,
|
|
296
202
|
) -> str:
|
|
203
|
+
"""Return the canonical absolute path to an existing scan-local file or directory of the requested kind."""
|
|
297
204
|
supplied = Path(value).expanduser()
|
|
298
205
|
if not supplied.is_absolute():
|
|
299
206
|
raise SystemExit(f"{label} must be an absolute path inside the scan directory.")
|
|
@@ -313,6 +220,7 @@ def deep_scan_path(
|
|
|
313
220
|
|
|
314
221
|
|
|
315
222
|
def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str:
|
|
223
|
+
"""Validate an absolute output path under the scan directory, allowing a new file under an existing parent."""
|
|
316
224
|
supplied = Path(value).expanduser()
|
|
317
225
|
if not supplied.is_absolute():
|
|
318
226
|
raise SystemExit(f"{label} must be an absolute path inside the scan directory.")
|
|
@@ -326,6 +234,7 @@ def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str:
|
|
|
326
234
|
|
|
327
235
|
|
|
328
236
|
def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]:
|
|
237
|
+
"""Publish the staged file and return the paths needed to finalize or undo the move."""
|
|
329
238
|
staged = Path(staged_path)
|
|
330
239
|
output = Path(output_path)
|
|
331
240
|
if staged == output:
|
|
@@ -343,6 +252,7 @@ def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path,
|
|
|
343
252
|
|
|
344
253
|
|
|
345
254
|
def rollback_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
255
|
+
"""Move the published file back to staging and restore the previous output if present."""
|
|
346
256
|
staged, output, backup = promotion
|
|
347
257
|
if output.exists():
|
|
348
258
|
os.replace(output, staged)
|
|
@@ -351,19 +261,14 @@ def rollback_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
|
351
261
|
|
|
352
262
|
|
|
353
263
|
def finish_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
264
|
+
"""Remove the previous output saved during promotion."""
|
|
354
265
|
backup = promotion[2]
|
|
355
266
|
if backup is not None:
|
|
356
267
|
backup.unlink(missing_ok=True)
|
|
357
268
|
|
|
358
269
|
|
|
359
|
-
def create_publication_copy(source: str | Path, destination: str | Path) -> None:
|
|
360
|
-
try:
|
|
361
|
-
os.link(source, destination)
|
|
362
|
-
except OSError:
|
|
363
|
-
shutil.copy2(source, destination)
|
|
364
|
-
|
|
365
|
-
|
|
366
270
|
def publication_matches_snapshot(publication: Path, snapshot: Path) -> bool:
|
|
271
|
+
"""Return whether the publication and snapshot identify the same file or contain identical bytes."""
|
|
367
272
|
try:
|
|
368
273
|
if publication.samefile(snapshot):
|
|
369
274
|
return True
|
|
@@ -393,17 +298,14 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]:
|
|
|
393
298
|
|
|
394
299
|
|
|
395
300
|
def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]:
|
|
301
|
+
"""Return persisted orchestration state including settings, workers, artifacts, and deduplication inputs.
|
|
302
|
+
|
|
303
|
+
Order workers by creation time and ID. Retain the recorded order of each
|
|
304
|
+
deduplication worker’s discovery inputs.
|
|
305
|
+
"""
|
|
396
306
|
run = require_deep_scan_run(connection, scan_id)
|
|
397
307
|
scan = dependencies().require_scan(connection, run["scan_id"])
|
|
398
|
-
worker_rows = connection
|
|
399
|
-
"""
|
|
400
|
-
SELECT *
|
|
401
|
-
FROM deep_scan_workers
|
|
402
|
-
WHERE scan_id = ?
|
|
403
|
-
ORDER BY created_at, id
|
|
404
|
-
""",
|
|
405
|
-
(run["scan_id"],),
|
|
406
|
-
)
|
|
308
|
+
worker_rows = saved_deep_workers(connection, scan)
|
|
407
309
|
input_rows = connection.execute(
|
|
408
310
|
"""
|
|
409
311
|
SELECT dedup_worker_id, discovery_worker_id, input_order
|
|
@@ -413,12 +315,13 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
413
315
|
""",
|
|
414
316
|
(run["scan_id"],),
|
|
415
317
|
)
|
|
318
|
+
manifest_path = archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
416
319
|
canonical_artifacts = None
|
|
417
320
|
if (
|
|
418
321
|
run["canonical_inventory_path"] is None
|
|
419
322
|
and run["status"] == "succeeded"
|
|
420
|
-
and
|
|
421
|
-
and
|
|
323
|
+
and manifest_path is not None
|
|
324
|
+
and manifest_path != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
422
325
|
and (Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "in_scope_files.txt").exists()
|
|
423
326
|
):
|
|
424
327
|
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
@@ -432,9 +335,12 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
432
335
|
"A capped Deep Scan without completed discoveries requires an empty "
|
|
433
336
|
"candidate ledger."
|
|
434
337
|
)
|
|
338
|
+
recipe = json.loads(scan["recipe_json"]) if scan["recipe_json"] is not None else {}
|
|
435
339
|
return {
|
|
436
340
|
"scanId": run["scan_id"],
|
|
437
341
|
"sdkOwnsCompletion": scan["recipe_json"] is not None,
|
|
342
|
+
"serviceTier": run["service_tier"],
|
|
343
|
+
**({"severityRubric": recipe["severityRubric"]} if "severityRubric" in recipe else {}),
|
|
438
344
|
"targetPath": scan["target_path"],
|
|
439
345
|
"scope": scan["scope"],
|
|
440
346
|
"includePaths": dependencies().requested_scan_paths(scan),
|
|
@@ -445,11 +351,17 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
445
351
|
else scan["user_context"]
|
|
446
352
|
),
|
|
447
353
|
"scanDir": scan["scan_dir"],
|
|
354
|
+
"workerUsageDirectory": str(worker_usage_directory(run["scan_id"])),
|
|
448
355
|
"schemaVersion": run["schema_version"],
|
|
449
356
|
"workflowVersion": run["workflow_version"],
|
|
450
357
|
"coordinatorGeneration": run["coordinator_generation"],
|
|
451
358
|
"status": run["status"],
|
|
452
359
|
"phase": run["phase"],
|
|
360
|
+
# Older coordinators could persist discovery on the parent before dispatching.
|
|
361
|
+
"discoveryStarted": (
|
|
362
|
+
PHASES.index(scan["phase"]) >= PHASES.index("discovery")
|
|
363
|
+
or run["phase"] in {"discovery", "reducing"}
|
|
364
|
+
),
|
|
453
365
|
"config": {
|
|
454
366
|
"workers": run["workers"],
|
|
455
367
|
"subagents": run["subagents"],
|
|
@@ -464,7 +376,7 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
464
376
|
"consecutiveErrors": run["consecutive_errors"],
|
|
465
377
|
"cancelRequested": bool(run["cancel_requested"]),
|
|
466
378
|
"canonicalArtifacts": canonical_artifacts,
|
|
467
|
-
"manifestPath":
|
|
379
|
+
"manifestPath": manifest_path,
|
|
468
380
|
"terminalReason": run["terminal_reason"],
|
|
469
381
|
"error": deep_scan_error(run),
|
|
470
382
|
"createdAt": run["created_at"],
|
|
@@ -483,8 +395,9 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
483
395
|
|
|
484
396
|
|
|
485
397
|
def independent_review_progress(
|
|
486
|
-
connection: sqlite3.Connection, scan_id: str
|
|
398
|
+
connection: sqlite3.Connection, scan_id: str, *, parent_running: bool
|
|
487
399
|
) -> dict[str, int | str] | None:
|
|
400
|
+
"""Return completed and allowed review counts and the consolidation state, or None for a missing run."""
|
|
488
401
|
run = connection.execute(
|
|
489
402
|
"""
|
|
490
403
|
SELECT completion_sequence, phase, updated_at, max_discovery_runs
|
|
@@ -495,21 +408,21 @@ def independent_review_progress(
|
|
|
495
408
|
).fetchone()
|
|
496
409
|
if run is None:
|
|
497
410
|
return None
|
|
498
|
-
active = connection.execute(
|
|
411
|
+
active, buffered = connection.execute(
|
|
499
412
|
"""
|
|
500
|
-
SELECT COUNT(*)
|
|
413
|
+
SELECT COUNT(*) FILTER (WHERE status IN ('queued', 'running')),
|
|
414
|
+
COUNT(*) FILTER (WHERE status = 'succeeded' AND merge_state = 'buffered')
|
|
501
415
|
FROM deep_scan_workers
|
|
502
|
-
WHERE scan_id = ?
|
|
503
|
-
AND kind = 'discovery'
|
|
504
|
-
AND status IN ('queued', 'running')
|
|
416
|
+
WHERE scan_id = ? AND kind = 'discovery'
|
|
505
417
|
""",
|
|
506
418
|
(scan_id,),
|
|
507
|
-
).fetchone()
|
|
419
|
+
).fetchone()
|
|
420
|
+
consolidating = run["phase"] in {"reducing", "terminal"} or (active == 0 and buffered > 0)
|
|
508
421
|
return {
|
|
509
422
|
"active": int(active),
|
|
510
423
|
"completed": int(run["completion_sequence"]),
|
|
511
424
|
"maximum": int(run["max_discovery_runs"]),
|
|
512
|
-
"consolidating":
|
|
425
|
+
"consolidating": parent_running and consolidating,
|
|
513
426
|
"updatedAt": str(run["updated_at"]),
|
|
514
427
|
}
|
|
515
428
|
|
|
@@ -549,10 +462,10 @@ def deep_scan_result(
|
|
|
549
462
|
def effective_deep_scan_config(
|
|
550
463
|
args: argparse.Namespace, workspace: sqlite3.Row | None = None
|
|
551
464
|
) -> dict[str, Any]:
|
|
465
|
+
"""Use persisted workspace settings when available, otherwise resolve the current launch configuration."""
|
|
552
466
|
if workspace is not None and workspace["deep_scan_config_json"] is not None:
|
|
553
467
|
return json.loads(workspace["deep_scan_config_json"])
|
|
554
|
-
|
|
555
|
-
return resolve_deep_scan_setup(available_parallelism, args.deep_scan_project_json)
|
|
468
|
+
return resolve_deep_scan_setup(project_json=args.deep_scan_project_json)
|
|
556
469
|
|
|
557
470
|
|
|
558
471
|
def ensure_deep_scan_run(
|
|
@@ -562,8 +475,10 @@ def ensure_deep_scan_run(
|
|
|
562
475
|
workflow_version: str,
|
|
563
476
|
timestamp: str,
|
|
564
477
|
*,
|
|
478
|
+
service_tier: str | None,
|
|
565
479
|
pending_target_project_path: str | None = None,
|
|
566
480
|
) -> sqlite3.Row:
|
|
481
|
+
"""Return the existing orchestration row or initialize one for a running Deep Scan with the supplied settings."""
|
|
567
482
|
existing = connection.execute(
|
|
568
483
|
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],)
|
|
569
484
|
).fetchone()
|
|
@@ -579,8 +494,8 @@ def ensure_deep_scan_run(
|
|
|
579
494
|
scan_id, schema_version, workflow_version, status, phase,
|
|
580
495
|
workers, subagents, stop_after_no_new, stop_after_consecutive_errors,
|
|
581
496
|
max_discovery_runs, max_time_hours, discovery_user_context_json,
|
|
582
|
-
created_at, updated_at, pending_target_project_path
|
|
583
|
-
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
497
|
+
created_at, updated_at, service_tier, pending_target_project_path
|
|
498
|
+
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
584
499
|
""",
|
|
585
500
|
(
|
|
586
501
|
scan["id"],
|
|
@@ -594,6 +509,7 @@ def ensure_deep_scan_run(
|
|
|
594
509
|
json.dumps(scan["user_context"]),
|
|
595
510
|
timestamp,
|
|
596
511
|
timestamp,
|
|
512
|
+
service_tier,
|
|
597
513
|
pending_target_project_path,
|
|
598
514
|
),
|
|
599
515
|
)
|
|
@@ -694,6 +610,8 @@ def begin_deep_scan_for_scan(
|
|
|
694
610
|
) -> dict[str, Any]:
|
|
695
611
|
scan_id = require_uuid(scan_id, "scan-id")
|
|
696
612
|
candidate = dependencies().require_scan(connection, scan_id)
|
|
613
|
+
model_thread_id = getattr(args, "model_thread_id", None)
|
|
614
|
+
require_tokenless_scan_owner(connection, candidate, model_thread_id)
|
|
697
615
|
workspace = dependencies().require_workspace(connection, candidate["workspace_id"])
|
|
698
616
|
if (
|
|
699
617
|
candidate["mode"] == "deep"
|
|
@@ -718,8 +636,17 @@ def begin_deep_scan_for_scan(
|
|
|
718
636
|
claimed_scan = connection.execute(
|
|
719
637
|
"UPDATE scans SET deep_scan_owner_thread_id = ?, updated_at = ? "
|
|
720
638
|
"WHERE id = ? AND deep_scan_owner_thread_id IS NULL "
|
|
721
|
-
"AND handoff_status = 'delivered' AND handoff_claim_token IS ?"
|
|
722
|
-
(
|
|
639
|
+
"AND handoff_status = 'delivered' AND handoff_claim_token IS ? "
|
|
640
|
+
"AND (handoff_claim_token IS NOT NULL OR continuation_thread_id = ? "
|
|
641
|
+
"OR (? IS NULL AND continuation_thread_id IS NULL))",
|
|
642
|
+
(
|
|
643
|
+
thread_id,
|
|
644
|
+
timestamp,
|
|
645
|
+
scan_id,
|
|
646
|
+
candidate["handoff_claim_token"],
|
|
647
|
+
thread_id,
|
|
648
|
+
model_thread_id,
|
|
649
|
+
),
|
|
723
650
|
)
|
|
724
651
|
if claimed_workspace.rowcount != 1 or claimed_scan.rowcount != 1:
|
|
725
652
|
raise SystemExit("A scan can only be orchestrated from its owning Codex thread.")
|
|
@@ -768,7 +695,14 @@ def begin_deep_scan_for_scan(
|
|
|
768
695
|
"UPDATE workspaces SET deep_scan_config_json = ? WHERE id = ? AND deep_scan_config_json IS NULL",
|
|
769
696
|
(json.dumps(configuration), workspace["id"]),
|
|
770
697
|
)
|
|
771
|
-
ensure_deep_scan_run(
|
|
698
|
+
ensure_deep_scan_run(
|
|
699
|
+
connection,
|
|
700
|
+
scan,
|
|
701
|
+
config,
|
|
702
|
+
workflow_version,
|
|
703
|
+
dependencies().now(),
|
|
704
|
+
service_tier=args.service_tier,
|
|
705
|
+
)
|
|
772
706
|
return deep_scan_result(connection, scan_id, start_disposition="created")
|
|
773
707
|
|
|
774
708
|
|
|
@@ -859,11 +793,8 @@ def begin_deep_scan_for_target(
|
|
|
859
793
|
workflow_version = optional_text(args.workflow_version, maximum=256)
|
|
860
794
|
if workflow_version is None:
|
|
861
795
|
raise SystemExit("workflow-version is required.")
|
|
862
|
-
|
|
863
|
-
target_root =
|
|
864
|
-
if target_root == target or target in target_root.parents:
|
|
865
|
-
raise SystemExit("The scan artifact directory must be outside the selected target.")
|
|
866
|
-
create_private_directory(target_root)
|
|
796
|
+
target_root = dependencies().scan_target_root(args.scan_root, target)
|
|
797
|
+
create_private_directory(target_root, scan_output=True)
|
|
867
798
|
model = optional_text(args.model, maximum=200)
|
|
868
799
|
reasoning_effort = optional_text(args.reasoning_effort, maximum=32)
|
|
869
800
|
workspace_id = str(uuid.uuid4())
|
|
@@ -917,12 +848,14 @@ def begin_deep_scan_for_target(
|
|
|
917
848
|
config,
|
|
918
849
|
workflow_version,
|
|
919
850
|
timestamp,
|
|
851
|
+
service_tier=args.service_tier,
|
|
920
852
|
pending_target_project_path=project_path,
|
|
921
853
|
)
|
|
922
854
|
return deep_scan_result(connection, scan_id, start_disposition="created")
|
|
923
855
|
|
|
924
856
|
|
|
925
857
|
def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
858
|
+
"""Begin orchestration for a registered scan or a target, returning the scan state and its start disposition."""
|
|
926
859
|
thread_id = optional_text(args.thread_id, maximum=512)
|
|
927
860
|
if thread_id is None:
|
|
928
861
|
raise SystemExit("thread-id is required.")
|
|
@@ -936,6 +869,7 @@ def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) ->
|
|
|
936
869
|
|
|
937
870
|
|
|
938
871
|
def get_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
872
|
+
"""Return the persisted orchestration result to the thread that owns the scan."""
|
|
939
873
|
scan, _ = require_owned_scan(connection, args.scan_id, args.thread_id)
|
|
940
874
|
return deep_scan_result(connection, scan["id"])
|
|
941
875
|
|
|
@@ -964,6 +898,7 @@ def coordinator_lease_is_live(
|
|
|
964
898
|
scan: sqlite3.Row,
|
|
965
899
|
timestamp: str,
|
|
966
900
|
) -> bool:
|
|
901
|
+
"""Check coordinator freshness using the lease generation and its applicable activity timestamps."""
|
|
967
902
|
if run["coordinator_generation"] == 1:
|
|
968
903
|
active_worker = connection.execute(
|
|
969
904
|
"""
|
|
@@ -1027,7 +962,8 @@ def claim_deep_scan_coordinator_locked(
|
|
|
1027
962
|
args.claim_token,
|
|
1028
963
|
error_message="Deep Scan orchestration is owned by another continuation.",
|
|
1029
964
|
)
|
|
1030
|
-
run
|
|
965
|
+
run = require_deep_scan_run(connection, scan_id)
|
|
966
|
+
require_running_deep_scan(run, scan)
|
|
1031
967
|
timestamp = dependencies().now()
|
|
1032
968
|
if args.coordinator_generation is not None:
|
|
1033
969
|
require_current_coordinator(run, args)
|
|
@@ -1061,6 +997,7 @@ def claim_deep_scan_coordinator_locked(
|
|
|
1061
997
|
def recover_expired_coordinator(
|
|
1062
998
|
connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str
|
|
1063
999
|
) -> None:
|
|
1000
|
+
"""Recover interrupted publications and reset active work so a replacement coordinator can resume."""
|
|
1064
1001
|
scan_id = run["scan_id"]
|
|
1065
1002
|
recover_candidate_ledger_publication(connection, scan_id)
|
|
1066
1003
|
legacy_generation = int(run["coordinator_generation"] == 1)
|
|
@@ -1136,6 +1073,7 @@ def recover_expired_coordinator(
|
|
|
1136
1073
|
|
|
1137
1074
|
|
|
1138
1075
|
def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None:
|
|
1076
|
+
"""Reconcile ledger backups with reducer snapshots after an interrupted publication."""
|
|
1139
1077
|
scan = dependencies().require_scan(connection, scan_id)
|
|
1140
1078
|
ledger = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl"
|
|
1141
1079
|
backups = sorted(
|
|
@@ -1173,26 +1111,11 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id
|
|
|
1173
1111
|
return
|
|
1174
1112
|
|
|
1175
1113
|
|
|
1176
|
-
def
|
|
1177
|
-
worker_id = require_uuid(worker_id, "worker-id")
|
|
1178
|
-
row = connection.execute(
|
|
1179
|
-
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
1180
|
-
).fetchone()
|
|
1181
|
-
if row is None:
|
|
1182
|
-
raise SystemExit("Codex Security Deep Scan worker not found.")
|
|
1183
|
-
return row
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
def require_running_deep_scan(
|
|
1187
|
-
connection: sqlite3.Connection, scan_id: str
|
|
1188
|
-
) -> tuple[sqlite3.Row, sqlite3.Row]:
|
|
1189
|
-
run = require_deep_scan_run(connection, scan_id)
|
|
1190
|
-
scan = dependencies().require_scan(connection, run["scan_id"])
|
|
1114
|
+
def require_running_deep_scan(run: sqlite3.Row, scan: sqlite3.Row) -> None:
|
|
1191
1115
|
if run["status"] != "running" or run["cancel_requested"]:
|
|
1192
1116
|
raise SystemExit("Only a running Deep Scan can update orchestration state.")
|
|
1193
1117
|
if scan["status"] != "running" or scan["canceled_at"] is not None:
|
|
1194
1118
|
raise SystemExit("Only a running scan can update Deep Scan orchestration state.")
|
|
1195
|
-
return run, scan
|
|
1196
1119
|
|
|
1197
1120
|
|
|
1198
1121
|
def require_worker_transition(current: str, requested: str) -> None:
|
|
@@ -1220,6 +1143,12 @@ def upsert_deep_scan_worker(
|
|
|
1220
1143
|
existing = connection.execute(
|
|
1221
1144
|
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
1222
1145
|
).fetchone()
|
|
1146
|
+
if args.terminal_failure and (
|
|
1147
|
+
args.status != "failed" or not optional_text(args.error_message)
|
|
1148
|
+
):
|
|
1149
|
+
raise SystemExit(
|
|
1150
|
+
"A terminal worker failure requires failed status and an error message."
|
|
1151
|
+
)
|
|
1223
1152
|
replaceable_failure_kind = args.replaceable_failure_kind
|
|
1224
1153
|
if replaceable_failure_kind is not None and (
|
|
1225
1154
|
args.kind != "discovery"
|
|
@@ -1244,7 +1173,7 @@ def upsert_deep_scan_worker(
|
|
|
1244
1173
|
and args.status in {"succeeded", "failed", "canceled"}
|
|
1245
1174
|
)
|
|
1246
1175
|
if not cleanup_update and not terminal_repeat:
|
|
1247
|
-
require_running_deep_scan(
|
|
1176
|
+
require_running_deep_scan(run, scan)
|
|
1248
1177
|
prompt_path = deep_scan_path(scan, args.prompt_path, "Worker prompt path", kind="file")
|
|
1249
1178
|
artifact_dir = deep_scan_path(
|
|
1250
1179
|
scan, args.artifact_dir, "Worker artifact directory", kind="directory"
|
|
@@ -1262,7 +1191,9 @@ def upsert_deep_scan_worker(
|
|
|
1262
1191
|
timestamp = dependencies().now()
|
|
1263
1192
|
if existing is None:
|
|
1264
1193
|
if args.kind == "dedup":
|
|
1265
|
-
raise SystemExit(
|
|
1194
|
+
raise SystemExit(
|
|
1195
|
+
"Deep Scan checkpoints are recorded with record-deep-scan-iteration."
|
|
1196
|
+
)
|
|
1266
1197
|
if args.status not in {"queued", "running"}:
|
|
1267
1198
|
raise SystemExit("A new Deep Scan worker must be queued or running.")
|
|
1268
1199
|
if (
|
|
@@ -1345,7 +1276,7 @@ def upsert_deep_scan_worker(
|
|
|
1345
1276
|
if args.status == "running" and attempt < 1:
|
|
1346
1277
|
raise SystemExit("A running Deep Scan worker attempt must be at least one.")
|
|
1347
1278
|
if args.kind == "dedup" and args.status == "succeeded":
|
|
1348
|
-
raise SystemExit("
|
|
1279
|
+
raise SystemExit("Deep Scan checkpoints are recorded with record-deep-scan-iteration.")
|
|
1349
1280
|
if args.kind == "discovery" and args.status == "succeeded" and result_manifest_path is None:
|
|
1350
1281
|
result_manifest_path = existing["result_manifest_path"]
|
|
1351
1282
|
if result_manifest_path is None:
|
|
@@ -1410,11 +1341,14 @@ def upsert_deep_scan_worker(
|
|
|
1410
1341
|
if error_message is None and args.status != "succeeded":
|
|
1411
1342
|
error_message = existing["error_message"]
|
|
1412
1343
|
started_at = existing["started_at"] or (timestamp if args.status == "running" else None)
|
|
1344
|
+
thread_id = optional_text(args.sdk_thread_id, maximum=512)
|
|
1345
|
+
if thread_id is None and attempt == existing["attempt"]:
|
|
1346
|
+
thread_id = existing["sdk_thread_id"]
|
|
1413
1347
|
connection.execute(
|
|
1414
1348
|
"""
|
|
1415
1349
|
UPDATE deep_scan_workers
|
|
1416
1350
|
SET status = ?, result_manifest_path = ?, attempt = ?,
|
|
1417
|
-
sdk_thread_id =
|
|
1351
|
+
sdk_thread_id = ?,
|
|
1418
1352
|
completion_sequence = ?, merge_state = ?,
|
|
1419
1353
|
error_message = ?,
|
|
1420
1354
|
started_at = ?, completed_at = ?, updated_at = ?
|
|
@@ -1424,7 +1358,7 @@ def upsert_deep_scan_worker(
|
|
|
1424
1358
|
args.status,
|
|
1425
1359
|
result_manifest_path or existing["result_manifest_path"],
|
|
1426
1360
|
attempt,
|
|
1427
|
-
|
|
1361
|
+
thread_id,
|
|
1428
1362
|
completion_sequence,
|
|
1429
1363
|
merge_state,
|
|
1430
1364
|
error_message,
|
|
@@ -1434,275 +1368,81 @@ def upsert_deep_scan_worker(
|
|
|
1434
1368
|
worker_id,
|
|
1435
1369
|
),
|
|
1436
1370
|
)
|
|
1371
|
+
if args.terminal_failure:
|
|
1372
|
+
# Persist the first terminal cause with its worker. An adopting coordinator
|
|
1373
|
+
# completes the normal failure path after active workers have settled.
|
|
1374
|
+
connection.execute(
|
|
1375
|
+
"UPDATE deep_scan_runs SET error_message = ? WHERE scan_id = ? AND error_message IS NULL",
|
|
1376
|
+
(error_message, scan_id),
|
|
1377
|
+
)
|
|
1437
1378
|
return deep_scan_result(connection, scan_id)
|
|
1438
1379
|
|
|
1439
1380
|
|
|
1440
|
-
def
|
|
1381
|
+
def record_deep_scan_iteration(
|
|
1441
1382
|
connection: sqlite3.Connection, args: argparse.Namespace
|
|
1442
1383
|
) -> dict[str, Any]:
|
|
1384
|
+
"""Commit the checkpoint and its input workers in one transaction, then return deep scan state.
|
|
1385
|
+
|
|
1386
|
+
Repeating the same iteration and inputs replays the saved result. Conflicting
|
|
1387
|
+
checkpoint or input-worker IDs raise SystemExit.
|
|
1388
|
+
"""
|
|
1443
1389
|
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1444
|
-
|
|
1390
|
+
checkpoint_id = require_uuid(args.worker_id, "worker-id")
|
|
1445
1391
|
input_ids = [require_uuid(value, "input-worker-id") for value in args.input_worker_id]
|
|
1446
|
-
if len(set(input_ids)) != len(input_ids):
|
|
1447
|
-
raise SystemExit("Dedup input worker IDs must be unique.")
|
|
1448
1392
|
connection.execute("BEGIN IMMEDIATE")
|
|
1449
1393
|
with connection:
|
|
1450
|
-
run
|
|
1394
|
+
run = require_deep_scan_run(connection, scan_id)
|
|
1395
|
+
scan = dependencies().require_scan(connection, scan_id)
|
|
1396
|
+
require_running_deep_scan(run, scan)
|
|
1451
1397
|
require_current_coordinator(run, args)
|
|
1452
|
-
|
|
1453
|
-
artifact_dir = deep_scan_path(
|
|
1454
|
-
scan, args.artifact_dir, "Dedup artifact directory", kind="directory"
|
|
1455
|
-
)
|
|
1398
|
+
result = deep_scan_path(scan, args.result_manifest_path, "Iteration result", kind="file")
|
|
1456
1399
|
existing = connection.execute(
|
|
1457
|
-
"SELECT
|
|
1400
|
+
"SELECT scan_id, result_manifest_path FROM deep_scan_workers WHERE id = ?",
|
|
1401
|
+
(checkpoint_id,),
|
|
1458
1402
|
).fetchone()
|
|
1459
1403
|
if existing is not None:
|
|
1460
|
-
|
|
1461
|
-
|
|
1404
|
+
if existing["scan_id"] != scan_id or existing["result_manifest_path"] != result:
|
|
1405
|
+
raise SystemExit("Iteration already belongs to another result.")
|
|
1406
|
+
recorded_ids = [
|
|
1407
|
+
row[0]
|
|
1462
1408
|
for row in connection.execute(
|
|
1463
|
-
""
|
|
1464
|
-
|
|
1465
|
-
FROM deep_scan_dedup_inputs
|
|
1466
|
-
WHERE dedup_worker_id = ?
|
|
1467
|
-
ORDER BY input_order
|
|
1468
|
-
""",
|
|
1469
|
-
(worker_id,),
|
|
1409
|
+
"SELECT discovery_worker_id FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order",
|
|
1410
|
+
(checkpoint_id,),
|
|
1470
1411
|
)
|
|
1471
1412
|
]
|
|
1472
|
-
if
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
and persisted_inputs == input_ids
|
|
1478
|
-
):
|
|
1479
|
-
connection.commit()
|
|
1480
|
-
return deep_scan_result(connection, scan_id)
|
|
1481
|
-
raise SystemExit("Dedup worker ID is already used by a different reducer claim.")
|
|
1482
|
-
active_reducer = connection.execute(
|
|
1483
|
-
"""
|
|
1484
|
-
SELECT 1 FROM deep_scan_workers
|
|
1485
|
-
WHERE scan_id = ? AND kind = 'dedup' AND status IN ('queued', 'running')
|
|
1486
|
-
""",
|
|
1487
|
-
(scan_id,),
|
|
1488
|
-
).fetchone()
|
|
1489
|
-
if active_reducer is not None:
|
|
1490
|
-
raise SystemExit("Only one Deep Scan dedup worker can run at a time.")
|
|
1491
|
-
buffered_ids = [
|
|
1492
|
-
row["id"]
|
|
1493
|
-
for row in connection.execute(
|
|
1494
|
-
"""
|
|
1495
|
-
SELECT id FROM deep_scan_workers
|
|
1496
|
-
WHERE scan_id = ? AND kind = 'discovery'
|
|
1497
|
-
AND status = 'succeeded' AND merge_state = 'buffered'
|
|
1498
|
-
ORDER BY completion_sequence
|
|
1499
|
-
""",
|
|
1500
|
-
(scan_id,),
|
|
1501
|
-
)
|
|
1502
|
-
]
|
|
1503
|
-
if input_ids != buffered_ids[: len(input_ids)]:
|
|
1504
|
-
raise SystemExit(
|
|
1505
|
-
"A Deep Scan dedup worker must claim an ordered prefix of buffered discovery "
|
|
1506
|
-
"results in completion order."
|
|
1507
|
-
)
|
|
1508
|
-
capped_singleton = (
|
|
1509
|
-
len(input_ids) == 1
|
|
1510
|
-
and (
|
|
1511
|
-
run["discovery_runs_dispatched"] >= run["max_discovery_runs"]
|
|
1512
|
-
or deep_scan_deadline_reached(run)
|
|
1513
|
-
)
|
|
1514
|
-
and connection.execute(
|
|
1515
|
-
"""
|
|
1516
|
-
SELECT 1 FROM deep_scan_workers
|
|
1517
|
-
WHERE scan_id = ? AND kind = 'discovery' AND status IN ('queued', 'running')
|
|
1518
|
-
LIMIT 1
|
|
1519
|
-
""",
|
|
1520
|
-
(scan_id,),
|
|
1521
|
-
).fetchone()
|
|
1522
|
-
is None
|
|
1523
|
-
)
|
|
1524
|
-
successful_reducer = connection.execute(
|
|
1525
|
-
"""
|
|
1526
|
-
SELECT 1 FROM deep_scan_workers
|
|
1527
|
-
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
1528
|
-
LIMIT 1
|
|
1529
|
-
""",
|
|
1530
|
-
(scan_id,),
|
|
1531
|
-
).fetchone()
|
|
1532
|
-
minimum_inputs = 1 if successful_reducer is not None or capped_singleton else 2
|
|
1533
|
-
if len(input_ids) < minimum_inputs:
|
|
1534
|
-
raise SystemExit(
|
|
1535
|
-
"The first Deep Scan dedup requires two buffered discovery results."
|
|
1536
|
-
if minimum_inputs == 2
|
|
1537
|
-
else "A Deep Scan dedup requires at least one buffered discovery result."
|
|
1538
|
-
)
|
|
1539
|
-
timestamp = dependencies().now()
|
|
1540
|
-
connection.execute(
|
|
1541
|
-
"""
|
|
1542
|
-
INSERT INTO deep_scan_workers (
|
|
1543
|
-
id, scan_id, kind, status, prompt_path, artifact_dir,
|
|
1544
|
-
created_at, updated_at
|
|
1545
|
-
) VALUES (?, ?, 'dedup', 'queued', ?, ?, ?, ?)
|
|
1546
|
-
""",
|
|
1547
|
-
(worker_id, scan_id, prompt_path, artifact_dir, timestamp, timestamp),
|
|
1548
|
-
)
|
|
1549
|
-
for input_order, input_id in enumerate(input_ids):
|
|
1550
|
-
connection.execute(
|
|
1551
|
-
"""
|
|
1552
|
-
INSERT INTO deep_scan_dedup_inputs (
|
|
1553
|
-
scan_id, dedup_worker_id, discovery_worker_id, input_order
|
|
1554
|
-
) VALUES (?, ?, ?, ?)
|
|
1555
|
-
""",
|
|
1556
|
-
(scan_id, worker_id, input_id, input_order),
|
|
1557
|
-
)
|
|
1558
|
-
connection.execute(
|
|
1559
|
-
f"""
|
|
1560
|
-
UPDATE deep_scan_workers
|
|
1561
|
-
SET merge_state = 'merging', updated_at = ?
|
|
1562
|
-
WHERE scan_id = ? AND id IN ({",".join("?" for _ in input_ids)})
|
|
1563
|
-
""",
|
|
1564
|
-
(timestamp, scan_id, *input_ids),
|
|
1565
|
-
)
|
|
1566
|
-
connection.execute(
|
|
1567
|
-
"UPDATE deep_scan_runs SET phase = 'reducing', updated_at = ? WHERE scan_id = ?",
|
|
1568
|
-
(timestamp, scan_id),
|
|
1569
|
-
)
|
|
1413
|
+
if recorded_ids != input_ids:
|
|
1414
|
+
raise SystemExit("Iteration already belongs to other completed scans.")
|
|
1415
|
+
return deep_scan_result(connection, scan_id)
|
|
1416
|
+
now = dependencies().now()
|
|
1417
|
+
# Historical scans use the same rows to bind the selected aggregate at publication.
|
|
1570
1418
|
connection.execute(
|
|
1571
|
-
"""
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
(timestamp, scan_id),
|
|
1419
|
+
"""INSERT INTO deep_scan_workers
|
|
1420
|
+
(id, scan_id, kind, status, prompt_path, artifact_dir, result_manifest_path,
|
|
1421
|
+
created_at, completed_at, updated_at)
|
|
1422
|
+
VALUES (?, ?, 'dedup', 'succeeded', ?, ?, ?, ?, ?, ?)""",
|
|
1423
|
+
(checkpoint_id, scan_id, result, str(Path(result).parent), result, now, now, now),
|
|
1577
1424
|
)
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1585
|
-
with dependencies().scan_completion_lock(scan_id):
|
|
1586
|
-
return commit_deep_scan_dedup_locked(connection, args, scan_id)
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
def commit_deep_scan_dedup_locked(
|
|
1590
|
-
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1591
|
-
) -> dict[str, Any]:
|
|
1592
|
-
worker_id = require_uuid(args.worker_id, "worker-id")
|
|
1593
|
-
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1594
|
-
publication_copy: Path | None = None
|
|
1595
|
-
connection.execute("BEGIN IMMEDIATE")
|
|
1596
|
-
try:
|
|
1597
|
-
run = require_deep_scan_run(connection, scan_id)
|
|
1598
|
-
require_current_coordinator(run, args)
|
|
1599
|
-
scan = dependencies().require_scan(connection, scan_id)
|
|
1600
|
-
worker = require_deep_scan_worker(connection, worker_id)
|
|
1601
|
-
if worker["scan_id"] != scan_id or worker["kind"] != "dedup":
|
|
1602
|
-
raise SystemExit("Dedup worker does not belong to this Deep Scan.")
|
|
1603
|
-
if worker["status"] == "succeeded":
|
|
1604
|
-
connection.commit()
|
|
1605
|
-
return deep_scan_result(connection, scan_id)
|
|
1606
|
-
require_running_deep_scan(connection, scan_id)
|
|
1607
|
-
if worker["status"] not in {"queued", "running"}:
|
|
1608
|
-
raise SystemExit("Only an active dedup worker can commit a result.")
|
|
1609
|
-
if args.candidate_ledger_path:
|
|
1610
|
-
candidate_ledger_path = deep_scan_path(
|
|
1611
|
-
scan,
|
|
1612
|
-
args.candidate_ledger_path,
|
|
1613
|
-
"Staged candidate ledger path",
|
|
1614
|
-
kind="file",
|
|
1615
|
-
)
|
|
1616
|
-
discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery"
|
|
1617
|
-
deep_scan_path(
|
|
1618
|
-
scan,
|
|
1619
|
-
str(discovery_dir / "in_scope_files.txt"),
|
|
1620
|
-
"Canonical in-scope inventory path",
|
|
1621
|
-
kind="file",
|
|
1622
|
-
)
|
|
1623
|
-
canonical_candidate_ledger_path = deep_scan_output_path(
|
|
1624
|
-
scan,
|
|
1625
|
-
str(discovery_dir / "candidate_ledger.jsonl"),
|
|
1626
|
-
"Canonical candidate ledger path",
|
|
1425
|
+
for index, worker_id in enumerate(input_ids):
|
|
1426
|
+
updated = connection.execute(
|
|
1427
|
+
"""UPDATE deep_scan_workers SET merge_state = 'merged', updated_at = ?
|
|
1428
|
+
WHERE scan_id = ? AND id = ? AND kind = 'discovery'
|
|
1429
|
+
AND status = 'succeeded' AND merge_state = 'buffered'""",
|
|
1430
|
+
(now, scan_id, worker_id),
|
|
1627
1431
|
)
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
canonical_candidate_ledger_path = None
|
|
1631
|
-
result_manifest_path = deep_scan_path(
|
|
1632
|
-
scan,
|
|
1633
|
-
args.result_manifest_path,
|
|
1634
|
-
"Dedup result manifest path",
|
|
1635
|
-
kind="file",
|
|
1636
|
-
)
|
|
1637
|
-
inputs = list(
|
|
1432
|
+
if updated.rowcount != 1:
|
|
1433
|
+
raise SystemExit("Iteration input is not an unmerged completed scan.")
|
|
1638
1434
|
connection.execute(
|
|
1639
|
-
""
|
|
1640
|
-
|
|
1641
|
-
FROM deep_scan_dedup_inputs AS inputs
|
|
1642
|
-
JOIN deep_scan_workers AS workers ON workers.id = inputs.discovery_worker_id
|
|
1643
|
-
WHERE inputs.dedup_worker_id = ?
|
|
1644
|
-
ORDER BY inputs.input_order
|
|
1645
|
-
""",
|
|
1646
|
-
(worker_id,),
|
|
1435
|
+
"INSERT INTO deep_scan_dedup_inputs VALUES (?, ?, ?, ?)",
|
|
1436
|
+
(scan_id, checkpoint_id, worker_id, index),
|
|
1647
1437
|
)
|
|
1648
|
-
)
|
|
1649
|
-
if not inputs or any(row["merge_state"] != "merging" for row in inputs):
|
|
1650
|
-
raise SystemExit("Dedup inputs are not in the claimed merging state.")
|
|
1651
|
-
if candidate_ledger_path and canonical_candidate_ledger_path:
|
|
1652
|
-
canonical_path = Path(canonical_candidate_ledger_path)
|
|
1653
|
-
publication_copy = canonical_path.with_name(
|
|
1654
|
-
f".{canonical_path.name}.{uuid.uuid4()}.publish"
|
|
1655
|
-
)
|
|
1656
|
-
create_publication_copy(candidate_ledger_path, publication_copy)
|
|
1657
|
-
promotion = promote_staged_file(
|
|
1658
|
-
str(publication_copy),
|
|
1659
|
-
canonical_candidate_ledger_path,
|
|
1660
|
-
)
|
|
1661
|
-
timestamp = dependencies().now()
|
|
1662
1438
|
connection.execute(
|
|
1663
|
-
""
|
|
1664
|
-
|
|
1665
|
-
SET merge_state = 'merged', updated_at = ?
|
|
1666
|
-
WHERE id IN (
|
|
1667
|
-
SELECT discovery_worker_id FROM deep_scan_dedup_inputs
|
|
1668
|
-
WHERE dedup_worker_id = ?
|
|
1669
|
-
)
|
|
1670
|
-
""",
|
|
1671
|
-
(timestamp, worker_id),
|
|
1439
|
+
"UPDATE deep_scan_runs SET phase = 'discovery', consecutive_no_new = ?, consecutive_errors = ?, updated_at = ? WHERE scan_id = ?",
|
|
1440
|
+
(args.no_new_streak, args.consecutive_errors, now, scan_id),
|
|
1672
1441
|
)
|
|
1673
1442
|
connection.execute(
|
|
1674
|
-
""
|
|
1675
|
-
|
|
1676
|
-
SET status = 'succeeded', result_manifest_path = ?,
|
|
1677
|
-
error_message = NULL, started_at = COALESCE(started_at, ?),
|
|
1678
|
-
completed_at = ?, updated_at = ?
|
|
1679
|
-
WHERE id = ?
|
|
1680
|
-
""",
|
|
1681
|
-
(result_manifest_path, timestamp, timestamp, timestamp, worker_id),
|
|
1443
|
+
"UPDATE scan_progress SET deep_review_pass = COALESCE(deep_review_pass, 0) + 1, updated_at = ? WHERE scan_id = ?",
|
|
1444
|
+
(now, scan_id),
|
|
1682
1445
|
)
|
|
1683
|
-
no_new_streak = (
|
|
1684
|
-
0 if args.new_findings_count > 0 else run["consecutive_no_new"] + len(inputs)
|
|
1685
|
-
)
|
|
1686
|
-
connection.execute(
|
|
1687
|
-
"""
|
|
1688
|
-
UPDATE deep_scan_runs
|
|
1689
|
-
SET phase = 'discovery', consecutive_no_new = ?, updated_at = ?
|
|
1690
|
-
WHERE scan_id = ?
|
|
1691
|
-
""",
|
|
1692
|
-
(no_new_streak, timestamp, scan_id),
|
|
1693
|
-
)
|
|
1694
|
-
connection.commit()
|
|
1695
|
-
except BaseException:
|
|
1696
|
-
connection.rollback()
|
|
1697
|
-
if promotion is not None:
|
|
1698
|
-
rollback_staged_file(promotion)
|
|
1699
|
-
if publication_copy is not None:
|
|
1700
|
-
publication_copy.unlink(missing_ok=True)
|
|
1701
|
-
raise
|
|
1702
|
-
if promotion is not None:
|
|
1703
|
-
finish_staged_file(promotion)
|
|
1704
|
-
if publication_copy is not None:
|
|
1705
|
-
publication_copy.unlink(missing_ok=True)
|
|
1706
1446
|
return deep_scan_result(connection, scan_id)
|
|
1707
1447
|
|
|
1708
1448
|
|
|
@@ -1715,10 +1455,10 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1715
1455
|
def finish_deep_scan_locked(
|
|
1716
1456
|
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1717
1457
|
) -> dict[str, Any]:
|
|
1718
|
-
omitted_worker_ids =
|
|
1458
|
+
omitted_worker_ids = {
|
|
1719
1459
|
require_uuid(value, "omitted-worker-id") for value in args.omitted_worker_id
|
|
1720
|
-
|
|
1721
|
-
if len(
|
|
1460
|
+
}
|
|
1461
|
+
if len(omitted_worker_ids) != len(args.omitted_worker_id):
|
|
1722
1462
|
raise SystemExit("Omitted Deep Scan worker IDs must be unique.")
|
|
1723
1463
|
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1724
1464
|
deadline_publication: (
|
|
@@ -1738,29 +1478,23 @@ def finish_deep_scan_locked(
|
|
|
1738
1478
|
)
|
|
1739
1479
|
standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
1740
1480
|
|
|
1741
|
-
buffered_worker_ids =
|
|
1481
|
+
buffered_worker_ids = {
|
|
1742
1482
|
row["id"]
|
|
1743
1483
|
for row in connection.execute(
|
|
1744
1484
|
"""
|
|
1745
1485
|
SELECT id
|
|
1746
1486
|
FROM deep_scan_workers
|
|
1747
1487
|
WHERE scan_id = ? AND kind = 'discovery' AND merge_state = 'buffered'
|
|
1748
|
-
ORDER BY completion_sequence, id
|
|
1749
1488
|
""",
|
|
1750
1489
|
(scan_id,),
|
|
1751
1490
|
)
|
|
1752
|
-
|
|
1753
|
-
# Historical succeeded runs retain their exact omitted-worker replay contract.
|
|
1754
|
-
omissions_match = (
|
|
1755
|
-
set(omitted_worker_ids) == set(buffered_worker_ids)
|
|
1756
|
-
if args.terminal_reason == "saturated" or run["status"] == "succeeded"
|
|
1757
|
-
else not omitted_worker_ids and not buffered_worker_ids
|
|
1758
|
-
)
|
|
1491
|
+
}
|
|
1759
1492
|
if run["status"] == "succeeded":
|
|
1760
1493
|
if (
|
|
1761
1494
|
run["terminal_reason"] != args.terminal_reason
|
|
1762
|
-
or
|
|
1763
|
-
|
|
1495
|
+
or archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
1496
|
+
not in {None, manifest_path}
|
|
1497
|
+
or omitted_worker_ids != buffered_worker_ids
|
|
1764
1498
|
):
|
|
1765
1499
|
raise SystemExit(
|
|
1766
1500
|
"Deep Scan terminal state is immutable; finish must exactly replay its "
|
|
@@ -1773,7 +1507,9 @@ def finish_deep_scan_locked(
|
|
|
1773
1507
|
)
|
|
1774
1508
|
connection.commit()
|
|
1775
1509
|
return deep_scan_result(connection, scan_id)
|
|
1776
|
-
require_running_deep_scan(
|
|
1510
|
+
require_running_deep_scan(run, scan)
|
|
1511
|
+
if run["error_message"] is not None:
|
|
1512
|
+
raise SystemExit(run["error_message"])
|
|
1777
1513
|
if (
|
|
1778
1514
|
args.terminal_reason == "saturated"
|
|
1779
1515
|
and run["consecutive_no_new"] < run["stop_after_no_new"]
|
|
@@ -1805,7 +1541,7 @@ def finish_deep_scan_locked(
|
|
|
1805
1541
|
raise SystemExit(
|
|
1806
1542
|
f"Deep Scan cannot finish without canonical discovery artifacts: {exc}"
|
|
1807
1543
|
) from exc
|
|
1808
|
-
|
|
1544
|
+
saved_iteration = connection.execute(
|
|
1809
1545
|
"""
|
|
1810
1546
|
SELECT 1 FROM deep_scan_workers
|
|
1811
1547
|
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
@@ -1823,8 +1559,8 @@ def finish_deep_scan_locked(
|
|
|
1823
1559
|
and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0
|
|
1824
1560
|
)
|
|
1825
1561
|
)
|
|
1826
|
-
if
|
|
1827
|
-
raise SystemExit("Deep Scan cannot finish without a
|
|
1562
|
+
if saved_iteration is None and not zero_discovery_deadline:
|
|
1563
|
+
raise SystemExit("Deep Scan cannot finish without a saved iteration.")
|
|
1828
1564
|
if args.terminal_reason == "saturated":
|
|
1829
1565
|
# Saturation cancels surplus workers, including failed cancellation writes.
|
|
1830
1566
|
cancel_active_workers(connection, scan_id, dependencies().now())
|
|
@@ -1834,7 +1570,7 @@ def finish_deep_scan_locked(
|
|
|
1834
1570
|
args.terminal_reason,
|
|
1835
1571
|
has_omitted_workers=bool(omitted_worker_ids),
|
|
1836
1572
|
)
|
|
1837
|
-
if args.terminal_reason == "saturated" and
|
|
1573
|
+
if args.terminal_reason == "saturated" and omitted_worker_ids != buffered_worker_ids:
|
|
1838
1574
|
raise SystemExit(
|
|
1839
1575
|
f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery "
|
|
1840
1576
|
"workers with --omitted-worker-id."
|
|
@@ -1865,7 +1601,6 @@ def finish_deep_scan_locked(
|
|
|
1865
1601
|
""",
|
|
1866
1602
|
(args.terminal_reason, manifest_path, timestamp, timestamp, scan_id),
|
|
1867
1603
|
)
|
|
1868
|
-
cancel_active_workers(connection, scan_id, timestamp)
|
|
1869
1604
|
connection.commit()
|
|
1870
1605
|
except BaseException:
|
|
1871
1606
|
connection.rollback()
|
|
@@ -1904,6 +1639,7 @@ def fail_deep_scan_locked(
|
|
|
1904
1639
|
run = require_deep_scan_run(connection, scan_id)
|
|
1905
1640
|
require_current_coordinator(run, args)
|
|
1906
1641
|
scan = dependencies().require_scan(connection, scan_id)
|
|
1642
|
+
recorded_manifest = archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
1907
1643
|
manifest_path = None
|
|
1908
1644
|
if args.manifest_path:
|
|
1909
1645
|
manifest_path = (
|
|
@@ -1916,14 +1652,14 @@ def fail_deep_scan_locked(
|
|
|
1916
1652
|
kind="file",
|
|
1917
1653
|
)
|
|
1918
1654
|
)
|
|
1919
|
-
elif
|
|
1920
|
-
manifest_path =
|
|
1655
|
+
elif recorded_manifest == str(Path(scan["scan_dir"]) / "scan-manifest.json"):
|
|
1656
|
+
manifest_path = recorded_manifest
|
|
1921
1657
|
if run["status"] in {"failed", "interrupted"} or scan["status"] == "failed":
|
|
1922
1658
|
if (
|
|
1923
1659
|
run["status"] == args.deep_status
|
|
1924
1660
|
and scan["status"] == "failed"
|
|
1925
1661
|
and run["error_message"] == message
|
|
1926
|
-
and
|
|
1662
|
+
and recorded_manifest == manifest_path
|
|
1927
1663
|
and scan["failure_message"] == message
|
|
1928
1664
|
):
|
|
1929
1665
|
connection.commit()
|
|
@@ -1942,6 +1678,9 @@ def fail_deep_scan_locked(
|
|
|
1942
1678
|
"status"
|
|
1943
1679
|
] != "running":
|
|
1944
1680
|
raise SystemExit("Only a running Deep Scan can be failed or interrupted.")
|
|
1681
|
+
# Cleanup can report a later error after a worker committed the terminal cause.
|
|
1682
|
+
if run["status"] == "running" and run["error_message"] is not None:
|
|
1683
|
+
message = run["error_message"]
|
|
1945
1684
|
if run["manifest_path"] not in {None, manifest_path}:
|
|
1946
1685
|
raise SystemExit("Deep Scan coordinator manifest path is immutable.")
|
|
1947
1686
|
if args.staged_manifest_path and manifest_path:
|
|
@@ -1963,20 +1702,9 @@ def fail_deep_scan_locked(
|
|
|
1963
1702
|
(args.deep_status, message, manifest_path, timestamp, timestamp, scan_id),
|
|
1964
1703
|
)
|
|
1965
1704
|
cancel_active_workers(connection, scan_id, timestamp)
|
|
1966
|
-
|
|
1967
|
-
"""
|
|
1968
|
-
UPDATE scans
|
|
1969
|
-
SET status = 'failed', failure_message = ?, completed_at = ?, updated_at = ?
|
|
1970
|
-
WHERE id = ? AND status = 'running'
|
|
1971
|
-
""",
|
|
1972
|
-
(message, timestamp, timestamp, scan_id),
|
|
1973
|
-
)
|
|
1974
|
-
if parent_update.rowcount != 1:
|
|
1705
|
+
if not saved_results.mark_running_scan_failed(connection, scan_id, message, timestamp):
|
|
1975
1706
|
raise SystemExit("Deep Scan failure could not be persisted to its parent scan.")
|
|
1976
|
-
|
|
1977
|
-
"UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?",
|
|
1978
|
-
(timestamp, scan_id),
|
|
1979
|
-
)
|
|
1707
|
+
saved_results.touch_scan_progress(connection, scan_id, timestamp)
|
|
1980
1708
|
connection.commit()
|
|
1981
1709
|
except BaseException:
|
|
1982
1710
|
connection.rollback()
|
|
@@ -2040,6 +1768,11 @@ def fail_from_parent_scan(
|
|
|
2040
1768
|
message: str | None,
|
|
2041
1769
|
timestamp: str,
|
|
2042
1770
|
) -> None:
|
|
1771
|
+
# Parent failure may supersede a prepared seal even after the coordinator succeeded.
|
|
1772
|
+
connection.execute(
|
|
1773
|
+
"UPDATE deep_scan_runs SET terminal_reviewed_surface_count = NULL WHERE scan_id = ?",
|
|
1774
|
+
(scan_id,),
|
|
1775
|
+
)
|
|
2043
1776
|
connection.execute(
|
|
2044
1777
|
"""
|
|
2045
1778
|
UPDATE deep_scan_runs
|
|
@@ -2056,8 +1789,8 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest
|
|
|
2056
1789
|
connection.execute(
|
|
2057
1790
|
"""
|
|
2058
1791
|
UPDATE deep_scan_runs
|
|
2059
|
-
SET status = 'canceled', phase = 'terminal', cancel_requested = 1,
|
|
2060
|
-
completed_at = ?, updated_at = ?
|
|
1792
|
+
SET status = 'canceled', phase = 'terminal', cancel_requested = 1, error_message = NULL,
|
|
1793
|
+
terminal_reviewed_surface_count = NULL, completed_at = ?, updated_at = ?
|
|
2061
1794
|
WHERE scan_id = ? AND status IN ('running', 'succeeded')
|
|
2062
1795
|
""",
|
|
2063
1796
|
(timestamp, timestamp, scan_id),
|
|
@@ -2079,6 +1812,7 @@ def cancel_active_workers(connection: sqlite3.Connection, scan_id: str, timestam
|
|
|
2079
1812
|
def other_running_deep_scans(
|
|
2080
1813
|
connection: sqlite3.Connection, current_scan_id: str
|
|
2081
1814
|
) -> list[dict[str, str]]:
|
|
1815
|
+
"""List other running Deep Scans ordered by most recent update, with target paths and phases."""
|
|
2082
1816
|
rows = connection.execute(
|
|
2083
1817
|
"""
|
|
2084
1818
|
SELECT id, target_path, phase, started_at, updated_at
|