@openai/codex-security 0.8.3 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +192 -151
- package/_bundled_plugin/.mcp.json +5 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
- package/_bundled_plugin/examples/completed-scan/report.md +11 -8
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
- package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
- package/_bundled_plugin/mcp/helpers.mjs +2 -11
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/package.json +1 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +5 -2
- package/_bundled_plugin/mcp/server.mjs +2 -11
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
- package/_bundled_plugin/references/default-severity-rubric.md +9 -4
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/finding-detail-fields.md +18 -18
- package/_bundled_plugin/references/sarif-adapter.md +8 -2
- package/_bundled_plugin/references/scan-artifacts.md +21 -12
- package/_bundled_plugin/references/scan-contract.md +27 -33
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
- package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
- package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +165 -342
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +153 -227
- package/_bundled_plugin/scripts/finding_preview.py +20 -0
- package/_bundled_plugin/scripts/launch_codex_security_mcp +1 -0
- package/_bundled_plugin/scripts/legacy_findings.py +148 -0
- package/_bundled_plugin/scripts/posix_scan_local_files.py +11 -0
- package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
- package/_bundled_plugin/scripts/report_projection.py +81 -43
- package/_bundled_plugin/scripts/review_state.py +8 -2
- package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +1 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
- package/_bundled_plugin/scripts/validate_scan_contract.py +7 -1
- package/_bundled_plugin/scripts/windows_scan_local_files.py +388 -17
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
- package/_bundled_plugin/scripts/workbench/handoff.py +47 -0
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +30 -51
- package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +9 -17
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
- package/_bundled_plugin/scripts/workbench/storage.py +71 -7
- package/_bundled_plugin/scripts/workbench_cli.py +30 -12
- package/_bundled_plugin/scripts/workbench_constants.py +10 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
- package/_bundled_plugin/scripts/workbench_db.py +369 -137
- package/_bundled_plugin/scripts/workbench_feedback.py +8 -7
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +4 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +12 -12
- package/_bundled_plugin/scripts/workbench_progress.py +10 -1
- package/_bundled_plugin/scripts/workbench_publication.py +14 -10
- package/_bundled_plugin/scripts/workbench_remediation.py +13 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +132 -77
- package/_bundled_plugin/scripts/workbench_scan_history.py +117 -55
- package/_bundled_plugin/scripts/workbench_scan_start.py +83 -24
- package/_bundled_plugin/scripts/workbench_scan_usage.py +644 -68
- package/_bundled_plugin/scripts/workbench_schema.py +44 -55
- package/_bundled_plugin/scripts/workbench_setup.py +13 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +18 -0
- package/_bundled_plugin/scripts/workbench_target.py +127 -56
- package/_bundled_plugin/scripts/workbench_target_state.py +3 -0
- package/_bundled_plugin/scripts/workbench_validation.py +43 -3
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -23
- package/_bundled_plugin/shared/workbench-migrations.json +34 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +5 -5
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
- package/_bundled_plugin/skills/fix-finding/SKILL.md +1 -1
- package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -25
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +3 -3
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +464 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +22 -1
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -2
- package/dist/api.d.ts +39 -4
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +774 -292
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -6
- package/dist/auth.js.map +1 -1
- package/dist/bedrock-oss-pricing.d.ts +6 -0
- package/dist/bedrock-oss-pricing.d.ts.map +1 -0
- package/dist/bedrock-oss-pricing.js +58 -0
- package/dist/bedrock-oss-pricing.js.map +1 -0
- package/dist/bedrock-pricing.d.ts +15 -0
- package/dist/bedrock-pricing.d.ts.map +1 -0
- package/dist/bedrock-pricing.js +117 -0
- package/dist/bedrock-pricing.js.map +1 -0
- package/dist/blackduck-findings.d.ts.map +1 -1
- package/dist/blackduck-findings.js +35 -20
- package/dist/blackduck-findings.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +4 -4
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/checkmarx-evidence.d.ts +19 -0
- package/dist/checkmarx-evidence.d.ts.map +1 -0
- package/dist/checkmarx-evidence.js +36 -0
- package/dist/checkmarx-evidence.js.map +1 -0
- package/dist/checkmarx-findings.d.ts +4 -0
- package/dist/checkmarx-findings.d.ts.map +1 -0
- package/dist/checkmarx-findings.js +27 -0
- package/dist/checkmarx-findings.js.map +1 -0
- package/dist/checkmarx-one.d.ts +4 -0
- package/dist/checkmarx-one.d.ts.map +1 -0
- package/dist/checkmarx-one.js +53 -0
- package/dist/checkmarx-one.js.map +1 -0
- package/dist/checkmarx-repository.d.ts +12 -0
- package/dist/checkmarx-repository.d.ts.map +1 -0
- package/dist/checkmarx-repository.js +181 -0
- package/dist/checkmarx-repository.js.map +1 -0
- package/dist/checkmarx-xml.d.ts +12 -0
- package/dist/checkmarx-xml.d.ts.map +1 -0
- package/dist/checkmarx-xml.js +196 -0
- package/dist/checkmarx-xml.js.map +1 -0
- package/dist/classify-severity-records.d.ts +1 -9
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +3 -13
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +2 -4
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli.d.ts +11 -5
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1734 -1238
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +2 -2
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +16 -3
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +27 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +100 -66
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts.map +1 -1
- package/dist/continuum-findings.js +80 -12
- package/dist/continuum-findings.js.map +1 -1
- package/dist/contract-values.d.ts +2 -0
- package/dist/contract-values.d.ts.map +1 -1
- package/dist/contract-values.js +12 -9
- package/dist/contract-values.js.map +1 -1
- package/dist/cost-model.d.ts +13 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +66 -112
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +42 -3
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +483 -104
- package/dist/cost.js.map +1 -1
- package/dist/custom-publish.d.ts +4 -0
- package/dist/custom-publish.d.ts.map +1 -1
- package/dist/custom-publish.js +40 -3
- package/dist/custom-publish.js.map +1 -1
- package/dist/deduplication/local.d.ts +2 -2
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +2 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +25 -5
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +3 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +1 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +0 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +22 -33
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan-defaults.d.ts +2 -0
- package/dist/deep-scan-defaults.d.ts.map +1 -1
- package/dist/deep-scan-defaults.js +28 -1
- package/dist/deep-scan-defaults.js.map +1 -1
- package/dist/deep-scan.d.ts +2 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +28 -4
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -1
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +1 -1
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts +3 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +103 -45
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +8 -1
- package/dist/errors.js.map +1 -1
- package/dist/external-findings-import.d.ts +5 -3
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +7 -6
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +7 -5
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +337 -89
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +8 -6
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +1318 -258
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +20 -2
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +148 -66
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +15 -3
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +42 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +5 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +23 -9
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +5 -4
- package/dist/feedback.js.map +1 -1
- package/dist/finding-prompt.d.ts +5 -0
- package/dist/finding-prompt.d.ts.map +1 -0
- package/dist/finding-prompt.js +32 -0
- package/dist/finding-prompt.js.map +1 -0
- package/dist/findings-client.d.ts +1 -0
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +27 -3
- package/dist/findings-client.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +0 -2
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +15 -4
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts +4 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +55 -30
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-git-remotes.d.ts +5 -0
- package/dist/hosted-git-remotes.d.ts.map +1 -0
- package/dist/hosted-git-remotes.js +285 -0
- package/dist/hosted-git-remotes.js.map +1 -0
- package/dist/hosted-git-repository.d.ts +28 -0
- package/dist/hosted-git-repository.d.ts.map +1 -0
- package/dist/hosted-git-repository.js +222 -0
- package/dist/hosted-git-repository.js.map +1 -0
- package/dist/hosted-scan-protocol.js +2 -2
- package/dist/hosted-scan-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +16 -11
- package/dist/hosted-scan.js.map +1 -1
- package/dist/hosted-scanner-http.d.ts +21 -0
- package/dist/hosted-scanner-http.d.ts.map +1 -0
- package/dist/hosted-scanner-http.js +204 -0
- package/dist/hosted-scanner-http.js.map +1 -0
- package/dist/hosted-scanner-v1.d.ts +6 -0
- package/dist/hosted-scanner-v1.d.ts.map +1 -0
- package/dist/hosted-scanner-v1.js +60 -0
- package/dist/hosted-scanner-v1.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +3 -0
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +18 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -2
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +47 -65
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +68 -5
- package/dist/knowledge-base.js.map +1 -1
- package/dist/local-git-metadata.d.ts +1 -1
- package/dist/local-git-metadata.d.ts.map +1 -1
- package/dist/local-git-metadata.js +1 -2
- package/dist/local-git-metadata.js.map +1 -1
- package/dist/multiscan.d.ts +6 -4
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +138 -65
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +2 -1
- package/dist/native-json-findings.d.ts.map +1 -1
- package/dist/native-json-findings.js +12 -2
- package/dist/native-json-findings.js.map +1 -1
- package/dist/opengrep-findings.d.ts +1 -0
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +17 -13
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/operation-telemetry.d.ts +104 -0
- package/dist/operation-telemetry.d.ts.map +1 -0
- package/dist/operation-telemetry.js +293 -0
- package/dist/operation-telemetry.js.map +1 -0
- package/dist/plugin-python-probe.d.ts +4 -0
- package/dist/plugin-python-probe.d.ts.map +1 -0
- package/dist/plugin-python-probe.js +10 -0
- package/dist/plugin-python-probe.js.map +1 -0
- package/dist/private-finding-import.d.ts +97 -2
- package/dist/private-finding-import.d.ts.map +1 -1
- package/dist/private-finding-import.js +58 -34
- package/dist/private-finding-import.js.map +1 -1
- package/dist/provider-profile.d.ts +1 -1
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +8 -1
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication-telemetry.d.ts +45 -0
- package/dist/publication-telemetry.d.ts.map +1 -0
- package/dist/publication-telemetry.js +50 -0
- package/dist/publication-telemetry.js.map +1 -0
- package/dist/publish.d.ts +4 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +42 -2
- package/dist/publish.js.map +1 -1
- package/dist/reader-selection.d.ts +2 -0
- package/dist/reader-selection.d.ts.map +1 -1
- package/dist/reader-selection.js +8 -0
- package/dist/reader-selection.js.map +1 -1
- package/dist/remediation.d.ts +1 -4
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -7
- package/dist/remediation.js.map +1 -1
- package/dist/request-metadata.d.ts +3 -0
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +9 -0
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +1 -1
- package/dist/result.js.map +1 -1
- package/dist/retry-windows-rename.d.ts +8 -0
- package/dist/retry-windows-rename.d.ts.map +1 -0
- package/dist/retry-windows-rename.js +24 -0
- package/dist/retry-windows-rename.js.map +1 -0
- package/dist/runtime.d.ts +20 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +313 -110
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +18 -15
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +141 -56
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +10 -5
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +60 -15
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +6 -6
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +75 -42
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +10 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +22 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +9 -19
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +12 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +106 -77
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-measurement.d.ts +26 -0
- package/dist/scan-measurement.d.ts.map +1 -0
- package/dist/scan-measurement.js +77 -0
- package/dist/scan-measurement.js.map +1 -0
- package/dist/scan-progress.d.ts +1 -0
- package/dist/scan-progress.d.ts.map +1 -1
- package/dist/scan-progress.js +3 -0
- package/dist/scan-progress.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +4 -1
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -1
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +1 -24
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +20 -2
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +200 -8
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +4 -1
- package/dist/scanner-repository-url.d.ts.map +1 -1
- package/dist/scanner-repository-url.js +10 -6
- package/dist/scanner-repository-url.js.map +1 -1
- package/dist/security-policy.d.ts +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +6 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +107 -43
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-guardian-findings.d.ts +38 -0
- package/dist/semgrep-guardian-findings.d.ts.map +1 -0
- package/dist/semgrep-guardian-findings.js +165 -0
- package/dist/semgrep-guardian-findings.js.map +1 -0
- package/dist/semgrep-platform-collector.d.ts +26 -0
- package/dist/semgrep-platform-collector.d.ts.map +1 -0
- package/dist/semgrep-platform-collector.js +108 -0
- package/dist/semgrep-platform-collector.js.map +1 -0
- package/dist/semgrep-platform-findings.d.ts +207 -6
- package/dist/semgrep-platform-findings.d.ts.map +1 -1
- package/dist/semgrep-platform-findings.js +983 -103
- package/dist/semgrep-platform-findings.js.map +1 -1
- package/dist/semgrep-platform-v1.d.ts +6 -0
- package/dist/semgrep-platform-v1.d.ts.map +1 -0
- package/dist/semgrep-platform-v1.js +83 -0
- package/dist/semgrep-platform-v1.js.map +1 -0
- package/dist/server/api.d.ts +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embedding-tokenizer.cdx.json +33 -0
- package/dist/server/embedding-tokenizer.d.ts +2 -0
- package/dist/server/embedding-tokenizer.d.ts.map +1 -0
- package/dist/server/embedding-tokenizer.js +322 -0
- package/dist/server/embedding-tokenizer.js.map +6 -0
- package/dist/server/embeddings.d.ts +16 -2
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +35 -5
- package/dist/server/embeddings.js.map +1 -1
- package/dist/session-lines.d.ts +5 -0
- package/dist/session-lines.d.ts.map +1 -0
- package/dist/session-lines.js +60 -0
- package/dist/session-lines.js.map +1 -0
- package/dist/severity-review.d.ts +3 -1
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +3 -2
- package/dist/severity-review.js.map +1 -1
- package/dist/skill-authentication.d.ts +1 -0
- package/dist/skill-authentication.d.ts.map +1 -1
- package/dist/skill-authentication.js +1 -0
- package/dist/skill-authentication.js.map +1 -1
- package/dist/snyk-findings.d.ts +7 -2
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +93 -70
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +76 -0
- package/dist/snyk-rest-findings.d.ts.map +1 -0
- package/dist/snyk-rest-findings.js +1051 -0
- package/dist/snyk-rest-findings.js.map +1 -0
- package/dist/snyk-rest-v1.d.ts +28 -0
- package/dist/snyk-rest-v1.d.ts.map +1 -0
- package/dist/snyk-rest-v1.js +132 -0
- package/dist/snyk-rest-v1.js.map +1 -0
- package/dist/snyk-token-origin.d.ts +4 -0
- package/dist/snyk-token-origin.d.ts.map +1 -0
- package/dist/snyk-token-origin.js +82 -0
- package/dist/snyk-token-origin.js.map +1 -0
- package/dist/sonar-collector.d.ts.map +1 -1
- package/dist/sonar-collector.js +0 -3
- package/dist/sonar-collector.js.map +1 -1
- package/dist/sonar-import.d.ts.map +1 -1
- package/dist/sonar-import.js +2 -4
- package/dist/sonar-import.js.map +1 -1
- package/dist/target-drift.d.ts +9 -0
- package/dist/target-drift.d.ts.map +1 -0
- package/dist/target-drift.js +27 -0
- package/dist/target-drift.js.map +1 -0
- package/dist/targets.d.ts +2 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +56 -20
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +9 -0
- package/dist/tenable-findings.d.ts.map +1 -0
- package/dist/tenable-findings.js +122 -0
- package/dist/tenable-findings.js.map +1 -0
- package/dist/terminal-stream.d.ts +13 -0
- package/dist/terminal-stream.d.ts.map +1 -0
- package/dist/terminal-stream.js +35 -0
- package/dist/terminal-stream.js.map +1 -0
- package/dist/trivy-findings.d.ts +19 -0
- package/dist/trivy-findings.d.ts.map +1 -0
- package/dist/trivy-findings.js +435 -0
- package/dist/trivy-findings.js.map +1 -0
- package/dist/trivy-import.d.ts +19 -0
- package/dist/trivy-import.d.ts.map +1 -0
- package/dist/trivy-import.js +151 -0
- package/dist/trivy-import.js.map +1 -0
- package/dist/validation-records.js +1 -1
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +2 -0
- package/dist/veracode-findings.d.ts.map +1 -1
- package/dist/veracode-findings.js +49 -11
- package/dist/veracode-findings.js.map +1 -1
- package/dist/vertex-pricing.d.ts +8 -0
- package/dist/vertex-pricing.d.ts.map +1 -0
- package/dist/vertex-pricing.js +20 -0
- package/dist/vertex-pricing.js.map +1 -0
- package/dist/windows-path.d.ts +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +26 -1
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-findings.d.ts +4 -0
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +57 -3
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +27 -12
- package/docs/cli.md +497 -550
- package/docs/dedupe-records.md +44 -24
- package/docs/findings-service.md +66 -39
- package/docs/hosted-remediation.md +14 -7
- package/docs/hosted-scans.md +50 -42
- package/docs/telemetry.md +236 -0
- package/docs/validate-records.md +78 -44
- package/package.json +18 -15
- package/schemas/external-findings.schema.json +3 -1
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
- package/dist/external-imports-feature.d.ts +0 -6
- package/dist/external-imports-feature.d.ts.map +0 -1
- package/dist/external-imports-feature.js +0 -51
- package/dist/external-imports-feature.js.map +0 -1
- package/dist/finding-normalization.d.ts +0 -5
- package/dist/finding-normalization.d.ts.map +0 -1
- package/dist/finding-normalization.js +0 -155
- package/dist/finding-normalization.js.map +0 -1
|
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
|
-
import shutil
|
|
9
8
|
import sqlite3
|
|
10
9
|
import sys
|
|
11
10
|
import uuid
|
|
@@ -16,8 +15,14 @@ from typing import Any
|
|
|
16
15
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
17
16
|
import workbench_saved_results as saved_results
|
|
18
17
|
from deep_scan_config import resolve_deep_scan_setup
|
|
19
|
-
from workbench.handoff import require_current_continuation
|
|
20
|
-
from workbench.storage import
|
|
18
|
+
from workbench.handoff import require_current_continuation, require_tokenless_scan_owner
|
|
19
|
+
from workbench.storage import (
|
|
20
|
+
archived_deep_path,
|
|
21
|
+
create_private_directory,
|
|
22
|
+
saved_deep_workers,
|
|
23
|
+
worker_usage_directory,
|
|
24
|
+
)
|
|
25
|
+
from workbench_constants import PHASES
|
|
21
26
|
from workbench_source_scopes import capture_source_scopes
|
|
22
27
|
from workbench_target import directory_snapshot_regular_file_count
|
|
23
28
|
from workbench_validation import optional_text, require_uuid, user_context_argument
|
|
@@ -42,6 +47,7 @@ def dependencies() -> Any:
|
|
|
42
47
|
|
|
43
48
|
|
|
44
49
|
def deep_scan_error(run: sqlite3.Row) -> str | None:
|
|
50
|
+
"""Return recorded error messages, publication error first, or None if both are absent."""
|
|
45
51
|
original = run["error_message"]
|
|
46
52
|
publication = run["publication_error_message"]
|
|
47
53
|
if not isinstance(publication, str):
|
|
@@ -68,6 +74,7 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit
|
|
|
68
74
|
|
|
69
75
|
|
|
70
76
|
def deep_scan_deadline_reached(run: sqlite3.Row) -> bool:
|
|
77
|
+
"""Return whether the time since run creation has reached its configured limit in hours."""
|
|
71
78
|
elapsed = _parse_timestamp(dependencies().now()) - _parse_timestamp(str(run["created_at"]))
|
|
72
79
|
return elapsed.total_seconds() / 3600 >= run["max_time_hours"]
|
|
73
80
|
|
|
@@ -75,6 +82,11 @@ def deep_scan_deadline_reached(run: sqlite3.Row) -> bool:
|
|
|
75
82
|
def require_deep_scan_ready_for_parent_completion(
|
|
76
83
|
connection: sqlite3.Connection, scan: sqlite3.Row
|
|
77
84
|
) -> None:
|
|
85
|
+
"""Require a succeeded discovery run with a saved manifest before a Deep Scan completes.
|
|
86
|
+
|
|
87
|
+
For a capped run, also verify its workers have finished. Raise SystemExit
|
|
88
|
+
when a completion requirement fails. Other scan modes can complete directly.
|
|
89
|
+
"""
|
|
78
90
|
if scan["mode"] != "deep":
|
|
79
91
|
return
|
|
80
92
|
run = connection.execute(
|
|
@@ -169,6 +181,7 @@ def require_deep_scan_workers_finished(
|
|
|
169
181
|
def require_owned_scan(
|
|
170
182
|
connection: sqlite3.Connection, scan_id: str, thread_id: str
|
|
171
183
|
) -> tuple[sqlite3.Row, sqlite3.Row]:
|
|
184
|
+
"""Return the scan and workspace rows or raise SystemExit unless thread_id matches the recorded owner."""
|
|
172
185
|
scan = dependencies().require_scan(connection, scan_id)
|
|
173
186
|
workspace = dependencies().require_workspace(connection, scan["workspace_id"])
|
|
174
187
|
owner = optional_text(thread_id, maximum=512)
|
|
@@ -187,6 +200,7 @@ def deep_scan_path(
|
|
|
187
200
|
*,
|
|
188
201
|
kind: str,
|
|
189
202
|
) -> str:
|
|
203
|
+
"""Return the canonical absolute path to an existing scan-local file or directory of the requested kind."""
|
|
190
204
|
supplied = Path(value).expanduser()
|
|
191
205
|
if not supplied.is_absolute():
|
|
192
206
|
raise SystemExit(f"{label} must be an absolute path inside the scan directory.")
|
|
@@ -206,6 +220,7 @@ def deep_scan_path(
|
|
|
206
220
|
|
|
207
221
|
|
|
208
222
|
def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str:
|
|
223
|
+
"""Validate an absolute output path under the scan directory, allowing a new file under an existing parent."""
|
|
209
224
|
supplied = Path(value).expanduser()
|
|
210
225
|
if not supplied.is_absolute():
|
|
211
226
|
raise SystemExit(f"{label} must be an absolute path inside the scan directory.")
|
|
@@ -219,6 +234,7 @@ def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str:
|
|
|
219
234
|
|
|
220
235
|
|
|
221
236
|
def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]:
|
|
237
|
+
"""Publish the staged file and return the paths needed to finalize or undo the move."""
|
|
222
238
|
staged = Path(staged_path)
|
|
223
239
|
output = Path(output_path)
|
|
224
240
|
if staged == output:
|
|
@@ -236,6 +252,7 @@ def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path,
|
|
|
236
252
|
|
|
237
253
|
|
|
238
254
|
def rollback_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
255
|
+
"""Move the published file back to staging and restore the previous output if present."""
|
|
239
256
|
staged, output, backup = promotion
|
|
240
257
|
if output.exists():
|
|
241
258
|
os.replace(output, staged)
|
|
@@ -244,19 +261,14 @@ def rollback_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
|
244
261
|
|
|
245
262
|
|
|
246
263
|
def finish_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
264
|
+
"""Remove the previous output saved during promotion."""
|
|
247
265
|
backup = promotion[2]
|
|
248
266
|
if backup is not None:
|
|
249
267
|
backup.unlink(missing_ok=True)
|
|
250
268
|
|
|
251
269
|
|
|
252
|
-
def create_publication_copy(source: str | Path, destination: str | Path) -> None:
|
|
253
|
-
try:
|
|
254
|
-
os.link(source, destination)
|
|
255
|
-
except OSError:
|
|
256
|
-
shutil.copy2(source, destination)
|
|
257
|
-
|
|
258
|
-
|
|
259
270
|
def publication_matches_snapshot(publication: Path, snapshot: Path) -> bool:
|
|
271
|
+
"""Return whether the publication and snapshot identify the same file or contain identical bytes."""
|
|
260
272
|
try:
|
|
261
273
|
if publication.samefile(snapshot):
|
|
262
274
|
return True
|
|
@@ -286,17 +298,14 @@ def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]:
|
|
|
286
298
|
|
|
287
299
|
|
|
288
300
|
def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]:
|
|
301
|
+
"""Return persisted orchestration state including settings, workers, artifacts, and deduplication inputs.
|
|
302
|
+
|
|
303
|
+
Order workers by creation time and ID. Retain the recorded order of each
|
|
304
|
+
deduplication worker’s discovery inputs.
|
|
305
|
+
"""
|
|
289
306
|
run = require_deep_scan_run(connection, scan_id)
|
|
290
307
|
scan = dependencies().require_scan(connection, run["scan_id"])
|
|
291
|
-
worker_rows = connection
|
|
292
|
-
"""
|
|
293
|
-
SELECT *
|
|
294
|
-
FROM deep_scan_workers
|
|
295
|
-
WHERE scan_id = ?
|
|
296
|
-
ORDER BY created_at, id
|
|
297
|
-
""",
|
|
298
|
-
(run["scan_id"],),
|
|
299
|
-
)
|
|
308
|
+
worker_rows = saved_deep_workers(connection, scan)
|
|
300
309
|
input_rows = connection.execute(
|
|
301
310
|
"""
|
|
302
311
|
SELECT dedup_worker_id, discovery_worker_id, input_order
|
|
@@ -306,12 +315,13 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
306
315
|
""",
|
|
307
316
|
(run["scan_id"],),
|
|
308
317
|
)
|
|
318
|
+
manifest_path = archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
309
319
|
canonical_artifacts = None
|
|
310
320
|
if (
|
|
311
321
|
run["canonical_inventory_path"] is None
|
|
312
322
|
and run["status"] == "succeeded"
|
|
313
|
-
and
|
|
314
|
-
and
|
|
323
|
+
and manifest_path is not None
|
|
324
|
+
and manifest_path != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
315
325
|
and (Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "in_scope_files.txt").exists()
|
|
316
326
|
):
|
|
317
327
|
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
@@ -341,11 +351,17 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
341
351
|
else scan["user_context"]
|
|
342
352
|
),
|
|
343
353
|
"scanDir": scan["scan_dir"],
|
|
354
|
+
"workerUsageDirectory": str(worker_usage_directory(run["scan_id"])),
|
|
344
355
|
"schemaVersion": run["schema_version"],
|
|
345
356
|
"workflowVersion": run["workflow_version"],
|
|
346
357
|
"coordinatorGeneration": run["coordinator_generation"],
|
|
347
358
|
"status": run["status"],
|
|
348
359
|
"phase": run["phase"],
|
|
360
|
+
# Older coordinators could persist discovery on the parent before dispatching.
|
|
361
|
+
"discoveryStarted": (
|
|
362
|
+
PHASES.index(scan["phase"]) >= PHASES.index("discovery")
|
|
363
|
+
or run["phase"] in {"discovery", "reducing"}
|
|
364
|
+
),
|
|
349
365
|
"config": {
|
|
350
366
|
"workers": run["workers"],
|
|
351
367
|
"subagents": run["subagents"],
|
|
@@ -360,7 +376,7 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
360
376
|
"consecutiveErrors": run["consecutive_errors"],
|
|
361
377
|
"cancelRequested": bool(run["cancel_requested"]),
|
|
362
378
|
"canonicalArtifacts": canonical_artifacts,
|
|
363
|
-
"manifestPath":
|
|
379
|
+
"manifestPath": manifest_path,
|
|
364
380
|
"terminalReason": run["terminal_reason"],
|
|
365
381
|
"error": deep_scan_error(run),
|
|
366
382
|
"createdAt": run["created_at"],
|
|
@@ -379,8 +395,9 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
379
395
|
|
|
380
396
|
|
|
381
397
|
def independent_review_progress(
|
|
382
|
-
connection: sqlite3.Connection, scan_id: str
|
|
398
|
+
connection: sqlite3.Connection, scan_id: str, *, parent_running: bool
|
|
383
399
|
) -> dict[str, int | str] | None:
|
|
400
|
+
"""Return completed and allowed review counts and the consolidation state, or None for a missing run."""
|
|
384
401
|
run = connection.execute(
|
|
385
402
|
"""
|
|
386
403
|
SELECT completion_sequence, phase, updated_at, max_discovery_runs
|
|
@@ -391,21 +408,21 @@ def independent_review_progress(
|
|
|
391
408
|
).fetchone()
|
|
392
409
|
if run is None:
|
|
393
410
|
return None
|
|
394
|
-
active = connection.execute(
|
|
411
|
+
active, buffered = connection.execute(
|
|
395
412
|
"""
|
|
396
|
-
SELECT COUNT(*)
|
|
413
|
+
SELECT COUNT(*) FILTER (WHERE status IN ('queued', 'running')),
|
|
414
|
+
COUNT(*) FILTER (WHERE status = 'succeeded' AND merge_state = 'buffered')
|
|
397
415
|
FROM deep_scan_workers
|
|
398
|
-
WHERE scan_id = ?
|
|
399
|
-
AND kind = 'discovery'
|
|
400
|
-
AND status IN ('queued', 'running')
|
|
416
|
+
WHERE scan_id = ? AND kind = 'discovery'
|
|
401
417
|
""",
|
|
402
418
|
(scan_id,),
|
|
403
|
-
).fetchone()
|
|
419
|
+
).fetchone()
|
|
420
|
+
consolidating = run["phase"] in {"reducing", "terminal"} or (active == 0 and buffered > 0)
|
|
404
421
|
return {
|
|
405
422
|
"active": int(active),
|
|
406
423
|
"completed": int(run["completion_sequence"]),
|
|
407
424
|
"maximum": int(run["max_discovery_runs"]),
|
|
408
|
-
"consolidating":
|
|
425
|
+
"consolidating": parent_running and consolidating,
|
|
409
426
|
"updatedAt": str(run["updated_at"]),
|
|
410
427
|
}
|
|
411
428
|
|
|
@@ -445,10 +462,10 @@ def deep_scan_result(
|
|
|
445
462
|
def effective_deep_scan_config(
|
|
446
463
|
args: argparse.Namespace, workspace: sqlite3.Row | None = None
|
|
447
464
|
) -> dict[str, Any]:
|
|
465
|
+
"""Use persisted workspace settings when available, otherwise resolve the current launch configuration."""
|
|
448
466
|
if workspace is not None and workspace["deep_scan_config_json"] is not None:
|
|
449
467
|
return json.loads(workspace["deep_scan_config_json"])
|
|
450
|
-
|
|
451
|
-
return resolve_deep_scan_setup(available_parallelism, args.deep_scan_project_json)
|
|
468
|
+
return resolve_deep_scan_setup(project_json=args.deep_scan_project_json)
|
|
452
469
|
|
|
453
470
|
|
|
454
471
|
def ensure_deep_scan_run(
|
|
@@ -461,6 +478,7 @@ def ensure_deep_scan_run(
|
|
|
461
478
|
service_tier: str | None,
|
|
462
479
|
pending_target_project_path: str | None = None,
|
|
463
480
|
) -> sqlite3.Row:
|
|
481
|
+
"""Return the existing orchestration row or initialize one for a running Deep Scan with the supplied settings."""
|
|
464
482
|
existing = connection.execute(
|
|
465
483
|
"SELECT * FROM deep_scan_runs WHERE scan_id = ?", (scan["id"],)
|
|
466
484
|
).fetchone()
|
|
@@ -592,6 +610,8 @@ def begin_deep_scan_for_scan(
|
|
|
592
610
|
) -> dict[str, Any]:
|
|
593
611
|
scan_id = require_uuid(scan_id, "scan-id")
|
|
594
612
|
candidate = dependencies().require_scan(connection, scan_id)
|
|
613
|
+
model_thread_id = getattr(args, "model_thread_id", None)
|
|
614
|
+
require_tokenless_scan_owner(connection, candidate, model_thread_id)
|
|
595
615
|
workspace = dependencies().require_workspace(connection, candidate["workspace_id"])
|
|
596
616
|
if (
|
|
597
617
|
candidate["mode"] == "deep"
|
|
@@ -616,8 +636,17 @@ def begin_deep_scan_for_scan(
|
|
|
616
636
|
claimed_scan = connection.execute(
|
|
617
637
|
"UPDATE scans SET deep_scan_owner_thread_id = ?, updated_at = ? "
|
|
618
638
|
"WHERE id = ? AND deep_scan_owner_thread_id IS NULL "
|
|
619
|
-
"AND handoff_status = 'delivered' AND handoff_claim_token IS ?"
|
|
620
|
-
(
|
|
639
|
+
"AND handoff_status = 'delivered' AND handoff_claim_token IS ? "
|
|
640
|
+
"AND (handoff_claim_token IS NOT NULL OR continuation_thread_id = ? "
|
|
641
|
+
"OR (? IS NULL AND continuation_thread_id IS NULL))",
|
|
642
|
+
(
|
|
643
|
+
thread_id,
|
|
644
|
+
timestamp,
|
|
645
|
+
scan_id,
|
|
646
|
+
candidate["handoff_claim_token"],
|
|
647
|
+
thread_id,
|
|
648
|
+
model_thread_id,
|
|
649
|
+
),
|
|
621
650
|
)
|
|
622
651
|
if claimed_workspace.rowcount != 1 or claimed_scan.rowcount != 1:
|
|
623
652
|
raise SystemExit("A scan can only be orchestrated from its owning Codex thread.")
|
|
@@ -764,11 +793,8 @@ def begin_deep_scan_for_target(
|
|
|
764
793
|
workflow_version = optional_text(args.workflow_version, maximum=256)
|
|
765
794
|
if workflow_version is None:
|
|
766
795
|
raise SystemExit("workflow-version is required.")
|
|
767
|
-
|
|
768
|
-
target_root =
|
|
769
|
-
if target_root == target or target in target_root.parents:
|
|
770
|
-
raise SystemExit("The scan artifact directory must be outside the selected target.")
|
|
771
|
-
create_private_directory(target_root)
|
|
796
|
+
target_root = dependencies().scan_target_root(args.scan_root, target)
|
|
797
|
+
create_private_directory(target_root, scan_output=True)
|
|
772
798
|
model = optional_text(args.model, maximum=200)
|
|
773
799
|
reasoning_effort = optional_text(args.reasoning_effort, maximum=32)
|
|
774
800
|
workspace_id = str(uuid.uuid4())
|
|
@@ -829,6 +855,7 @@ def begin_deep_scan_for_target(
|
|
|
829
855
|
|
|
830
856
|
|
|
831
857
|
def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
858
|
+
"""Begin orchestration for a registered scan or a target, returning the scan state and its start disposition."""
|
|
832
859
|
thread_id = optional_text(args.thread_id, maximum=512)
|
|
833
860
|
if thread_id is None:
|
|
834
861
|
raise SystemExit("thread-id is required.")
|
|
@@ -842,6 +869,7 @@ def begin_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) ->
|
|
|
842
869
|
|
|
843
870
|
|
|
844
871
|
def get_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
872
|
+
"""Return the persisted orchestration result to the thread that owns the scan."""
|
|
845
873
|
scan, _ = require_owned_scan(connection, args.scan_id, args.thread_id)
|
|
846
874
|
return deep_scan_result(connection, scan["id"])
|
|
847
875
|
|
|
@@ -870,6 +898,7 @@ def coordinator_lease_is_live(
|
|
|
870
898
|
scan: sqlite3.Row,
|
|
871
899
|
timestamp: str,
|
|
872
900
|
) -> bool:
|
|
901
|
+
"""Check coordinator freshness using the lease generation and its applicable activity timestamps."""
|
|
873
902
|
if run["coordinator_generation"] == 1:
|
|
874
903
|
active_worker = connection.execute(
|
|
875
904
|
"""
|
|
@@ -933,7 +962,8 @@ def claim_deep_scan_coordinator_locked(
|
|
|
933
962
|
args.claim_token,
|
|
934
963
|
error_message="Deep Scan orchestration is owned by another continuation.",
|
|
935
964
|
)
|
|
936
|
-
run
|
|
965
|
+
run = require_deep_scan_run(connection, scan_id)
|
|
966
|
+
require_running_deep_scan(run, scan)
|
|
937
967
|
timestamp = dependencies().now()
|
|
938
968
|
if args.coordinator_generation is not None:
|
|
939
969
|
require_current_coordinator(run, args)
|
|
@@ -967,6 +997,7 @@ def claim_deep_scan_coordinator_locked(
|
|
|
967
997
|
def recover_expired_coordinator(
|
|
968
998
|
connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str
|
|
969
999
|
) -> None:
|
|
1000
|
+
"""Recover interrupted publications and reset active work so a replacement coordinator can resume."""
|
|
970
1001
|
scan_id = run["scan_id"]
|
|
971
1002
|
recover_candidate_ledger_publication(connection, scan_id)
|
|
972
1003
|
legacy_generation = int(run["coordinator_generation"] == 1)
|
|
@@ -1042,6 +1073,7 @@ def recover_expired_coordinator(
|
|
|
1042
1073
|
|
|
1043
1074
|
|
|
1044
1075
|
def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None:
|
|
1076
|
+
"""Reconcile ledger backups with reducer snapshots after an interrupted publication."""
|
|
1045
1077
|
scan = dependencies().require_scan(connection, scan_id)
|
|
1046
1078
|
ledger = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl"
|
|
1047
1079
|
backups = sorted(
|
|
@@ -1079,26 +1111,11 @@ def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id
|
|
|
1079
1111
|
return
|
|
1080
1112
|
|
|
1081
1113
|
|
|
1082
|
-
def
|
|
1083
|
-
worker_id = require_uuid(worker_id, "worker-id")
|
|
1084
|
-
row = connection.execute(
|
|
1085
|
-
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
1086
|
-
).fetchone()
|
|
1087
|
-
if row is None:
|
|
1088
|
-
raise SystemExit("Codex Security Deep Scan worker not found.")
|
|
1089
|
-
return row
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
def require_running_deep_scan(
|
|
1093
|
-
connection: sqlite3.Connection, scan_id: str
|
|
1094
|
-
) -> tuple[sqlite3.Row, sqlite3.Row]:
|
|
1095
|
-
run = require_deep_scan_run(connection, scan_id)
|
|
1096
|
-
scan = dependencies().require_scan(connection, run["scan_id"])
|
|
1114
|
+
def require_running_deep_scan(run: sqlite3.Row, scan: sqlite3.Row) -> None:
|
|
1097
1115
|
if run["status"] != "running" or run["cancel_requested"]:
|
|
1098
1116
|
raise SystemExit("Only a running Deep Scan can update orchestration state.")
|
|
1099
1117
|
if scan["status"] != "running" or scan["canceled_at"] is not None:
|
|
1100
1118
|
raise SystemExit("Only a running scan can update Deep Scan orchestration state.")
|
|
1101
|
-
return run, scan
|
|
1102
1119
|
|
|
1103
1120
|
|
|
1104
1121
|
def require_worker_transition(current: str, requested: str) -> None:
|
|
@@ -1126,6 +1143,12 @@ def upsert_deep_scan_worker(
|
|
|
1126
1143
|
existing = connection.execute(
|
|
1127
1144
|
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
1128
1145
|
).fetchone()
|
|
1146
|
+
if args.terminal_failure and (
|
|
1147
|
+
args.status != "failed" or not optional_text(args.error_message)
|
|
1148
|
+
):
|
|
1149
|
+
raise SystemExit(
|
|
1150
|
+
"A terminal worker failure requires failed status and an error message."
|
|
1151
|
+
)
|
|
1129
1152
|
replaceable_failure_kind = args.replaceable_failure_kind
|
|
1130
1153
|
if replaceable_failure_kind is not None and (
|
|
1131
1154
|
args.kind != "discovery"
|
|
@@ -1150,7 +1173,7 @@ def upsert_deep_scan_worker(
|
|
|
1150
1173
|
and args.status in {"succeeded", "failed", "canceled"}
|
|
1151
1174
|
)
|
|
1152
1175
|
if not cleanup_update and not terminal_repeat:
|
|
1153
|
-
require_running_deep_scan(
|
|
1176
|
+
require_running_deep_scan(run, scan)
|
|
1154
1177
|
prompt_path = deep_scan_path(scan, args.prompt_path, "Worker prompt path", kind="file")
|
|
1155
1178
|
artifact_dir = deep_scan_path(
|
|
1156
1179
|
scan, args.artifact_dir, "Worker artifact directory", kind="directory"
|
|
@@ -1168,7 +1191,9 @@ def upsert_deep_scan_worker(
|
|
|
1168
1191
|
timestamp = dependencies().now()
|
|
1169
1192
|
if existing is None:
|
|
1170
1193
|
if args.kind == "dedup":
|
|
1171
|
-
raise SystemExit(
|
|
1194
|
+
raise SystemExit(
|
|
1195
|
+
"Deep Scan checkpoints are recorded with record-deep-scan-iteration."
|
|
1196
|
+
)
|
|
1172
1197
|
if args.status not in {"queued", "running"}:
|
|
1173
1198
|
raise SystemExit("A new Deep Scan worker must be queued or running.")
|
|
1174
1199
|
if (
|
|
@@ -1251,7 +1276,7 @@ def upsert_deep_scan_worker(
|
|
|
1251
1276
|
if args.status == "running" and attempt < 1:
|
|
1252
1277
|
raise SystemExit("A running Deep Scan worker attempt must be at least one.")
|
|
1253
1278
|
if args.kind == "dedup" and args.status == "succeeded":
|
|
1254
|
-
raise SystemExit("
|
|
1279
|
+
raise SystemExit("Deep Scan checkpoints are recorded with record-deep-scan-iteration.")
|
|
1255
1280
|
if args.kind == "discovery" and args.status == "succeeded" and result_manifest_path is None:
|
|
1256
1281
|
result_manifest_path = existing["result_manifest_path"]
|
|
1257
1282
|
if result_manifest_path is None:
|
|
@@ -1316,11 +1341,14 @@ def upsert_deep_scan_worker(
|
|
|
1316
1341
|
if error_message is None and args.status != "succeeded":
|
|
1317
1342
|
error_message = existing["error_message"]
|
|
1318
1343
|
started_at = existing["started_at"] or (timestamp if args.status == "running" else None)
|
|
1344
|
+
thread_id = optional_text(args.sdk_thread_id, maximum=512)
|
|
1345
|
+
if thread_id is None and attempt == existing["attempt"]:
|
|
1346
|
+
thread_id = existing["sdk_thread_id"]
|
|
1319
1347
|
connection.execute(
|
|
1320
1348
|
"""
|
|
1321
1349
|
UPDATE deep_scan_workers
|
|
1322
1350
|
SET status = ?, result_manifest_path = ?, attempt = ?,
|
|
1323
|
-
sdk_thread_id =
|
|
1351
|
+
sdk_thread_id = ?,
|
|
1324
1352
|
completion_sequence = ?, merge_state = ?,
|
|
1325
1353
|
error_message = ?,
|
|
1326
1354
|
started_at = ?, completed_at = ?, updated_at = ?
|
|
@@ -1330,7 +1358,7 @@ def upsert_deep_scan_worker(
|
|
|
1330
1358
|
args.status,
|
|
1331
1359
|
result_manifest_path or existing["result_manifest_path"],
|
|
1332
1360
|
attempt,
|
|
1333
|
-
|
|
1361
|
+
thread_id,
|
|
1334
1362
|
completion_sequence,
|
|
1335
1363
|
merge_state,
|
|
1336
1364
|
error_message,
|
|
@@ -1340,275 +1368,81 @@ def upsert_deep_scan_worker(
|
|
|
1340
1368
|
worker_id,
|
|
1341
1369
|
),
|
|
1342
1370
|
)
|
|
1371
|
+
if args.terminal_failure:
|
|
1372
|
+
# Persist the first terminal cause with its worker. An adopting coordinator
|
|
1373
|
+
# completes the normal failure path after active workers have settled.
|
|
1374
|
+
connection.execute(
|
|
1375
|
+
"UPDATE deep_scan_runs SET error_message = ? WHERE scan_id = ? AND error_message IS NULL",
|
|
1376
|
+
(error_message, scan_id),
|
|
1377
|
+
)
|
|
1343
1378
|
return deep_scan_result(connection, scan_id)
|
|
1344
1379
|
|
|
1345
1380
|
|
|
1346
|
-
def
|
|
1381
|
+
def record_deep_scan_iteration(
|
|
1347
1382
|
connection: sqlite3.Connection, args: argparse.Namespace
|
|
1348
1383
|
) -> dict[str, Any]:
|
|
1384
|
+
"""Commit the checkpoint and its input workers in one transaction, then return deep scan state.
|
|
1385
|
+
|
|
1386
|
+
Repeating the same iteration and inputs replays the saved result. Conflicting
|
|
1387
|
+
checkpoint or input-worker IDs raise SystemExit.
|
|
1388
|
+
"""
|
|
1349
1389
|
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1350
|
-
|
|
1390
|
+
checkpoint_id = require_uuid(args.worker_id, "worker-id")
|
|
1351
1391
|
input_ids = [require_uuid(value, "input-worker-id") for value in args.input_worker_id]
|
|
1352
|
-
if len(set(input_ids)) != len(input_ids):
|
|
1353
|
-
raise SystemExit("Dedup input worker IDs must be unique.")
|
|
1354
1392
|
connection.execute("BEGIN IMMEDIATE")
|
|
1355
1393
|
with connection:
|
|
1356
|
-
run
|
|
1394
|
+
run = require_deep_scan_run(connection, scan_id)
|
|
1395
|
+
scan = dependencies().require_scan(connection, scan_id)
|
|
1396
|
+
require_running_deep_scan(run, scan)
|
|
1357
1397
|
require_current_coordinator(run, args)
|
|
1358
|
-
|
|
1359
|
-
artifact_dir = deep_scan_path(
|
|
1360
|
-
scan, args.artifact_dir, "Dedup artifact directory", kind="directory"
|
|
1361
|
-
)
|
|
1398
|
+
result = deep_scan_path(scan, args.result_manifest_path, "Iteration result", kind="file")
|
|
1362
1399
|
existing = connection.execute(
|
|
1363
|
-
"SELECT
|
|
1400
|
+
"SELECT scan_id, result_manifest_path FROM deep_scan_workers WHERE id = ?",
|
|
1401
|
+
(checkpoint_id,),
|
|
1364
1402
|
).fetchone()
|
|
1365
1403
|
if existing is not None:
|
|
1366
|
-
|
|
1367
|
-
|
|
1404
|
+
if existing["scan_id"] != scan_id or existing["result_manifest_path"] != result:
|
|
1405
|
+
raise SystemExit("Iteration already belongs to another result.")
|
|
1406
|
+
recorded_ids = [
|
|
1407
|
+
row[0]
|
|
1368
1408
|
for row in connection.execute(
|
|
1369
|
-
""
|
|
1370
|
-
|
|
1371
|
-
FROM deep_scan_dedup_inputs
|
|
1372
|
-
WHERE dedup_worker_id = ?
|
|
1373
|
-
ORDER BY input_order
|
|
1374
|
-
""",
|
|
1375
|
-
(worker_id,),
|
|
1409
|
+
"SELECT discovery_worker_id FROM deep_scan_dedup_inputs WHERE dedup_worker_id = ? ORDER BY input_order",
|
|
1410
|
+
(checkpoint_id,),
|
|
1376
1411
|
)
|
|
1377
1412
|
]
|
|
1378
|
-
if
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
and persisted_inputs == input_ids
|
|
1384
|
-
):
|
|
1385
|
-
connection.commit()
|
|
1386
|
-
return deep_scan_result(connection, scan_id)
|
|
1387
|
-
raise SystemExit("Dedup worker ID is already used by a different reducer claim.")
|
|
1388
|
-
active_reducer = connection.execute(
|
|
1389
|
-
"""
|
|
1390
|
-
SELECT 1 FROM deep_scan_workers
|
|
1391
|
-
WHERE scan_id = ? AND kind = 'dedup' AND status IN ('queued', 'running')
|
|
1392
|
-
""",
|
|
1393
|
-
(scan_id,),
|
|
1394
|
-
).fetchone()
|
|
1395
|
-
if active_reducer is not None:
|
|
1396
|
-
raise SystemExit("Only one Deep Scan dedup worker can run at a time.")
|
|
1397
|
-
buffered_ids = [
|
|
1398
|
-
row["id"]
|
|
1399
|
-
for row in connection.execute(
|
|
1400
|
-
"""
|
|
1401
|
-
SELECT id FROM deep_scan_workers
|
|
1402
|
-
WHERE scan_id = ? AND kind = 'discovery'
|
|
1403
|
-
AND status = 'succeeded' AND merge_state = 'buffered'
|
|
1404
|
-
ORDER BY completion_sequence
|
|
1405
|
-
""",
|
|
1406
|
-
(scan_id,),
|
|
1407
|
-
)
|
|
1408
|
-
]
|
|
1409
|
-
if input_ids != buffered_ids[: len(input_ids)]:
|
|
1410
|
-
raise SystemExit(
|
|
1411
|
-
"A Deep Scan dedup worker must claim an ordered prefix of buffered discovery "
|
|
1412
|
-
"results in completion order."
|
|
1413
|
-
)
|
|
1414
|
-
capped_singleton = (
|
|
1415
|
-
len(input_ids) == 1
|
|
1416
|
-
and (
|
|
1417
|
-
run["discovery_runs_dispatched"] >= run["max_discovery_runs"]
|
|
1418
|
-
or deep_scan_deadline_reached(run)
|
|
1419
|
-
)
|
|
1420
|
-
and connection.execute(
|
|
1421
|
-
"""
|
|
1422
|
-
SELECT 1 FROM deep_scan_workers
|
|
1423
|
-
WHERE scan_id = ? AND kind = 'discovery' AND status IN ('queued', 'running')
|
|
1424
|
-
LIMIT 1
|
|
1425
|
-
""",
|
|
1426
|
-
(scan_id,),
|
|
1427
|
-
).fetchone()
|
|
1428
|
-
is None
|
|
1429
|
-
)
|
|
1430
|
-
successful_reducer = connection.execute(
|
|
1431
|
-
"""
|
|
1432
|
-
SELECT 1 FROM deep_scan_workers
|
|
1433
|
-
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
1434
|
-
LIMIT 1
|
|
1435
|
-
""",
|
|
1436
|
-
(scan_id,),
|
|
1437
|
-
).fetchone()
|
|
1438
|
-
minimum_inputs = 1 if successful_reducer is not None or capped_singleton else 2
|
|
1439
|
-
if len(input_ids) < minimum_inputs:
|
|
1440
|
-
raise SystemExit(
|
|
1441
|
-
"The first Deep Scan dedup requires two buffered discovery results."
|
|
1442
|
-
if minimum_inputs == 2
|
|
1443
|
-
else "A Deep Scan dedup requires at least one buffered discovery result."
|
|
1444
|
-
)
|
|
1445
|
-
timestamp = dependencies().now()
|
|
1446
|
-
connection.execute(
|
|
1447
|
-
"""
|
|
1448
|
-
INSERT INTO deep_scan_workers (
|
|
1449
|
-
id, scan_id, kind, status, prompt_path, artifact_dir,
|
|
1450
|
-
created_at, updated_at
|
|
1451
|
-
) VALUES (?, ?, 'dedup', 'queued', ?, ?, ?, ?)
|
|
1452
|
-
""",
|
|
1453
|
-
(worker_id, scan_id, prompt_path, artifact_dir, timestamp, timestamp),
|
|
1454
|
-
)
|
|
1455
|
-
for input_order, input_id in enumerate(input_ids):
|
|
1456
|
-
connection.execute(
|
|
1457
|
-
"""
|
|
1458
|
-
INSERT INTO deep_scan_dedup_inputs (
|
|
1459
|
-
scan_id, dedup_worker_id, discovery_worker_id, input_order
|
|
1460
|
-
) VALUES (?, ?, ?, ?)
|
|
1461
|
-
""",
|
|
1462
|
-
(scan_id, worker_id, input_id, input_order),
|
|
1463
|
-
)
|
|
1464
|
-
connection.execute(
|
|
1465
|
-
f"""
|
|
1466
|
-
UPDATE deep_scan_workers
|
|
1467
|
-
SET merge_state = 'merging', updated_at = ?
|
|
1468
|
-
WHERE scan_id = ? AND id IN ({",".join("?" for _ in input_ids)})
|
|
1469
|
-
""",
|
|
1470
|
-
(timestamp, scan_id, *input_ids),
|
|
1471
|
-
)
|
|
1413
|
+
if recorded_ids != input_ids:
|
|
1414
|
+
raise SystemExit("Iteration already belongs to other completed scans.")
|
|
1415
|
+
return deep_scan_result(connection, scan_id)
|
|
1416
|
+
now = dependencies().now()
|
|
1417
|
+
# Historical scans use the same rows to bind the selected aggregate at publication.
|
|
1472
1418
|
connection.execute(
|
|
1473
|
-
"
|
|
1474
|
-
|
|
1419
|
+
"""INSERT INTO deep_scan_workers
|
|
1420
|
+
(id, scan_id, kind, status, prompt_path, artifact_dir, result_manifest_path,
|
|
1421
|
+
created_at, completed_at, updated_at)
|
|
1422
|
+
VALUES (?, ?, 'dedup', 'succeeded', ?, ?, ?, ?, ?, ?)""",
|
|
1423
|
+
(checkpoint_id, scan_id, result, str(Path(result).parent), result, now, now, now),
|
|
1475
1424
|
)
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
(timestamp, scan_id),
|
|
1483
|
-
)
|
|
1484
|
-
return deep_scan_result(connection, scan_id)
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
def commit_deep_scan_dedup(
|
|
1488
|
-
connection: sqlite3.Connection, args: argparse.Namespace
|
|
1489
|
-
) -> dict[str, Any]:
|
|
1490
|
-
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1491
|
-
with dependencies().scan_completion_lock(scan_id):
|
|
1492
|
-
return commit_deep_scan_dedup_locked(connection, args, scan_id)
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
def commit_deep_scan_dedup_locked(
|
|
1496
|
-
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1497
|
-
) -> dict[str, Any]:
|
|
1498
|
-
worker_id = require_uuid(args.worker_id, "worker-id")
|
|
1499
|
-
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1500
|
-
publication_copy: Path | None = None
|
|
1501
|
-
connection.execute("BEGIN IMMEDIATE")
|
|
1502
|
-
try:
|
|
1503
|
-
run = require_deep_scan_run(connection, scan_id)
|
|
1504
|
-
require_current_coordinator(run, args)
|
|
1505
|
-
scan = dependencies().require_scan(connection, scan_id)
|
|
1506
|
-
worker = require_deep_scan_worker(connection, worker_id)
|
|
1507
|
-
if worker["scan_id"] != scan_id or worker["kind"] != "dedup":
|
|
1508
|
-
raise SystemExit("Dedup worker does not belong to this Deep Scan.")
|
|
1509
|
-
if worker["status"] == "succeeded":
|
|
1510
|
-
connection.commit()
|
|
1511
|
-
return deep_scan_result(connection, scan_id)
|
|
1512
|
-
require_running_deep_scan(connection, scan_id)
|
|
1513
|
-
if worker["status"] not in {"queued", "running"}:
|
|
1514
|
-
raise SystemExit("Only an active dedup worker can commit a result.")
|
|
1515
|
-
if args.candidate_ledger_path:
|
|
1516
|
-
candidate_ledger_path = deep_scan_path(
|
|
1517
|
-
scan,
|
|
1518
|
-
args.candidate_ledger_path,
|
|
1519
|
-
"Staged candidate ledger path",
|
|
1520
|
-
kind="file",
|
|
1521
|
-
)
|
|
1522
|
-
discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery"
|
|
1523
|
-
deep_scan_path(
|
|
1524
|
-
scan,
|
|
1525
|
-
str(discovery_dir / "in_scope_files.txt"),
|
|
1526
|
-
"Canonical in-scope inventory path",
|
|
1527
|
-
kind="file",
|
|
1528
|
-
)
|
|
1529
|
-
canonical_candidate_ledger_path = deep_scan_output_path(
|
|
1530
|
-
scan,
|
|
1531
|
-
str(discovery_dir / "candidate_ledger.jsonl"),
|
|
1532
|
-
"Canonical candidate ledger path",
|
|
1425
|
+
for index, worker_id in enumerate(input_ids):
|
|
1426
|
+
updated = connection.execute(
|
|
1427
|
+
"""UPDATE deep_scan_workers SET merge_state = 'merged', updated_at = ?
|
|
1428
|
+
WHERE scan_id = ? AND id = ? AND kind = 'discovery'
|
|
1429
|
+
AND status = 'succeeded' AND merge_state = 'buffered'""",
|
|
1430
|
+
(now, scan_id, worker_id),
|
|
1533
1431
|
)
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
canonical_candidate_ledger_path = None
|
|
1537
|
-
result_manifest_path = deep_scan_path(
|
|
1538
|
-
scan,
|
|
1539
|
-
args.result_manifest_path,
|
|
1540
|
-
"Dedup result manifest path",
|
|
1541
|
-
kind="file",
|
|
1542
|
-
)
|
|
1543
|
-
inputs = list(
|
|
1432
|
+
if updated.rowcount != 1:
|
|
1433
|
+
raise SystemExit("Iteration input is not an unmerged completed scan.")
|
|
1544
1434
|
connection.execute(
|
|
1545
|
-
""
|
|
1546
|
-
|
|
1547
|
-
FROM deep_scan_dedup_inputs AS inputs
|
|
1548
|
-
JOIN deep_scan_workers AS workers ON workers.id = inputs.discovery_worker_id
|
|
1549
|
-
WHERE inputs.dedup_worker_id = ?
|
|
1550
|
-
ORDER BY inputs.input_order
|
|
1551
|
-
""",
|
|
1552
|
-
(worker_id,),
|
|
1553
|
-
)
|
|
1554
|
-
)
|
|
1555
|
-
if not inputs or any(row["merge_state"] != "merging" for row in inputs):
|
|
1556
|
-
raise SystemExit("Dedup inputs are not in the claimed merging state.")
|
|
1557
|
-
if candidate_ledger_path and canonical_candidate_ledger_path:
|
|
1558
|
-
canonical_path = Path(canonical_candidate_ledger_path)
|
|
1559
|
-
publication_copy = canonical_path.with_name(
|
|
1560
|
-
f".{canonical_path.name}.{uuid.uuid4()}.publish"
|
|
1435
|
+
"INSERT INTO deep_scan_dedup_inputs VALUES (?, ?, ?, ?)",
|
|
1436
|
+
(scan_id, checkpoint_id, worker_id, index),
|
|
1561
1437
|
)
|
|
1562
|
-
create_publication_copy(candidate_ledger_path, publication_copy)
|
|
1563
|
-
promotion = promote_staged_file(
|
|
1564
|
-
str(publication_copy),
|
|
1565
|
-
canonical_candidate_ledger_path,
|
|
1566
|
-
)
|
|
1567
|
-
timestamp = dependencies().now()
|
|
1568
1438
|
connection.execute(
|
|
1569
|
-
""
|
|
1570
|
-
|
|
1571
|
-
SET merge_state = 'merged', updated_at = ?
|
|
1572
|
-
WHERE id IN (
|
|
1573
|
-
SELECT discovery_worker_id FROM deep_scan_dedup_inputs
|
|
1574
|
-
WHERE dedup_worker_id = ?
|
|
1575
|
-
)
|
|
1576
|
-
""",
|
|
1577
|
-
(timestamp, worker_id),
|
|
1578
|
-
)
|
|
1579
|
-
connection.execute(
|
|
1580
|
-
"""
|
|
1581
|
-
UPDATE deep_scan_workers
|
|
1582
|
-
SET status = 'succeeded', result_manifest_path = ?,
|
|
1583
|
-
error_message = NULL, started_at = COALESCE(started_at, ?),
|
|
1584
|
-
completed_at = ?, updated_at = ?
|
|
1585
|
-
WHERE id = ?
|
|
1586
|
-
""",
|
|
1587
|
-
(result_manifest_path, timestamp, timestamp, timestamp, worker_id),
|
|
1588
|
-
)
|
|
1589
|
-
no_new_streak = (
|
|
1590
|
-
0 if args.new_findings_count > 0 else run["consecutive_no_new"] + len(inputs)
|
|
1439
|
+
"UPDATE deep_scan_runs SET phase = 'discovery', consecutive_no_new = ?, consecutive_errors = ?, updated_at = ? WHERE scan_id = ?",
|
|
1440
|
+
(args.no_new_streak, args.consecutive_errors, now, scan_id),
|
|
1591
1441
|
)
|
|
1592
1442
|
connection.execute(
|
|
1593
|
-
""
|
|
1594
|
-
|
|
1595
|
-
SET phase = 'discovery', consecutive_no_new = ?, updated_at = ?
|
|
1596
|
-
WHERE scan_id = ?
|
|
1597
|
-
""",
|
|
1598
|
-
(no_new_streak, timestamp, scan_id),
|
|
1443
|
+
"UPDATE scan_progress SET deep_review_pass = COALESCE(deep_review_pass, 0) + 1, updated_at = ? WHERE scan_id = ?",
|
|
1444
|
+
(now, scan_id),
|
|
1599
1445
|
)
|
|
1600
|
-
connection.commit()
|
|
1601
|
-
except BaseException:
|
|
1602
|
-
connection.rollback()
|
|
1603
|
-
if promotion is not None:
|
|
1604
|
-
rollback_staged_file(promotion)
|
|
1605
|
-
if publication_copy is not None:
|
|
1606
|
-
publication_copy.unlink(missing_ok=True)
|
|
1607
|
-
raise
|
|
1608
|
-
if promotion is not None:
|
|
1609
|
-
finish_staged_file(promotion)
|
|
1610
|
-
if publication_copy is not None:
|
|
1611
|
-
publication_copy.unlink(missing_ok=True)
|
|
1612
1446
|
return deep_scan_result(connection, scan_id)
|
|
1613
1447
|
|
|
1614
1448
|
|
|
@@ -1621,10 +1455,10 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1621
1455
|
def finish_deep_scan_locked(
|
|
1622
1456
|
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1623
1457
|
) -> dict[str, Any]:
|
|
1624
|
-
omitted_worker_ids =
|
|
1458
|
+
omitted_worker_ids = {
|
|
1625
1459
|
require_uuid(value, "omitted-worker-id") for value in args.omitted_worker_id
|
|
1626
|
-
|
|
1627
|
-
if len(
|
|
1460
|
+
}
|
|
1461
|
+
if len(omitted_worker_ids) != len(args.omitted_worker_id):
|
|
1628
1462
|
raise SystemExit("Omitted Deep Scan worker IDs must be unique.")
|
|
1629
1463
|
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1630
1464
|
deadline_publication: (
|
|
@@ -1644,29 +1478,23 @@ def finish_deep_scan_locked(
|
|
|
1644
1478
|
)
|
|
1645
1479
|
standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
1646
1480
|
|
|
1647
|
-
buffered_worker_ids =
|
|
1481
|
+
buffered_worker_ids = {
|
|
1648
1482
|
row["id"]
|
|
1649
1483
|
for row in connection.execute(
|
|
1650
1484
|
"""
|
|
1651
1485
|
SELECT id
|
|
1652
1486
|
FROM deep_scan_workers
|
|
1653
1487
|
WHERE scan_id = ? AND kind = 'discovery' AND merge_state = 'buffered'
|
|
1654
|
-
ORDER BY completion_sequence, id
|
|
1655
1488
|
""",
|
|
1656
1489
|
(scan_id,),
|
|
1657
1490
|
)
|
|
1658
|
-
|
|
1659
|
-
# Historical succeeded runs retain their exact omitted-worker replay contract.
|
|
1660
|
-
omissions_match = (
|
|
1661
|
-
set(omitted_worker_ids) == set(buffered_worker_ids)
|
|
1662
|
-
if args.terminal_reason == "saturated" or run["status"] == "succeeded"
|
|
1663
|
-
else not omitted_worker_ids and not buffered_worker_ids
|
|
1664
|
-
)
|
|
1491
|
+
}
|
|
1665
1492
|
if run["status"] == "succeeded":
|
|
1666
1493
|
if (
|
|
1667
1494
|
run["terminal_reason"] != args.terminal_reason
|
|
1668
|
-
or
|
|
1669
|
-
|
|
1495
|
+
or archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
1496
|
+
not in {None, manifest_path}
|
|
1497
|
+
or omitted_worker_ids != buffered_worker_ids
|
|
1670
1498
|
):
|
|
1671
1499
|
raise SystemExit(
|
|
1672
1500
|
"Deep Scan terminal state is immutable; finish must exactly replay its "
|
|
@@ -1679,7 +1507,9 @@ def finish_deep_scan_locked(
|
|
|
1679
1507
|
)
|
|
1680
1508
|
connection.commit()
|
|
1681
1509
|
return deep_scan_result(connection, scan_id)
|
|
1682
|
-
require_running_deep_scan(
|
|
1510
|
+
require_running_deep_scan(run, scan)
|
|
1511
|
+
if run["error_message"] is not None:
|
|
1512
|
+
raise SystemExit(run["error_message"])
|
|
1683
1513
|
if (
|
|
1684
1514
|
args.terminal_reason == "saturated"
|
|
1685
1515
|
and run["consecutive_no_new"] < run["stop_after_no_new"]
|
|
@@ -1711,7 +1541,7 @@ def finish_deep_scan_locked(
|
|
|
1711
1541
|
raise SystemExit(
|
|
1712
1542
|
f"Deep Scan cannot finish without canonical discovery artifacts: {exc}"
|
|
1713
1543
|
) from exc
|
|
1714
|
-
|
|
1544
|
+
saved_iteration = connection.execute(
|
|
1715
1545
|
"""
|
|
1716
1546
|
SELECT 1 FROM deep_scan_workers
|
|
1717
1547
|
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
@@ -1729,8 +1559,8 @@ def finish_deep_scan_locked(
|
|
|
1729
1559
|
and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0
|
|
1730
1560
|
)
|
|
1731
1561
|
)
|
|
1732
|
-
if
|
|
1733
|
-
raise SystemExit("Deep Scan cannot finish without a
|
|
1562
|
+
if saved_iteration is None and not zero_discovery_deadline:
|
|
1563
|
+
raise SystemExit("Deep Scan cannot finish without a saved iteration.")
|
|
1734
1564
|
if args.terminal_reason == "saturated":
|
|
1735
1565
|
# Saturation cancels surplus workers, including failed cancellation writes.
|
|
1736
1566
|
cancel_active_workers(connection, scan_id, dependencies().now())
|
|
@@ -1740,7 +1570,7 @@ def finish_deep_scan_locked(
|
|
|
1740
1570
|
args.terminal_reason,
|
|
1741
1571
|
has_omitted_workers=bool(omitted_worker_ids),
|
|
1742
1572
|
)
|
|
1743
|
-
if args.terminal_reason == "saturated" and
|
|
1573
|
+
if args.terminal_reason == "saturated" and omitted_worker_ids != buffered_worker_ids:
|
|
1744
1574
|
raise SystemExit(
|
|
1745
1575
|
f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery "
|
|
1746
1576
|
"workers with --omitted-worker-id."
|
|
@@ -1771,7 +1601,6 @@ def finish_deep_scan_locked(
|
|
|
1771
1601
|
""",
|
|
1772
1602
|
(args.terminal_reason, manifest_path, timestamp, timestamp, scan_id),
|
|
1773
1603
|
)
|
|
1774
|
-
cancel_active_workers(connection, scan_id, timestamp)
|
|
1775
1604
|
connection.commit()
|
|
1776
1605
|
except BaseException:
|
|
1777
1606
|
connection.rollback()
|
|
@@ -1810,6 +1639,7 @@ def fail_deep_scan_locked(
|
|
|
1810
1639
|
run = require_deep_scan_run(connection, scan_id)
|
|
1811
1640
|
require_current_coordinator(run, args)
|
|
1812
1641
|
scan = dependencies().require_scan(connection, scan_id)
|
|
1642
|
+
recorded_manifest = archived_deep_path(Path(scan["scan_dir"]), run["manifest_path"])
|
|
1813
1643
|
manifest_path = None
|
|
1814
1644
|
if args.manifest_path:
|
|
1815
1645
|
manifest_path = (
|
|
@@ -1822,14 +1652,14 @@ def fail_deep_scan_locked(
|
|
|
1822
1652
|
kind="file",
|
|
1823
1653
|
)
|
|
1824
1654
|
)
|
|
1825
|
-
elif
|
|
1826
|
-
manifest_path =
|
|
1655
|
+
elif recorded_manifest == str(Path(scan["scan_dir"]) / "scan-manifest.json"):
|
|
1656
|
+
manifest_path = recorded_manifest
|
|
1827
1657
|
if run["status"] in {"failed", "interrupted"} or scan["status"] == "failed":
|
|
1828
1658
|
if (
|
|
1829
1659
|
run["status"] == args.deep_status
|
|
1830
1660
|
and scan["status"] == "failed"
|
|
1831
1661
|
and run["error_message"] == message
|
|
1832
|
-
and
|
|
1662
|
+
and recorded_manifest == manifest_path
|
|
1833
1663
|
and scan["failure_message"] == message
|
|
1834
1664
|
):
|
|
1835
1665
|
connection.commit()
|
|
@@ -1848,6 +1678,9 @@ def fail_deep_scan_locked(
|
|
|
1848
1678
|
"status"
|
|
1849
1679
|
] != "running":
|
|
1850
1680
|
raise SystemExit("Only a running Deep Scan can be failed or interrupted.")
|
|
1681
|
+
# Cleanup can report a later error after a worker committed the terminal cause.
|
|
1682
|
+
if run["status"] == "running" and run["error_message"] is not None:
|
|
1683
|
+
message = run["error_message"]
|
|
1851
1684
|
if run["manifest_path"] not in {None, manifest_path}:
|
|
1852
1685
|
raise SystemExit("Deep Scan coordinator manifest path is immutable.")
|
|
1853
1686
|
if args.staged_manifest_path and manifest_path:
|
|
@@ -1869,20 +1702,9 @@ def fail_deep_scan_locked(
|
|
|
1869
1702
|
(args.deep_status, message, manifest_path, timestamp, timestamp, scan_id),
|
|
1870
1703
|
)
|
|
1871
1704
|
cancel_active_workers(connection, scan_id, timestamp)
|
|
1872
|
-
|
|
1873
|
-
"""
|
|
1874
|
-
UPDATE scans
|
|
1875
|
-
SET status = 'failed', failure_message = ?, completed_at = ?, updated_at = ?
|
|
1876
|
-
WHERE id = ? AND status = 'running'
|
|
1877
|
-
""",
|
|
1878
|
-
(message, timestamp, timestamp, scan_id),
|
|
1879
|
-
)
|
|
1880
|
-
if parent_update.rowcount != 1:
|
|
1705
|
+
if not saved_results.mark_running_scan_failed(connection, scan_id, message, timestamp):
|
|
1881
1706
|
raise SystemExit("Deep Scan failure could not be persisted to its parent scan.")
|
|
1882
|
-
|
|
1883
|
-
"UPDATE scan_progress SET updated_at = ? WHERE scan_id = ?",
|
|
1884
|
-
(timestamp, scan_id),
|
|
1885
|
-
)
|
|
1707
|
+
saved_results.touch_scan_progress(connection, scan_id, timestamp)
|
|
1886
1708
|
connection.commit()
|
|
1887
1709
|
except BaseException:
|
|
1888
1710
|
connection.rollback()
|
|
@@ -1967,7 +1789,7 @@ def cancel_from_parent_scan(connection: sqlite3.Connection, scan_id: str, timest
|
|
|
1967
1789
|
connection.execute(
|
|
1968
1790
|
"""
|
|
1969
1791
|
UPDATE deep_scan_runs
|
|
1970
|
-
SET status = 'canceled', phase = 'terminal', cancel_requested = 1,
|
|
1792
|
+
SET status = 'canceled', phase = 'terminal', cancel_requested = 1, error_message = NULL,
|
|
1971
1793
|
terminal_reviewed_surface_count = NULL, completed_at = ?, updated_at = ?
|
|
1972
1794
|
WHERE scan_id = ? AND status IN ('running', 'succeeded')
|
|
1973
1795
|
""",
|
|
@@ -1990,6 +1812,7 @@ def cancel_active_workers(connection: sqlite3.Connection, scan_id: str, timestam
|
|
|
1990
1812
|
def other_running_deep_scans(
|
|
1991
1813
|
connection: sqlite3.Connection, current_scan_id: str
|
|
1992
1814
|
) -> list[dict[str, str]]:
|
|
1815
|
+
"""List other running Deep Scans ordered by most recent update, with target paths and phases."""
|
|
1993
1816
|
rows = connection.execute(
|
|
1994
1817
|
"""
|
|
1995
1818
|
SELECT id, target_path, phase, started_at, updated_at
|