@openai/codex-security 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -788
- package/_bundled_plugin/.codex-plugin/plugin.json +2 -0
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/references/artifact-storage.md +1 -1
- package/_bundled_plugin/references/baseline-auditor.md +18 -0
- package/_bundled_plugin/references/core-scan.md +5 -42
- package/_bundled_plugin/references/final-report.md +1 -1
- package/_bundled_plugin/references/focused-investigator.md +13 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -2
- package/_bundled_plugin/references/scan-runtime.md +4 -2
- package/_bundled_plugin/references/security-guidance.md +14 -1
- package/_bundled_plugin/references/validate-scan-findings.md +3 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +29 -12
- package/_bundled_plugin/scripts/dispatch_inventory_tool +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +29 -3
- package/_bundled_plugin/scripts/launch_codex_security_mcp +54 -16
- package/_bundled_plugin/scripts/posix_scan_local_files.py +26 -1
- package/_bundled_plugin/scripts/snapshot_sqlite.py +8 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +216 -25
- package/_bundled_plugin/scripts/workbench/handoff.py +55 -25
- package/_bundled_plugin/scripts/workbench/storage.py +25 -5
- package/_bundled_plugin/scripts/workbench_cli.py +24 -2
- package/_bundled_plugin/scripts/workbench_constants.py +2 -0
- package/_bundled_plugin/scripts/workbench_db.py +244 -217
- package/_bundled_plugin/scripts/workbench_feedback.py +49 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +226 -29
- package/_bundled_plugin/scripts/workbench_progress.py +159 -3
- package/_bundled_plugin/scripts/workbench_publication.py +2 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +142 -2
- package/_bundled_plugin/scripts/workbench_saved_results.py +48 -8
- package/_bundled_plugin/scripts/workbench_scan_history.py +281 -119
- package/_bundled_plugin/scripts/workbench_scan_start.py +143 -9
- package/_bundled_plugin/scripts/workbench_scan_usage.py +36 -10
- package/_bundled_plugin/scripts/workbench_schema.py +220 -14
- package/_bundled_plugin/scripts/workbench_setup.py +1 -1
- package/_bundled_plugin/scripts/workbench_target.py +471 -84
- package/_bundled_plugin/scripts/workbench_target_state.py +1108 -22
- package/_bundled_plugin/shared/workbench-history-repairs.json +60 -14
- package/_bundled_plugin/shared/workbench-migrations.json +66 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +1 -0
- package/_bundled_plugin/skills/publish-findings/SKILL.md +24 -160
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +2 -2
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +3 -58
- package/_bundled_plugin/skills/publish-findings/scripts/collect_continuum.py +224 -0
- package/_bundled_plugin/skills/publish-findings/scripts/collector_paths.py +63 -0
- package/_bundled_plugin/skills/security-scan/SKILL.md +9 -46
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -21
- package/_bundled_plugin/skills/security-scan/references/managed-scan.md +11 -0
- package/_bundled_plugin/skills/security-scan/references/scan-results.md +15 -0
- package/_bundled_plugin/skills/security-scan/references/scoped-paths.md +23 -0
- package/_bundled_plugin/skills/security-scan/references/sdk-files.md +24 -0
- package/bin/codex-security.mjs +2 -2
- package/dist/api.d.ts +6 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +437 -154
- package/dist/api.js.map +1 -1
- package/dist/armorcode-findings.d.ts +9 -0
- package/dist/armorcode-findings.d.ts.map +1 -0
- package/dist/armorcode-findings.js +145 -0
- package/dist/armorcode-findings.js.map +1 -0
- package/dist/artifact-export.d.ts +2 -0
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +29 -13
- package/dist/artifact-export.js.map +1 -1
- package/dist/auth.d.ts +6 -2
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +33 -12
- package/dist/auth.js.map +1 -1
- package/dist/batch-cost.d.ts +18 -0
- package/dist/batch-cost.d.ts.map +1 -0
- package/dist/batch-cost.js +41 -0
- package/dist/batch-cost.js.map +1 -0
- package/dist/blackduck-findings.d.ts +1 -1
- package/dist/blackduck-findings.d.ts.map +1 -1
- package/dist/blackduck-findings.js +5 -18
- package/dist/blackduck-findings.js.map +1 -1
- package/dist/bundled-plugin.d.ts +4 -0
- package/dist/bundled-plugin.d.ts.map +1 -0
- package/dist/bundled-plugin.js +23 -0
- package/dist/bundled-plugin.js.map +1 -0
- package/dist/classify-scan-severity.d.ts +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +11 -3
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +9 -6
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-argument.d.ts +4 -0
- package/dist/cli-argument.d.ts.map +1 -0
- package/dist/cli-argument.js +17 -0
- package/dist/cli-argument.js.map +1 -0
- package/dist/cli-help.d.ts +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +71 -29
- package/dist/cli-help.js.map +1 -1
- package/dist/cli-hook.d.ts +3 -0
- package/dist/cli-hook.d.ts.map +1 -0
- package/dist/cli-hook.js +87 -0
- package/dist/cli-hook.js.map +1 -0
- package/dist/cli-integrations.d.ts +13 -0
- package/dist/cli-integrations.d.ts.map +1 -0
- package/dist/cli-integrations.js +333 -0
- package/dist/cli-integrations.js.map +1 -0
- package/dist/cli-mcp.d.ts +15 -0
- package/dist/cli-mcp.d.ts.map +1 -0
- package/dist/cli-mcp.js +178 -0
- package/dist/cli-mcp.js.map +1 -0
- package/dist/cli-token-view.d.ts +10 -0
- package/dist/cli-token-view.d.ts.map +1 -0
- package/dist/cli-token-view.js +41 -0
- package/dist/cli-token-view.js.map +1 -0
- package/dist/cli.d.ts +5 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1763 -894
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +33 -31
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-sdk-environment.d.ts +6 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -1
- package/dist/codex-sdk-environment.js +19 -0
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/command-words.d.ts +3 -0
- package/dist/command-words.d.ts.map +1 -0
- package/dist/command-words.js +46 -0
- package/dist/command-words.js.map +1 -0
- package/dist/component-scan.d.ts +3 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +40 -7
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +30 -1
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts +1 -1
- package/dist/continuum-findings.d.ts.map +1 -1
- package/dist/continuum-findings.js +93 -31
- package/dist/continuum-findings.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +1 -45
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +29 -19
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/local.d.ts +3 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +8 -1
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +2 -1
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +2 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +16 -15
- package/dist/deep-scan.js.map +1 -1
- package/dist/endor-findings.d.ts +2 -2
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +6 -23
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +3 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -3
- package/dist/errors.js.map +1 -1
- package/dist/executable-path.d.ts +3 -0
- package/dist/executable-path.d.ts.map +1 -1
- package/dist/executable-path.js +26 -0
- package/dist/executable-path.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +12 -1
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +66 -4
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +234 -53
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +6 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +326 -1633
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +1 -0
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +31 -51
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +1 -1
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +12 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +3 -11
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +29 -176
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/finding-prompt.d.ts.map +1 -1
- package/dist/finding-prompt.js +2 -2
- package/dist/finding-prompt.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +3 -4
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +19 -287
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-remediation.d.ts +1 -0
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +23 -3
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +19 -35
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +34 -27
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts +11 -0
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js.map +1 -1
- package/dist/local-git-metadata.d.ts +5 -2
- package/dist/local-git-metadata.d.ts.map +1 -1
- package/dist/local-git-metadata.js +8 -4
- package/dist/local-git-metadata.js.map +1 -1
- package/dist/multiscan.d.ts +4 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +79 -37
- package/dist/multiscan.js.map +1 -1
- package/dist/opengrep-findings.d.ts +4 -2
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +39 -22
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/private-finding-import.d.ts +8 -13
- package/dist/private-finding-import.d.ts.map +1 -1
- package/dist/private-finding-import.js +69 -129
- package/dist/private-finding-import.js.map +1 -1
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +53 -13
- package/dist/project-config.js.map +1 -1
- package/dist/prompt-files.d.ts +7 -2
- package/dist/prompt-files.d.ts.map +1 -1
- package/dist/prompt-files.js +42 -20
- package/dist/prompt-files.js.map +1 -1
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +4 -9
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication.d.ts +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js.map +1 -1
- package/dist/record.d.ts +1 -5
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +3 -51
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -1
- package/dist/remediation.js.map +1 -1
- package/dist/runtime.d.ts +35 -7
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +778 -153
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +6 -1
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +267 -115
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-archive-path.d.ts +2 -0
- package/dist/scan-archive-path.d.ts.map +1 -0
- package/dist/scan-archive-path.js +5 -0
- package/dist/scan-archive-path.js.map +1 -0
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +6 -2
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +1 -0
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +18 -2
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +4 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts +1 -0
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +25 -9
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +28 -4
- package/dist/scan-logs.js.map +1 -1
- package/dist/scanner-import.d.ts +5 -11
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +16 -89
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +26 -6
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +14 -48
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-platform-findings.d.ts +20 -70
- package/dist/semgrep-platform-findings.d.ts.map +1 -1
- package/dist/semgrep-platform-findings.js +116 -333
- package/dist/semgrep-platform-findings.js.map +1 -1
- package/dist/snyk-findings.d.ts +2 -0
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +5 -8
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +3 -20
- package/dist/snyk-rest-findings.d.ts.map +1 -1
- package/dist/snyk-rest-findings.js +45 -217
- package/dist/snyk-rest-findings.js.map +1 -1
- package/dist/sonar-findings.d.ts +1 -1
- package/dist/sonar-findings.d.ts.map +1 -1
- package/dist/sonar-findings.js +2 -4
- package/dist/sonar-findings.js.map +1 -1
- package/dist/sonar-import.d.ts +3 -0
- package/dist/sonar-import.d.ts.map +1 -1
- package/dist/sonar-import.js +28 -30
- package/dist/sonar-import.js.map +1 -1
- package/dist/stored-finding-projection.d.ts +6 -0
- package/dist/stored-finding-projection.d.ts.map +1 -0
- package/dist/stored-finding-projection.js +104 -0
- package/dist/stored-finding-projection.js.map +1 -0
- package/dist/stored-findings-publication.d.ts +23 -0
- package/dist/stored-findings-publication.d.ts.map +1 -0
- package/dist/stored-findings-publication.js +169 -0
- package/dist/stored-findings-publication.js.map +1 -0
- package/dist/targets.d.ts +6 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +76 -23
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +1 -4
- package/dist/tenable-findings.d.ts.map +1 -1
- package/dist/tenable-findings.js.map +1 -1
- package/dist/trivy-findings.d.ts.map +1 -1
- package/dist/trivy-findings.js +0 -11
- package/dist/trivy-findings.js.map +1 -1
- package/dist/trusted-executable.d.ts +6 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +140 -37
- package/dist/trusted-executable.js.map +1 -1
- package/dist/veracode-findings.d.ts +1 -3
- package/dist/veracode-findings.d.ts.map +1 -1
- package/dist/veracode-findings.js +13 -38
- package/dist/veracode-findings.js.map +1 -1
- package/dist/version.d.ts +3 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +184 -32
- package/dist/version.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -2
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +14 -92
- package/dist/wiz-findings.js.map +1 -1
- package/docs/cli.md +373 -106
- package/docs/findings-service.md +22 -9
- package/docs/hosted-remediation.md +13 -4
- package/docs/sdk.md +903 -0
- package/package.json +10 -6
- package/schemas/external-findings.schema.json +2 -1
- package/dist/hosted-scanner-v1.d.ts +0 -6
- package/dist/hosted-scanner-v1.d.ts.map +0 -1
- package/dist/hosted-scanner-v1.js +0 -60
- package/dist/hosted-scanner-v1.js.map +0 -1
- package/dist/reader-selection.d.ts +0 -15
- package/dist/reader-selection.d.ts.map +0 -1
- package/dist/reader-selection.js +0 -89
- package/dist/reader-selection.js.map +0 -1
- package/dist/semgrep-platform-v1.d.ts +0 -6
- package/dist/semgrep-platform-v1.d.ts.map +0 -1
- package/dist/semgrep-platform-v1.js +0 -83
- package/dist/semgrep-platform-v1.js.map +0 -1
- package/dist/snyk-rest-v1.d.ts +0 -28
- package/dist/snyk-rest-v1.d.ts.map +0 -1
- package/dist/snyk-rest-v1.js +0 -132
- package/dist/snyk-rest-v1.js.map +0 -1
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
|
+
import errno
|
|
6
7
|
import json
|
|
7
8
|
import os
|
|
8
9
|
import re
|
|
@@ -11,6 +12,7 @@ import sys
|
|
|
11
12
|
import tempfile
|
|
12
13
|
from collections.abc import Callable, Iterator
|
|
13
14
|
from contextlib import contextmanager
|
|
15
|
+
from dataclasses import dataclass
|
|
14
16
|
from datetime import datetime, timezone
|
|
15
17
|
from pathlib import Path
|
|
16
18
|
from typing import Any
|
|
@@ -19,7 +21,12 @@ from typing import Any
|
|
|
19
21
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
20
22
|
from filesystem_identity import serialize_filesystem_identity
|
|
21
23
|
from finalize_scan_contract import write_scan_local_bytes
|
|
22
|
-
from workbench.storage import
|
|
24
|
+
from workbench.storage import (
|
|
25
|
+
archive_source_directory,
|
|
26
|
+
create_private_temporary_directory,
|
|
27
|
+
short_archive_prefix,
|
|
28
|
+
state_dir,
|
|
29
|
+
)
|
|
23
30
|
from workbench_feedback import get_scan_feedback
|
|
24
31
|
from workbench_target import (
|
|
25
32
|
directory_content_digest,
|
|
@@ -58,6 +65,7 @@ def scan_target_identity(
|
|
|
58
65
|
diff_target: dict[str, str] | None,
|
|
59
66
|
*,
|
|
60
67
|
metadata: os.stat_result | None = None,
|
|
68
|
+
expected_digest: str | None = None,
|
|
61
69
|
) -> tuple[str, str | None, int | str, int | str]:
|
|
62
70
|
"""Return revision, content digest, device ID, and inode for scan registration.
|
|
63
71
|
|
|
@@ -72,7 +80,7 @@ def scan_target_identity(
|
|
|
72
80
|
snapshot_digest = (
|
|
73
81
|
directory_content_digest(target)
|
|
74
82
|
if revision == "unversioned"
|
|
75
|
-
else worktree_content_digest(target)
|
|
83
|
+
else worktree_content_digest(target, expected_digest=expected_digest)
|
|
76
84
|
)
|
|
77
85
|
return (
|
|
78
86
|
revision,
|
|
@@ -82,6 +90,25 @@ def scan_target_identity(
|
|
|
82
90
|
)
|
|
83
91
|
|
|
84
92
|
|
|
93
|
+
@dataclass(frozen=True)
|
|
94
|
+
class WindowsResumeTarget:
|
|
95
|
+
path: Path
|
|
96
|
+
snapshot: tuple[str, str | None, int | str, int | str]
|
|
97
|
+
native_id: tuple[int, int] | None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def prepare_cli_scan_target(
|
|
101
|
+
target: Path, diff_target: dict[str, str] | None, mode: str, launch_kind: str
|
|
102
|
+
) -> tuple[tuple[str, str | None, int | str, int | str], WindowsResumeTarget | None]:
|
|
103
|
+
if os.name == "nt" and mode in {"standard", "deep"} and launch_kind == "scan":
|
|
104
|
+
from windows_scan_local_files import pinned_resume_checkout
|
|
105
|
+
|
|
106
|
+
with pinned_resume_checkout(target) as native:
|
|
107
|
+
snapshot = scan_target_identity(target, diff_target)
|
|
108
|
+
return snapshot, WindowsResumeTarget(target, snapshot, native)
|
|
109
|
+
return scan_target_identity(target, diff_target), None
|
|
110
|
+
|
|
111
|
+
|
|
85
112
|
def scan_diff_identity(
|
|
86
113
|
diff_target: dict[str, str] | None,
|
|
87
114
|
) -> tuple[str | None, str | None, str | None, str | None]:
|
|
@@ -129,7 +156,7 @@ def archive_scan(
|
|
|
129
156
|
if archived_scan_dir is not None and (
|
|
130
157
|
not args.archive_existing
|
|
131
158
|
or archived_scan_dir.parent != scan_dir.parent
|
|
132
|
-
or
|
|
159
|
+
or archive_source_directory(archived_scan_dir, scan_dir) != scan_dir
|
|
133
160
|
):
|
|
134
161
|
raise SystemExit("The archived scan must be a previous sibling of the scan directory.")
|
|
135
162
|
|
|
@@ -179,9 +206,17 @@ def archive_scan(
|
|
|
179
206
|
)
|
|
180
207
|
if before_archive is not None:
|
|
181
208
|
before_archive()
|
|
182
|
-
|
|
183
|
-
tempfile.mkdtemp(
|
|
184
|
-
|
|
209
|
+
try:
|
|
210
|
+
reserved = tempfile.mkdtemp(
|
|
211
|
+
prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent
|
|
212
|
+
)
|
|
213
|
+
except OSError as error:
|
|
214
|
+
if os.name != "posix" or error.errno != errno.ENAMETOOLONG:
|
|
215
|
+
raise
|
|
216
|
+
reserved = tempfile.mkdtemp(
|
|
217
|
+
prefix=short_archive_prefix(scan_dir.name), dir=scan_dir.parent
|
|
218
|
+
)
|
|
219
|
+
archived_scan_dir = Path(reserved).resolve()
|
|
185
220
|
archived_scan_dir.rmdir()
|
|
186
221
|
scan_dir.rename(archived_scan_dir)
|
|
187
222
|
moved = True
|
|
@@ -269,6 +304,95 @@ def archive_scan(
|
|
|
269
304
|
raise
|
|
270
305
|
|
|
271
306
|
|
|
307
|
+
def restore_cli_scan_archive(
|
|
308
|
+
connection: sqlite3.Connection,
|
|
309
|
+
args: argparse.Namespace,
|
|
310
|
+
canonical_directory: Callable[[Path], Path],
|
|
311
|
+
) -> dict[str, str]:
|
|
312
|
+
supplied_scan_dir = Path(args.scan_dir).expanduser().absolute()
|
|
313
|
+
supplied_archive = Path(args.archived_scan_dir).expanduser().absolute()
|
|
314
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
315
|
+
try:
|
|
316
|
+
scan_dir = supplied_scan_dir.resolve()
|
|
317
|
+
archived_scan_dir = supplied_archive.resolve()
|
|
318
|
+
archived_owner = connection.execute(
|
|
319
|
+
"SELECT id FROM scans WHERE scan_dir = ?", (str(archived_scan_dir),)
|
|
320
|
+
).fetchone()
|
|
321
|
+
previous_scan = connection.execute(
|
|
322
|
+
"SELECT id FROM scans WHERE scan_dir = ?", (str(scan_dir),)
|
|
323
|
+
).fetchone()
|
|
324
|
+
previous_scan_id = previous_scan["id"] if previous_scan is not None else None
|
|
325
|
+
if archived_owner is not None:
|
|
326
|
+
disposition = "already-recorded"
|
|
327
|
+
elif previous_scan_id != args.previous_scan_id:
|
|
328
|
+
disposition = "ownership-changed"
|
|
329
|
+
else:
|
|
330
|
+
archived_scan_dir = canonical_directory(supplied_archive)
|
|
331
|
+
if (
|
|
332
|
+
os.path.normcase(supplied_scan_dir) != os.path.normcase(scan_dir)
|
|
333
|
+
or archived_scan_dir.parent != scan_dir.parent
|
|
334
|
+
or not archived_scan_dir.name.startswith(f"{scan_dir.name}.previous-")
|
|
335
|
+
):
|
|
336
|
+
raise SystemExit(
|
|
337
|
+
"The archived scan must be a previous sibling of the scan directory."
|
|
338
|
+
)
|
|
339
|
+
try:
|
|
340
|
+
supplied_scan_dir.lstat()
|
|
341
|
+
except FileNotFoundError:
|
|
342
|
+
pass
|
|
343
|
+
else:
|
|
344
|
+
canonical_directory(supplied_scan_dir).rmdir()
|
|
345
|
+
archived_scan_dir.rename(scan_dir)
|
|
346
|
+
disposition = "restored"
|
|
347
|
+
connection.commit()
|
|
348
|
+
except BaseException:
|
|
349
|
+
connection.rollback()
|
|
350
|
+
raise
|
|
351
|
+
return {
|
|
352
|
+
"disposition": disposition,
|
|
353
|
+
"scanDir": str(scan_dir),
|
|
354
|
+
"archivedScanDir": str(archived_scan_dir),
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
@contextmanager
|
|
359
|
+
def register_windows_resume_output(
|
|
360
|
+
connection: sqlite3.Connection,
|
|
361
|
+
scan_id: str,
|
|
362
|
+
scan_dir: Path,
|
|
363
|
+
target: WindowsResumeTarget,
|
|
364
|
+
before_create: Callable[[], None],
|
|
365
|
+
) -> Iterator[None]:
|
|
366
|
+
from windows_scan_local_files import create_deep_resume_output, pinned_resume_checkout
|
|
367
|
+
|
|
368
|
+
def committed() -> bool:
|
|
369
|
+
# A signal can arrive after commit. The registered scan then owns its child.
|
|
370
|
+
return (
|
|
371
|
+
not connection.in_transaction
|
|
372
|
+
and connection.execute("SELECT 1 FROM scans WHERE id = ?", (scan_id,)).fetchone()
|
|
373
|
+
is not None
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
with pinned_resume_checkout(target.path) as current:
|
|
377
|
+
if current != target.native_id or (
|
|
378
|
+
current is None and scan_target_identity(target.path, None) != target.snapshot
|
|
379
|
+
):
|
|
380
|
+
raise SystemExit(
|
|
381
|
+
"The selected scan target changed while the scan was starting. Try again."
|
|
382
|
+
)
|
|
383
|
+
before_create()
|
|
384
|
+
with create_deep_resume_output(scan_dir, committed) as identity:
|
|
385
|
+
connection.execute(
|
|
386
|
+
"UPDATE scans SET windows_resume_output_identity = ?, windows_resume_target_identity = ? WHERE id = ?",
|
|
387
|
+
(
|
|
388
|
+
json.dumps(identity),
|
|
389
|
+
json.dumps(target.native_id) if target.native_id is not None else None,
|
|
390
|
+
scan_id,
|
|
391
|
+
),
|
|
392
|
+
)
|
|
393
|
+
yield
|
|
394
|
+
|
|
395
|
+
|
|
272
396
|
def insert_running_scan(
|
|
273
397
|
connection: sqlite3.Connection,
|
|
274
398
|
*,
|
|
@@ -278,6 +402,7 @@ def insert_running_scan(
|
|
|
278
402
|
scope: str,
|
|
279
403
|
diff_target: dict[str, str] | None,
|
|
280
404
|
target_identity: tuple[str, str | None, int | str, int | str],
|
|
405
|
+
repository_generation: str | None,
|
|
281
406
|
source_scopes: dict[str, Any],
|
|
282
407
|
target_root: Path,
|
|
283
408
|
target_summary: str | None,
|
|
@@ -290,6 +415,7 @@ def insert_running_scan(
|
|
|
290
415
|
surface: str | None = "plugin",
|
|
291
416
|
package_version: str | None = None,
|
|
292
417
|
launch_kind: str = "scan",
|
|
418
|
+
hosted: bool = False,
|
|
293
419
|
) -> str:
|
|
294
420
|
revision = target_identity[0]
|
|
295
421
|
native_scan = scan_dir is None
|
|
@@ -307,19 +433,21 @@ def insert_running_scan(
|
|
|
307
433
|
connection.execute(
|
|
308
434
|
"""
|
|
309
435
|
INSERT INTO scans (
|
|
310
|
-
id, workspace_id, target_id,
|
|
436
|
+
id, workspace_id, target_id, repository_generation, target_path,
|
|
437
|
+
target_revision, target_snapshot_digest,
|
|
311
438
|
target_device, target_inode, source_scopes_json, scope, mode, user_context,
|
|
312
439
|
deep_scan_owner_thread_id, diff_target_kind, diff_base_revision,
|
|
313
440
|
diff_head_revision, diff_content_digest, target_summary, scan_dir, model,
|
|
314
441
|
reasoning_effort, status, phase, handoff_status, started_at, created_at, updated_at,
|
|
315
442
|
launch_surface, launch_package_version, launch_plugin_version, launch_kind
|
|
316
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
317
|
-
'running',
|
|
443
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
444
|
+
'running', ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
318
445
|
""",
|
|
319
446
|
(
|
|
320
447
|
scan_id,
|
|
321
448
|
workspace["id"],
|
|
322
449
|
workspace["target_id"],
|
|
450
|
+
repository_generation,
|
|
323
451
|
str(target),
|
|
324
452
|
*target_identity,
|
|
325
453
|
json.dumps(source_scopes, allow_nan=False, separators=(",", ":"), sort_keys=True),
|
|
@@ -332,6 +460,12 @@ def insert_running_scan(
|
|
|
332
460
|
str(scan_dir),
|
|
333
461
|
optional_text(model, maximum=200),
|
|
334
462
|
optional_text(reasoning_effort, maximum=32),
|
|
463
|
+
"threat_model"
|
|
464
|
+
if not hosted
|
|
465
|
+
and launch_kind == "scan"
|
|
466
|
+
and workspace["default_mode"] == "standard"
|
|
467
|
+
and handoff_status == "delivered"
|
|
468
|
+
else "preflight",
|
|
335
469
|
handoff_status,
|
|
336
470
|
timestamp,
|
|
337
471
|
timestamp,
|
|
@@ -17,10 +17,10 @@ from dataclasses import dataclass
|
|
|
17
17
|
from datetime import datetime, timezone
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
from typing import Any, Mapping
|
|
20
|
-
from urllib.parse import quote
|
|
21
20
|
|
|
22
21
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
23
|
-
from
|
|
22
|
+
from snapshot_sqlite import readonly_sqlite_uri
|
|
23
|
+
from workbench.storage import archive_source_directory, worker_usage_directory
|
|
24
24
|
|
|
25
25
|
TOKEN_FIELDS = {
|
|
26
26
|
"input_tokens": "inputTokens",
|
|
@@ -336,6 +336,12 @@ def collect_scan_usage(
|
|
|
336
336
|
started_at=started_at,
|
|
337
337
|
completed_at=stopped_at,
|
|
338
338
|
completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
|
|
339
|
+
resume_owner=(
|
|
340
|
+
scan["mode"] == "standard"
|
|
341
|
+
and "recipe_json" in scan.keys()
|
|
342
|
+
and scan["recipe_json"] is not None
|
|
343
|
+
and session.thread_id == scan["continuation_thread_id"]
|
|
344
|
+
),
|
|
339
345
|
is_scan_owner=(
|
|
340
346
|
scan["mode"] == "deep"
|
|
341
347
|
and session.thread_id == scan["deep_scan_owner_thread_id"]
|
|
@@ -607,9 +613,9 @@ def _scan_worker_directories(
|
|
|
607
613
|
|
|
608
614
|
|
|
609
615
|
def _codex_state_database() -> Path | None:
|
|
610
|
-
configured_database = os.environ.get("CODEX_STATE_DB", "")
|
|
611
|
-
if configured_database:
|
|
612
|
-
path = Path(
|
|
616
|
+
configured_database = os.environ.get("CODEX_STATE_DB", "")
|
|
617
|
+
if configured_database.strip():
|
|
618
|
+
path = Path(os.path.expanduser(configured_database))
|
|
613
619
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
614
620
|
|
|
615
621
|
configured_home = os.environ.get("CODEX_HOME", "")
|
|
@@ -654,7 +660,7 @@ def _discover_rollout_sessions(
|
|
|
654
660
|
completed_at: datetime | None,
|
|
655
661
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
656
662
|
database = sqlite3.connect(
|
|
657
|
-
|
|
663
|
+
readonly_sqlite_uri(state_database),
|
|
658
664
|
uri=True,
|
|
659
665
|
timeout=STATE_DATABASE_TIMEOUT_SECONDS,
|
|
660
666
|
)
|
|
@@ -708,10 +714,12 @@ def _discover_rollout_sessions(
|
|
|
708
714
|
for path in worker_directories
|
|
709
715
|
if Path(path).is_relative_to(scan_directory)
|
|
710
716
|
}
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
original
|
|
717
|
+
originals = {
|
|
718
|
+
original
|
|
719
|
+
for path in bound_directories | {Path(path) for path in worker_directories}
|
|
720
|
+
if (original := archive_source_directory(scan_directory, path)) is not None
|
|
721
|
+
}
|
|
722
|
+
for original in originals:
|
|
715
723
|
candidates = {original / relative: path for relative, path in relatives.items()}
|
|
716
724
|
# Older archives kept absolute discovery paths in their original directory.
|
|
717
725
|
if (
|
|
@@ -924,6 +932,7 @@ def _read_rollout_usage(
|
|
|
924
932
|
completed_at: datetime | None,
|
|
925
933
|
completed_turn_id: str | None = None,
|
|
926
934
|
is_scan_owner: bool = False,
|
|
935
|
+
resume_owner: bool = False,
|
|
927
936
|
) -> RolloutUsage:
|
|
928
937
|
result = RolloutUsage(_empty_token_usage(), set(), None, {})
|
|
929
938
|
total = result.tokens
|
|
@@ -945,6 +954,7 @@ def _read_rollout_usage(
|
|
|
945
954
|
started_turns: set[str] = set()
|
|
946
955
|
open_turns: set[str] = set()
|
|
947
956
|
closed_turns: set[str] = set()
|
|
957
|
+
failed_turns: set[str] = set()
|
|
948
958
|
measured_turns: set[str] = set()
|
|
949
959
|
|
|
950
960
|
def past_scan_end(timestamp: datetime, turn_id: object) -> bool:
|
|
@@ -1075,6 +1085,8 @@ def _read_rollout_usage(
|
|
|
1075
1085
|
if timestamp >= started_at:
|
|
1076
1086
|
if terminal_turn_id in started_turns:
|
|
1077
1087
|
closed_turns.add(terminal_turn_id)
|
|
1088
|
+
if payload.get("type") == "task_complete" and payload.get("error"):
|
|
1089
|
+
failed_turns.add(terminal_turn_id)
|
|
1078
1090
|
if payload.get("type") == "turn_aborted":
|
|
1079
1091
|
# Later successful work does not recover interrupted request usage.
|
|
1080
1092
|
warnings.add("turn_interrupted")
|
|
@@ -1141,6 +1153,20 @@ def _read_rollout_usage(
|
|
|
1141
1153
|
warnings.add("token_record_incomplete")
|
|
1142
1154
|
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
1143
1155
|
_add_token_usage(response_usage, usage)
|
|
1156
|
+
if (
|
|
1157
|
+
resume_owner
|
|
1158
|
+
and receipt_turn_id == current_turn_id
|
|
1159
|
+
and current_turn_id in started_turns
|
|
1160
|
+
and (open_turns - {current_turn_id} or failed_turns)
|
|
1161
|
+
):
|
|
1162
|
+
# Matching thread totals account for turns left open or failed before resume.
|
|
1163
|
+
cumulative = _usage_counts(payload.get("thread_token_usage"))
|
|
1164
|
+
if cumulative is not None and all(
|
|
1165
|
+
cumulative[key] == total[key] + response_usage[key]
|
|
1166
|
+
for key in TOKEN_FIELDS.values()
|
|
1167
|
+
):
|
|
1168
|
+
open_turns.intersection_update({current_turn_id})
|
|
1169
|
+
closed_turns.difference_update(failed_turns - {current_turn_id})
|
|
1144
1170
|
continue
|
|
1145
1171
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
1146
1172
|
continue
|
|
@@ -52,6 +52,7 @@ def apply_migrations(
|
|
|
52
52
|
backfill_security_targets: Callable[[sqlite3.Connection], None],
|
|
53
53
|
*,
|
|
54
54
|
immediate: bool = False,
|
|
55
|
+
discovery_roots: tuple[Path, ...] = (),
|
|
55
56
|
) -> None:
|
|
56
57
|
"""Apply missing schema versions and repair supported historical migrations.
|
|
57
58
|
|
|
@@ -59,9 +60,17 @@ def apply_migrations(
|
|
|
59
60
|
Set immediate to acquire the SQLite write lock up front. The backfill callback
|
|
60
61
|
links older scans and workspaces to stable targets when required.
|
|
61
62
|
"""
|
|
63
|
+
from workbench_target import scoped_git_discovery
|
|
64
|
+
|
|
62
65
|
connection.commit()
|
|
63
66
|
connection.execute("BEGIN IMMEDIATE" if immediate else "BEGIN")
|
|
64
|
-
with
|
|
67
|
+
with (
|
|
68
|
+
connection,
|
|
69
|
+
scoped_git_discovery(
|
|
70
|
+
lambda: [*migration_target_paths(connection), *discovery_roots],
|
|
71
|
+
recorded_git_roots=lambda: recorded_git_checkout_roots(connection),
|
|
72
|
+
),
|
|
73
|
+
):
|
|
65
74
|
connection.execute(
|
|
66
75
|
"""
|
|
67
76
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
@@ -83,8 +92,33 @@ def apply_migrations(
|
|
|
83
92
|
apply_repair_statement(connection, statement.replace("\n", " "))
|
|
84
93
|
elif version == 16:
|
|
85
94
|
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
95
|
+
elif version == 67:
|
|
96
|
+
should_backfill_targets = (
|
|
97
|
+
repair_repository_identity_migration(connection)
|
|
98
|
+
or version not in applied
|
|
99
|
+
or should_backfill_targets
|
|
100
|
+
)
|
|
86
101
|
elif version in applied:
|
|
87
|
-
if version in (
|
|
102
|
+
if version in (
|
|
103
|
+
2,
|
|
104
|
+
12,
|
|
105
|
+
13,
|
|
106
|
+
26,
|
|
107
|
+
28,
|
|
108
|
+
31,
|
|
109
|
+
32,
|
|
110
|
+
42,
|
|
111
|
+
47,
|
|
112
|
+
50,
|
|
113
|
+
54,
|
|
114
|
+
55,
|
|
115
|
+
58,
|
|
116
|
+
59,
|
|
117
|
+
60,
|
|
118
|
+
61,
|
|
119
|
+
69,
|
|
120
|
+
70,
|
|
121
|
+
):
|
|
88
122
|
repair_schema(connection, version)
|
|
89
123
|
elif version == 11:
|
|
90
124
|
repair_deep_scan_migration(connection)
|
|
@@ -96,6 +130,10 @@ def apply_migrations(
|
|
|
96
130
|
connection.execute(statement)
|
|
97
131
|
if version == 38:
|
|
98
132
|
migrate_finding_workflow_results(connection)
|
|
133
|
+
elif version == 68:
|
|
134
|
+
from workbench_target_state import upgrade_linux_repository_generations
|
|
135
|
+
|
|
136
|
+
upgrade_linux_repository_generations(connection)
|
|
99
137
|
if version not in applied:
|
|
100
138
|
connection.execute(
|
|
101
139
|
"INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
|
|
@@ -107,6 +145,65 @@ def apply_migrations(
|
|
|
107
145
|
backfill_security_targets(connection)
|
|
108
146
|
|
|
109
147
|
|
|
148
|
+
def migration_target_paths(connection: sqlite3.Connection) -> list[Path]:
|
|
149
|
+
paths: set[Path] = set()
|
|
150
|
+
for table, column in (
|
|
151
|
+
("security_targets", "current_path"),
|
|
152
|
+
("workspaces", "target_path"),
|
|
153
|
+
("scans", "target_path"),
|
|
154
|
+
):
|
|
155
|
+
columns = {row["name"] for row in connection.execute(f"PRAGMA table_info({table})")}
|
|
156
|
+
if column in columns:
|
|
157
|
+
condition = (
|
|
158
|
+
" AND (target_path_unvalidated = 0 OR target_id IS NOT NULL)"
|
|
159
|
+
if table == "workspaces" and "target_path_unvalidated" in columns
|
|
160
|
+
else ""
|
|
161
|
+
)
|
|
162
|
+
paths.update(
|
|
163
|
+
Path(row[0])
|
|
164
|
+
for row in connection.execute(
|
|
165
|
+
f"SELECT DISTINCT {column} FROM {table} WHERE {column} IS NOT NULL{condition}"
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
return list(paths)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def recorded_git_checkout_roots(connection: sqlite3.Connection) -> set[Path] | None:
|
|
172
|
+
"""Return saved boundaries, or None if any historical target lacks one."""
|
|
173
|
+
target_columns = {
|
|
174
|
+
row["name"] for row in connection.execute("PRAGMA table_info(security_targets)")
|
|
175
|
+
}
|
|
176
|
+
boundaries = (
|
|
177
|
+
{
|
|
178
|
+
Path(target): Path(root)
|
|
179
|
+
for target, root in connection.execute(
|
|
180
|
+
"SELECT current_path, git_checkout_root FROM security_targets "
|
|
181
|
+
"WHERE git_checkout_root IS NOT NULL"
|
|
182
|
+
)
|
|
183
|
+
}
|
|
184
|
+
if "git_checkout_root" in target_columns
|
|
185
|
+
else {}
|
|
186
|
+
)
|
|
187
|
+
# Even an unversioned scan may have come from an unborn Git checkout.
|
|
188
|
+
if any(path not in boundaries for path in migration_target_paths(connection)):
|
|
189
|
+
return None
|
|
190
|
+
roots = set(boundaries.values())
|
|
191
|
+
columns = {row["name"] for row in connection.execute("PRAGMA table_info(workspaces)")}
|
|
192
|
+
if "target_path_unvalidated" in columns:
|
|
193
|
+
saved = (
|
|
194
|
+
"rejected_checkout_roots_json" if "rejected_checkout_roots_json" in columns else "NULL"
|
|
195
|
+
)
|
|
196
|
+
for (root,) in connection.execute(
|
|
197
|
+
f"SELECT DISTINCT value FROM workspaces LEFT JOIN json_each({saved}) "
|
|
198
|
+
f"WHERE {saved} IS NOT NULL OR "
|
|
199
|
+
"(target_path IS NOT NULL AND target_path_unvalidated = 1 AND target_id IS NULL)"
|
|
200
|
+
):
|
|
201
|
+
if root is None:
|
|
202
|
+
return None
|
|
203
|
+
roots.add(Path(root))
|
|
204
|
+
return roots
|
|
205
|
+
|
|
206
|
+
|
|
110
207
|
def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: str) -> None:
|
|
111
208
|
for query in _REPAIR_PLAN:
|
|
112
209
|
actions = connection.execute("\n".join(query), {"history": _REPAIR_HISTORY}).fetchall()
|
|
@@ -181,22 +278,131 @@ def repair_stable_targets_migration(connection: sqlite3.Connection) -> bool:
|
|
|
181
278
|
and "target_id" in scan_columns
|
|
182
279
|
and existing_objects == {"security_targets", "scans_by_target"}
|
|
183
280
|
):
|
|
184
|
-
|
|
281
|
+
validated_workspace = (
|
|
282
|
+
"workspace.target_path_unvalidated = 0 AND "
|
|
283
|
+
if "target_path_unvalidated" in workspace_columns
|
|
284
|
+
else ""
|
|
285
|
+
)
|
|
286
|
+
needs_backfill = bool(
|
|
287
|
+
connection.execute(
|
|
288
|
+
f"""
|
|
289
|
+
SELECT EXISTS (
|
|
290
|
+
SELECT 1 FROM workspaces AS workspace
|
|
291
|
+
WHERE ({validated_workspace}workspace.target_id IS NULL
|
|
292
|
+
AND workspace.target_path IS NOT NULL)
|
|
293
|
+
OR (
|
|
294
|
+
workspace.target_id IS NOT NULL
|
|
295
|
+
AND NOT EXISTS (
|
|
296
|
+
SELECT 1 FROM security_targets
|
|
297
|
+
WHERE security_targets.id = workspace.target_id
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
) OR EXISTS (
|
|
301
|
+
SELECT 1 FROM scans AS scan
|
|
302
|
+
WHERE scan.target_id IS NULL
|
|
303
|
+
OR NOT EXISTS (
|
|
304
|
+
SELECT 1 FROM security_targets
|
|
305
|
+
WHERE security_targets.id = scan.target_id
|
|
306
|
+
)
|
|
307
|
+
)
|
|
308
|
+
"""
|
|
309
|
+
).fetchone()[0]
|
|
310
|
+
)
|
|
311
|
+
if not needs_backfill:
|
|
312
|
+
return False
|
|
313
|
+
else:
|
|
314
|
+
repair_schema(connection, 16)
|
|
185
315
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
316
|
+
for table in ("workspaces", "scans"):
|
|
317
|
+
connection.execute(
|
|
318
|
+
f"""
|
|
319
|
+
UPDATE {table}
|
|
320
|
+
SET target_id = NULL
|
|
321
|
+
WHERE target_id IS NOT NULL
|
|
322
|
+
AND NOT EXISTS (
|
|
323
|
+
SELECT 1 FROM security_targets
|
|
324
|
+
WHERE security_targets.id = {table}.target_id
|
|
325
|
+
)
|
|
326
|
+
"""
|
|
327
|
+
)
|
|
197
328
|
return True
|
|
198
329
|
|
|
199
330
|
|
|
331
|
+
def repair_repository_identity_migration(connection: sqlite3.Connection) -> bool:
|
|
332
|
+
columns = {row["name"] for row in connection.execute("PRAGMA table_info(security_targets)")}
|
|
333
|
+
index_exists = connection.execute(
|
|
334
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'index' "
|
|
335
|
+
"AND name = 'security_targets_by_repository_identity'"
|
|
336
|
+
).fetchone()
|
|
337
|
+
identity_changed = "repository_identity" not in columns or index_exists is None
|
|
338
|
+
if (
|
|
339
|
+
"repository_identity" in columns
|
|
340
|
+
and connection.execute(
|
|
341
|
+
"SELECT 1 FROM schema_migrations WHERE version = 67 AND name = ?",
|
|
342
|
+
("persist repository identities",),
|
|
343
|
+
).fetchone()
|
|
344
|
+
is not None
|
|
345
|
+
and not any(
|
|
346
|
+
row["name"] == "repository_generation"
|
|
347
|
+
for row in connection.execute("PRAGMA table_info(scans)")
|
|
348
|
+
)
|
|
349
|
+
):
|
|
350
|
+
from workbench_target_state import normalize_pre_release_repository_identities
|
|
351
|
+
|
|
352
|
+
normalize_pre_release_repository_identities(connection)
|
|
353
|
+
migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 67)
|
|
354
|
+
for statement in sql_statements(migration_sql):
|
|
355
|
+
if statement.startswith(("ALTER TABLE security_targets", "ALTER TABLE scans")):
|
|
356
|
+
add_migration_column(connection, statement)
|
|
357
|
+
continue
|
|
358
|
+
for prefix in ("CREATE UNIQUE INDEX ", "CREATE INDEX ", "CREATE TRIGGER "):
|
|
359
|
+
if statement.startswith(prefix):
|
|
360
|
+
statement = statement.replace(prefix, f"{prefix}IF NOT EXISTS ", 1)
|
|
361
|
+
break
|
|
362
|
+
connection.execute(statement)
|
|
363
|
+
|
|
364
|
+
from workbench_scan_history import _scan_completion_order
|
|
365
|
+
|
|
366
|
+
if connection.execute(
|
|
367
|
+
"SELECT 1 FROM scans WHERE status = 'complete' AND completion_sequence IS NULL LIMIT 1"
|
|
368
|
+
).fetchone():
|
|
369
|
+
# SQL preserves the recorded sequence. Both projections omit it so the merge
|
|
370
|
+
# compares each pending row with the timestamp of the next recorded completion.
|
|
371
|
+
recorded = connection.execute(
|
|
372
|
+
"SELECT id, started_at, completed_at FROM scans "
|
|
373
|
+
"WHERE status = 'complete' AND completion_sequence IS NOT NULL "
|
|
374
|
+
"ORDER BY completion_sequence"
|
|
375
|
+
).fetchall()
|
|
376
|
+
unsequenced = connection.execute(
|
|
377
|
+
"SELECT id, started_at, completed_at FROM scans "
|
|
378
|
+
"WHERE status = 'complete' AND completion_sequence IS NULL"
|
|
379
|
+
).fetchall()
|
|
380
|
+
pending = iter(sorted(unsequenced, key=_scan_completion_order))
|
|
381
|
+
candidate = next(pending, None)
|
|
382
|
+
completed = []
|
|
383
|
+
for scan in recorded:
|
|
384
|
+
while candidate is not None and _scan_completion_order(
|
|
385
|
+
candidate
|
|
386
|
+
) <= _scan_completion_order(scan):
|
|
387
|
+
completed.append(candidate)
|
|
388
|
+
candidate = next(pending, None)
|
|
389
|
+
completed.append(scan)
|
|
390
|
+
if candidate is not None:
|
|
391
|
+
completed.append(candidate)
|
|
392
|
+
completed.extend(pending)
|
|
393
|
+
connection.execute("UPDATE scans SET completion_sequence = NULL WHERE status = 'complete'")
|
|
394
|
+
sequence = connection.execute(
|
|
395
|
+
"SELECT COALESCE(MAX(completion_sequence), 0) FROM scans"
|
|
396
|
+
).fetchone()[0]
|
|
397
|
+
for scan in completed:
|
|
398
|
+
sequence += 1
|
|
399
|
+
connection.execute(
|
|
400
|
+
"UPDATE scans SET completion_sequence = ? WHERE id = ?",
|
|
401
|
+
(sequence, scan["id"]),
|
|
402
|
+
)
|
|
403
|
+
return identity_changed
|
|
404
|
+
|
|
405
|
+
|
|
200
406
|
def repair_schema(connection: sqlite3.Connection, version: int) -> None:
|
|
201
407
|
for statement in _STATEMENTS[version]:
|
|
202
408
|
apply_repair_statement(connection, statement)
|
|
@@ -76,7 +76,7 @@ def require_diff_target(
|
|
|
76
76
|
if kind == "working_tree":
|
|
77
77
|
base = resolve_git_commit(target, base_revision or "HEAD", "Working-tree base")
|
|
78
78
|
head = resolve_git_commit(target, head_revision or current_head, "Working-tree HEAD")
|
|
79
|
-
current_digest = worktree_content_digest(target)
|
|
79
|
+
current_digest = worktree_content_digest(target, expected_digest=content_digest)
|
|
80
80
|
if base != current_head or head != current_head:
|
|
81
81
|
raise SystemExit(
|
|
82
82
|
"Repository HEAD changed after these working-tree changes were selected. "
|