@openai/codex-security 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -9
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/core-scan.md +2 -0
- package/_bundled_plugin/references/final-report.md +5 -1
- package/_bundled_plugin/references/investigation-time-budget.md +11 -0
- package/_bundled_plugin/references/sarif-adapter.md +1 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -6
- package/_bundled_plugin/references/scan-runtime.md +15 -0
- package/_bundled_plugin/references/threat-model.md +2 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +6 -2
- package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +92 -44
- package/_bundled_plugin/scripts/finding_preview.py +5 -3
- package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
- package/_bundled_plugin/scripts/generate_rank_input.py +13 -6
- package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
- package/_bundled_plugin/scripts/report_projection.py +2 -1
- package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
- package/_bundled_plugin/scripts/workbench_cli.py +5 -10
- package/_bundled_plugin/scripts/workbench_constants.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +68 -39
- package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
- package/_bundled_plugin/scripts/workbench_progress.py +0 -50
- package/_bundled_plugin/scripts/workbench_publication.py +4 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
- package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
- package/_bundled_plugin/scripts/workbench_scan_history.py +82 -80
- package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
- package/_bundled_plugin/scripts/workbench_scan_usage.py +246 -38
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
- package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
- package/_bundled_plugin/scripts/workbench_target.py +28 -12
- package/_bundled_plugin/scripts/workbench_validation.py +52 -13
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
- package/_bundled_plugin/shared/workbench-migrations.json +15 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/dist/api.d.ts +4 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +215 -93
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +2 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +38 -23
- package/dist/artifact-export.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +16 -18
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity-records.d.ts +76 -0
- package/dist/classify-severity-records.d.ts.map +1 -0
- package/dist/classify-severity-records.js +141 -0
- package/dist/classify-severity-records.js.map +1 -0
- package/dist/classify-severity.d.ts +1 -23
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -24
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-daybreak.d.ts +5 -0
- package/dist/cli-daybreak.d.ts.map +1 -0
- package/dist/cli-daybreak.js +89 -0
- package/dist/cli-daybreak.js.map +1 -0
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +2 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +957 -738
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +1 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.js +2 -2
- package/dist/codeowners.js.map +1 -1
- package/dist/codex-sdk-environment.js +1 -1
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +3 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.js +2 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -5
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +5 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +65 -17
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +7 -2
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +2 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +17 -8
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +2 -5
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +8 -6
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.js +1 -1
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/retry.d.ts.map +1 -1
- package/dist/deduplication/retry.js +5 -0
- package/dist/deduplication/retry.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +15 -10
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +4 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +21 -1
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan.d.ts +6 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +8 -6
- package/dist/deep-scan.js.map +1 -1
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +26 -6
- package/dist/findings-import.js.map +1 -1
- package/dist/git-remote.d.ts +10 -0
- package/dist/git-remote.d.ts.map +1 -0
- package/dist/git-remote.js +24 -0
- package/dist/git-remote.js.map +1 -0
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +5 -7
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -0
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +15 -0
- package/dist/host-records-protocol.d.ts.map +1 -0
- package/dist/host-records-protocol.js +15 -0
- package/dist/host-records-protocol.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +5 -0
- package/dist/hosted-remediation-protocol.d.ts.map +1 -0
- package/dist/hosted-remediation-protocol.js +26 -0
- package/dist/hosted-remediation-protocol.js.map +1 -0
- package/dist/hosted-remediation.d.ts +89 -0
- package/dist/hosted-remediation.d.ts.map +1 -0
- package/dist/hosted-remediation.js +122 -0
- package/dist/hosted-remediation.js.map +1 -0
- package/dist/hosted-scan.d.ts +1 -6
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +2 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.d.ts +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +5 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +87 -7
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +65 -29
- package/dist/multiscan.js.map +1 -1
- package/dist/patch-publication.d.ts +26 -0
- package/dist/patch-publication.d.ts.map +1 -0
- package/dist/patch-publication.js +105 -0
- package/dist/patch-publication.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +4 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +29 -15
- package/dist/project-config.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +135 -19
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +48 -25
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +63 -0
- package/dist/remediation.d.ts.map +1 -0
- package/dist/remediation.js +110 -0
- package/dist/remediation.js.map +1 -0
- package/dist/request-metadata.d.ts +17 -7
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +43 -8
- package/dist/request-metadata.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +104 -36
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +16 -4
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.js +8 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +15 -9
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +12 -6
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +5 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +9 -8
- package/dist/scan-executor.js.map +1 -1
- package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
- package/dist/scan-progress.d.ts.map +1 -0
- package/dist/scan-progress.js +138 -0
- package/dist/scan-progress.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +7 -1
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-time-budget.d.ts +9 -0
- package/dist/scan-time-budget.d.ts.map +1 -0
- package/dist/scan-time-budget.js +16 -0
- package/dist/scan-time-budget.js.map +1 -0
- package/dist/security-policy.js +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +4 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +14 -2
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-review.d.ts +29 -0
- package/dist/severity-review.d.ts.map +1 -0
- package/dist/severity-review.js +47 -0
- package/dist/severity-review.js.map +1 -0
- package/dist/severity-store.js +4 -1
- package/dist/severity-store.js.map +1 -1
- package/dist/timestamps.d.ts +3 -0
- package/dist/timestamps.d.ts.map +1 -0
- package/dist/timestamps.js +14 -0
- package/dist/timestamps.js.map +1 -0
- package/dist/validation-records.d.ts +38 -0
- package/dist/validation-records.d.ts.map +1 -0
- package/dist/validation-records.js +76 -0
- package/dist/validation-records.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +17 -4
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts.map +1 -1
- package/dist/workbench-arguments.js +9 -3
- package/dist/workbench-arguments.js.map +1 -1
- package/dist/worker-progress.d.ts +13 -7
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +12 -44
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +89 -0
- package/docs/cli.md +352 -45
- package/docs/findings-service.md +7 -2
- package/docs/hosted-remediation.md +117 -0
- package/docs/hosted-scans.md +8 -3
- package/docs/validate-records.md +69 -0
- package/package.json +13 -10
- package/schemas/project-config.schema.json +7 -1
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
- package/_bundled_plugin/references/config-preflight.md +0 -93
- package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
- package/_bundled_plugin/scripts/config_preflight.py +0 -911
- package/dist/deep-progress.d.ts.map +0 -1
- package/dist/deep-progress.js +0 -92
- package/dist/deep-progress.js.map +0 -1
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
"""Bind optional finding excerpts to the Git objects selected at scan start."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sqlite3
|
|
10
|
+
import stat
|
|
11
|
+
import sys
|
|
12
|
+
from functools import cache
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any
|
|
15
|
+
from unicodedata import normalize
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
|
+
from workbench_target import (
|
|
19
|
+
clean_worktree_content_digest,
|
|
20
|
+
git_bytes,
|
|
21
|
+
git_worktree_context,
|
|
22
|
+
)
|
|
23
|
+
from workbench_validation import reject_non_finite_json
|
|
24
|
+
|
|
25
|
+
OBJECT_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def expected_target_kinds(scan: sqlite3.Row) -> list[str]:
|
|
29
|
+
if scan["mode"] == "diff":
|
|
30
|
+
return ["git_diff"]
|
|
31
|
+
if scan["target_revision"] == "unversioned":
|
|
32
|
+
return ["directory_snapshot"]
|
|
33
|
+
if scan["target_snapshot_digest"] is None:
|
|
34
|
+
return ["git_worktree", "git_revision"]
|
|
35
|
+
if scan["target_snapshot_digest"] == clean_worktree_content_digest():
|
|
36
|
+
return ["git_revision"]
|
|
37
|
+
return ["git_worktree"]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def requested_scan_paths(scan: sqlite3.Row) -> list[str]:
|
|
41
|
+
if "recipe_json" in scan.keys() and scan["recipe_json"] is not None:
|
|
42
|
+
recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
|
|
43
|
+
target = recipe["target"]
|
|
44
|
+
if target["kind"] == "paths":
|
|
45
|
+
return target["paths"]
|
|
46
|
+
return [scan["scope"]]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def public_scan_recipe(scan: sqlite3.Row) -> dict[str, Any]:
|
|
50
|
+
recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
|
|
51
|
+
recipe.pop("_codexSecurityFileScopes", None)
|
|
52
|
+
return recipe
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def normalized_path_component(value: str) -> str:
|
|
56
|
+
return normalize("NFC", normalize("NFD", value).casefold())
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def relative_path(value: str) -> PurePosixPath | None:
|
|
60
|
+
path = PurePosixPath(value)
|
|
61
|
+
return None if "\\" in value or path.is_absolute() or ".." in path.parts else path
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def safe_source_path(target: Path, value: str) -> Path | None:
|
|
65
|
+
parsed = relative_path(value)
|
|
66
|
+
if parsed is None:
|
|
67
|
+
return None
|
|
68
|
+
try:
|
|
69
|
+
path = (target / parsed.as_posix()).resolve()
|
|
70
|
+
path.relative_to(target)
|
|
71
|
+
return path
|
|
72
|
+
except (OSError, RuntimeError, ValueError):
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def offline_git_bytes(repository: Path, *arguments: str) -> bytes | None:
|
|
77
|
+
names = ("GIT_NO_LAZY_FETCH", "GIT_ALLOW_PROTOCOL", "GIT_NO_REPLACE_OBJECTS")
|
|
78
|
+
previous = {name: os.environ.get(name) for name in names}
|
|
79
|
+
os.environ["GIT_NO_LAZY_FETCH"] = "1"
|
|
80
|
+
os.environ["GIT_ALLOW_PROTOCOL"] = ""
|
|
81
|
+
os.environ["GIT_NO_REPLACE_OBJECTS"] = "1"
|
|
82
|
+
try:
|
|
83
|
+
return git_bytes(repository, *arguments)
|
|
84
|
+
finally:
|
|
85
|
+
for name, value in previous.items():
|
|
86
|
+
if value is None:
|
|
87
|
+
os.environ.pop(name, None)
|
|
88
|
+
else:
|
|
89
|
+
os.environ[name] = value
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@cache
|
|
93
|
+
def tree_entries(repository: Path, tree: str) -> dict[str, tuple[tuple[str, str, str], ...]] | None:
|
|
94
|
+
content = offline_git_bytes(repository, "ls-tree", "-z", tree)
|
|
95
|
+
if content is None:
|
|
96
|
+
return None
|
|
97
|
+
entries: dict[str, list[tuple[str, str, str]]] = {}
|
|
98
|
+
for record in content.split(b"\0"):
|
|
99
|
+
if not record:
|
|
100
|
+
continue
|
|
101
|
+
metadata, separator, name = record.partition(b"\t")
|
|
102
|
+
fields = metadata.split(b" ")
|
|
103
|
+
if not separator or len(fields) != 3:
|
|
104
|
+
return None
|
|
105
|
+
mode, object_type, raw_object = fields
|
|
106
|
+
object_id = raw_object.decode("ascii")
|
|
107
|
+
if not OBJECT_ID.fullmatch(object_id):
|
|
108
|
+
return None
|
|
109
|
+
kind = (
|
|
110
|
+
"directory"
|
|
111
|
+
if mode == b"040000" and object_type == b"tree"
|
|
112
|
+
else "file"
|
|
113
|
+
if mode in {b"100644", b"100755"} and object_type == b"blob"
|
|
114
|
+
else "other"
|
|
115
|
+
)
|
|
116
|
+
decoded_name = os.fsdecode(name)
|
|
117
|
+
entries.setdefault(normalized_path_component(decoded_name), []).append(
|
|
118
|
+
(decoded_name, kind, object_id)
|
|
119
|
+
)
|
|
120
|
+
return {name: tuple(matches) for name, matches in entries.items()}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def exact_tree_path(
|
|
124
|
+
repository: Path, tree: str, value: str, *, require_unambiguous: bool = False
|
|
125
|
+
) -> tuple[str, str, str] | None:
|
|
126
|
+
path = relative_path(value)
|
|
127
|
+
if path is None:
|
|
128
|
+
return None
|
|
129
|
+
kind, object_id = "directory", tree
|
|
130
|
+
for name in path.parts:
|
|
131
|
+
if kind != "directory":
|
|
132
|
+
return None
|
|
133
|
+
entries = tree_entries(repository, object_id) or {}
|
|
134
|
+
aliases = entries.get(normalized_path_component(name), ())
|
|
135
|
+
if require_unambiguous and len(aliases) != 1:
|
|
136
|
+
return None
|
|
137
|
+
entry = next((entry for entry in aliases if entry[0] == name), None)
|
|
138
|
+
if entry is None:
|
|
139
|
+
return None
|
|
140
|
+
_, kind, object_id = entry
|
|
141
|
+
return path.as_posix(), kind, object_id
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def ordinary_path(metadata: os.stat_result) -> bool:
|
|
145
|
+
return not (
|
|
146
|
+
getattr(metadata, "st_file_attributes", 0)
|
|
147
|
+
& getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
|
|
148
|
+
) and (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def filesystem_alias(selected: Path, candidate: Path) -> bool:
|
|
152
|
+
if normalized_path_component(selected.name) != normalized_path_component(candidate.name):
|
|
153
|
+
return False
|
|
154
|
+
try:
|
|
155
|
+
if not all(ordinary_path(path.lstat()) for path in (selected, candidate)):
|
|
156
|
+
return False
|
|
157
|
+
if not selected.samefile(candidate):
|
|
158
|
+
return False
|
|
159
|
+
with os.scandir(selected.parent) as entries:
|
|
160
|
+
return (
|
|
161
|
+
sum(
|
|
162
|
+
normalized_path_component(entry.name)
|
|
163
|
+
== normalized_path_component(selected.name)
|
|
164
|
+
for entry in entries
|
|
165
|
+
)
|
|
166
|
+
== 1
|
|
167
|
+
)
|
|
168
|
+
except OSError:
|
|
169
|
+
return False
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def existing_tree_path(
|
|
173
|
+
repository: Path, tree: str, value: str, filesystem_root: Path
|
|
174
|
+
) -> tuple[str, str, str] | None:
|
|
175
|
+
path = relative_path(value)
|
|
176
|
+
if path is None:
|
|
177
|
+
return None
|
|
178
|
+
canonical: list[str] = []
|
|
179
|
+
kind, object_id = "directory", tree
|
|
180
|
+
for index, name in enumerate(path.parts, start=1):
|
|
181
|
+
if kind != "directory":
|
|
182
|
+
return None
|
|
183
|
+
entries = tree_entries(repository, object_id) or {}
|
|
184
|
+
aliases = entries.get(normalized_path_component(name), ())
|
|
185
|
+
selected = filesystem_root / PurePosixPath(*path.parts[:index])
|
|
186
|
+
matches = []
|
|
187
|
+
try:
|
|
188
|
+
if not ordinary_path(selected.lstat()):
|
|
189
|
+
return None
|
|
190
|
+
for entry in aliases:
|
|
191
|
+
candidate = filesystem_root / PurePosixPath(*canonical, entry[0])
|
|
192
|
+
# Every colliding Git name needs a filesystem witness. Missing
|
|
193
|
+
# names cannot establish which entry the user selected.
|
|
194
|
+
if not ordinary_path(candidate.lstat()):
|
|
195
|
+
return None
|
|
196
|
+
if selected.samefile(candidate) and (
|
|
197
|
+
entry[0] == name or filesystem_alias(selected, candidate)
|
|
198
|
+
):
|
|
199
|
+
matches.append(entry)
|
|
200
|
+
except OSError:
|
|
201
|
+
return None
|
|
202
|
+
if len(matches) != 1:
|
|
203
|
+
return None
|
|
204
|
+
name, kind, object_id = matches[0]
|
|
205
|
+
canonical.append(name)
|
|
206
|
+
return PurePosixPath(*canonical).as_posix(), kind, object_id
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def target_tree(target: Path, revision: str) -> tuple[Path, str] | None:
|
|
210
|
+
repository, prefix = git_worktree_context(target)
|
|
211
|
+
raw_tree = offline_git_bytes(repository, "rev-parse", "--verify", f"{revision}^{{tree}}")
|
|
212
|
+
tree = raw_tree.decode("ascii").strip() if raw_tree is not None else ""
|
|
213
|
+
if not OBJECT_ID.fullmatch(tree):
|
|
214
|
+
return None
|
|
215
|
+
selected = existing_tree_path(repository, tree, prefix, repository)
|
|
216
|
+
if selected is None or selected[1] != "directory":
|
|
217
|
+
return None
|
|
218
|
+
return repository, selected[2]
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def capture_source_scopes(
|
|
222
|
+
target: Path,
|
|
223
|
+
target_identity: tuple[str, str | None, int | str, int | str],
|
|
224
|
+
paths: list[str],
|
|
225
|
+
*,
|
|
226
|
+
diff_target_kind: str | None = None,
|
|
227
|
+
) -> dict[str, Any]:
|
|
228
|
+
revision, snapshot = target_identity[:2]
|
|
229
|
+
result: dict[str, Any] = {"version": 1, "revision": revision, "scopes": []}
|
|
230
|
+
# HEAD cannot represent the uncommitted bytes reviewed by a working-tree scan.
|
|
231
|
+
if diff_target_kind == "working_tree":
|
|
232
|
+
return result
|
|
233
|
+
if revision == "unversioned" or (
|
|
234
|
+
snapshot is not None and snapshot != clean_worktree_content_digest()
|
|
235
|
+
):
|
|
236
|
+
return result
|
|
237
|
+
try:
|
|
238
|
+
repository, _ = git_worktree_context(target)
|
|
239
|
+
if offline_git_bytes(repository, "replace", "--list") != b"":
|
|
240
|
+
# A revision alone does not identify the replacement view scanned.
|
|
241
|
+
return result
|
|
242
|
+
context = target_tree(target, revision)
|
|
243
|
+
if context is None:
|
|
244
|
+
return result
|
|
245
|
+
repository, tree = context
|
|
246
|
+
result["targetTree"] = tree
|
|
247
|
+
for requested in paths:
|
|
248
|
+
selected = safe_source_path(target, requested)
|
|
249
|
+
if selected is None:
|
|
250
|
+
continue
|
|
251
|
+
canonical = existing_tree_path(repository, tree, requested, target)
|
|
252
|
+
if canonical is None or canonical[1] not in {"file", "directory"}:
|
|
253
|
+
continue
|
|
254
|
+
try:
|
|
255
|
+
metadata = selected.stat()
|
|
256
|
+
except OSError:
|
|
257
|
+
continue
|
|
258
|
+
if (canonical[1] == "directory") != stat.S_ISDIR(metadata.st_mode):
|
|
259
|
+
continue
|
|
260
|
+
result["scopes"].append(
|
|
261
|
+
{
|
|
262
|
+
"path": PurePosixPath(requested).as_posix(),
|
|
263
|
+
"canonicalPath": canonical[0],
|
|
264
|
+
"kind": canonical[1],
|
|
265
|
+
"objectId": canonical[2],
|
|
266
|
+
}
|
|
267
|
+
)
|
|
268
|
+
except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
|
|
269
|
+
# Excerpts are optional. Failure to bind a scope never widens it and
|
|
270
|
+
# must not prevent the scan itself from starting.
|
|
271
|
+
pass
|
|
272
|
+
return result
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def load_source_scopes(
|
|
276
|
+
scan: sqlite3.Row, target: Path, requested: list[str]
|
|
277
|
+
) -> tuple[Path, str, list[dict[str, str]]] | None:
|
|
278
|
+
if "diff_target_kind" in scan.keys() and scan["diff_target_kind"] == "working_tree":
|
|
279
|
+
return None
|
|
280
|
+
saved = scan["source_scopes_json"] if "source_scopes_json" in scan.keys() else None
|
|
281
|
+
paths = {PurePosixPath(path).as_posix() for path in requested}
|
|
282
|
+
if saved is not None:
|
|
283
|
+
metadata = json.loads(saved)
|
|
284
|
+
if (
|
|
285
|
+
not isinstance(metadata, dict)
|
|
286
|
+
or metadata.get("version") != 1
|
|
287
|
+
or metadata.get("revision") != scan["target_revision"]
|
|
288
|
+
):
|
|
289
|
+
return None
|
|
290
|
+
tree = metadata.get("targetTree")
|
|
291
|
+
records = metadata.get("scopes")
|
|
292
|
+
if (
|
|
293
|
+
not isinstance(tree, str)
|
|
294
|
+
or not OBJECT_ID.fullmatch(tree)
|
|
295
|
+
or not isinstance(records, list)
|
|
296
|
+
):
|
|
297
|
+
return None
|
|
298
|
+
scopes = [
|
|
299
|
+
record
|
|
300
|
+
for record in records
|
|
301
|
+
if isinstance(record, dict)
|
|
302
|
+
and isinstance(record.get("path"), str)
|
|
303
|
+
and record.get("path") in paths
|
|
304
|
+
and isinstance(record.get("canonicalPath"), str)
|
|
305
|
+
and relative_path(record["canonicalPath"]) is not None
|
|
306
|
+
and record.get("kind") in ("file", "directory")
|
|
307
|
+
and isinstance(record.get("objectId"), str)
|
|
308
|
+
and OBJECT_ID.fullmatch(record["objectId"])
|
|
309
|
+
]
|
|
310
|
+
return (git_worktree_context(target)[0], tree, scopes) if scopes else None
|
|
311
|
+
|
|
312
|
+
# Exact, unambiguous historical entries establish legacy scope kinds.
|
|
313
|
+
# Keep any previously recorded file kinds as an additional constraint.
|
|
314
|
+
recipe = scan["recipe_json"] if "recipe_json" in scan.keys() else None
|
|
315
|
+
legacy_recipe = json.loads(recipe) if recipe else None
|
|
316
|
+
legacy_files = (
|
|
317
|
+
legacy_recipe.get("_codexSecurityFileScopes") if isinstance(legacy_recipe, dict) else None
|
|
318
|
+
)
|
|
319
|
+
repository, _ = git_worktree_context(target)
|
|
320
|
+
if offline_git_bytes(repository, "replace", "--list") != b"":
|
|
321
|
+
# Old records do not say which replacement view was scanned.
|
|
322
|
+
return None
|
|
323
|
+
context = target_tree(target, scan["target_revision"])
|
|
324
|
+
if context is None:
|
|
325
|
+
return None
|
|
326
|
+
repository, tree = context
|
|
327
|
+
scopes = []
|
|
328
|
+
for path in paths:
|
|
329
|
+
entry = exact_tree_path(repository, tree, path, require_unambiguous=True)
|
|
330
|
+
if entry is None or entry[1] not in {"file", "directory"}:
|
|
331
|
+
continue
|
|
332
|
+
kind = entry[1]
|
|
333
|
+
if isinstance(legacy_files, list) and kind != (
|
|
334
|
+
"file" if path in legacy_files else "directory"
|
|
335
|
+
):
|
|
336
|
+
continue
|
|
337
|
+
scopes.append(
|
|
338
|
+
{
|
|
339
|
+
"path": path,
|
|
340
|
+
"canonicalPath": path,
|
|
341
|
+
"kind": kind,
|
|
342
|
+
"objectId": entry[2],
|
|
343
|
+
}
|
|
344
|
+
)
|
|
345
|
+
return (repository, tree, scopes) if scopes else None
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def source_object_for_path(
|
|
349
|
+
repository: Path, tree: str, target: Path, value: str, scope: dict[str, str]
|
|
350
|
+
) -> str | None:
|
|
351
|
+
path = relative_path(value)
|
|
352
|
+
if path is None or safe_source_path(target, value) is None:
|
|
353
|
+
return None
|
|
354
|
+
prefixes = {PurePosixPath(scope[name]).parts for name in ("path", "canonicalPath")}
|
|
355
|
+
suffix: tuple[str, ...] | None = None
|
|
356
|
+
for prefix in sorted(prefixes, key=len, reverse=True):
|
|
357
|
+
if path.parts[: len(prefix)] == prefix:
|
|
358
|
+
suffix = path.parts[len(prefix) :]
|
|
359
|
+
break
|
|
360
|
+
if suffix is None:
|
|
361
|
+
for length in {len(prefix) for prefix in prefixes}:
|
|
362
|
+
if length > len(path.parts) or scope["kind"] == "file" and length != len(path.parts):
|
|
363
|
+
continue
|
|
364
|
+
selected = existing_tree_path(
|
|
365
|
+
repository, tree, PurePosixPath(*path.parts[:length]).as_posix(), target
|
|
366
|
+
)
|
|
367
|
+
if selected == (scope["canonicalPath"], scope["kind"], scope["objectId"]):
|
|
368
|
+
suffix = path.parts[length:]
|
|
369
|
+
break
|
|
370
|
+
if suffix is None:
|
|
371
|
+
return None
|
|
372
|
+
if scope["kind"] == "file":
|
|
373
|
+
return scope["objectId"] if not suffix else None
|
|
374
|
+
relative = PurePosixPath(*suffix).as_posix()
|
|
375
|
+
entry = exact_tree_path(repository, scope["objectId"], relative, require_unambiguous=True)
|
|
376
|
+
if entry is None:
|
|
377
|
+
selected = safe_source_path(target, scope["canonicalPath"])
|
|
378
|
+
if selected is not None:
|
|
379
|
+
entry = existing_tree_path(repository, scope["objectId"], relative, selected)
|
|
380
|
+
return entry[2] if entry is not None and entry[1] == "file" else None
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def main() -> None:
|
|
384
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
if __name__ == "__main__":
|
|
388
|
+
main()
|
|
@@ -24,6 +24,7 @@ from rank_preview import DEFAULT_PREVIEW_READ_BYTES, is_binary_sample
|
|
|
24
24
|
from windows_scan_local_files import copy_directory_junction
|
|
25
25
|
from workbench_constants import GIT_REPOSITORY_ENVIRONMENT
|
|
26
26
|
|
|
27
|
+
SOURCE_METADATA_DIRECTORIES = {".git", ".svn"}
|
|
27
28
|
_WINDOWS = os.name == "nt"
|
|
28
29
|
|
|
29
30
|
|
|
@@ -87,7 +88,10 @@ def git_blob_samples(
|
|
|
87
88
|
if not object_names:
|
|
88
89
|
return []
|
|
89
90
|
|
|
90
|
-
|
|
91
|
+
# Git before 2.38 rejects -z, so only names with line breaks need it.
|
|
92
|
+
nul_input = any("\n" in name or "\r" in name for name in object_names)
|
|
93
|
+
separator = b"\0" if nul_input else b"\n"
|
|
94
|
+
request = separator.join(os.fsencode(name) for name in object_names) + separator
|
|
91
95
|
read_fd, write_fd = os.pipe()
|
|
92
96
|
with (
|
|
93
97
|
ThreadPoolExecutor(max_workers=1) as reader,
|
|
@@ -106,7 +110,7 @@ def git_blob_samples(
|
|
|
106
110
|
target,
|
|
107
111
|
"cat-file",
|
|
108
112
|
"--batch",
|
|
109
|
-
"-z",
|
|
113
|
+
*(["-z"] if nul_input else []),
|
|
110
114
|
text=False,
|
|
111
115
|
input_data=request,
|
|
112
116
|
git_dir=git_dir,
|
|
@@ -444,10 +448,16 @@ def worktree_content_digest_for_context(
|
|
|
444
448
|
) -> str:
|
|
445
449
|
digest = hashlib.sha256()
|
|
446
450
|
update_digest_field(digest, b"format", b"codex-security-snapshot/v1")
|
|
451
|
+
source_pathspecs = [
|
|
452
|
+
f":(literal){pathspec}",
|
|
453
|
+
":(top,glob,exclude)**/.svn",
|
|
454
|
+
":(top,glob,exclude)**/.svn/**",
|
|
455
|
+
]
|
|
447
456
|
tracked_ok = _update_digest_field_from_git(
|
|
448
457
|
digest,
|
|
449
458
|
b"tracked-diff",
|
|
450
459
|
repository,
|
|
460
|
+
"--no-literal-pathspecs",
|
|
451
461
|
"diff",
|
|
452
462
|
"--binary",
|
|
453
463
|
"--full-index",
|
|
@@ -456,18 +466,19 @@ def worktree_content_digest_for_context(
|
|
|
456
466
|
"--ignore-submodules=none",
|
|
457
467
|
"HEAD",
|
|
458
468
|
"--",
|
|
459
|
-
|
|
469
|
+
*source_pathspecs,
|
|
460
470
|
git_dir=git_dir,
|
|
461
471
|
work_tree=work_tree,
|
|
462
472
|
)
|
|
463
473
|
untracked = git_bytes(
|
|
464
474
|
repository,
|
|
475
|
+
"--no-literal-pathspecs",
|
|
465
476
|
"ls-files",
|
|
466
477
|
"--others",
|
|
467
478
|
"--exclude-standard",
|
|
468
479
|
"-z",
|
|
469
480
|
"--",
|
|
470
|
-
|
|
481
|
+
*source_pathspecs,
|
|
471
482
|
git_dir=git_dir,
|
|
472
483
|
work_tree=work_tree,
|
|
473
484
|
)
|
|
@@ -670,6 +681,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
|
|
|
670
681
|
nested_worktrees: set[tuple[int, int]] = set()
|
|
671
682
|
for entry in (entry for entry in listed.split(b"\0") if entry):
|
|
672
683
|
relative = Path(os.fsdecode(entry[2:]))
|
|
684
|
+
if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
|
|
685
|
+
continue
|
|
673
686
|
if scope_depth:
|
|
674
687
|
if len(relative.parts) <= scope_depth:
|
|
675
688
|
continue
|
|
@@ -741,6 +754,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
|
|
|
741
754
|
while pending:
|
|
742
755
|
children: list[Path] = []
|
|
743
756
|
for path in pending:
|
|
757
|
+
if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(target).parts):
|
|
758
|
+
continue
|
|
744
759
|
try:
|
|
745
760
|
path = _directory_link_boundary(target, path, linked_prefixes)
|
|
746
761
|
metadata = path.lstat()
|
|
@@ -761,7 +776,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
|
|
|
761
776
|
continue
|
|
762
777
|
candidates: dict[bytes, Path] = {}
|
|
763
778
|
for path in children:
|
|
764
|
-
if os.path.normcase(path.name)
|
|
779
|
+
if os.path.normcase(path.name) in SOURCE_METADATA_DIRECTORIES:
|
|
765
780
|
continue
|
|
766
781
|
try:
|
|
767
782
|
metadata = path.lstat()
|
|
@@ -809,7 +824,7 @@ def source_directory_snapshot_paths(
|
|
|
809
824
|
parent = Path(directory)
|
|
810
825
|
for name in directories[:]:
|
|
811
826
|
path = parent / name
|
|
812
|
-
if name
|
|
827
|
+
if name in SOURCE_METADATA_DIRECTORIES or path in excluded:
|
|
813
828
|
directories.remove(name)
|
|
814
829
|
continue
|
|
815
830
|
paths.append(path)
|
|
@@ -817,7 +832,9 @@ def source_directory_snapshot_paths(
|
|
|
817
832
|
if getattr(path.lstat(), "st_reparse_tag", 0) & 0x20000000:
|
|
818
833
|
directories.remove(name)
|
|
819
834
|
paths.extend(
|
|
820
|
-
parent / name
|
|
835
|
+
parent / name
|
|
836
|
+
for name in files
|
|
837
|
+
if name not in SOURCE_METADATA_DIRECTORIES and parent / name not in excluded
|
|
821
838
|
)
|
|
822
839
|
return sorted(paths)
|
|
823
840
|
|
|
@@ -840,7 +857,7 @@ def directory_content_digest(
|
|
|
840
857
|
else git_directory_snapshot_paths(target)
|
|
841
858
|
)
|
|
842
859
|
if paths is None:
|
|
843
|
-
paths = source_directory_snapshot_paths(target)
|
|
860
|
+
paths = source_directory_snapshot_paths(target, excluded)
|
|
844
861
|
digest = hashlib.sha256()
|
|
845
862
|
update_digest_field(digest, b"format", b"codex-security-directory/v1")
|
|
846
863
|
for path in paths:
|
|
@@ -981,9 +998,8 @@ def copy_git_worktree_files(
|
|
|
981
998
|
continue
|
|
982
999
|
source_path = repository / relative
|
|
983
1000
|
try:
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
relative = source_path.relative_to(repository)
|
|
1001
|
+
source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
|
|
1002
|
+
relative = source_path.relative_to(repository)
|
|
987
1003
|
metadata = source_path.lstat()
|
|
988
1004
|
except FileNotFoundError:
|
|
989
1005
|
continue
|
|
@@ -1056,7 +1072,7 @@ def git_target_metadata(target: Path) -> dict[str, Any]:
|
|
|
1056
1072
|
branch = git_output(target, "symbolic-ref", "--quiet", "--short", "HEAD")
|
|
1057
1073
|
metadata.update({"branch": branch, "detachedHead": revision is not None and branch is None})
|
|
1058
1074
|
if revision is not None:
|
|
1059
|
-
subject = git_bytes(target, "show", "-s", "--format=%s", "HEAD")
|
|
1075
|
+
subject = git_bytes(target, "show", "-s", "--no-show-signature", "--format=%s", "HEAD")
|
|
1060
1076
|
metadata.update(
|
|
1061
1077
|
{
|
|
1062
1078
|
"commitSubject": (subject or b"").decode("utf-8", errors="replace").strip() or None,
|
|
@@ -16,10 +16,11 @@ from typing import Any
|
|
|
16
16
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
17
17
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
18
|
import finalize_scan_contract as finalizer
|
|
19
|
-
from workbench_scan_usage import TOKEN_FIELDS
|
|
19
|
+
from workbench_scan_usage import TOKEN_FIELDS, valid_account_rate_limit
|
|
20
20
|
from workbench_scan_usage import (
|
|
21
21
|
_reject_nonstandard_json_number as reject_nonstandard_json_number,
|
|
22
22
|
)
|
|
23
|
+
from workbench_scan_usage import _timestamp as usage_timestamp
|
|
23
24
|
|
|
24
25
|
reject_non_finite_json = finalizer._reject_non_finite_json
|
|
25
26
|
|
|
@@ -68,16 +69,19 @@ SCAN_USAGE_TOKEN_KEYS = tuple(TOKEN_FIELDS.values())
|
|
|
68
69
|
|
|
69
70
|
def _valid_legacy_scan_cost(cost: object) -> bool:
|
|
70
71
|
token_keys = ("inputTokens", "cachedInputTokens", "cacheWriteInputTokens", "outputTokens")
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
72
|
+
try:
|
|
73
|
+
return (
|
|
74
|
+
isinstance(cost, dict)
|
|
75
|
+
and isinstance(cost.get("model"), str)
|
|
76
|
+
and bool(cost["model"])
|
|
77
|
+
and all(type(cost.get(key)) is int and cost[key] >= 0 for key in token_keys)
|
|
78
|
+
and cost["cachedInputTokens"] + cost["cacheWriteInputTokens"] <= cost["inputTokens"]
|
|
79
|
+
and type(cost.get("estimatedUsd")) in (int, float)
|
|
80
|
+
and math.isfinite(cost["estimatedUsd"])
|
|
81
|
+
and cost["estimatedUsd"] >= 0
|
|
82
|
+
)
|
|
83
|
+
except OverflowError:
|
|
84
|
+
return False
|
|
81
85
|
|
|
82
86
|
|
|
83
87
|
def _valid_measured_scan_usage(usage: object) -> bool:
|
|
@@ -86,7 +90,7 @@ def _valid_measured_scan_usage(usage: object) -> bool:
|
|
|
86
90
|
coverage = usage.get("coverage")
|
|
87
91
|
thread_count = usage.get("threadCount")
|
|
88
92
|
if (
|
|
89
|
-
coverage not in
|
|
93
|
+
coverage not in ("complete", "partial", "unavailable")
|
|
90
94
|
or usage.get("source") != "codex_rollout"
|
|
91
95
|
or type(thread_count) is not int
|
|
92
96
|
or thread_count < 0
|
|
@@ -116,9 +120,15 @@ def _valid_measured_scan_usage(usage: object) -> bool:
|
|
|
116
120
|
or len(set(warnings)) != len(warnings)
|
|
117
121
|
):
|
|
118
122
|
return False
|
|
123
|
+
if "accountRateLimits" in usage:
|
|
124
|
+
limits = usage["accountRateLimits"]
|
|
125
|
+
if not isinstance(limits, list) or not all(
|
|
126
|
+
valid_account_rate_limit(limit) for limit in limits
|
|
127
|
+
):
|
|
128
|
+
return False
|
|
119
129
|
if coverage == "unavailable":
|
|
120
130
|
return thread_count == 0 and set(usage).issubset(
|
|
121
|
-
{"coverage", "source", "threadCount", "warnings", "completion"}
|
|
131
|
+
{"coverage", "source", "threadCount", "warnings", "completion", "accountRateLimits"}
|
|
122
132
|
)
|
|
123
133
|
|
|
124
134
|
allowed_keys = {
|
|
@@ -128,6 +138,11 @@ def _valid_measured_scan_usage(usage: object) -> bool:
|
|
|
128
138
|
"missingThreadCount",
|
|
129
139
|
"warnings",
|
|
130
140
|
"completion",
|
|
141
|
+
"byRole",
|
|
142
|
+
"window",
|
|
143
|
+
"observedAt",
|
|
144
|
+
"tokensUpdatedAt",
|
|
145
|
+
"accountRateLimits",
|
|
131
146
|
*SCAN_USAGE_TOKEN_KEYS,
|
|
132
147
|
}
|
|
133
148
|
if thread_count == 0 or not set(usage).issubset(allowed_keys):
|
|
@@ -144,6 +159,30 @@ def _valid_measured_scan_usage(usage: object) -> bool:
|
|
|
144
159
|
return False
|
|
145
160
|
if coverage == "partial" and not (warnings or missing):
|
|
146
161
|
return False
|
|
162
|
+
for field in ("observedAt", "tokensUpdatedAt"):
|
|
163
|
+
if field in usage and usage_timestamp(usage[field]) is None:
|
|
164
|
+
return False
|
|
165
|
+
if "window" in usage:
|
|
166
|
+
window = usage["window"]
|
|
167
|
+
if not isinstance(window, dict) or set(window) != {"startedAt", "endedAt"}:
|
|
168
|
+
return False
|
|
169
|
+
start, end = (usage_timestamp(window[key]) for key in ("startedAt", "endedAt"))
|
|
170
|
+
if start is None or end is None or start > end:
|
|
171
|
+
return False
|
|
172
|
+
if "byRole" in usage:
|
|
173
|
+
roles = usage["byRole"]
|
|
174
|
+
keys = {*SCAN_USAGE_TOKEN_KEYS, "threadCount"}
|
|
175
|
+
if not isinstance(roles, dict) or set(roles) != {"parent", "workers"}:
|
|
176
|
+
return False
|
|
177
|
+
if any(
|
|
178
|
+
not isinstance(role, dict)
|
|
179
|
+
or set(role) != keys
|
|
180
|
+
or any(type(role[key]) is not int or role[key] < 0 for key in keys)
|
|
181
|
+
for role in roles.values()
|
|
182
|
+
):
|
|
183
|
+
return False
|
|
184
|
+
if any(sum(role[key] for role in roles.values()) != usage[key] for key in keys):
|
|
185
|
+
return False
|
|
147
186
|
return True
|
|
148
187
|
|
|
149
188
|
|
|
@@ -1,4 +1,23 @@
|
|
|
1
1
|
[
|
|
2
|
+
[
|
|
3
|
+
"-- Move source-scope previews before the shared history repairs.",
|
|
4
|
+
"WITH input AS (SELECT :history AS history)",
|
|
5
|
+
"SELECT 'sql' AS operation,",
|
|
6
|
+
" 'UPDATE schema_migrations SET version = 50 WHERE version = ' || version AS value",
|
|
7
|
+
"FROM schema_migrations",
|
|
8
|
+
"WHERE version IN (34, 40, 41, 42, 43, 44, 48)",
|
|
9
|
+
" AND name = 'persist authorized source excerpt scopes';"
|
|
10
|
+
],
|
|
11
|
+
[
|
|
12
|
+
"WITH history AS (",
|
|
13
|
+
" SELECT json_extract(value, '$.version') AS version,",
|
|
14
|
+
" json_extract(value, '$.name') AS name FROM json_each(:history)",
|
|
15
|
+
")",
|
|
16
|
+
"SELECT 'error' AS operation,",
|
|
17
|
+
" 'The Codex Security database has an unsupported source-scope migration history.' AS value",
|
|
18
|
+
"FROM schema_migrations JOIN history USING (version)",
|
|
19
|
+
"WHERE version IN (43, 50) AND schema_migrations.name != history.name;"
|
|
20
|
+
],
|
|
2
21
|
[
|
|
3
22
|
"-- mirror and early-moves: relocate numbers before checking profiles.",
|
|
4
23
|
"WITH",
|
|
@@ -410,5 +410,20 @@
|
|
|
410
410
|
"statements": [
|
|
411
411
|
"CREATE INDEX finding_issue_receipts_by_destination_issue\nON finding_issue_receipts(destination_key, external_id, finding_id);"
|
|
412
412
|
]
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
"version": 50,
|
|
416
|
+
"name": "persist authorized source excerpt scopes",
|
|
417
|
+
"statements": ["ALTER TABLE scans ADD COLUMN source_scopes_json TEXT;"]
|
|
418
|
+
},
|
|
419
|
+
{
|
|
420
|
+
"version": 51,
|
|
421
|
+
"name": "preserve deep scan worker thread history",
|
|
422
|
+
"statements": [
|
|
423
|
+
"CREATE TABLE deep_scan_worker_threads (\n scan_id TEXT NOT NULL REFERENCES scans(id) ON DELETE CASCADE,\n worker_id TEXT NOT NULL,\n sdk_thread_id TEXT NOT NULL,\n PRIMARY KEY (scan_id, worker_id, sdk_thread_id)\n);",
|
|
424
|
+
"INSERT INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\nSELECT scan_id, id, sdk_thread_id FROM deep_scan_workers\nWHERE sdk_thread_id IS NOT NULL;",
|
|
425
|
+
"CREATE TRIGGER record_deep_scan_worker_thread_insert\nAFTER INSERT ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;",
|
|
426
|
+
"CREATE TRIGGER record_deep_scan_worker_thread_update\nAFTER UPDATE OF sdk_thread_id ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;"
|
|
427
|
+
]
|
|
413
428
|
}
|
|
414
429
|
]
|