@openai/codex-security 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/README.md +45 -9
  2. package/_bundled_plugin/.codex-plugin/plugin.json +5 -5
  3. package/_bundled_plugin/.mcp.json +1 -0
  4. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  5. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  6. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  7. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +13 -20
  8. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  9. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  10. package/_bundled_plugin/references/core-scan.md +2 -0
  11. package/_bundled_plugin/references/final-report.md +5 -1
  12. package/_bundled_plugin/references/investigation-time-budget.md +11 -0
  13. package/_bundled_plugin/references/sarif-adapter.md +1 -0
  14. package/_bundled_plugin/references/scan-prologue.md +2 -6
  15. package/_bundled_plugin/references/scan-runtime.md +15 -0
  16. package/_bundled_plugin/references/threat-model.md +2 -0
  17. package/_bundled_plugin/schemas/coverage.schema.json +45 -126
  18. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +5 -1
  19. package/_bundled_plugin/schemas/findings.schema.json +140 -427
  20. package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +106 -119
  21. package/_bundled_plugin/schemas/scan-manifest.schema.json +73 -188
  22. package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
  23. package/_bundled_plugin/scripts/finalize_scan_contract.py +87 -49
  24. package/_bundled_plugin/scripts/finding_preview.py +5 -3
  25. package/_bundled_plugin/scripts/generate_rank_input.py +26 -106
  26. package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
  27. package/_bundled_plugin/scripts/report_projection.py +35 -13
  28. package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
  29. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
  30. package/_bundled_plugin/scripts/workbench_cli.py +9 -9
  31. package/_bundled_plugin/scripts/workbench_constants.py +4 -0
  32. package/_bundled_plugin/scripts/workbench_db.py +103 -45
  33. package/_bundled_plugin/scripts/workbench_progress.py +0 -50
  34. package/_bundled_plugin/scripts/workbench_publication.py +216 -247
  35. package/_bundled_plugin/scripts/workbench_scan_history.py +3 -2
  36. package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
  37. package/_bundled_plugin/scripts/workbench_scan_usage.py +360 -37
  38. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  39. package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
  40. package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
  41. package/_bundled_plugin/scripts/workbench_target.py +7 -5
  42. package/_bundled_plugin/scripts/workbench_validation.py +52 -2
  43. package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
  44. package/_bundled_plugin/shared/workbench-migrations.json +22 -0
  45. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
  46. package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
  47. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
  48. package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
  49. package/_bundled_plugin/skills/security-scan/SKILL.md +4 -2
  50. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
  51. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -3
  52. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +2 -0
  53. package/_bundled_plugin/skills/track-findings/SKILL.md +8 -4
  54. package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
  55. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +1 -1
  56. package/dist/api.d.ts +6 -2
  57. package/dist/api.d.ts.map +1 -1
  58. package/dist/api.js +266 -39
  59. package/dist/api.js.map +1 -1
  60. package/dist/artifact-export.d.ts +2 -1
  61. package/dist/artifact-export.d.ts.map +1 -1
  62. package/dist/artifact-export.js +38 -23
  63. package/dist/artifact-export.js.map +1 -1
  64. package/dist/classify-severity-records.d.ts +76 -0
  65. package/dist/classify-severity-records.d.ts.map +1 -0
  66. package/dist/classify-severity-records.js +141 -0
  67. package/dist/classify-severity-records.js.map +1 -0
  68. package/dist/classify-severity.d.ts +1 -23
  69. package/dist/classify-severity.d.ts.map +1 -1
  70. package/dist/classify-severity.js +3 -24
  71. package/dist/classify-severity.js.map +1 -1
  72. package/dist/cli-help.d.ts.map +1 -1
  73. package/dist/cli-help.js +2 -0
  74. package/dist/cli-help.js.map +1 -1
  75. package/dist/cli.d.ts +4 -2
  76. package/dist/cli.d.ts.map +1 -1
  77. package/dist/cli.js +148 -149
  78. package/dist/cli.js.map +1 -1
  79. package/dist/cloud-publish.js +1 -1
  80. package/dist/cloud-publish.js.map +1 -1
  81. package/dist/codex-sdk-environment.js +1 -1
  82. package/dist/codex-sdk-environment.js.map +1 -1
  83. package/dist/component-plan.js +1 -1
  84. package/dist/component-plan.js.map +1 -1
  85. package/dist/component-scan.d.ts.map +1 -1
  86. package/dist/component-scan.js +5 -4
  87. package/dist/component-scan.js.map +1 -1
  88. package/dist/config.d.ts +12 -0
  89. package/dist/config.d.ts.map +1 -1
  90. package/dist/config.js +10 -4
  91. package/dist/config.js.map +1 -1
  92. package/dist/contract.d.ts +6 -0
  93. package/dist/contract.d.ts.map +1 -1
  94. package/dist/contract.js +69 -20
  95. package/dist/contract.js.map +1 -1
  96. package/dist/cost-model.d.ts.map +1 -1
  97. package/dist/cost-model.js +7 -2
  98. package/dist/cost-model.js.map +1 -1
  99. package/dist/custom-validation-prompt.js +3 -3
  100. package/dist/custom-validation-prompt.js.map +1 -1
  101. package/dist/deduplication/codex-review.d.ts +1 -5
  102. package/dist/deduplication/codex-review.d.ts.map +1 -1
  103. package/dist/deduplication/codex-review.js +2 -2
  104. package/dist/deduplication/codex-review.js.map +1 -1
  105. package/dist/deduplication/deduplication-prompts.d.ts +1 -1
  106. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  107. package/dist/deduplication/deduplication-prompts.js +2 -2
  108. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  109. package/dist/deduplication/deduplication-reviewer.d.ts +1 -1
  110. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  111. package/dist/deduplication/deduplication-reviewer.js +1 -1
  112. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  113. package/dist/deduplication/records-protocol.d.ts +3 -8
  114. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  115. package/dist/deduplication/records-protocol.js +11 -171
  116. package/dist/deduplication/records-protocol.js.map +1 -1
  117. package/dist/deduplication/records.js +1 -1
  118. package/dist/deduplication/records.js.map +1 -1
  119. package/dist/deduplication/scan.d.ts.map +1 -1
  120. package/dist/deduplication/scan.js +10 -9
  121. package/dist/deduplication/scan.js.map +1 -1
  122. package/dist/deep-config.d.ts +4 -1
  123. package/dist/deep-config.d.ts.map +1 -1
  124. package/dist/deep-config.js +21 -1
  125. package/dist/deep-config.js.map +1 -1
  126. package/dist/deep-progress.d.ts +15 -1
  127. package/dist/deep-progress.d.ts.map +1 -1
  128. package/dist/deep-progress.js +15 -16
  129. package/dist/deep-progress.js.map +1 -1
  130. package/dist/executable-path.d.ts +2 -0
  131. package/dist/executable-path.d.ts.map +1 -0
  132. package/dist/executable-path.js +12 -0
  133. package/dist/executable-path.js.map +1 -0
  134. package/dist/feedback.d.ts +1 -4
  135. package/dist/feedback.d.ts.map +1 -1
  136. package/dist/feedback.js +1 -1
  137. package/dist/feedback.js.map +1 -1
  138. package/dist/findings-import.d.ts +1 -1
  139. package/dist/findings-import.d.ts.map +1 -1
  140. package/dist/findings-import.js +3 -3
  141. package/dist/findings-import.js.map +1 -1
  142. package/dist/git-remote.d.ts +10 -0
  143. package/dist/git-remote.d.ts.map +1 -0
  144. package/dist/git-remote.js +24 -0
  145. package/dist/git-remote.js.map +1 -0
  146. package/dist/github.d.ts.map +1 -1
  147. package/dist/github.js +5 -7
  148. package/dist/github.js.map +1 -1
  149. package/dist/host-protocol.d.ts +19 -0
  150. package/dist/host-protocol.d.ts.map +1 -0
  151. package/dist/host-protocol.js +270 -0
  152. package/dist/host-protocol.js.map +1 -0
  153. package/dist/host-records-protocol.d.ts +15 -0
  154. package/dist/host-records-protocol.d.ts.map +1 -0
  155. package/dist/host-records-protocol.js +15 -0
  156. package/dist/host-records-protocol.js.map +1 -0
  157. package/dist/hosted-scan-protocol.d.ts +4 -0
  158. package/dist/hosted-scan-protocol.d.ts.map +1 -0
  159. package/dist/hosted-scan-protocol.js +25 -0
  160. package/dist/hosted-scan-protocol.js.map +1 -0
  161. package/dist/hosted-scan.d.ts +112 -0
  162. package/dist/hosted-scan.d.ts.map +1 -0
  163. package/dist/hosted-scan.js +260 -0
  164. package/dist/hosted-scan.js.map +1 -0
  165. package/dist/index.d.ts +7 -0
  166. package/dist/index.d.ts.map +1 -1
  167. package/dist/index.js +3 -0
  168. package/dist/index.js.map +1 -1
  169. package/dist/knowledge-base.d.ts.map +1 -1
  170. package/dist/knowledge-base.js +87 -7
  171. package/dist/knowledge-base.js.map +1 -1
  172. package/dist/multiscan.d.ts.map +1 -1
  173. package/dist/multiscan.js +108 -36
  174. package/dist/multiscan.js.map +1 -1
  175. package/dist/patch-publication.d.ts +26 -0
  176. package/dist/patch-publication.d.ts.map +1 -0
  177. package/dist/patch-publication.js +105 -0
  178. package/dist/patch-publication.js.map +1 -0
  179. package/dist/patch-tui.d.ts.map +1 -1
  180. package/dist/patch-tui.js +4 -1
  181. package/dist/patch-tui.js.map +1 -1
  182. package/dist/project-config-schema.d.ts +2 -0
  183. package/dist/project-config-schema.d.ts.map +1 -1
  184. package/dist/project-config-schema.js +1 -0
  185. package/dist/project-config-schema.js.map +1 -1
  186. package/dist/project-config.d.ts +1 -0
  187. package/dist/project-config.d.ts.map +1 -1
  188. package/dist/project-config.js +16 -6
  189. package/dist/project-config.js.map +1 -1
  190. package/dist/publication-store.d.ts.map +1 -1
  191. package/dist/publication-store.js +73 -63
  192. package/dist/publication-store.js.map +1 -1
  193. package/dist/publication.d.ts +0 -2
  194. package/dist/publication.d.ts.map +1 -1
  195. package/dist/publication.js +135 -24
  196. package/dist/publication.js.map +1 -1
  197. package/dist/publish.d.ts.map +1 -1
  198. package/dist/publish.js +39 -25
  199. package/dist/publish.js.map +1 -1
  200. package/dist/runtime.d.ts +5 -2
  201. package/dist/runtime.d.ts.map +1 -1
  202. package/dist/runtime.js +86 -58
  203. package/dist/runtime.js.map +1 -1
  204. package/dist/scan-executor.d.ts +104 -0
  205. package/dist/scan-executor.d.ts.map +1 -0
  206. package/dist/scan-executor.js +62 -0
  207. package/dist/scan-executor.js.map +1 -0
  208. package/dist/scan-settings.d.ts +1 -0
  209. package/dist/scan-settings.d.ts.map +1 -1
  210. package/dist/scan-settings.js +7 -1
  211. package/dist/scan-settings.js.map +1 -1
  212. package/dist/scan-time-budget.d.ts +9 -0
  213. package/dist/scan-time-budget.d.ts.map +1 -0
  214. package/dist/scan-time-budget.js +16 -0
  215. package/dist/scan-time-budget.js.map +1 -0
  216. package/dist/severity-review.d.ts +29 -0
  217. package/dist/severity-review.d.ts.map +1 -0
  218. package/dist/severity-review.js +47 -0
  219. package/dist/severity-review.js.map +1 -0
  220. package/dist/targets.d.ts.map +1 -1
  221. package/dist/targets.js +17 -25
  222. package/dist/targets.js.map +1 -1
  223. package/dist/validation-records.d.ts +38 -0
  224. package/dist/validation-records.d.ts.map +1 -0
  225. package/dist/validation-records.js +76 -0
  226. package/dist/validation-records.js.map +1 -0
  227. package/dist/version.d.ts.map +1 -1
  228. package/dist/version.js +4 -2
  229. package/dist/version.js.map +1 -1
  230. package/dist/worker-progress.d.ts +43 -15
  231. package/dist/worker-progress.d.ts.map +1 -1
  232. package/dist/worker-progress.js +22 -80
  233. package/dist/worker-progress.js.map +1 -1
  234. package/docs/classify-severity-records.md +89 -0
  235. package/docs/cli.md +281 -35
  236. package/docs/hosted-scans.md +242 -0
  237. package/docs/validate-records.md +69 -0
  238. package/package.json +18 -14
  239. package/schemas/hosted-scan-v2.fixture.json +145 -0
  240. package/schemas/project-config.schema.json +7 -1
  241. package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
  242. package/_bundled_plugin/references/config-preflight.md +0 -93
  243. package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
  244. package/_bundled_plugin/scripts/config_preflight.py +0 -911
@@ -8,9 +8,16 @@ import sys
8
8
  from pathlib import Path, PurePosixPath
9
9
  from typing import Any
10
10
 
11
- # Some plugin hosts launch Python with safe-path isolation enabled.
12
11
  sys.path.insert(0, str(Path(__file__).resolve().parent))
13
- from workbench_target import clean_worktree_content_digest, git_bytes
12
+ from workbench_source_scopes import (
13
+ load_source_scopes,
14
+ normalized_path_component,
15
+ offline_git_bytes,
16
+ relative_path,
17
+ safe_source_path,
18
+ source_object_for_path,
19
+ )
20
+ from workbench_target import clean_worktree_content_digest
14
21
 
15
22
  CONTEXT_LINES = 3
16
23
  MAX_BYTES = 16_000
@@ -21,23 +28,70 @@ def finding_source_excerpt(
21
28
  scan: sqlite3.Row,
22
29
  target: Path | None,
23
30
  locations: list[dict[str, Any]],
31
+ scopes: list[str],
24
32
  ) -> str | None:
25
- if target is None or not locations:
33
+ if target is None or not locations or scan["target_revision"] == "unversioned":
26
34
  return None
27
- location = next(
28
- (
29
- candidate
30
- for candidate in locations
31
- if "root_control" in str(candidate.get("role") or "").lower()
32
- ),
33
- locations[0],
35
+ snapshot = scan["target_snapshot_digest"]
36
+ if snapshot is not None and snapshot != clean_worktree_content_digest():
37
+ return None
38
+ locations = [
39
+ location
40
+ for location in locations
41
+ if isinstance(path := location.get("path"), str)
42
+ and isinstance(location.get("startLine"), int)
43
+ and safe_source_path(target, path) is not None
44
+ ]
45
+ if not locations:
46
+ return None
47
+ try:
48
+ context = load_source_scopes(scan, target, scopes)
49
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
50
+ return None
51
+ if context is None:
52
+ return None
53
+ repository, tree, records = context
54
+ indexed: dict[tuple[str, ...], list[dict[str, str]]] = {}
55
+ for record in records:
56
+ for name in ("path", "canonicalPath"):
57
+ parts = tuple(
58
+ normalized_path_component(part) for part in PurePosixPath(record[name]).parts
59
+ )
60
+ indexed.setdefault(parts, []).append(record)
61
+ lengths = sorted({len(parts) for parts in indexed})
62
+
63
+ def source_object(path: str) -> str | None:
64
+ parsed = relative_path(path)
65
+ if parsed is None:
66
+ return None
67
+ parts = tuple(normalized_path_component(part) for part in parsed.parts)
68
+ try:
69
+ for length in lengths:
70
+ if length > len(parts):
71
+ break
72
+ for record in indexed.get(parts[:length], []):
73
+ object_id = source_object_for_path(repository, tree, target, path, record)
74
+ if object_id is not None:
75
+ return object_id
76
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
77
+ # An unavailable source object leaves the optional excerpt absent.
78
+ pass
79
+ return None
80
+
81
+ locations.sort(
82
+ key=lambda location: (
83
+ location.get("role") != "root_control",
84
+ "root_control" not in str(location.get("role") or "").lower(),
85
+ )
34
86
  )
35
- path = location.get("path")
36
- start_line = location.get("startLine")
37
- end_line = location.get("endLine")
38
- if not isinstance(path, str) or not isinstance(start_line, int):
87
+ for location in locations:
88
+ object_id = source_object(location["path"])
89
+ if object_id is not None:
90
+ break
91
+ else:
39
92
  return None
40
- source = scanned_source_text(scan, target, path)
93
+ start_line, end_line = location["startLine"], location.get("endLine")
94
+ source = scanned_source_text(repository, object_id)
41
95
  if not source or "\0" in source:
42
96
  return None
43
97
  # Match location validation and SARIF hashing: CR, LF, and CRLF are line breaks.
@@ -58,46 +112,15 @@ def finding_source_excerpt(
58
112
  f"{line_number:>{width}} {lines[line_number - 1]}"
59
113
  for line_number in range(excerpt_start, excerpt_end + 1)
60
114
  )
61
- encoded = excerpt.encode("utf-8")[:MAX_BYTES]
62
- return encoded.decode("utf-8", errors="ignore")
115
+ return excerpt.encode("utf-8")[:MAX_BYTES].decode("utf-8", errors="ignore")
63
116
 
64
117
 
65
- def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | None:
66
- if safe_source_path(target, path) is None:
67
- return None
68
- revision = scan["target_revision"]
69
- if revision == "unversioned":
70
- return None
71
- snapshot_digest = scan["target_snapshot_digest"]
72
- if (
73
- scan["diff_target_kind"] == "working_tree"
74
- and scan["diff_content_digest"] != clean_worktree_content_digest()
75
- ):
76
- return None
77
- if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
78
- return None
79
- if (
80
- scan["diff_target_kind"] == "working_tree"
81
- and scan["diff_content_digest"] != clean_worktree_content_digest()
82
- ):
83
- return None
84
- object_name = f"{revision}:./{path}"
85
- content = git_bytes(target, "cat-file", "blob", object_name)
86
- return content.decode("utf-8", errors="replace") if content is not None else None
87
-
88
-
89
- def safe_source_path(target: Path, relative_path: str) -> Path | None:
90
- if "\\" in relative_path:
91
- return None
92
- parsed = PurePosixPath(relative_path)
93
- if parsed.is_absolute() or ".." in parsed.parts:
94
- return None
118
+ def scanned_source_text(repository: Path, object_id: str) -> str | None:
95
119
  try:
96
- path = (target / parsed.as_posix()).resolve()
97
- path.relative_to(target)
98
- except (OSError, RuntimeError, ValueError):
120
+ content = offline_git_bytes(repository, "cat-file", "blob", object_id)
121
+ except (OSError, RuntimeError, SystemExit):
99
122
  return None
100
- return path
123
+ return content.decode("utf-8", errors="replace") if content is not None else None
101
124
 
102
125
 
103
126
  if __name__ == "__main__":
@@ -0,0 +1,388 @@
1
+ """Bind optional finding excerpts to the Git objects selected at scan start."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import os
8
+ import re
9
+ import sqlite3
10
+ import stat
11
+ import sys
12
+ from functools import cache
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import Any
15
+ from unicodedata import normalize
16
+
17
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
18
+ from workbench_target import (
19
+ clean_worktree_content_digest,
20
+ git_bytes,
21
+ git_worktree_context,
22
+ )
23
+ from workbench_validation import reject_non_finite_json
24
+
25
+ OBJECT_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
26
+
27
+
28
+ def expected_target_kinds(scan: sqlite3.Row) -> list[str]:
29
+ if scan["mode"] == "diff":
30
+ return ["git_diff"]
31
+ if scan["target_revision"] == "unversioned":
32
+ return ["directory_snapshot"]
33
+ if scan["target_snapshot_digest"] is None:
34
+ return ["git_worktree", "git_revision"]
35
+ if scan["target_snapshot_digest"] == clean_worktree_content_digest():
36
+ return ["git_revision"]
37
+ return ["git_worktree"]
38
+
39
+
40
+ def requested_scan_paths(scan: sqlite3.Row) -> list[str]:
41
+ if "recipe_json" in scan.keys() and scan["recipe_json"] is not None:
42
+ recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
43
+ target = recipe["target"]
44
+ if target["kind"] == "paths":
45
+ return target["paths"]
46
+ return [scan["scope"]]
47
+
48
+
49
+ def public_scan_recipe(scan: sqlite3.Row) -> dict[str, Any]:
50
+ recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
51
+ recipe.pop("_codexSecurityFileScopes", None)
52
+ return recipe
53
+
54
+
55
+ def normalized_path_component(value: str) -> str:
56
+ return normalize("NFC", normalize("NFD", value).casefold())
57
+
58
+
59
+ def relative_path(value: str) -> PurePosixPath | None:
60
+ path = PurePosixPath(value)
61
+ return None if "\\" in value or path.is_absolute() or ".." in path.parts else path
62
+
63
+
64
+ def safe_source_path(target: Path, value: str) -> Path | None:
65
+ parsed = relative_path(value)
66
+ if parsed is None:
67
+ return None
68
+ try:
69
+ path = (target / parsed.as_posix()).resolve()
70
+ path.relative_to(target)
71
+ return path
72
+ except (OSError, RuntimeError, ValueError):
73
+ return None
74
+
75
+
76
+ def offline_git_bytes(repository: Path, *arguments: str) -> bytes | None:
77
+ names = ("GIT_NO_LAZY_FETCH", "GIT_ALLOW_PROTOCOL", "GIT_NO_REPLACE_OBJECTS")
78
+ previous = {name: os.environ.get(name) for name in names}
79
+ os.environ["GIT_NO_LAZY_FETCH"] = "1"
80
+ os.environ["GIT_ALLOW_PROTOCOL"] = ""
81
+ os.environ["GIT_NO_REPLACE_OBJECTS"] = "1"
82
+ try:
83
+ return git_bytes(repository, *arguments)
84
+ finally:
85
+ for name, value in previous.items():
86
+ if value is None:
87
+ os.environ.pop(name, None)
88
+ else:
89
+ os.environ[name] = value
90
+
91
+
92
+ @cache
93
+ def tree_entries(repository: Path, tree: str) -> dict[str, tuple[tuple[str, str, str], ...]] | None:
94
+ content = offline_git_bytes(repository, "ls-tree", "-z", tree)
95
+ if content is None:
96
+ return None
97
+ entries: dict[str, list[tuple[str, str, str]]] = {}
98
+ for record in content.split(b"\0"):
99
+ if not record:
100
+ continue
101
+ metadata, separator, name = record.partition(b"\t")
102
+ fields = metadata.split(b" ")
103
+ if not separator or len(fields) != 3:
104
+ return None
105
+ mode, object_type, raw_object = fields
106
+ object_id = raw_object.decode("ascii")
107
+ if not OBJECT_ID.fullmatch(object_id):
108
+ return None
109
+ kind = (
110
+ "directory"
111
+ if mode == b"040000" and object_type == b"tree"
112
+ else "file"
113
+ if mode in {b"100644", b"100755"} and object_type == b"blob"
114
+ else "other"
115
+ )
116
+ decoded_name = os.fsdecode(name)
117
+ entries.setdefault(normalized_path_component(decoded_name), []).append(
118
+ (decoded_name, kind, object_id)
119
+ )
120
+ return {name: tuple(matches) for name, matches in entries.items()}
121
+
122
+
123
+ def exact_tree_path(
124
+ repository: Path, tree: str, value: str, *, require_unambiguous: bool = False
125
+ ) -> tuple[str, str, str] | None:
126
+ path = relative_path(value)
127
+ if path is None:
128
+ return None
129
+ kind, object_id = "directory", tree
130
+ for name in path.parts:
131
+ if kind != "directory":
132
+ return None
133
+ entries = tree_entries(repository, object_id) or {}
134
+ aliases = entries.get(normalized_path_component(name), ())
135
+ if require_unambiguous and len(aliases) != 1:
136
+ return None
137
+ entry = next((entry for entry in aliases if entry[0] == name), None)
138
+ if entry is None:
139
+ return None
140
+ _, kind, object_id = entry
141
+ return path.as_posix(), kind, object_id
142
+
143
+
144
+ def ordinary_path(metadata: os.stat_result) -> bool:
145
+ return not (
146
+ getattr(metadata, "st_file_attributes", 0)
147
+ & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
148
+ ) and (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode))
149
+
150
+
151
+ def filesystem_alias(selected: Path, candidate: Path) -> bool:
152
+ if normalized_path_component(selected.name) != normalized_path_component(candidate.name):
153
+ return False
154
+ try:
155
+ if not all(ordinary_path(path.lstat()) for path in (selected, candidate)):
156
+ return False
157
+ if not selected.samefile(candidate):
158
+ return False
159
+ with os.scandir(selected.parent) as entries:
160
+ return (
161
+ sum(
162
+ normalized_path_component(entry.name)
163
+ == normalized_path_component(selected.name)
164
+ for entry in entries
165
+ )
166
+ == 1
167
+ )
168
+ except OSError:
169
+ return False
170
+
171
+
172
+ def existing_tree_path(
173
+ repository: Path, tree: str, value: str, filesystem_root: Path
174
+ ) -> tuple[str, str, str] | None:
175
+ path = relative_path(value)
176
+ if path is None:
177
+ return None
178
+ canonical: list[str] = []
179
+ kind, object_id = "directory", tree
180
+ for index, name in enumerate(path.parts, start=1):
181
+ if kind != "directory":
182
+ return None
183
+ entries = tree_entries(repository, object_id) or {}
184
+ aliases = entries.get(normalized_path_component(name), ())
185
+ selected = filesystem_root / PurePosixPath(*path.parts[:index])
186
+ matches = []
187
+ try:
188
+ if not ordinary_path(selected.lstat()):
189
+ return None
190
+ for entry in aliases:
191
+ candidate = filesystem_root / PurePosixPath(*canonical, entry[0])
192
+ # Every colliding Git name needs a filesystem witness. Missing
193
+ # names cannot establish which entry the user selected.
194
+ if not ordinary_path(candidate.lstat()):
195
+ return None
196
+ if selected.samefile(candidate) and (
197
+ entry[0] == name or filesystem_alias(selected, candidate)
198
+ ):
199
+ matches.append(entry)
200
+ except OSError:
201
+ return None
202
+ if len(matches) != 1:
203
+ return None
204
+ name, kind, object_id = matches[0]
205
+ canonical.append(name)
206
+ return PurePosixPath(*canonical).as_posix(), kind, object_id
207
+
208
+
209
+ def target_tree(target: Path, revision: str) -> tuple[Path, str] | None:
210
+ repository, prefix = git_worktree_context(target)
211
+ raw_tree = offline_git_bytes(repository, "rev-parse", "--verify", f"{revision}^{{tree}}")
212
+ tree = raw_tree.decode("ascii").strip() if raw_tree is not None else ""
213
+ if not OBJECT_ID.fullmatch(tree):
214
+ return None
215
+ selected = existing_tree_path(repository, tree, prefix, repository)
216
+ if selected is None or selected[1] != "directory":
217
+ return None
218
+ return repository, selected[2]
219
+
220
+
221
+ def capture_source_scopes(
222
+ target: Path,
223
+ target_identity: tuple[str, str | None, int | str, int | str],
224
+ paths: list[str],
225
+ *,
226
+ diff_target_kind: str | None = None,
227
+ ) -> dict[str, Any]:
228
+ revision, snapshot = target_identity[:2]
229
+ result: dict[str, Any] = {"version": 1, "revision": revision, "scopes": []}
230
+ # HEAD cannot represent the uncommitted bytes reviewed by a working-tree scan.
231
+ if diff_target_kind == "working_tree":
232
+ return result
233
+ if revision == "unversioned" or (
234
+ snapshot is not None and snapshot != clean_worktree_content_digest()
235
+ ):
236
+ return result
237
+ try:
238
+ repository, _ = git_worktree_context(target)
239
+ if offline_git_bytes(repository, "replace", "--list") != b"":
240
+ # A revision alone does not identify the replacement view scanned.
241
+ return result
242
+ context = target_tree(target, revision)
243
+ if context is None:
244
+ return result
245
+ repository, tree = context
246
+ result["targetTree"] = tree
247
+ for requested in paths:
248
+ selected = safe_source_path(target, requested)
249
+ if selected is None:
250
+ continue
251
+ canonical = existing_tree_path(repository, tree, requested, target)
252
+ if canonical is None or canonical[1] not in {"file", "directory"}:
253
+ continue
254
+ try:
255
+ metadata = selected.stat()
256
+ except OSError:
257
+ continue
258
+ if (canonical[1] == "directory") != stat.S_ISDIR(metadata.st_mode):
259
+ continue
260
+ result["scopes"].append(
261
+ {
262
+ "path": PurePosixPath(requested).as_posix(),
263
+ "canonicalPath": canonical[0],
264
+ "kind": canonical[1],
265
+ "objectId": canonical[2],
266
+ }
267
+ )
268
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
269
+ # Excerpts are optional. Failure to bind a scope never widens it and
270
+ # must not prevent the scan itself from starting.
271
+ pass
272
+ return result
273
+
274
+
275
+ def load_source_scopes(
276
+ scan: sqlite3.Row, target: Path, requested: list[str]
277
+ ) -> tuple[Path, str, list[dict[str, str]]] | None:
278
+ if "diff_target_kind" in scan.keys() and scan["diff_target_kind"] == "working_tree":
279
+ return None
280
+ saved = scan["source_scopes_json"] if "source_scopes_json" in scan.keys() else None
281
+ paths = {PurePosixPath(path).as_posix() for path in requested}
282
+ if saved is not None:
283
+ metadata = json.loads(saved)
284
+ if (
285
+ not isinstance(metadata, dict)
286
+ or metadata.get("version") != 1
287
+ or metadata.get("revision") != scan["target_revision"]
288
+ ):
289
+ return None
290
+ tree = metadata.get("targetTree")
291
+ records = metadata.get("scopes")
292
+ if (
293
+ not isinstance(tree, str)
294
+ or not OBJECT_ID.fullmatch(tree)
295
+ or not isinstance(records, list)
296
+ ):
297
+ return None
298
+ scopes = [
299
+ record
300
+ for record in records
301
+ if isinstance(record, dict)
302
+ and isinstance(record.get("path"), str)
303
+ and record.get("path") in paths
304
+ and isinstance(record.get("canonicalPath"), str)
305
+ and relative_path(record["canonicalPath"]) is not None
306
+ and record.get("kind") in ("file", "directory")
307
+ and isinstance(record.get("objectId"), str)
308
+ and OBJECT_ID.fullmatch(record["objectId"])
309
+ ]
310
+ return (git_worktree_context(target)[0], tree, scopes) if scopes else None
311
+
312
+ # Exact, unambiguous historical entries establish legacy scope kinds.
313
+ # Keep any previously recorded file kinds as an additional constraint.
314
+ recipe = scan["recipe_json"] if "recipe_json" in scan.keys() else None
315
+ legacy_recipe = json.loads(recipe) if recipe else None
316
+ legacy_files = (
317
+ legacy_recipe.get("_codexSecurityFileScopes") if isinstance(legacy_recipe, dict) else None
318
+ )
319
+ repository, _ = git_worktree_context(target)
320
+ if offline_git_bytes(repository, "replace", "--list") != b"":
321
+ # Old records do not say which replacement view was scanned.
322
+ return None
323
+ context = target_tree(target, scan["target_revision"])
324
+ if context is None:
325
+ return None
326
+ repository, tree = context
327
+ scopes = []
328
+ for path in paths:
329
+ entry = exact_tree_path(repository, tree, path, require_unambiguous=True)
330
+ if entry is None or entry[1] not in {"file", "directory"}:
331
+ continue
332
+ kind = entry[1]
333
+ if isinstance(legacy_files, list) and kind != (
334
+ "file" if path in legacy_files else "directory"
335
+ ):
336
+ continue
337
+ scopes.append(
338
+ {
339
+ "path": path,
340
+ "canonicalPath": path,
341
+ "kind": kind,
342
+ "objectId": entry[2],
343
+ }
344
+ )
345
+ return (repository, tree, scopes) if scopes else None
346
+
347
+
348
+ def source_object_for_path(
349
+ repository: Path, tree: str, target: Path, value: str, scope: dict[str, str]
350
+ ) -> str | None:
351
+ path = relative_path(value)
352
+ if path is None or safe_source_path(target, value) is None:
353
+ return None
354
+ prefixes = {PurePosixPath(scope[name]).parts for name in ("path", "canonicalPath")}
355
+ suffix: tuple[str, ...] | None = None
356
+ for prefix in sorted(prefixes, key=len, reverse=True):
357
+ if path.parts[: len(prefix)] == prefix:
358
+ suffix = path.parts[len(prefix) :]
359
+ break
360
+ if suffix is None:
361
+ for length in {len(prefix) for prefix in prefixes}:
362
+ if length > len(path.parts) or scope["kind"] == "file" and length != len(path.parts):
363
+ continue
364
+ selected = existing_tree_path(
365
+ repository, tree, PurePosixPath(*path.parts[:length]).as_posix(), target
366
+ )
367
+ if selected == (scope["canonicalPath"], scope["kind"], scope["objectId"]):
368
+ suffix = path.parts[length:]
369
+ break
370
+ if suffix is None:
371
+ return None
372
+ if scope["kind"] == "file":
373
+ return scope["objectId"] if not suffix else None
374
+ relative = PurePosixPath(*suffix).as_posix()
375
+ entry = exact_tree_path(repository, scope["objectId"], relative, require_unambiguous=True)
376
+ if entry is None:
377
+ selected = safe_source_path(target, scope["canonicalPath"])
378
+ if selected is not None:
379
+ entry = existing_tree_path(repository, scope["objectId"], relative, selected)
380
+ return entry[2] if entry is not None and entry[1] == "file" else None
381
+
382
+
383
+ def main() -> None:
384
+ argparse.ArgumentParser(description=__doc__).parse_args()
385
+
386
+
387
+ if __name__ == "__main__":
388
+ main()
@@ -87,7 +87,10 @@ def git_blob_samples(
87
87
  if not object_names:
88
88
  return []
89
89
 
90
- request = b"\0".join(os.fsencode(name) for name in object_names) + b"\0"
90
+ # Git before 2.38 rejects -z, so only names with line breaks need it.
91
+ nul_input = any("\n" in name or "\r" in name for name in object_names)
92
+ separator = b"\0" if nul_input else b"\n"
93
+ request = separator.join(os.fsencode(name) for name in object_names) + separator
91
94
  read_fd, write_fd = os.pipe()
92
95
  with (
93
96
  ThreadPoolExecutor(max_workers=1) as reader,
@@ -106,7 +109,7 @@ def git_blob_samples(
106
109
  target,
107
110
  "cat-file",
108
111
  "--batch",
109
- "-z",
112
+ *(["-z"] if nul_input else []),
110
113
  text=False,
111
114
  input_data=request,
112
115
  git_dir=git_dir,
@@ -981,9 +984,8 @@ def copy_git_worktree_files(
981
984
  continue
982
985
  source_path = repository / relative
983
986
  try:
984
- if _WINDOWS:
985
- source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
986
- relative = source_path.relative_to(repository)
987
+ source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
988
+ relative = source_path.relative_to(repository)
987
989
  metadata = source_path.lstat()
988
990
  except FileNotFoundError:
989
991
  continue
@@ -16,10 +16,11 @@ from typing import Any
16
16
  # Some plugin hosts launch Python with safe-path isolation enabled.
17
17
  sys.path.insert(0, str(Path(__file__).resolve().parent))
18
18
  import finalize_scan_contract as finalizer
19
- from workbench_scan_usage import TOKEN_FIELDS
19
+ from workbench_scan_usage import TOKEN_FIELDS, valid_account_rate_limit
20
20
  from workbench_scan_usage import (
21
21
  _reject_nonstandard_json_number as reject_nonstandard_json_number,
22
22
  )
23
+ from workbench_scan_usage import _timestamp as usage_timestamp
23
24
 
24
25
  reject_non_finite_json = finalizer._reject_non_finite_json
25
26
 
@@ -92,6 +93,19 @@ def _valid_measured_scan_usage(usage: object) -> bool:
92
93
  or thread_count < 0
93
94
  ):
94
95
  return False
96
+ completion = usage.get("completion")
97
+ if "completion" in usage and (
98
+ not isinstance(completion, dict)
99
+ or set(completion) != {"threadId", "turnId", "finalized"}
100
+ or not isinstance(completion["threadId"], str)
101
+ or not completion["threadId"]
102
+ or not isinstance(completion["turnId"], str)
103
+ or not completion["turnId"]
104
+ or type(completion["finalized"]) is not bool
105
+ or not completion["finalized"]
106
+ and coverage == "complete"
107
+ ):
108
+ return False
95
109
  warnings = usage.get("warnings", [])
96
110
  if (
97
111
  not isinstance(warnings, list)
@@ -105,7 +119,7 @@ def _valid_measured_scan_usage(usage: object) -> bool:
105
119
  return False
106
120
  if coverage == "unavailable":
107
121
  return thread_count == 0 and set(usage).issubset(
108
- {"coverage", "source", "threadCount", "warnings"}
122
+ {"coverage", "source", "threadCount", "warnings", "completion"}
109
123
  )
110
124
 
111
125
  allowed_keys = {
@@ -114,6 +128,12 @@ def _valid_measured_scan_usage(usage: object) -> bool:
114
128
  "threadCount",
115
129
  "missingThreadCount",
116
130
  "warnings",
131
+ "completion",
132
+ "byRole",
133
+ "window",
134
+ "observedAt",
135
+ "tokensUpdatedAt",
136
+ "accountRateLimits",
117
137
  *SCAN_USAGE_TOKEN_KEYS,
118
138
  }
119
139
  if thread_count == 0 or not set(usage).issubset(allowed_keys):
@@ -130,6 +150,36 @@ def _valid_measured_scan_usage(usage: object) -> bool:
130
150
  return False
131
151
  if coverage == "partial" and not (warnings or missing):
132
152
  return False
153
+ for field in ("observedAt", "tokensUpdatedAt"):
154
+ if field in usage and usage_timestamp(usage[field]) is None:
155
+ return False
156
+ if "window" in usage:
157
+ window = usage["window"]
158
+ if not isinstance(window, dict) or set(window) != {"startedAt", "endedAt"}:
159
+ return False
160
+ start, end = (usage_timestamp(window[key]) for key in ("startedAt", "endedAt"))
161
+ if start is None or end is None or start > end:
162
+ return False
163
+ if "byRole" in usage:
164
+ roles = usage["byRole"]
165
+ keys = {*SCAN_USAGE_TOKEN_KEYS, "threadCount"}
166
+ if not isinstance(roles, dict) or set(roles) != {"parent", "workers"}:
167
+ return False
168
+ if any(
169
+ not isinstance(role, dict)
170
+ or set(role) != keys
171
+ or any(type(role[key]) is not int or role[key] < 0 for key in keys)
172
+ for role in roles.values()
173
+ ):
174
+ return False
175
+ if any(sum(role[key] for role in roles.values()) != usage[key] for key in keys):
176
+ return False
177
+ if "accountRateLimits" in usage:
178
+ limits = usage["accountRateLimits"]
179
+ if not isinstance(limits, list) or not all(
180
+ valid_account_rate_limit(limit) for limit in limits
181
+ ):
182
+ return False
133
183
  return True
134
184
 
135
185
 
@@ -1,4 +1,23 @@
1
1
  [
2
+ [
3
+ "-- Move source-scope previews before the shared history repairs.",
4
+ "WITH input AS (SELECT :history AS history)",
5
+ "SELECT 'sql' AS operation,",
6
+ " 'UPDATE schema_migrations SET version = 50 WHERE version = ' || version AS value",
7
+ "FROM schema_migrations",
8
+ "WHERE version IN (34, 40, 41, 42, 43, 44, 48)",
9
+ " AND name = 'persist authorized source excerpt scopes';"
10
+ ],
11
+ [
12
+ "WITH history AS (",
13
+ " SELECT json_extract(value, '$.version') AS version,",
14
+ " json_extract(value, '$.name') AS name FROM json_each(:history)",
15
+ ")",
16
+ "SELECT 'error' AS operation,",
17
+ " 'The Codex Security database has an unsupported source-scope migration history.' AS value",
18
+ "FROM schema_migrations JOIN history USING (version)",
19
+ "WHERE version IN (43, 50) AND schema_migrations.name != history.name;"
20
+ ],
2
21
  [
3
22
  "-- mirror and early-moves: relocate numbers before checking profiles.",
4
23
  "WITH",