@openai/codex-security 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (290) hide show
  1. package/README.md +52 -9
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  5. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  6. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
  7. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  8. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  9. package/_bundled_plugin/references/core-scan.md +2 -0
  10. package/_bundled_plugin/references/final-report.md +5 -1
  11. package/_bundled_plugin/references/investigation-time-budget.md +11 -0
  12. package/_bundled_plugin/references/sarif-adapter.md +1 -0
  13. package/_bundled_plugin/references/scan-prologue.md +2 -6
  14. package/_bundled_plugin/references/scan-runtime.md +15 -0
  15. package/_bundled_plugin/references/threat-model.md +2 -0
  16. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +6 -2
  17. package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
  18. package/_bundled_plugin/scripts/finalize_scan_contract.py +92 -44
  19. package/_bundled_plugin/scripts/finding_preview.py +5 -3
  20. package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
  21. package/_bundled_plugin/scripts/generate_rank_input.py +13 -6
  22. package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
  23. package/_bundled_plugin/scripts/report_projection.py +2 -1
  24. package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
  25. package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
  26. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
  27. package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
  28. package/_bundled_plugin/scripts/workbench_cli.py +5 -10
  29. package/_bundled_plugin/scripts/workbench_constants.py +4 -0
  30. package/_bundled_plugin/scripts/workbench_db.py +68 -39
  31. package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
  32. package/_bundled_plugin/scripts/workbench_progress.py +0 -50
  33. package/_bundled_plugin/scripts/workbench_publication.py +4 -3
  34. package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
  35. package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
  36. package/_bundled_plugin/scripts/workbench_scan_history.py +82 -80
  37. package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
  38. package/_bundled_plugin/scripts/workbench_scan_usage.py +246 -38
  39. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  40. package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
  41. package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
  42. package/_bundled_plugin/scripts/workbench_target.py +28 -12
  43. package/_bundled_plugin/scripts/workbench_validation.py +52 -13
  44. package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
  45. package/_bundled_plugin/shared/workbench-migrations.json +15 -0
  46. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
  47. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
  48. package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
  49. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  50. package/dist/api.d.ts +4 -3
  51. package/dist/api.d.ts.map +1 -1
  52. package/dist/api.js +215 -93
  53. package/dist/api.js.map +1 -1
  54. package/dist/artifact-export.d.ts +2 -1
  55. package/dist/artifact-export.d.ts.map +1 -1
  56. package/dist/artifact-export.js +38 -23
  57. package/dist/artifact-export.js.map +1 -1
  58. package/dist/classify-scan-severity.d.ts.map +1 -1
  59. package/dist/classify-scan-severity.js +16 -18
  60. package/dist/classify-scan-severity.js.map +1 -1
  61. package/dist/classify-severity-records.d.ts +76 -0
  62. package/dist/classify-severity-records.d.ts.map +1 -0
  63. package/dist/classify-severity-records.js +141 -0
  64. package/dist/classify-severity-records.js.map +1 -0
  65. package/dist/classify-severity.d.ts +1 -23
  66. package/dist/classify-severity.d.ts.map +1 -1
  67. package/dist/classify-severity.js +3 -24
  68. package/dist/classify-severity.js.map +1 -1
  69. package/dist/cli-daybreak.d.ts +5 -0
  70. package/dist/cli-daybreak.d.ts.map +1 -0
  71. package/dist/cli-daybreak.js +89 -0
  72. package/dist/cli-daybreak.js.map +1 -0
  73. package/dist/cli-help.d.ts.map +1 -1
  74. package/dist/cli-help.js +2 -0
  75. package/dist/cli-help.js.map +1 -1
  76. package/dist/cli.d.ts +9 -3
  77. package/dist/cli.d.ts.map +1 -1
  78. package/dist/cli.js +957 -738
  79. package/dist/cli.js.map +1 -1
  80. package/dist/cloud-publish.js +1 -1
  81. package/dist/cloud-publish.js.map +1 -1
  82. package/dist/codeowners.js +2 -2
  83. package/dist/codeowners.js.map +1 -1
  84. package/dist/codex-sdk-environment.js +1 -1
  85. package/dist/codex-sdk-environment.js.map +1 -1
  86. package/dist/component-plan.d.ts.map +1 -1
  87. package/dist/component-plan.js +3 -2
  88. package/dist/component-plan.js.map +1 -1
  89. package/dist/component-scan.js +2 -2
  90. package/dist/component-scan.js.map +1 -1
  91. package/dist/config.d.ts +12 -0
  92. package/dist/config.d.ts.map +1 -1
  93. package/dist/config.js +15 -5
  94. package/dist/config.js.map +1 -1
  95. package/dist/contract.d.ts +5 -0
  96. package/dist/contract.d.ts.map +1 -1
  97. package/dist/contract.js +65 -17
  98. package/dist/contract.js.map +1 -1
  99. package/dist/cost-model.d.ts.map +1 -1
  100. package/dist/cost-model.js +7 -2
  101. package/dist/cost-model.js.map +1 -1
  102. package/dist/cost.d.ts +2 -1
  103. package/dist/cost.d.ts.map +1 -1
  104. package/dist/cost.js +17 -8
  105. package/dist/cost.js.map +1 -1
  106. package/dist/custom-validation-prompt.js +3 -3
  107. package/dist/custom-validation-prompt.js.map +1 -1
  108. package/dist/custom-validation.d.ts.map +1 -1
  109. package/dist/custom-validation.js +2 -5
  110. package/dist/custom-validation.js.map +1 -1
  111. package/dist/deduplication/codex-review.d.ts.map +1 -1
  112. package/dist/deduplication/codex-review.js +8 -6
  113. package/dist/deduplication/codex-review.js.map +1 -1
  114. package/dist/deduplication/deduplication-prompts.d.ts +1 -1
  115. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  116. package/dist/deduplication/deduplication-prompts.js +2 -2
  117. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  118. package/dist/deduplication/local.d.ts.map +1 -1
  119. package/dist/deduplication/local.js +5 -2
  120. package/dist/deduplication/local.js.map +1 -1
  121. package/dist/deduplication/records.js +1 -1
  122. package/dist/deduplication/records.js.map +1 -1
  123. package/dist/deduplication/retry.d.ts.map +1 -1
  124. package/dist/deduplication/retry.js +5 -0
  125. package/dist/deduplication/retry.js.map +1 -1
  126. package/dist/deduplication/scan.d.ts.map +1 -1
  127. package/dist/deduplication/scan.js +15 -10
  128. package/dist/deduplication/scan.js.map +1 -1
  129. package/dist/deep-config.d.ts +4 -1
  130. package/dist/deep-config.d.ts.map +1 -1
  131. package/dist/deep-config.js +21 -1
  132. package/dist/deep-config.js.map +1 -1
  133. package/dist/deep-scan.d.ts +6 -0
  134. package/dist/deep-scan.d.ts.map +1 -1
  135. package/dist/deep-scan.js +8 -6
  136. package/dist/deep-scan.js.map +1 -1
  137. package/dist/findings-import.d.ts +1 -1
  138. package/dist/findings-import.d.ts.map +1 -1
  139. package/dist/findings-import.js +26 -6
  140. package/dist/findings-import.js.map +1 -1
  141. package/dist/git-remote.d.ts +10 -0
  142. package/dist/git-remote.d.ts.map +1 -0
  143. package/dist/git-remote.js +24 -0
  144. package/dist/git-remote.js.map +1 -0
  145. package/dist/github.d.ts.map +1 -1
  146. package/dist/github.js +5 -7
  147. package/dist/github.js.map +1 -1
  148. package/dist/host-protocol.d.ts +1 -0
  149. package/dist/host-protocol.d.ts.map +1 -1
  150. package/dist/host-protocol.js +2 -2
  151. package/dist/host-protocol.js.map +1 -1
  152. package/dist/host-records-protocol.d.ts +15 -0
  153. package/dist/host-records-protocol.d.ts.map +1 -0
  154. package/dist/host-records-protocol.js +15 -0
  155. package/dist/host-records-protocol.js.map +1 -0
  156. package/dist/hosted-remediation-protocol.d.ts +5 -0
  157. package/dist/hosted-remediation-protocol.d.ts.map +1 -0
  158. package/dist/hosted-remediation-protocol.js +26 -0
  159. package/dist/hosted-remediation-protocol.js.map +1 -0
  160. package/dist/hosted-remediation.d.ts +89 -0
  161. package/dist/hosted-remediation.d.ts.map +1 -0
  162. package/dist/hosted-remediation.js +122 -0
  163. package/dist/hosted-remediation.js.map +1 -0
  164. package/dist/hosted-scan.d.ts +1 -6
  165. package/dist/hosted-scan.d.ts.map +1 -1
  166. package/dist/hosted-scan.js +2 -2
  167. package/dist/hosted-scan.js.map +1 -1
  168. package/dist/import-scan.d.ts +1 -0
  169. package/dist/import-scan.d.ts.map +1 -1
  170. package/dist/import-scan.js +5 -1
  171. package/dist/import-scan.js.map +1 -1
  172. package/dist/index.d.ts +9 -1
  173. package/dist/index.d.ts.map +1 -1
  174. package/dist/index.js +4 -0
  175. package/dist/index.js.map +1 -1
  176. package/dist/knowledge-base.d.ts.map +1 -1
  177. package/dist/knowledge-base.js +87 -7
  178. package/dist/knowledge-base.js.map +1 -1
  179. package/dist/multiscan.d.ts.map +1 -1
  180. package/dist/multiscan.js +65 -29
  181. package/dist/multiscan.js.map +1 -1
  182. package/dist/patch-publication.d.ts +26 -0
  183. package/dist/patch-publication.d.ts.map +1 -0
  184. package/dist/patch-publication.js +105 -0
  185. package/dist/patch-publication.js.map +1 -0
  186. package/dist/patch-tui.d.ts.map +1 -1
  187. package/dist/patch-tui.js +4 -1
  188. package/dist/patch-tui.js.map +1 -1
  189. package/dist/project-config-schema.d.ts +2 -0
  190. package/dist/project-config-schema.d.ts.map +1 -1
  191. package/dist/project-config-schema.js +1 -0
  192. package/dist/project-config-schema.js.map +1 -1
  193. package/dist/project-config.d.ts +1 -0
  194. package/dist/project-config.d.ts.map +1 -1
  195. package/dist/project-config.js +29 -15
  196. package/dist/project-config.js.map +1 -1
  197. package/dist/publication.d.ts.map +1 -1
  198. package/dist/publication.js +135 -19
  199. package/dist/publication.js.map +1 -1
  200. package/dist/publish.d.ts +2 -0
  201. package/dist/publish.d.ts.map +1 -1
  202. package/dist/publish.js +48 -25
  203. package/dist/publish.js.map +1 -1
  204. package/dist/remediation.d.ts +63 -0
  205. package/dist/remediation.d.ts.map +1 -0
  206. package/dist/remediation.js +110 -0
  207. package/dist/remediation.js.map +1 -0
  208. package/dist/request-metadata.d.ts +17 -7
  209. package/dist/request-metadata.d.ts.map +1 -1
  210. package/dist/request-metadata.js +43 -8
  211. package/dist/request-metadata.js.map +1 -1
  212. package/dist/runtime.d.ts.map +1 -1
  213. package/dist/runtime.js +104 -36
  214. package/dist/runtime.js.map +1 -1
  215. package/dist/saved-scan-bootstrap.d.ts +1 -0
  216. package/dist/saved-scan-bootstrap.d.ts.map +1 -1
  217. package/dist/saved-scan-bootstrap.js +16 -4
  218. package/dist/saved-scan-bootstrap.js.map +1 -1
  219. package/dist/scan-activity.js +8 -8
  220. package/dist/scan-activity.js.map +1 -1
  221. package/dist/scan-comparison.d.ts.map +1 -1
  222. package/dist/scan-comparison.js +15 -9
  223. package/dist/scan-comparison.js.map +1 -1
  224. package/dist/scan-dashboard.d.ts.map +1 -1
  225. package/dist/scan-dashboard.js +12 -6
  226. package/dist/scan-dashboard.js.map +1 -1
  227. package/dist/scan-executor.d.ts +5 -1
  228. package/dist/scan-executor.d.ts.map +1 -1
  229. package/dist/scan-executor.js +9 -8
  230. package/dist/scan-executor.js.map +1 -1
  231. package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
  232. package/dist/scan-progress.d.ts.map +1 -0
  233. package/dist/scan-progress.js +138 -0
  234. package/dist/scan-progress.js.map +1 -0
  235. package/dist/scan-settings.d.ts +1 -0
  236. package/dist/scan-settings.d.ts.map +1 -1
  237. package/dist/scan-settings.js +7 -1
  238. package/dist/scan-settings.js.map +1 -1
  239. package/dist/scan-time-budget.d.ts +9 -0
  240. package/dist/scan-time-budget.d.ts.map +1 -0
  241. package/dist/scan-time-budget.js +16 -0
  242. package/dist/scan-time-budget.js.map +1 -0
  243. package/dist/security-policy.js +1 -1
  244. package/dist/security-policy.js.map +1 -1
  245. package/dist/server/embeddings.d.ts.map +1 -1
  246. package/dist/server/embeddings.js +4 -0
  247. package/dist/server/embeddings.js.map +1 -1
  248. package/dist/server/sqlite-store.d.ts.map +1 -1
  249. package/dist/server/sqlite-store.js +14 -2
  250. package/dist/server/sqlite-store.js.map +1 -1
  251. package/dist/severity-review.d.ts +29 -0
  252. package/dist/severity-review.d.ts.map +1 -0
  253. package/dist/severity-review.js +47 -0
  254. package/dist/severity-review.js.map +1 -0
  255. package/dist/severity-store.js +4 -1
  256. package/dist/severity-store.js.map +1 -1
  257. package/dist/timestamps.d.ts +3 -0
  258. package/dist/timestamps.d.ts.map +1 -0
  259. package/dist/timestamps.js +14 -0
  260. package/dist/timestamps.js.map +1 -0
  261. package/dist/validation-records.d.ts +38 -0
  262. package/dist/validation-records.d.ts.map +1 -0
  263. package/dist/validation-records.js +76 -0
  264. package/dist/validation-records.js.map +1 -0
  265. package/dist/version.d.ts +1 -1
  266. package/dist/version.d.ts.map +1 -1
  267. package/dist/version.js +17 -4
  268. package/dist/version.js.map +1 -1
  269. package/dist/workbench-arguments.d.ts.map +1 -1
  270. package/dist/workbench-arguments.js +9 -3
  271. package/dist/workbench-arguments.js.map +1 -1
  272. package/dist/worker-progress.d.ts +13 -7
  273. package/dist/worker-progress.d.ts.map +1 -1
  274. package/dist/worker-progress.js +12 -44
  275. package/dist/worker-progress.js.map +1 -1
  276. package/docs/classify-severity-records.md +89 -0
  277. package/docs/cli.md +352 -45
  278. package/docs/findings-service.md +7 -2
  279. package/docs/hosted-remediation.md +117 -0
  280. package/docs/hosted-scans.md +8 -3
  281. package/docs/validate-records.md +69 -0
  282. package/package.json +13 -10
  283. package/schemas/project-config.schema.json +7 -1
  284. package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
  285. package/_bundled_plugin/references/config-preflight.md +0 -93
  286. package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
  287. package/_bundled_plugin/scripts/config_preflight.py +0 -911
  288. package/dist/deep-progress.d.ts.map +0 -1
  289. package/dist/deep-progress.js +0 -92
  290. package/dist/deep-progress.js.map +0 -1
@@ -0,0 +1,388 @@
1
+ """Bind optional finding excerpts to the Git objects selected at scan start."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import os
8
+ import re
9
+ import sqlite3
10
+ import stat
11
+ import sys
12
+ from functools import cache
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import Any
15
+ from unicodedata import normalize
16
+
17
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
18
+ from workbench_target import (
19
+ clean_worktree_content_digest,
20
+ git_bytes,
21
+ git_worktree_context,
22
+ )
23
+ from workbench_validation import reject_non_finite_json
24
+
25
+ OBJECT_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
26
+
27
+
28
+ def expected_target_kinds(scan: sqlite3.Row) -> list[str]:
29
+ if scan["mode"] == "diff":
30
+ return ["git_diff"]
31
+ if scan["target_revision"] == "unversioned":
32
+ return ["directory_snapshot"]
33
+ if scan["target_snapshot_digest"] is None:
34
+ return ["git_worktree", "git_revision"]
35
+ if scan["target_snapshot_digest"] == clean_worktree_content_digest():
36
+ return ["git_revision"]
37
+ return ["git_worktree"]
38
+
39
+
40
+ def requested_scan_paths(scan: sqlite3.Row) -> list[str]:
41
+ if "recipe_json" in scan.keys() and scan["recipe_json"] is not None:
42
+ recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
43
+ target = recipe["target"]
44
+ if target["kind"] == "paths":
45
+ return target["paths"]
46
+ return [scan["scope"]]
47
+
48
+
49
+ def public_scan_recipe(scan: sqlite3.Row) -> dict[str, Any]:
50
+ recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
51
+ recipe.pop("_codexSecurityFileScopes", None)
52
+ return recipe
53
+
54
+
55
+ def normalized_path_component(value: str) -> str:
56
+ return normalize("NFC", normalize("NFD", value).casefold())
57
+
58
+
59
+ def relative_path(value: str) -> PurePosixPath | None:
60
+ path = PurePosixPath(value)
61
+ return None if "\\" in value or path.is_absolute() or ".." in path.parts else path
62
+
63
+
64
+ def safe_source_path(target: Path, value: str) -> Path | None:
65
+ parsed = relative_path(value)
66
+ if parsed is None:
67
+ return None
68
+ try:
69
+ path = (target / parsed.as_posix()).resolve()
70
+ path.relative_to(target)
71
+ return path
72
+ except (OSError, RuntimeError, ValueError):
73
+ return None
74
+
75
+
76
+ def offline_git_bytes(repository: Path, *arguments: str) -> bytes | None:
77
+ names = ("GIT_NO_LAZY_FETCH", "GIT_ALLOW_PROTOCOL", "GIT_NO_REPLACE_OBJECTS")
78
+ previous = {name: os.environ.get(name) for name in names}
79
+ os.environ["GIT_NO_LAZY_FETCH"] = "1"
80
+ os.environ["GIT_ALLOW_PROTOCOL"] = ""
81
+ os.environ["GIT_NO_REPLACE_OBJECTS"] = "1"
82
+ try:
83
+ return git_bytes(repository, *arguments)
84
+ finally:
85
+ for name, value in previous.items():
86
+ if value is None:
87
+ os.environ.pop(name, None)
88
+ else:
89
+ os.environ[name] = value
90
+
91
+
92
+ @cache
93
+ def tree_entries(repository: Path, tree: str) -> dict[str, tuple[tuple[str, str, str], ...]] | None:
94
+ content = offline_git_bytes(repository, "ls-tree", "-z", tree)
95
+ if content is None:
96
+ return None
97
+ entries: dict[str, list[tuple[str, str, str]]] = {}
98
+ for record in content.split(b"\0"):
99
+ if not record:
100
+ continue
101
+ metadata, separator, name = record.partition(b"\t")
102
+ fields = metadata.split(b" ")
103
+ if not separator or len(fields) != 3:
104
+ return None
105
+ mode, object_type, raw_object = fields
106
+ object_id = raw_object.decode("ascii")
107
+ if not OBJECT_ID.fullmatch(object_id):
108
+ return None
109
+ kind = (
110
+ "directory"
111
+ if mode == b"040000" and object_type == b"tree"
112
+ else "file"
113
+ if mode in {b"100644", b"100755"} and object_type == b"blob"
114
+ else "other"
115
+ )
116
+ decoded_name = os.fsdecode(name)
117
+ entries.setdefault(normalized_path_component(decoded_name), []).append(
118
+ (decoded_name, kind, object_id)
119
+ )
120
+ return {name: tuple(matches) for name, matches in entries.items()}
121
+
122
+
123
+ def exact_tree_path(
124
+ repository: Path, tree: str, value: str, *, require_unambiguous: bool = False
125
+ ) -> tuple[str, str, str] | None:
126
+ path = relative_path(value)
127
+ if path is None:
128
+ return None
129
+ kind, object_id = "directory", tree
130
+ for name in path.parts:
131
+ if kind != "directory":
132
+ return None
133
+ entries = tree_entries(repository, object_id) or {}
134
+ aliases = entries.get(normalized_path_component(name), ())
135
+ if require_unambiguous and len(aliases) != 1:
136
+ return None
137
+ entry = next((entry for entry in aliases if entry[0] == name), None)
138
+ if entry is None:
139
+ return None
140
+ _, kind, object_id = entry
141
+ return path.as_posix(), kind, object_id
142
+
143
+
144
+ def ordinary_path(metadata: os.stat_result) -> bool:
145
+ return not (
146
+ getattr(metadata, "st_file_attributes", 0)
147
+ & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
148
+ ) and (stat.S_ISDIR(metadata.st_mode) or stat.S_ISREG(metadata.st_mode))
149
+
150
+
151
+ def filesystem_alias(selected: Path, candidate: Path) -> bool:
152
+ if normalized_path_component(selected.name) != normalized_path_component(candidate.name):
153
+ return False
154
+ try:
155
+ if not all(ordinary_path(path.lstat()) for path in (selected, candidate)):
156
+ return False
157
+ if not selected.samefile(candidate):
158
+ return False
159
+ with os.scandir(selected.parent) as entries:
160
+ return (
161
+ sum(
162
+ normalized_path_component(entry.name)
163
+ == normalized_path_component(selected.name)
164
+ for entry in entries
165
+ )
166
+ == 1
167
+ )
168
+ except OSError:
169
+ return False
170
+
171
+
172
+ def existing_tree_path(
173
+ repository: Path, tree: str, value: str, filesystem_root: Path
174
+ ) -> tuple[str, str, str] | None:
175
+ path = relative_path(value)
176
+ if path is None:
177
+ return None
178
+ canonical: list[str] = []
179
+ kind, object_id = "directory", tree
180
+ for index, name in enumerate(path.parts, start=1):
181
+ if kind != "directory":
182
+ return None
183
+ entries = tree_entries(repository, object_id) or {}
184
+ aliases = entries.get(normalized_path_component(name), ())
185
+ selected = filesystem_root / PurePosixPath(*path.parts[:index])
186
+ matches = []
187
+ try:
188
+ if not ordinary_path(selected.lstat()):
189
+ return None
190
+ for entry in aliases:
191
+ candidate = filesystem_root / PurePosixPath(*canonical, entry[0])
192
+ # Every colliding Git name needs a filesystem witness. Missing
193
+ # names cannot establish which entry the user selected.
194
+ if not ordinary_path(candidate.lstat()):
195
+ return None
196
+ if selected.samefile(candidate) and (
197
+ entry[0] == name or filesystem_alias(selected, candidate)
198
+ ):
199
+ matches.append(entry)
200
+ except OSError:
201
+ return None
202
+ if len(matches) != 1:
203
+ return None
204
+ name, kind, object_id = matches[0]
205
+ canonical.append(name)
206
+ return PurePosixPath(*canonical).as_posix(), kind, object_id
207
+
208
+
209
+ def target_tree(target: Path, revision: str) -> tuple[Path, str] | None:
210
+ repository, prefix = git_worktree_context(target)
211
+ raw_tree = offline_git_bytes(repository, "rev-parse", "--verify", f"{revision}^{{tree}}")
212
+ tree = raw_tree.decode("ascii").strip() if raw_tree is not None else ""
213
+ if not OBJECT_ID.fullmatch(tree):
214
+ return None
215
+ selected = existing_tree_path(repository, tree, prefix, repository)
216
+ if selected is None or selected[1] != "directory":
217
+ return None
218
+ return repository, selected[2]
219
+
220
+
221
+ def capture_source_scopes(
222
+ target: Path,
223
+ target_identity: tuple[str, str | None, int | str, int | str],
224
+ paths: list[str],
225
+ *,
226
+ diff_target_kind: str | None = None,
227
+ ) -> dict[str, Any]:
228
+ revision, snapshot = target_identity[:2]
229
+ result: dict[str, Any] = {"version": 1, "revision": revision, "scopes": []}
230
+ # HEAD cannot represent the uncommitted bytes reviewed by a working-tree scan.
231
+ if diff_target_kind == "working_tree":
232
+ return result
233
+ if revision == "unversioned" or (
234
+ snapshot is not None and snapshot != clean_worktree_content_digest()
235
+ ):
236
+ return result
237
+ try:
238
+ repository, _ = git_worktree_context(target)
239
+ if offline_git_bytes(repository, "replace", "--list") != b"":
240
+ # A revision alone does not identify the replacement view scanned.
241
+ return result
242
+ context = target_tree(target, revision)
243
+ if context is None:
244
+ return result
245
+ repository, tree = context
246
+ result["targetTree"] = tree
247
+ for requested in paths:
248
+ selected = safe_source_path(target, requested)
249
+ if selected is None:
250
+ continue
251
+ canonical = existing_tree_path(repository, tree, requested, target)
252
+ if canonical is None or canonical[1] not in {"file", "directory"}:
253
+ continue
254
+ try:
255
+ metadata = selected.stat()
256
+ except OSError:
257
+ continue
258
+ if (canonical[1] == "directory") != stat.S_ISDIR(metadata.st_mode):
259
+ continue
260
+ result["scopes"].append(
261
+ {
262
+ "path": PurePosixPath(requested).as_posix(),
263
+ "canonicalPath": canonical[0],
264
+ "kind": canonical[1],
265
+ "objectId": canonical[2],
266
+ }
267
+ )
268
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
269
+ # Excerpts are optional. Failure to bind a scope never widens it and
270
+ # must not prevent the scan itself from starting.
271
+ pass
272
+ return result
273
+
274
+
275
+ def load_source_scopes(
276
+ scan: sqlite3.Row, target: Path, requested: list[str]
277
+ ) -> tuple[Path, str, list[dict[str, str]]] | None:
278
+ if "diff_target_kind" in scan.keys() and scan["diff_target_kind"] == "working_tree":
279
+ return None
280
+ saved = scan["source_scopes_json"] if "source_scopes_json" in scan.keys() else None
281
+ paths = {PurePosixPath(path).as_posix() for path in requested}
282
+ if saved is not None:
283
+ metadata = json.loads(saved)
284
+ if (
285
+ not isinstance(metadata, dict)
286
+ or metadata.get("version") != 1
287
+ or metadata.get("revision") != scan["target_revision"]
288
+ ):
289
+ return None
290
+ tree = metadata.get("targetTree")
291
+ records = metadata.get("scopes")
292
+ if (
293
+ not isinstance(tree, str)
294
+ or not OBJECT_ID.fullmatch(tree)
295
+ or not isinstance(records, list)
296
+ ):
297
+ return None
298
+ scopes = [
299
+ record
300
+ for record in records
301
+ if isinstance(record, dict)
302
+ and isinstance(record.get("path"), str)
303
+ and record.get("path") in paths
304
+ and isinstance(record.get("canonicalPath"), str)
305
+ and relative_path(record["canonicalPath"]) is not None
306
+ and record.get("kind") in ("file", "directory")
307
+ and isinstance(record.get("objectId"), str)
308
+ and OBJECT_ID.fullmatch(record["objectId"])
309
+ ]
310
+ return (git_worktree_context(target)[0], tree, scopes) if scopes else None
311
+
312
+ # Exact, unambiguous historical entries establish legacy scope kinds.
313
+ # Keep any previously recorded file kinds as an additional constraint.
314
+ recipe = scan["recipe_json"] if "recipe_json" in scan.keys() else None
315
+ legacy_recipe = json.loads(recipe) if recipe else None
316
+ legacy_files = (
317
+ legacy_recipe.get("_codexSecurityFileScopes") if isinstance(legacy_recipe, dict) else None
318
+ )
319
+ repository, _ = git_worktree_context(target)
320
+ if offline_git_bytes(repository, "replace", "--list") != b"":
321
+ # Old records do not say which replacement view was scanned.
322
+ return None
323
+ context = target_tree(target, scan["target_revision"])
324
+ if context is None:
325
+ return None
326
+ repository, tree = context
327
+ scopes = []
328
+ for path in paths:
329
+ entry = exact_tree_path(repository, tree, path, require_unambiguous=True)
330
+ if entry is None or entry[1] not in {"file", "directory"}:
331
+ continue
332
+ kind = entry[1]
333
+ if isinstance(legacy_files, list) and kind != (
334
+ "file" if path in legacy_files else "directory"
335
+ ):
336
+ continue
337
+ scopes.append(
338
+ {
339
+ "path": path,
340
+ "canonicalPath": path,
341
+ "kind": kind,
342
+ "objectId": entry[2],
343
+ }
344
+ )
345
+ return (repository, tree, scopes) if scopes else None
346
+
347
+
348
+ def source_object_for_path(
349
+ repository: Path, tree: str, target: Path, value: str, scope: dict[str, str]
350
+ ) -> str | None:
351
+ path = relative_path(value)
352
+ if path is None or safe_source_path(target, value) is None:
353
+ return None
354
+ prefixes = {PurePosixPath(scope[name]).parts for name in ("path", "canonicalPath")}
355
+ suffix: tuple[str, ...] | None = None
356
+ for prefix in sorted(prefixes, key=len, reverse=True):
357
+ if path.parts[: len(prefix)] == prefix:
358
+ suffix = path.parts[len(prefix) :]
359
+ break
360
+ if suffix is None:
361
+ for length in {len(prefix) for prefix in prefixes}:
362
+ if length > len(path.parts) or scope["kind"] == "file" and length != len(path.parts):
363
+ continue
364
+ selected = existing_tree_path(
365
+ repository, tree, PurePosixPath(*path.parts[:length]).as_posix(), target
366
+ )
367
+ if selected == (scope["canonicalPath"], scope["kind"], scope["objectId"]):
368
+ suffix = path.parts[length:]
369
+ break
370
+ if suffix is None:
371
+ return None
372
+ if scope["kind"] == "file":
373
+ return scope["objectId"] if not suffix else None
374
+ relative = PurePosixPath(*suffix).as_posix()
375
+ entry = exact_tree_path(repository, scope["objectId"], relative, require_unambiguous=True)
376
+ if entry is None:
377
+ selected = safe_source_path(target, scope["canonicalPath"])
378
+ if selected is not None:
379
+ entry = existing_tree_path(repository, scope["objectId"], relative, selected)
380
+ return entry[2] if entry is not None and entry[1] == "file" else None
381
+
382
+
383
+ def main() -> None:
384
+ argparse.ArgumentParser(description=__doc__).parse_args()
385
+
386
+
387
+ if __name__ == "__main__":
388
+ main()
@@ -24,6 +24,7 @@ from rank_preview import DEFAULT_PREVIEW_READ_BYTES, is_binary_sample
24
24
  from windows_scan_local_files import copy_directory_junction
25
25
  from workbench_constants import GIT_REPOSITORY_ENVIRONMENT
26
26
 
27
+ SOURCE_METADATA_DIRECTORIES = {".git", ".svn"}
27
28
  _WINDOWS = os.name == "nt"
28
29
 
29
30
 
@@ -87,7 +88,10 @@ def git_blob_samples(
87
88
  if not object_names:
88
89
  return []
89
90
 
90
- request = b"\0".join(os.fsencode(name) for name in object_names) + b"\0"
91
+ # Git before 2.38 rejects -z, so only names with line breaks need it.
92
+ nul_input = any("\n" in name or "\r" in name for name in object_names)
93
+ separator = b"\0" if nul_input else b"\n"
94
+ request = separator.join(os.fsencode(name) for name in object_names) + separator
91
95
  read_fd, write_fd = os.pipe()
92
96
  with (
93
97
  ThreadPoolExecutor(max_workers=1) as reader,
@@ -106,7 +110,7 @@ def git_blob_samples(
106
110
  target,
107
111
  "cat-file",
108
112
  "--batch",
109
- "-z",
113
+ *(["-z"] if nul_input else []),
110
114
  text=False,
111
115
  input_data=request,
112
116
  git_dir=git_dir,
@@ -444,10 +448,16 @@ def worktree_content_digest_for_context(
444
448
  ) -> str:
445
449
  digest = hashlib.sha256()
446
450
  update_digest_field(digest, b"format", b"codex-security-snapshot/v1")
451
+ source_pathspecs = [
452
+ f":(literal){pathspec}",
453
+ ":(top,glob,exclude)**/.svn",
454
+ ":(top,glob,exclude)**/.svn/**",
455
+ ]
447
456
  tracked_ok = _update_digest_field_from_git(
448
457
  digest,
449
458
  b"tracked-diff",
450
459
  repository,
460
+ "--no-literal-pathspecs",
451
461
  "diff",
452
462
  "--binary",
453
463
  "--full-index",
@@ -456,18 +466,19 @@ def worktree_content_digest_for_context(
456
466
  "--ignore-submodules=none",
457
467
  "HEAD",
458
468
  "--",
459
- pathspec,
469
+ *source_pathspecs,
460
470
  git_dir=git_dir,
461
471
  work_tree=work_tree,
462
472
  )
463
473
  untracked = git_bytes(
464
474
  repository,
475
+ "--no-literal-pathspecs",
465
476
  "ls-files",
466
477
  "--others",
467
478
  "--exclude-standard",
468
479
  "-z",
469
480
  "--",
470
- pathspec,
481
+ *source_pathspecs,
471
482
  git_dir=git_dir,
472
483
  work_tree=work_tree,
473
484
  )
@@ -670,6 +681,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
670
681
  nested_worktrees: set[tuple[int, int]] = set()
671
682
  for entry in (entry for entry in listed.split(b"\0") if entry):
672
683
  relative = Path(os.fsdecode(entry[2:]))
684
+ if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
685
+ continue
673
686
  if scope_depth:
674
687
  if len(relative.parts) <= scope_depth:
675
688
  continue
@@ -741,6 +754,8 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
741
754
  while pending:
742
755
  children: list[Path] = []
743
756
  for path in pending:
757
+ if not SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(target).parts):
758
+ continue
744
759
  try:
745
760
  path = _directory_link_boundary(target, path, linked_prefixes)
746
761
  metadata = path.lstat()
@@ -761,7 +776,7 @@ def git_directory_snapshot_paths(target: Path) -> list[Path] | None:
761
776
  continue
762
777
  candidates: dict[bytes, Path] = {}
763
778
  for path in children:
764
- if os.path.normcase(path.name) == ".git":
779
+ if os.path.normcase(path.name) in SOURCE_METADATA_DIRECTORIES:
765
780
  continue
766
781
  try:
767
782
  metadata = path.lstat()
@@ -809,7 +824,7 @@ def source_directory_snapshot_paths(
809
824
  parent = Path(directory)
810
825
  for name in directories[:]:
811
826
  path = parent / name
812
- if name == ".git" or path in excluded:
827
+ if name in SOURCE_METADATA_DIRECTORIES or path in excluded:
813
828
  directories.remove(name)
814
829
  continue
815
830
  paths.append(path)
@@ -817,7 +832,9 @@ def source_directory_snapshot_paths(
817
832
  if getattr(path.lstat(), "st_reparse_tag", 0) & 0x20000000:
818
833
  directories.remove(name)
819
834
  paths.extend(
820
- parent / name for name in files if name != ".git" and parent / name not in excluded
835
+ parent / name
836
+ for name in files
837
+ if name not in SOURCE_METADATA_DIRECTORIES and parent / name not in excluded
821
838
  )
822
839
  return sorted(paths)
823
840
 
@@ -840,7 +857,7 @@ def directory_content_digest(
840
857
  else git_directory_snapshot_paths(target)
841
858
  )
842
859
  if paths is None:
843
- paths = source_directory_snapshot_paths(target)
860
+ paths = source_directory_snapshot_paths(target, excluded)
844
861
  digest = hashlib.sha256()
845
862
  update_digest_field(digest, b"format", b"codex-security-directory/v1")
846
863
  for path in paths:
@@ -981,9 +998,8 @@ def copy_git_worktree_files(
981
998
  continue
982
999
  source_path = repository / relative
983
1000
  try:
984
- if _WINDOWS:
985
- source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
986
- relative = source_path.relative_to(repository)
1001
+ source_path = _directory_link_boundary(repository, source_path, linked_prefixes)
1002
+ relative = source_path.relative_to(repository)
987
1003
  metadata = source_path.lstat()
988
1004
  except FileNotFoundError:
989
1005
  continue
@@ -1056,7 +1072,7 @@ def git_target_metadata(target: Path) -> dict[str, Any]:
1056
1072
  branch = git_output(target, "symbolic-ref", "--quiet", "--short", "HEAD")
1057
1073
  metadata.update({"branch": branch, "detachedHead": revision is not None and branch is None})
1058
1074
  if revision is not None:
1059
- subject = git_bytes(target, "show", "-s", "--format=%s", "HEAD")
1075
+ subject = git_bytes(target, "show", "-s", "--no-show-signature", "--format=%s", "HEAD")
1060
1076
  metadata.update(
1061
1077
  {
1062
1078
  "commitSubject": (subject or b"").decode("utf-8", errors="replace").strip() or None,
@@ -16,10 +16,11 @@ from typing import Any
16
16
  # Some plugin hosts launch Python with safe-path isolation enabled.
17
17
  sys.path.insert(0, str(Path(__file__).resolve().parent))
18
18
  import finalize_scan_contract as finalizer
19
- from workbench_scan_usage import TOKEN_FIELDS
19
+ from workbench_scan_usage import TOKEN_FIELDS, valid_account_rate_limit
20
20
  from workbench_scan_usage import (
21
21
  _reject_nonstandard_json_number as reject_nonstandard_json_number,
22
22
  )
23
+ from workbench_scan_usage import _timestamp as usage_timestamp
23
24
 
24
25
  reject_non_finite_json = finalizer._reject_non_finite_json
25
26
 
@@ -68,16 +69,19 @@ SCAN_USAGE_TOKEN_KEYS = tuple(TOKEN_FIELDS.values())
68
69
 
69
70
  def _valid_legacy_scan_cost(cost: object) -> bool:
70
71
  token_keys = ("inputTokens", "cachedInputTokens", "cacheWriteInputTokens", "outputTokens")
71
- return (
72
- isinstance(cost, dict)
73
- and isinstance(cost.get("model"), str)
74
- and bool(cost["model"])
75
- and all(type(cost.get(key)) is int and cost[key] >= 0 for key in token_keys)
76
- and cost["cachedInputTokens"] + cost["cacheWriteInputTokens"] <= cost["inputTokens"]
77
- and type(cost.get("estimatedUsd")) in (int, float)
78
- and math.isfinite(cost["estimatedUsd"])
79
- and cost["estimatedUsd"] >= 0
80
- )
72
+ try:
73
+ return (
74
+ isinstance(cost, dict)
75
+ and isinstance(cost.get("model"), str)
76
+ and bool(cost["model"])
77
+ and all(type(cost.get(key)) is int and cost[key] >= 0 for key in token_keys)
78
+ and cost["cachedInputTokens"] + cost["cacheWriteInputTokens"] <= cost["inputTokens"]
79
+ and type(cost.get("estimatedUsd")) in (int, float)
80
+ and math.isfinite(cost["estimatedUsd"])
81
+ and cost["estimatedUsd"] >= 0
82
+ )
83
+ except OverflowError:
84
+ return False
81
85
 
82
86
 
83
87
  def _valid_measured_scan_usage(usage: object) -> bool:
@@ -86,7 +90,7 @@ def _valid_measured_scan_usage(usage: object) -> bool:
86
90
  coverage = usage.get("coverage")
87
91
  thread_count = usage.get("threadCount")
88
92
  if (
89
- coverage not in {"complete", "partial", "unavailable"}
93
+ coverage not in ("complete", "partial", "unavailable")
90
94
  or usage.get("source") != "codex_rollout"
91
95
  or type(thread_count) is not int
92
96
  or thread_count < 0
@@ -116,9 +120,15 @@ def _valid_measured_scan_usage(usage: object) -> bool:
116
120
  or len(set(warnings)) != len(warnings)
117
121
  ):
118
122
  return False
123
+ if "accountRateLimits" in usage:
124
+ limits = usage["accountRateLimits"]
125
+ if not isinstance(limits, list) or not all(
126
+ valid_account_rate_limit(limit) for limit in limits
127
+ ):
128
+ return False
119
129
  if coverage == "unavailable":
120
130
  return thread_count == 0 and set(usage).issubset(
121
- {"coverage", "source", "threadCount", "warnings", "completion"}
131
+ {"coverage", "source", "threadCount", "warnings", "completion", "accountRateLimits"}
122
132
  )
123
133
 
124
134
  allowed_keys = {
@@ -128,6 +138,11 @@ def _valid_measured_scan_usage(usage: object) -> bool:
128
138
  "missingThreadCount",
129
139
  "warnings",
130
140
  "completion",
141
+ "byRole",
142
+ "window",
143
+ "observedAt",
144
+ "tokensUpdatedAt",
145
+ "accountRateLimits",
131
146
  *SCAN_USAGE_TOKEN_KEYS,
132
147
  }
133
148
  if thread_count == 0 or not set(usage).issubset(allowed_keys):
@@ -144,6 +159,30 @@ def _valid_measured_scan_usage(usage: object) -> bool:
144
159
  return False
145
160
  if coverage == "partial" and not (warnings or missing):
146
161
  return False
162
+ for field in ("observedAt", "tokensUpdatedAt"):
163
+ if field in usage and usage_timestamp(usage[field]) is None:
164
+ return False
165
+ if "window" in usage:
166
+ window = usage["window"]
167
+ if not isinstance(window, dict) or set(window) != {"startedAt", "endedAt"}:
168
+ return False
169
+ start, end = (usage_timestamp(window[key]) for key in ("startedAt", "endedAt"))
170
+ if start is None or end is None or start > end:
171
+ return False
172
+ if "byRole" in usage:
173
+ roles = usage["byRole"]
174
+ keys = {*SCAN_USAGE_TOKEN_KEYS, "threadCount"}
175
+ if not isinstance(roles, dict) or set(roles) != {"parent", "workers"}:
176
+ return False
177
+ if any(
178
+ not isinstance(role, dict)
179
+ or set(role) != keys
180
+ or any(type(role[key]) is not int or role[key] < 0 for key in keys)
181
+ for role in roles.values()
182
+ ):
183
+ return False
184
+ if any(sum(role[key] for role in roles.values()) != usage[key] for key in keys):
185
+ return False
147
186
  return True
148
187
 
149
188
 
@@ -1,4 +1,23 @@
1
1
  [
2
+ [
3
+ "-- Move source-scope previews before the shared history repairs.",
4
+ "WITH input AS (SELECT :history AS history)",
5
+ "SELECT 'sql' AS operation,",
6
+ " 'UPDATE schema_migrations SET version = 50 WHERE version = ' || version AS value",
7
+ "FROM schema_migrations",
8
+ "WHERE version IN (34, 40, 41, 42, 43, 44, 48)",
9
+ " AND name = 'persist authorized source excerpt scopes';"
10
+ ],
11
+ [
12
+ "WITH history AS (",
13
+ " SELECT json_extract(value, '$.version') AS version,",
14
+ " json_extract(value, '$.name') AS name FROM json_each(:history)",
15
+ ")",
16
+ "SELECT 'error' AS operation,",
17
+ " 'The Codex Security database has an unsupported source-scope migration history.' AS value",
18
+ "FROM schema_migrations JOIN history USING (version)",
19
+ "WHERE version IN (43, 50) AND schema_migrations.name != history.name;"
20
+ ],
2
21
  [
3
22
  "-- mirror and early-moves: relocate numbers before checking profiles.",
4
23
  "WITH",
@@ -410,5 +410,20 @@
410
410
  "statements": [
411
411
  "CREATE INDEX finding_issue_receipts_by_destination_issue\nON finding_issue_receipts(destination_key, external_id, finding_id);"
412
412
  ]
413
+ },
414
+ {
415
+ "version": 50,
416
+ "name": "persist authorized source excerpt scopes",
417
+ "statements": ["ALTER TABLE scans ADD COLUMN source_scopes_json TEXT;"]
418
+ },
419
+ {
420
+ "version": 51,
421
+ "name": "preserve deep scan worker thread history",
422
+ "statements": [
423
+ "CREATE TABLE deep_scan_worker_threads (\n scan_id TEXT NOT NULL REFERENCES scans(id) ON DELETE CASCADE,\n worker_id TEXT NOT NULL,\n sdk_thread_id TEXT NOT NULL,\n PRIMARY KEY (scan_id, worker_id, sdk_thread_id)\n);",
424
+ "INSERT INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\nSELECT scan_id, id, sdk_thread_id FROM deep_scan_workers\nWHERE sdk_thread_id IS NOT NULL;",
425
+ "CREATE TRIGGER record_deep_scan_worker_thread_insert\nAFTER INSERT ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;",
426
+ "CREATE TRIGGER record_deep_scan_worker_thread_update\nAFTER UPDATE OF sdk_thread_id ON deep_scan_workers\nWHEN NEW.sdk_thread_id IS NOT NULL\nBEGIN\n INSERT OR IGNORE INTO deep_scan_worker_threads (scan_id, worker_id, sdk_thread_id)\n VALUES (NEW.scan_id, NEW.id, NEW.sdk_thread_id);\nEND;"
427
+ ]
413
428
  }
414
429
  ]