@openai/codex-security 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +68 -37
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  5. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
  6. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  7. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  8. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  9. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  10. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  11. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  12. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  13. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  14. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  15. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  16. package/_bundled_plugin/references/access-advisory.md +11 -0
  17. package/_bundled_plugin/references/core-scan.md +30 -77
  18. package/_bundled_plugin/references/final-report.md +45 -189
  19. package/_bundled_plugin/references/scan-contract.md +2 -0
  20. package/_bundled_plugin/references/scan-prologue.md +1 -1
  21. package/_bundled_plugin/references/scan-runtime.md +1 -5
  22. package/_bundled_plugin/references/threat-model.md +33 -41
  23. package/_bundled_plugin/schemas/coverage.schema.json +12 -1
  24. package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
  25. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
  26. package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
  27. package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
  28. package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
  29. package/_bundled_plugin/scripts/report_projection.py +44 -15
  30. package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
  31. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
  32. package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
  33. package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
  34. package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
  35. package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
  36. package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
  37. package/_bundled_plugin/scripts/workbench_cli.py +3 -0
  38. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
  39. package/_bundled_plugin/scripts/workbench_db.py +51 -22
  40. package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
  41. package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
  42. package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
  43. package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
  44. package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
  45. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  46. package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
  47. package/_bundled_plugin/scripts/workbench_target.py +142 -17
  48. package/_bundled_plugin/scripts/workbench_validation.py +9 -2
  49. package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
  50. package/_bundled_plugin/shared/workbench-migrations.json +28 -0
  51. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
  52. package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
  53. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
  54. package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
  55. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
  56. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
  57. package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
  58. package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
  59. package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
  60. package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
  61. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
  62. package/_bundled_plugin/skills/publish-findings/SKILL.md +28 -24
  63. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
  64. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
  65. package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
  66. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
  67. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
  68. package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
  69. package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
  70. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
  71. package/_bundled_plugin/skills/validation/SKILL.md +42 -104
  72. package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
  73. package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
  74. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
  75. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
  76. package/dist/api.d.ts +5 -3
  77. package/dist/api.d.ts.map +1 -1
  78. package/dist/api.js +341 -37
  79. package/dist/api.js.map +1 -1
  80. package/dist/auth.d.ts.map +1 -1
  81. package/dist/auth.js +4 -1
  82. package/dist/auth.js.map +1 -1
  83. package/dist/cli.d.ts +2 -2
  84. package/dist/cli.d.ts.map +1 -1
  85. package/dist/cli.js +674 -419
  86. package/dist/cli.js.map +1 -1
  87. package/dist/component-scan.d.ts +8 -2
  88. package/dist/component-scan.d.ts.map +1 -1
  89. package/dist/component-scan.js +2 -0
  90. package/dist/component-scan.js.map +1 -1
  91. package/dist/contract.d.ts +2 -0
  92. package/dist/contract.d.ts.map +1 -1
  93. package/dist/contract.js +67 -33
  94. package/dist/contract.js.map +1 -1
  95. package/dist/cost.d.ts +18 -0
  96. package/dist/cost.d.ts.map +1 -1
  97. package/dist/cost.js +107 -12
  98. package/dist/cost.js.map +1 -1
  99. package/dist/coverage-presentation.d.ts +6 -0
  100. package/dist/coverage-presentation.d.ts.map +1 -0
  101. package/dist/coverage-presentation.js +34 -0
  102. package/dist/coverage-presentation.js.map +1 -0
  103. package/dist/custom-validation-prompt.js +3 -3
  104. package/dist/custom-validation-prompt.js.map +1 -1
  105. package/dist/custom-validation.d.ts.map +1 -1
  106. package/dist/custom-validation.js +30 -4
  107. package/dist/custom-validation.js.map +1 -1
  108. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  109. package/dist/deduplication/checkpointed-review.js +3 -1
  110. package/dist/deduplication/checkpointed-review.js.map +1 -1
  111. package/dist/deduplication/scan.d.ts.map +1 -1
  112. package/dist/deduplication/scan.js +7 -5
  113. package/dist/deduplication/scan.js.map +1 -1
  114. package/dist/endor-findings.d.ts +4 -3
  115. package/dist/endor-findings.d.ts.map +1 -1
  116. package/dist/endor-findings.js +16 -11
  117. package/dist/endor-findings.js.map +1 -1
  118. package/dist/external-findings-import.d.ts.map +1 -1
  119. package/dist/external-findings-import.js +13 -5
  120. package/dist/external-findings-import.js.map +1 -1
  121. package/dist/external-findings-local.d.ts.map +1 -1
  122. package/dist/external-findings-local.js +8 -1
  123. package/dist/external-findings-local.js.map +1 -1
  124. package/dist/external-findings-publish.d.ts +1 -1
  125. package/dist/external-findings-publish.d.ts.map +1 -1
  126. package/dist/external-findings-publish.js +11 -3
  127. package/dist/external-findings-publish.js.map +1 -1
  128. package/dist/external-findings-read.d.ts +11 -1
  129. package/dist/external-findings-read.d.ts.map +1 -1
  130. package/dist/external-findings-read.js +15 -6
  131. package/dist/external-findings-read.js.map +1 -1
  132. package/dist/external-import-contract.d.ts +3 -1
  133. package/dist/external-import-contract.d.ts.map +1 -1
  134. package/dist/external-import-contract.js +33 -25
  135. package/dist/external-import-contract.js.map +1 -1
  136. package/dist/external-imports-feature.d.ts +6 -0
  137. package/dist/external-imports-feature.d.ts.map +1 -0
  138. package/dist/external-imports-feature.js +51 -0
  139. package/dist/external-imports-feature.js.map +1 -0
  140. package/dist/finding-workflow.d.ts +22 -5
  141. package/dist/finding-workflow.d.ts.map +1 -1
  142. package/dist/finding-workflow.js +21 -9
  143. package/dist/finding-workflow.js.map +1 -1
  144. package/dist/github-advisories.d.ts.map +1 -1
  145. package/dist/github-advisories.js +3 -1
  146. package/dist/github-advisories.js.map +1 -1
  147. package/dist/github-findings.d.ts +4 -3
  148. package/dist/github-findings.d.ts.map +1 -1
  149. package/dist/github-findings.js +16 -14
  150. package/dist/github-findings.js.map +1 -1
  151. package/dist/hosted-remediation.d.ts.map +1 -1
  152. package/dist/hosted-remediation.js +19 -2
  153. package/dist/hosted-remediation.js.map +1 -1
  154. package/dist/hosted-scan.d.ts +1 -1
  155. package/dist/hosted-scan.d.ts.map +1 -1
  156. package/dist/hosted-scan.js +0 -5
  157. package/dist/hosted-scan.js.map +1 -1
  158. package/dist/import-scan.js +1 -1
  159. package/dist/import-scan.js.map +1 -1
  160. package/dist/index.d.ts +3 -3
  161. package/dist/index.d.ts.map +1 -1
  162. package/dist/index.js +1 -1
  163. package/dist/index.js.map +1 -1
  164. package/dist/knowledge-base.d.ts +1 -0
  165. package/dist/knowledge-base.d.ts.map +1 -1
  166. package/dist/knowledge-base.js +17 -8
  167. package/dist/knowledge-base.js.map +1 -1
  168. package/dist/models.d.ts +1 -0
  169. package/dist/models.d.ts.map +1 -1
  170. package/dist/models.js.map +1 -1
  171. package/dist/owner-evidence.d.ts +2 -0
  172. package/dist/owner-evidence.d.ts.map +1 -1
  173. package/dist/owner-evidence.js +2 -1
  174. package/dist/owner-evidence.js.map +1 -1
  175. package/dist/publish.d.ts +2 -0
  176. package/dist/publish.d.ts.map +1 -1
  177. package/dist/publish.js +1 -1
  178. package/dist/publish.js.map +1 -1
  179. package/dist/remediation.d.ts +1 -25
  180. package/dist/remediation.d.ts.map +1 -1
  181. package/dist/remediation.js +3 -24
  182. package/dist/remediation.js.map +1 -1
  183. package/dist/runtime.d.ts.map +1 -1
  184. package/dist/runtime.js +7 -7
  185. package/dist/runtime.js.map +1 -1
  186. package/dist/scan-dashboard.d.ts +5 -3
  187. package/dist/scan-dashboard.d.ts.map +1 -1
  188. package/dist/scan-dashboard.js +79 -26
  189. package/dist/scan-dashboard.js.map +1 -1
  190. package/dist/scan-history-renderer.d.ts.map +1 -1
  191. package/dist/scan-history-renderer.js +8 -2
  192. package/dist/scan-history-renderer.js.map +1 -1
  193. package/dist/scan-owner-suggestions.d.ts +1 -1
  194. package/dist/scan-owner-suggestions.d.ts.map +1 -1
  195. package/dist/scan-owner-suggestions.js +4 -4
  196. package/dist/scan-owner-suggestions.js.map +1 -1
  197. package/dist/scan-usage-display.d.ts +8 -0
  198. package/dist/scan-usage-display.d.ts.map +1 -0
  199. package/dist/scan-usage-display.js +46 -0
  200. package/dist/scan-usage-display.js.map +1 -0
  201. package/dist/scanner-import.d.ts +1 -1
  202. package/dist/scanner-import.d.ts.map +1 -1
  203. package/dist/scanner-import.js +7 -3
  204. package/dist/scanner-import.js.map +1 -1
  205. package/dist/security-policy.d.ts.map +1 -1
  206. package/dist/security-policy.js +1 -20
  207. package/dist/security-policy.js.map +1 -1
  208. package/dist/semgrep-findings.d.ts +4 -3
  209. package/dist/semgrep-findings.d.ts.map +1 -1
  210. package/dist/semgrep-findings.js +9 -9
  211. package/dist/semgrep-findings.js.map +1 -1
  212. package/dist/skill-authentication.d.ts +18 -0
  213. package/dist/skill-authentication.d.ts.map +1 -0
  214. package/dist/skill-authentication.js +82 -0
  215. package/dist/skill-authentication.js.map +1 -0
  216. package/dist/snyk-findings.d.ts +5 -4
  217. package/dist/snyk-findings.d.ts.map +1 -1
  218. package/dist/snyk-findings.js +18 -17
  219. package/dist/snyk-findings.js.map +1 -1
  220. package/dist/suggest-owners.d.ts +4 -1
  221. package/dist/suggest-owners.d.ts.map +1 -1
  222. package/dist/suggest-owners.js +53 -21
  223. package/dist/suggest-owners.js.map +1 -1
  224. package/dist/targets.d.ts.map +1 -1
  225. package/dist/targets.js +4 -1
  226. package/dist/targets.js.map +1 -1
  227. package/dist/wiz-csv.d.ts +2 -1
  228. package/dist/wiz-csv.d.ts.map +1 -1
  229. package/dist/wiz-csv.js +3 -3
  230. package/dist/wiz-csv.js.map +1 -1
  231. package/dist/wiz-findings.d.ts +2 -1
  232. package/dist/wiz-findings.d.ts.map +1 -1
  233. package/dist/wiz-findings.js +27 -23
  234. package/dist/wiz-findings.js.map +1 -1
  235. package/docs/classify-severity-records.md +3 -4
  236. package/docs/cli.md +155 -38
  237. package/docs/findings-service.md +119 -118
  238. package/docs/hosted-remediation.md +19 -17
  239. package/docs/hosted-scans.md +37 -11
  240. package/package.json +8 -8
  241. package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
  242. package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
  243. package/_bundled_plugin/scripts/rank_preview.py +0 -129
@@ -1,350 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Generate and post-process Codex Security scan worklists.
3
-
4
- This script stays deliberately model-free:
5
-
6
- - `make-repo-scope-input` lists every explicitly scoped repository file.
7
- - `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
8
- worklist from Git changed paths. It supports committed revision diffs and
9
- local working-tree patches.
10
-
11
- Candidate selection uses repository scope, native tool ignore rules, and binary detection,
12
- not filename or directory classifications. Diff inputs use bounded source previews.
13
- """
14
-
15
- from __future__ import annotations
16
-
17
- import argparse
18
- import json
19
- import os
20
- import subprocess
21
- import sys
22
- from collections.abc import Iterable
23
- from pathlib import Path
24
-
25
- # Some plugin hosts launch Python with safe-path isolation enabled.
26
- sys.path.insert(0, str(Path(__file__).resolve().parent))
27
- from generate_in_scope_files import windows_stream_component
28
- from rank_preview import (
29
- DEFAULT_PREVIEW_BYTES,
30
- is_binary_file,
31
- preview_for,
32
- preview_for_bytes,
33
- )
34
- from workbench_target import (
35
- SOURCE_METADATA_DIRECTORIES,
36
- git_blob_samples,
37
- git_command,
38
- git_directory_snapshot_paths,
39
- )
40
-
41
- JsonRow = dict[str, object]
42
-
43
-
44
- def parse_args() -> argparse.Namespace:
45
- parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
46
- subparsers = parser.add_subparsers(dest="command", required=True)
47
-
48
- scoped = subparsers.add_parser(
49
- "make-repo-scope-input",
50
- help="List every explicitly scoped file without ranking or reading its contents.",
51
- )
52
- scoped.add_argument("--repo", required=True, help="Repository root.")
53
- scoped.add_argument(
54
- "--scopes-file",
55
- required=True,
56
- help="JSON array of repository-relative files and directories to scan together.",
57
- )
58
- scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
59
-
60
- diff = subparsers.add_parser(
61
- "make-diff-rank-input",
62
- help="Create rank_input.jsonl from Git changed text files.",
63
- )
64
- diff.add_argument("--repo", required=True, help="Repository root.")
65
- diff.add_argument("--base", required=True, help="Git diff base revision.")
66
- diff.add_argument(
67
- "--mode",
68
- choices=("revisions", "local-patch"),
69
- default="revisions",
70
- help="Git diff mode: committed revisions or staged plus unstaged local patch.",
71
- )
72
- diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
73
- diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
74
- diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")
75
- diff.add_argument(
76
- "--preview-bytes",
77
- type=int,
78
- default=DEFAULT_PREVIEW_BYTES,
79
- help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
80
- )
81
-
82
- return parser.parse_args()
83
-
84
-
85
- def resolve_scope(repo: Path, scope: str) -> Path:
86
- scope_path = Path(scope)
87
- stream = windows_stream_component(scope_path)
88
- if stream is not None:
89
- raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")
90
- if not scope_path.is_absolute():
91
- scope_path = repo / scope_path
92
- repository = repo.resolve()
93
- try:
94
- relative = scope_path.relative_to(repository)
95
- except ValueError as exc:
96
- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
97
- ancestor = repository
98
- for part in relative.parts:
99
- if part == "..":
100
- if ancestor == repository:
101
- raise SystemExit(f"Scope must be inside repo: {scope_path}")
102
- ancestor = ancestor.parent
103
- continue
104
- ancestor /= part
105
- try:
106
- metadata = ancestor.stat(follow_symlinks=False)
107
- except OSError as exc:
108
- raise SystemExit(f"Scope path not found: {ancestor}") from exc
109
- if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
110
- raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
111
- scope_path = scope_path.resolve()
112
- repo_resolved = repo.resolve()
113
- try:
114
- scope_path.relative_to(repo_resolved)
115
- except ValueError as exc:
116
- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
117
- if not scope_path.is_dir() and not scope_path.is_file():
118
- raise SystemExit(f"Scope path not found: {scope_path}")
119
- return scope_path
120
-
121
-
122
- def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
123
- output.parent.mkdir(parents=True, exist_ok=True)
124
- with output.open("w", encoding="utf-8") as handle:
125
- for row in rows:
126
- handle.write(json.dumps(row, ensure_ascii=True, separators=(",", ":")))
127
- handle.write("\n")
128
-
129
-
130
- def load_scopes_file(scopes_file: Path) -> list[str]:
131
- try:
132
- loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
133
- except (OSError, UnicodeError, json.JSONDecodeError) as exc:
134
- raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc
135
- if (
136
- not isinstance(loaded, list)
137
- or not loaded
138
- or any(not isinstance(scope, str) or not scope for scope in loaded)
139
- ):
140
- raise SystemExit(f"Scopes file must contain a non-empty JSON string array: {scopes_file}")
141
- return loaded
142
-
143
-
144
- def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
145
- """Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
146
-
147
- Git retains tracked files and applies its standard exclusions to untracked files.
148
- Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
149
- Explicit file scopes bypass directory ignore rules.
150
- """
151
- if scope_path.is_file():
152
- return (scope_path,)
153
- git_candidates = git_directory_snapshot_paths(scope_path)
154
- if git_candidates is not None:
155
- return git_candidates
156
-
157
- command = [
158
- "rg",
159
- "--files",
160
- "--hidden",
161
- "--no-require-git",
162
- "--null",
163
- # Also exclude descendants when the scope starts inside metadata.
164
- "--glob",
165
- "!**/{.git,.svn}",
166
- "--glob",
167
- "!**/{.git,.svn}/**",
168
- "--",
169
- str(scope_path.relative_to(repo)),
170
- ]
171
- try:
172
- result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
173
- except OSError as exc:
174
- ignore_names = (".gitignore", ".ignore", ".rgignore")
175
- ancestors = (scope_path, *scope_path.parents)
176
- has_ignore_rules = (
177
- any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
178
- or any(
179
- (ancestor / name).is_file()
180
- for ancestor in ancestors
181
- if ancestor == repo or repo in ancestor.parents
182
- for name in ignore_names
183
- )
184
- or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
185
- )
186
- if has_ignore_rules:
187
- raise SystemExit(
188
- "Could not safely enumerate ignored scoped files without Git or ripgrep."
189
- ) from exc
190
- return scope_path.rglob("*")
191
-
192
- if result.returncode not in (0, 1):
193
- detail = result.stderr.decode("utf-8", errors="replace").strip()
194
- raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
195
- return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
196
-
197
-
198
- def make_repo_scope_input(args: argparse.Namespace) -> None:
199
- repo = Path(args.repo).expanduser().resolve()
200
- if not repo.is_dir():
201
- raise SystemExit(f"Repo path not found: {repo}")
202
-
203
- scopes = load_scopes_file(Path(args.scopes_file).expanduser())
204
- rows_by_path: dict[str, JsonRow] = {}
205
- for scope in scopes:
206
- scope_path = resolve_scope(repo, scope)
207
- candidates = scope_candidates(repo, scope_path)
208
- for path in candidates:
209
- try:
210
- if path.is_symlink() or not path.is_file():
211
- continue
212
- relative = path.resolve(strict=True).relative_to(repo)
213
- except (OSError, ValueError):
214
- continue
215
- if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
216
- continue
217
- rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()})
218
-
219
- rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
220
- output = Path(args.out).expanduser()
221
- write_jsonl(output, rows)
222
- print(f"Wrote {len(rows)} scoped paths to {output}")
223
-
224
-
225
- def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
226
- """Return changed regular files from the selected side of each change."""
227
- result = git_command(
228
- repo,
229
- "diff",
230
- "--ignore-submodules=all",
231
- "--raw",
232
- "-z",
233
- "--diff-filter=ACMRDT",
234
- *diff_args,
235
- "--",
236
- text=False,
237
- )
238
- result.check_returncode()
239
- fields = result.stdout.split(b"\0")
240
- if fields and not fields[-1]:
241
- fields.pop()
242
-
243
- changed: list[tuple[Path, str]] = []
244
- index = 0
245
- while index < len(fields):
246
- metadata = fields[index].split()
247
- status = chr(metadata[-1][0])
248
- index += 1
249
- if status in {"C", "R"}:
250
- index += 1
251
- path = repo / os.fsdecode(fields[index])
252
- index += 1
253
- selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
254
- if selected_mode.startswith(b"100"):
255
- changed.append((path, status))
256
- return changed
257
-
258
-
259
- def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
260
- if mode == "revisions":
261
- return run_git_changed_paths(repo, [f"{base}..{head}"])
262
- if mode == "local-patch":
263
- unstaged = run_git_changed_paths(repo, [base])
264
- staged = run_git_changed_paths(repo, ["--cached", base])
265
- untracked = git_command(
266
- repo,
267
- "ls-files",
268
- "--others",
269
- "--exclude-standard",
270
- "-z",
271
- text=False,
272
- )
273
- untracked.check_returncode()
274
- combined = dict(staged)
275
- combined.update(unstaged)
276
- combined.update(
277
- (repo / os.fsdecode(relative), "A")
278
- for relative in untracked.stdout.split(b"\0")
279
- if relative and not relative.endswith(b"/")
280
- )
281
- return sorted(
282
- (path, status)
283
- for path, status in combined.items()
284
- if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
285
- and (status == "D" or (not path.is_symlink() and path.is_file()))
286
- )
287
- raise SystemExit(f"Unknown diff mode: {mode}")
288
-
289
-
290
- def make_diff_rank_input(args: argparse.Namespace) -> None:
291
- repo = Path(args.repo).expanduser().resolve()
292
- if not repo.is_dir():
293
- raise SystemExit(f"Repo path not found: {repo}")
294
-
295
- changed = git_changed_paths(repo, args.base, args.head, args.mode)
296
- revision_refs = {
297
- path.relative_to(repo): (
298
- f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
299
- )
300
- for path, status in changed
301
- if args.mode == "revisions" or status == "D"
302
- }
303
- revision_samples = dict(
304
- zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
305
- )
306
-
307
- rows: list[JsonRow] = []
308
- for path, status in changed:
309
- rel = path.relative_to(repo)
310
-
311
- preview = ""
312
- if args.mode == "revisions" or status == "D":
313
- sample = revision_samples[rel]
314
- if sample is None:
315
- revision = args.base if status == "D" else args.head
316
- raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
317
- content, is_binary = sample
318
- if is_binary:
319
- continue
320
- if status != "D":
321
- preview, _ = preview_for_bytes(content, args.preview_bytes)
322
- elif not path.is_symlink() and path.is_file():
323
- try:
324
- path.resolve(strict=True).relative_to(repo)
325
- except (OSError, ValueError):
326
- preview = ""
327
- else:
328
- preview, is_binary = preview_for(path, args.preview_bytes)
329
- if is_binary or is_binary_file(path):
330
- continue
331
- rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
332
-
333
- rows.sort(key=lambda row: str(row["path"]))
334
- output = Path(args.out).expanduser()
335
- write_jsonl(output, rows)
336
- print(f"Wrote {len(rows)} rows to {output}")
337
-
338
-
339
- def main() -> None:
340
- args = parse_args()
341
- if args.command == "make-repo-scope-input":
342
- make_repo_scope_input(args)
343
- elif args.command == "make-diff-rank-input":
344
- make_diff_rank_input(args)
345
- else:
346
- raise SystemExit(f"Unknown command: {args.command}")
347
-
348
-
349
- if __name__ == "__main__":
350
- main()
@@ -1,129 +0,0 @@
1
- """Build bounded source previews for rank inputs regardless of language."""
2
-
3
- from __future__ import annotations
4
-
5
- import argparse
6
- from bisect import bisect_right
7
- from pathlib import Path
8
-
9
- DEFAULT_PREVIEW_BYTES = 1024
10
- DEFAULT_PREVIEW_READ_BYTES = 64 * 1024
11
- PREVIEW_HEAD_LINES = 12
12
- PREVIEW_SAMPLE_LINES = 10
13
- _UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
14
-
15
-
16
- def _decode_source(data: bytes) -> str:
17
- encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8-sig"
18
- return data.decode(encoding, errors="ignore")
19
-
20
-
21
- def is_binary_sample(data: bytes) -> bool:
22
- if data.startswith(_UTF16_BOMS):
23
- return "\0" in _decode_source(data)
24
- return b"\0" in data
25
-
26
-
27
- def is_binary_file(path: Path) -> bool:
28
- """Classify the whole file in bounded chunks."""
29
- try:
30
- with path.open("rb") as source:
31
- sample = source.read(DEFAULT_PREVIEW_READ_BYTES)
32
- if is_binary_sample(sample):
33
- return True
34
- bom = sample[:2] if sample.startswith(_UTF16_BOMS) else b""
35
- while chunk := source.read(DEFAULT_PREVIEW_READ_BYTES):
36
- # Even-sized reads preserve UTF-16 code-unit alignment and byte order.
37
- if is_binary_sample(bom + chunk) if bom else b"\0" in chunk:
38
- return True
39
- except OSError:
40
- return True
41
- return False
42
-
43
-
44
- def truncate_utf8(text: str, max_bytes: int) -> str:
45
- if max_bytes <= 0:
46
- return ""
47
- encoded = text.encode("utf-8")
48
- if len(encoded) <= max_bytes:
49
- return text
50
- return encoded[:max_bytes].decode("utf-8", errors="ignore")
51
-
52
-
53
- def select_preview_lines(lines: list[str]) -> list[str]:
54
- nonblank = [line for line in lines if line.strip()]
55
- head = nonblank[:PREVIEW_HEAD_LINES]
56
- remainder = nonblank[PREVIEW_HEAD_LINES:]
57
- if len(remainder) <= PREVIEW_SAMPLE_LINES:
58
- return [*head, *remainder]
59
-
60
- last_index = len(remainder) - 1
61
- sampled = [
62
- remainder[index * last_index // (PREVIEW_SAMPLE_LINES - 1)]
63
- for index in range(PREVIEW_SAMPLE_LINES)
64
- ]
65
- return [*head, "...", *sampled]
66
-
67
-
68
- def fit_preview_lines(lines: list[str], max_bytes: int) -> str:
69
- if not lines or max_bytes <= 0:
70
- return ""
71
-
72
- full_preview = "\n".join(lines)
73
- if len(full_preview.encode("utf-8")) <= max_bytes:
74
- return full_preview
75
-
76
- def render(line_bytes: int) -> str:
77
- return "\n".join(
78
- line if line == "..." else truncate_utf8(line, line_bytes) for line in lines
79
- )
80
-
81
- high = max(len(line.encode("utf-8")) for line in lines)
82
- limit = bisect_right(
83
- range(high + 1), max_bytes, key=lambda size: len(render(size).encode("utf-8"))
84
- )
85
- best = render(limit - 1) if limit else ""
86
- if best.strip() not in {"", "..."}:
87
- return best
88
- return truncate_utf8(full_preview, max_bytes)
89
-
90
-
91
- def preview_for(
92
- path: Path, preview_bytes: int, *, max_read_bytes: int = DEFAULT_PREVIEW_READ_BYTES
93
- ) -> tuple[str, bool]:
94
- """Preview the first 64 KiB by default, including the binary-detection sample."""
95
- if max_read_bytes <= 0:
96
- raise ValueError("max_read_bytes must be positive")
97
- try:
98
- with path.open("rb") as source:
99
- sample = source.read(min(4096, max_read_bytes))
100
- if is_binary_sample(sample):
101
- return "", True
102
- remaining = source.read(max_read_bytes - len(sample))
103
- data = sample + remaining
104
- except OSError:
105
- return "", True
106
- return preview_for_bytes(data, preview_bytes)
107
-
108
-
109
- def preview_for_bytes(data: bytes, preview_bytes: int) -> tuple[str, bool]:
110
- if is_binary_sample(data):
111
- return "", True
112
- lines = _decode_source(data).replace("\r\n", "\n").replace("\r", "\n").split("\n")
113
- start, end = 0, len(lines)
114
- while start < end and not lines[start].strip():
115
- start += 1
116
- while end > start and not lines[end - 1].strip():
117
- end -= 1
118
- lines = lines[start:end]
119
- text = "\n".join(lines)
120
- if len(text.encode("utf-8")) <= preview_bytes:
121
- return text, False
122
- return fit_preview_lines(select_preview_lines(lines), preview_bytes), False
123
-
124
-
125
- if __name__ == "__main__":
126
- parser = argparse.ArgumentParser(
127
- description="Build bounded source previews for Codex Security rank inputs."
128
- )
129
- parser.parse_args()