@openai/codex-security 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +306 -0
  3. package/_bundled_plugin/.app.json +20 -0
  4. package/_bundled_plugin/.codex-plugin/plugin.json +41 -0
  5. package/_bundled_plugin/.mcp.json +20 -0
  6. package/_bundled_plugin/assets/logo.png +0 -0
  7. package/_bundled_plugin/examples/completed-scan/coverage.json +22 -0
  8. package/_bundled_plugin/examples/completed-scan/findings.json +51 -0
  9. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +43 -0
  10. package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
  11. package/_bundled_plugin/mcp/server.mjs +24 -0
  12. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  13. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  14. package/_bundled_plugin/preflight/capability-profiles.toml +179 -0
  15. package/_bundled_plugin/references/config-preflight.md +115 -0
  16. package/_bundled_plugin/references/final-report.md +205 -0
  17. package/_bundled_plugin/references/finding-detail-fields.md +162 -0
  18. package/_bundled_plugin/references/sarif-adapter.md +26 -0
  19. package/_bundled_plugin/references/scan-artifacts.md +116 -0
  20. package/_bundled_plugin/references/scan-contract.md +164 -0
  21. package/_bundled_plugin/references/security-guidance.md +15 -0
  22. package/_bundled_plugin/references/shared-hard-rules.md +15 -0
  23. package/_bundled_plugin/references/static-finding-assessment.md +75 -0
  24. package/_bundled_plugin/schemas/coverage.schema.json +228 -0
  25. package/_bundled_plugin/schemas/findings.schema.json +359 -0
  26. package/_bundled_plugin/schemas/scan-manifest.schema.json +332 -0
  27. package/_bundled_plugin/scripts/config_preflight.py +901 -0
  28. package/_bundled_plugin/scripts/deep_scan_config.py +108 -0
  29. package/_bundled_plugin/scripts/deep_scan_workbench.py +1547 -0
  30. package/_bundled_plugin/scripts/filesystem_identity.py +30 -0
  31. package/_bundled_plugin/scripts/finalize_scan_contract.py +2208 -0
  32. package/_bundled_plugin/scripts/finding_preview.py +270 -0
  33. package/_bundled_plugin/scripts/generate_rank_input.py +1013 -0
  34. package/_bundled_plugin/scripts/normalize_candidates.py +320 -0
  35. package/_bundled_plugin/scripts/rank_preview.py +968 -0
  36. package/_bundled_plugin/scripts/report_projection.py +844 -0
  37. package/_bundled_plugin/scripts/resolve_security_md.py +158 -0
  38. package/_bundled_plugin/scripts/snapshot_sqlite.py +26 -0
  39. package/_bundled_plugin/scripts/validate_report_format.py +138 -0
  40. package/_bundled_plugin/scripts/validate_scan_contract.py +108 -0
  41. package/_bundled_plugin/scripts/validate_tracking_source.py +92 -0
  42. package/_bundled_plugin/scripts/windows_scan_local_files.py +612 -0
  43. package/_bundled_plugin/scripts/workbench/__init__.py +1 -0
  44. package/_bundled_plugin/scripts/workbench/handoff.py +238 -0
  45. package/_bundled_plugin/scripts/workbench_cli.py +325 -0
  46. package/_bundled_plugin/scripts/workbench_constants.py +81 -0
  47. package/_bundled_plugin/scripts/workbench_db.py +3728 -0
  48. package/_bundled_plugin/scripts/workbench_native_indexes.py +242 -0
  49. package/_bundled_plugin/scripts/workbench_progress.py +226 -0
  50. package/_bundled_plugin/scripts/workbench_remediation.py +125 -0
  51. package/_bundled_plugin/scripts/workbench_scan_history.py +829 -0
  52. package/_bundled_plugin/scripts/workbench_scan_start.py +149 -0
  53. package/_bundled_plugin/scripts/workbench_schema.py +714 -0
  54. package/_bundled_plugin/scripts/workbench_source_excerpt.py +99 -0
  55. package/_bundled_plugin/scripts/workbench_target.py +469 -0
  56. package/_bundled_plugin/scripts/workbench_target_state.py +63 -0
  57. package/_bundled_plugin/scripts/workbench_validation.py +294 -0
  58. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +117 -0
  59. package/_bundled_plugin/skills/attack-path-analysis/agents/openai.yaml +4 -0
  60. package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +76 -0
  61. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +153 -0
  62. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +166 -0
  63. package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +4 -0
  64. package/_bundled_plugin/skills/define-security-policy/SKILL.md +93 -0
  65. package/_bundled_plugin/skills/define-security-policy/agents/openai.yaml +4 -0
  66. package/_bundled_plugin/skills/finding-discovery/SKILL.md +155 -0
  67. package/_bundled_plugin/skills/finding-discovery/agents/openai.yaml +4 -0
  68. package/_bundled_plugin/skills/fix-finding/SKILL.md +112 -0
  69. package/_bundled_plugin/skills/fix-finding/agents/openai.yaml +4 -0
  70. package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +240 -0
  71. package/_bundled_plugin/skills/propose-security-hardening/agents/openai.yaml +4 -0
  72. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +424 -0
  73. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +167 -0
  74. package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +4 -0
  75. package/_bundled_plugin/skills/security-scan/SKILL.md +42 -0
  76. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +4 -0
  77. package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +48 -0
  78. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +85 -0
  79. package/_bundled_plugin/skills/threat-model/SKILL.md +59 -0
  80. package/_bundled_plugin/skills/threat-model/agents/openai.yaml +4 -0
  81. package/_bundled_plugin/skills/threat-model/references/threat-model-guidance.md +48 -0
  82. package/_bundled_plugin/skills/track-findings/SKILL.md +222 -0
  83. package/_bundled_plugin/skills/track-findings/agents/openai.yaml +6 -0
  84. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +54 -0
  85. package/_bundled_plugin/skills/track-findings/references/jira.md +77 -0
  86. package/_bundled_plugin/skills/triage-finding/SKILL.md +342 -0
  87. package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +4 -0
  88. package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +168 -0
  89. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +62 -0
  90. package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +108 -0
  91. package/_bundled_plugin/skills/validation/SKILL.md +110 -0
  92. package/_bundled_plugin/skills/validation/agents/openai.yaml +4 -0
  93. package/_bundled_plugin/skills/validation/references/validation-guidance.md +122 -0
  94. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +263 -0
  95. package/_bundled_plugin/skills/vulnerability-writeup/agents/openai.yaml +4 -0
  96. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +104 -0
  97. package/bin/codex-security.mjs +26 -0
  98. package/dist/api.d.ts +112 -0
  99. package/dist/api.d.ts.map +1 -0
  100. package/dist/api.js +1051 -0
  101. package/dist/api.js.map +1 -0
  102. package/dist/auth.d.ts +29 -0
  103. package/dist/auth.d.ts.map +1 -0
  104. package/dist/auth.js +246 -0
  105. package/dist/auth.js.map +1 -0
  106. package/dist/bulk-scan-discovery.d.ts +35 -0
  107. package/dist/bulk-scan-discovery.d.ts.map +1 -0
  108. package/dist/bulk-scan-discovery.js +217 -0
  109. package/dist/bulk-scan-discovery.js.map +1 -0
  110. package/dist/cli.d.ts +67 -0
  111. package/dist/cli.d.ts.map +1 -0
  112. package/dist/cli.js +2128 -0
  113. package/dist/cli.js.map +1 -0
  114. package/dist/config.d.ts +19 -0
  115. package/dist/config.d.ts.map +1 -0
  116. package/dist/config.js +199 -0
  117. package/dist/config.js.map +1 -0
  118. package/dist/contract.d.ts +21 -0
  119. package/dist/contract.d.ts.map +1 -0
  120. package/dist/contract.js +848 -0
  121. package/dist/contract.js.map +1 -0
  122. package/dist/cost.d.ts +30 -0
  123. package/dist/cost.d.ts.map +1 -0
  124. package/dist/cost.js +280 -0
  125. package/dist/cost.js.map +1 -0
  126. package/dist/errors.d.ts +38 -0
  127. package/dist/errors.d.ts.map +1 -0
  128. package/dist/errors.js +52 -0
  129. package/dist/errors.js.map +1 -0
  130. package/dist/index.d.ts +22 -0
  131. package/dist/index.d.ts.map +1 -0
  132. package/dist/index.js +11 -0
  133. package/dist/index.js.map +1 -0
  134. package/dist/knowledge-base.d.ts +7 -0
  135. package/dist/knowledge-base.d.ts.map +1 -0
  136. package/dist/knowledge-base.js +181 -0
  137. package/dist/knowledge-base.js.map +1 -0
  138. package/dist/models.d.ts +231 -0
  139. package/dist/models.d.ts.map +1 -0
  140. package/dist/models.js +3 -0
  141. package/dist/models.js.map +1 -0
  142. package/dist/multiscan.d.ts +30 -0
  143. package/dist/multiscan.d.ts.map +1 -0
  144. package/dist/multiscan.js +397 -0
  145. package/dist/multiscan.js.map +1 -0
  146. package/dist/result.d.ts +39 -0
  147. package/dist/result.d.ts.map +1 -0
  148. package/dist/result.js +71 -0
  149. package/dist/result.js.map +1 -0
  150. package/dist/runtime.d.ts +69 -0
  151. package/dist/runtime.d.ts.map +1 -0
  152. package/dist/runtime.js +997 -0
  153. package/dist/runtime.js.map +1 -0
  154. package/dist/scan-comparison.d.ts +42 -0
  155. package/dist/scan-comparison.d.ts.map +1 -0
  156. package/dist/scan-comparison.js +153 -0
  157. package/dist/scan-comparison.js.map +1 -0
  158. package/dist/scan-history-renderer.d.ts +13 -0
  159. package/dist/scan-history-renderer.d.ts.map +1 -0
  160. package/dist/scan-history-renderer.js +297 -0
  161. package/dist/scan-history-renderer.js.map +1 -0
  162. package/dist/targets.d.ts +39 -0
  163. package/dist/targets.d.ts.map +1 -0
  164. package/dist/targets.js +295 -0
  165. package/dist/targets.js.map +1 -0
  166. package/dist/trusted-executable.d.ts +6 -0
  167. package/dist/trusted-executable.d.ts.map +1 -0
  168. package/dist/trusted-executable.js +77 -0
  169. package/dist/trusted-executable.js.map +1 -0
  170. package/dist/version.d.ts +6 -0
  171. package/dist/version.d.ts.map +1 -0
  172. package/dist/version.js +42 -0
  173. package/dist/version.js.map +1 -0
  174. package/dist/worker-progress.d.ts +13 -0
  175. package/dist/worker-progress.d.ts.map +1 -0
  176. package/dist/worker-progress.js +113 -0
  177. package/dist/worker-progress.js.map +1 -0
  178. package/package.json +74 -0
@@ -0,0 +1,1013 @@
1
+ #!/usr/bin/env python3
2
+ """Generate and post-process Codex Security scan worklists.
3
+
4
+ This script stays deliberately model-free:
5
+
6
+ - `make-repo-rank-input` creates the deterministic repository or scoped-path
7
+ JSONL candidate worklist that ranking subagents consume.
8
+ - `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
9
+ worklist from Git changed paths. It supports committed revision diffs and
10
+ local working-tree patches.
11
+ - `make-rank-shards` partitions the ranking input into deterministic shards.
12
+ - `make-rank-pool-plan` assigns those shards to a deterministic bounded worker
13
+ pool.
14
+ - `validate-rank-worker` validates one worker slot and emits a content-bound
15
+ completion receipt.
16
+ - `validate-rank-shard` validates one completed worker output before the
17
+ coordinator accepts it.
18
+ - `validate-rank-pool` validates the pool plan and every assigned shard output.
19
+ - `merge-rank-outputs` validates and combines worker-local shard outputs.
20
+ - `copy-deep-review-input` copies every candidate into the deep-review worklist
21
+ for exhaustive mode.
22
+ - `select-deep-review-input` selects the ranked rows for deep review.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import argparse
28
+ import hashlib
29
+ import json
30
+ import re
31
+ import subprocess
32
+ import sys
33
+ from collections import Counter
34
+ from collections.abc import Callable
35
+ from pathlib import Path
36
+
37
+ # Some plugin hosts launch Python with safe-path isolation enabled.
38
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
39
+ from rank_preview import DEFAULT_PREVIEW_BYTES, TEXT_CODE_EXTENSIONS, preview_for
40
+
41
+ EXCLUDED_DIRS = {
42
+ ".cache",
43
+ ".circleci",
44
+ ".devcontainer",
45
+ ".git",
46
+ ".github",
47
+ ".idea",
48
+ ".mypy_cache",
49
+ ".pytest_cache",
50
+ ".ruff_cache",
51
+ ".tox",
52
+ ".venv",
53
+ ".vscode",
54
+ "__pycache__",
55
+ "bench",
56
+ "benchmark",
57
+ "bintest",
58
+ "build",
59
+ "build_config",
60
+ "build_configs",
61
+ "build-tools",
62
+ "build_tools",
63
+ "ci",
64
+ "coverage",
65
+ "deps",
66
+ "dev",
67
+ "dist",
68
+ "doc",
69
+ "docs",
70
+ "example",
71
+ "examples",
72
+ "external",
73
+ "extern",
74
+ "fixture",
75
+ "fixtures",
76
+ "generated",
77
+ "node_modules",
78
+ "sample",
79
+ "samples",
80
+ "target",
81
+ "test",
82
+ "tests",
83
+ "testing",
84
+ "third-party",
85
+ "third_party",
86
+ "tmp",
87
+ "vendor",
88
+ }
89
+
90
+ EXCLUDED_FILENAMES = {
91
+ ".DS_Store",
92
+ "CHANGELOG",
93
+ "CHANGELOG.md",
94
+ "CONTRIBUTING.md",
95
+ "Dockerfile",
96
+ "Gemfile",
97
+ "Gemfile.lock",
98
+ "LICENSE",
99
+ "LICENSE.md",
100
+ "Makefile",
101
+ "NEWS",
102
+ "NEWS.md",
103
+ "NOTICE",
104
+ "README",
105
+ "README.md",
106
+ "README.rst",
107
+ "Rakefile",
108
+ "SECURITY.md",
109
+ "TODO",
110
+ "TODO.md",
111
+ "docker-compose.yml",
112
+ "package-lock.json",
113
+ "pnpm-lock.yaml",
114
+ "yarn.lock",
115
+ }
116
+
117
+ SHARD_INPUT_GLOB = "rank-shard-*.input.jsonl"
118
+ SHARD_OUTPUT_GLOB = "rank-shard-*.output.jsonl"
119
+ SHARD_INPUT_PATTERN = re.compile(r"^rank-shard-([0-9]{4,})\.input\.jsonl$")
120
+ DIRECT_SCOPE_PREVIEW_READ_BYTES = 64 * 1024
121
+ RANK_POOL_PLAN_SCHEMA_VERSION = 1
122
+ RANK_POOL_STRATEGY = "round_robin"
123
+ RANK_POOL_WORKER_CAP = 6
124
+ JsonRow = dict[str, object]
125
+ RowValidator = Callable[[JsonRow, Path, int], None]
126
+ RankWorkerAssignment = tuple[int, list[str], list[str]]
127
+
128
+
129
+ def parse_args() -> argparse.Namespace:
130
+ parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
131
+ subparsers = parser.add_subparsers(dest="command", required=True)
132
+
133
+ make = subparsers.add_parser(
134
+ "make-repo-rank-input",
135
+ help="Create rank_input.jsonl for subagent-based file ranking.",
136
+ )
137
+ make.add_argument("--repo", required=True, help="Repository root.")
138
+ make.add_argument(
139
+ "--scope",
140
+ default=".",
141
+ help="Path within the repository to scan. Defaults to the repository root.",
142
+ )
143
+ make.add_argument(
144
+ "--scopes-file",
145
+ help="JSON array of repository-relative files and directories to scan together.",
146
+ )
147
+ make.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
148
+ make.add_argument("--area", default="", help="Area label. Defaults to scope.")
149
+ make.add_argument(
150
+ "--preview-bytes",
151
+ type=int,
152
+ default=DEFAULT_PREVIEW_BYTES,
153
+ help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
154
+ )
155
+
156
+ bind = subparsers.add_parser(
157
+ "bind-repo-scopes",
158
+ help="Copy SDK scoped-path targets into the unsealed manifest and coverage documents.",
159
+ )
160
+ bind.add_argument("--scopes-file", required=True, help="JSON array of requested scopes.")
161
+ bind.add_argument("--manifest", required=True, help="Unsealed scan-manifest.json path.")
162
+ bind.add_argument("--coverage", required=True, help="Unsealed coverage.json path.")
163
+
164
+ diff = subparsers.add_parser(
165
+ "make-diff-rank-input",
166
+ help="Create rank_input.jsonl from Git changed source-like files.",
167
+ )
168
+ diff.add_argument("--repo", required=True, help="Repository root.")
169
+ diff.add_argument("--base", required=True, help="Git diff base revision.")
170
+ diff.add_argument(
171
+ "--mode",
172
+ choices=("revisions", "local-patch"),
173
+ default="revisions",
174
+ help="Git diff mode: committed revisions or staged plus unstaged local patch.",
175
+ )
176
+ diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
177
+ diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
178
+ diff.add_argument("--area", default="diff", help="Area label for ranking rows.")
179
+ diff.add_argument(
180
+ "--preview-bytes",
181
+ type=int,
182
+ default=DEFAULT_PREVIEW_BYTES,
183
+ help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
184
+ )
185
+
186
+ shards = subparsers.add_parser(
187
+ "make-rank-shards",
188
+ help="Partition rank_input.jsonl into deterministic worker input shards.",
189
+ )
190
+ shards.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
191
+ shards.add_argument("--out-dir", required=True, help="Directory for worker input shards.")
192
+ shards.add_argument(
193
+ "--max-rows",
194
+ type=int,
195
+ default=150,
196
+ help="Maximum rows per shard. Defaults to 150.",
197
+ )
198
+
199
+ pool_plan = subparsers.add_parser(
200
+ "make-rank-pool-plan",
201
+ help="Assign rank shards to a deterministic bounded worker pool.",
202
+ )
203
+ pool_plan.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
204
+ pool_plan.add_argument(
205
+ "--usable-worker-slots",
206
+ required=True,
207
+ type=int,
208
+ help="Usable ranking-worker slots reported by capability preflight; capped at 6.",
209
+ )
210
+ pool_plan.add_argument("--out", required=True, help="Output rank_worker_assignments.json path.")
211
+
212
+ validate_shard = subparsers.add_parser(
213
+ "validate-rank-shard",
214
+ help="Validate one worker output against its rank input shard.",
215
+ )
216
+ validate_shard.add_argument("--input", required=True, help="Worker rank input shard.")
217
+ validate_shard.add_argument("--output", required=True, help="Worker rank output shard.")
218
+
219
+ validate_worker = subparsers.add_parser(
220
+ "validate-rank-worker",
221
+ help="Validate one assigned ranking-worker slot and emit its completion receipt.",
222
+ )
223
+ validate_worker.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
224
+ validate_worker.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
225
+ validate_worker.add_argument(
226
+ "--slot",
227
+ required=True,
228
+ type=int,
229
+ help="One-based ranking-worker slot from the rank pool plan.",
230
+ )
231
+
232
+ validate_pool = subparsers.add_parser(
233
+ "validate-rank-pool",
234
+ help="Validate a rank pool plan and every assigned shard output.",
235
+ )
236
+ validate_pool.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
237
+ validate_pool.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
238
+
239
+ merge = subparsers.add_parser(
240
+ "merge-rank-outputs",
241
+ help="Validate worker shard outputs and create rank_output.jsonl.",
242
+ )
243
+ merge.add_argument("--rank-input", required=True, help="Authoritative rank input JSONL.")
244
+ merge.add_argument("--shard-dir", required=True, help="Directory of input and output shards.")
245
+ merge.add_argument("--out", required=True, help="Output rank_output.jsonl path.")
246
+
247
+ copy = subparsers.add_parser(
248
+ "copy-deep-review-input",
249
+ help="Create deep_review_input.jsonl directly from rank_input.jsonl.",
250
+ )
251
+ copy.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
252
+ copy.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
253
+
254
+ select = subparsers.add_parser(
255
+ "select-deep-review-input",
256
+ help="Create deep_review_input.jsonl from worker-produced rank_output.jsonl.",
257
+ )
258
+ select.add_argument("--rank-output", required=True, help="Worker ranking output JSONL.")
259
+ select.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
260
+ select.add_argument(
261
+ "--top-percent",
262
+ type=int,
263
+ default=100,
264
+ help="Percent of included files to keep for deep review.",
265
+ )
266
+ return parser.parse_args()
267
+
268
+
269
+ def path_is_excluded(path: Path) -> bool:
270
+ if any(part in EXCLUDED_DIRS for part in path.parts):
271
+ return True
272
+ if path.name in EXCLUDED_FILENAMES:
273
+ return True
274
+ return path.name.endswith((".min.js", ".map"))
275
+
276
+
277
+ def resolve_scope(repo: Path, scope: str, *, expand_user: bool = True) -> Path:
278
+ scope_path = Path(scope).expanduser() if expand_user else Path(scope)
279
+ if not scope_path.is_absolute():
280
+ scope_path = repo / scope_path
281
+ scope_path = scope_path.resolve()
282
+ repo_resolved = repo.resolve()
283
+ try:
284
+ scope_path.relative_to(repo_resolved)
285
+ except ValueError as exc:
286
+ raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
287
+ if not scope_path.is_dir() and not scope_path.is_file():
288
+ raise SystemExit(f"Scope path not found: {scope_path}")
289
+ return scope_path
290
+
291
+
292
+ def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
293
+ output.parent.mkdir(parents=True, exist_ok=True)
294
+ with output.open("w", encoding="utf-8") as handle:
295
+ for row in rows:
296
+ handle.write(json.dumps(row, ensure_ascii=True, separators=(",", ":")))
297
+ handle.write("\n")
298
+
299
+
300
+ def write_json(output: Path, payload: dict[str, object]) -> None:
301
+ output.parent.mkdir(parents=True, exist_ok=True)
302
+ output.write_text(
303
+ json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
304
+ encoding="utf-8",
305
+ )
306
+
307
+
308
+ def load_scopes_file(scopes_file: Path) -> list[str]:
309
+ try:
310
+ loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
311
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
312
+ raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc
313
+ if (
314
+ not isinstance(loaded, list)
315
+ or not loaded
316
+ or any(not isinstance(scope, str) or not scope for scope in loaded)
317
+ ):
318
+ raise SystemExit(f"Scopes file must contain a non-empty JSON string array: {scopes_file}")
319
+ return loaded
320
+
321
+
322
+ def load_jsonl(path: Path, label: str, validator: RowValidator) -> list[JsonRow]:
323
+ if not path.exists():
324
+ raise SystemExit(f"{label} missing: {path}")
325
+
326
+ rows: list[JsonRow] = []
327
+ with path.open(encoding="utf-8") as handle:
328
+ for line_number, raw_line in enumerate(handle, start=1):
329
+ if not raw_line.strip():
330
+ raise SystemExit(f"{path}:{line_number}: blank JSONL rows are not allowed")
331
+ try:
332
+ parsed: object = json.loads(raw_line)
333
+ except json.JSONDecodeError as exc:
334
+ raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc
335
+ if not isinstance(parsed, dict):
336
+ raise SystemExit(f"{path}:{line_number}: expected a JSON object")
337
+ row = {str(key): value for key, value in parsed.items()}
338
+ validator(row, path, line_number)
339
+ rows.append(row)
340
+ return rows
341
+
342
+
343
+ def require_exact_fields(row: JsonRow, expected: set[str], path: Path, line_number: int) -> None:
344
+ actual = set(row)
345
+ if actual != expected:
346
+ missing = sorted(expected - actual)
347
+ unexpected = sorted(actual - expected)
348
+ details: list[str] = []
349
+ if missing:
350
+ details.append(f"missing fields {missing}")
351
+ if unexpected:
352
+ details.append(f"unexpected fields {unexpected}")
353
+ raise SystemExit(f"{path}:{line_number}: {'; '.join(details)}")
354
+
355
+
356
+ def require_string(
357
+ row: JsonRow, field: str, path: Path, line_number: int, *, allow_empty: bool
358
+ ) -> None:
359
+ value = row[field]
360
+ if not isinstance(value, str) or (not allow_empty and not value.strip()):
361
+ requirement = "a string" if allow_empty else "a non-empty string"
362
+ raise SystemExit(f"{path}:{line_number}: {field} must be {requirement}")
363
+
364
+
365
+ def validate_rank_input_row(row: JsonRow, path: Path, line_number: int) -> None:
366
+ require_exact_fields(row, {"path", "area", "preview"}, path, line_number)
367
+ require_string(row, "path", path, line_number, allow_empty=False)
368
+ require_string(row, "area", path, line_number, allow_empty=True)
369
+ require_string(row, "preview", path, line_number, allow_empty=True)
370
+
371
+
372
+ def validate_rank_output_row(row: JsonRow, path: Path, line_number: int) -> None:
373
+ require_exact_fields(row, {"path", "area", "score", "include", "reason"}, path, line_number)
374
+ require_string(row, "path", path, line_number, allow_empty=False)
375
+ require_string(row, "area", path, line_number, allow_empty=True)
376
+ score = row["score"]
377
+ if isinstance(score, bool) or not isinstance(score, int):
378
+ raise SystemExit(f"{path}:{line_number}: score must be an integer from 1 through 10")
379
+ if not 1 <= score <= 10:
380
+ raise SystemExit(f"{path}:{line_number}: score must be from 1 through 10")
381
+ if not isinstance(row["include"], bool):
382
+ raise SystemExit(f"{path}:{line_number}: include must be a boolean")
383
+ require_string(row, "reason", path, line_number, allow_empty=False)
384
+
385
+
386
+ def require_unique_paths(rows: list[JsonRow], label: str) -> None:
387
+ seen: set[str] = set()
388
+ duplicates: set[str] = set()
389
+ for row in rows:
390
+ path = str(row["path"])
391
+ if path in seen:
392
+ duplicates.add(path)
393
+ seen.add(path)
394
+ if duplicates:
395
+ raise SystemExit(f"{label} contains duplicate paths: {sorted(duplicates)}")
396
+
397
+
398
+ def make_repo_rank_input(args: argparse.Namespace) -> None:
399
+ repo = Path(args.repo).expanduser().resolve()
400
+ if not repo.is_dir():
401
+ raise SystemExit(f"Repo path not found: {repo}")
402
+ scopes = [args.scope]
403
+ explicit_scopes = args.scopes_file is not None
404
+ if explicit_scopes:
405
+ scopes = load_scopes_file(Path(args.scopes_file).expanduser())
406
+
407
+ resolved_scopes = [
408
+ resolve_scope(repo, scope, expand_user=not explicit_scopes) for scope in scopes
409
+ ]
410
+ directly_requested_files = {
411
+ scope_abs for scope_abs in resolved_scopes if explicit_scopes and scope_abs.is_file()
412
+ }
413
+ rows_by_path: dict[str, JsonRow] = {}
414
+ for scope_abs in resolved_scopes:
415
+ scope_rel = scope_abs.relative_to(repo)
416
+ area = args.area or scope_rel.as_posix()
417
+ candidates = (scope_abs,) if scope_abs.is_file() else scope_abs.rglob("*")
418
+ for path in candidates:
419
+ try:
420
+ if path.is_symlink() or not path.is_file():
421
+ continue
422
+ path.resolve(strict=True).relative_to(repo)
423
+ except (OSError, ValueError):
424
+ continue
425
+ rel = path.relative_to(repo)
426
+ directly_requested = path in directly_requested_files
427
+ excluded_path = (
428
+ path.relative_to(scope_abs if scope_abs.is_dir() else scope_abs.parent)
429
+ if explicit_scopes
430
+ else rel
431
+ )
432
+ if not directly_requested and (
433
+ path_is_excluded(excluded_path) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS
434
+ ):
435
+ continue
436
+
437
+ if (
438
+ directly_requested
439
+ and path.suffix.lower() not in TEXT_CODE_EXTENSIONS
440
+ and path.name not in EXCLUDED_FILENAMES
441
+ ):
442
+ preview = ""
443
+ else:
444
+ preview, is_binary = preview_for(
445
+ path,
446
+ args.preview_bytes,
447
+ max_read_bytes=DIRECT_SCOPE_PREVIEW_READ_BYTES if directly_requested else None,
448
+ )
449
+ if is_binary and not directly_requested:
450
+ continue
451
+ rows_by_path.setdefault(
452
+ rel.as_posix(),
453
+ {"path": rel.as_posix(), "area": area, "preview": preview},
454
+ )
455
+
456
+ rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
457
+ output = Path(args.out).expanduser()
458
+ write_jsonl(output, rows)
459
+ print(f"Wrote {len(rows)} rows to {output}")
460
+
461
+
462
+ def bind_repo_scopes(args: argparse.Namespace) -> None:
463
+ scopes = load_scopes_file(Path(args.scopes_file).expanduser())
464
+ manifest_path = Path(args.manifest).expanduser()
465
+ coverage_path = Path(args.coverage).expanduser()
466
+ try:
467
+ manifest: object = json.loads(manifest_path.read_text(encoding="utf-8"))
468
+ coverage: object = json.loads(coverage_path.read_text(encoding="utf-8"))
469
+ if not isinstance(manifest, dict) or not isinstance(coverage, dict):
470
+ raise ValueError("expected JSON objects")
471
+ scan = manifest.get("scan")
472
+ if not isinstance(scan, dict):
473
+ raise ValueError("manifest.scan must be an object")
474
+ scope = scan.get("scope")
475
+ if not isinstance(scope, dict):
476
+ raise ValueError("manifest.scan.scope must be an object")
477
+ except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
478
+ raise SystemExit("Unable to bind requested scopes into the scan contract") from exc
479
+ scope["includePaths"] = scopes
480
+ coverage["includePaths"] = scopes
481
+ manifest_path.write_text(
482
+ json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
483
+ )
484
+ coverage_path.write_text(
485
+ json.dumps(coverage, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
486
+ )
487
+ print(f"Bound {len(scopes)} requested scopes into the scan contract")
488
+
489
+
490
+ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
491
+ result = subprocess.run(
492
+ [
493
+ "git",
494
+ "-C",
495
+ str(repo),
496
+ "diff",
497
+ "--name-status",
498
+ "-z",
499
+ "--diff-filter=ACMRD",
500
+ *diff_args,
501
+ ],
502
+ check=True,
503
+ capture_output=True,
504
+ text=True,
505
+ )
506
+ fields = result.stdout.split("\0")
507
+ if fields and not fields[-1]:
508
+ fields.pop()
509
+
510
+ changed: list[tuple[Path, str]] = []
511
+ index = 0
512
+ while index < len(fields):
513
+ status = fields[index][0]
514
+ index += 1
515
+ if status in {"C", "R"}:
516
+ index += 1
517
+ path = repo / fields[index]
518
+ index += 1
519
+ changed.append((path, status))
520
+ return changed
521
+
522
+
523
+ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
524
+ if mode == "revisions":
525
+ return run_git_changed_paths(repo, [f"{base}..{head}"])
526
+ if mode == "local-patch":
527
+ unstaged = run_git_changed_paths(repo, [base])
528
+ staged = run_git_changed_paths(repo, ["--cached", base])
529
+ combined = dict(staged)
530
+ combined.update(unstaged)
531
+ return sorted(combined.items())
532
+ raise SystemExit(f"Unknown diff mode: {mode}")
533
+
534
+
535
+ def make_diff_rank_input(args: argparse.Namespace) -> None:
536
+ repo = Path(args.repo).expanduser().resolve()
537
+ if not repo.is_dir():
538
+ raise SystemExit(f"Repo path not found: {repo}")
539
+
540
+ rows: list[JsonRow] = []
541
+ for path, status in git_changed_paths(repo, args.base, args.head, args.mode):
542
+ rel = path.relative_to(repo)
543
+ if path_is_excluded(rel) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS:
544
+ continue
545
+
546
+ if status == "D":
547
+ preview = ""
548
+ elif path.is_file():
549
+ preview, is_binary = preview_for(path, args.preview_bytes)
550
+ if is_binary:
551
+ continue
552
+ else:
553
+ preview = ""
554
+ rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
555
+
556
+ rows.sort(key=lambda row: str(row["path"]))
557
+ output = Path(args.out).expanduser()
558
+ write_jsonl(output, rows)
559
+ print(f"Wrote {len(rows)} rows to {output}")
560
+
561
+
562
+ def make_rank_shards(args: argparse.Namespace) -> None:
563
+ if args.max_rows < 1:
564
+ raise SystemExit("--max-rows must be at least 1")
565
+
566
+ rank_input = Path(args.rank_input).expanduser()
567
+ rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
568
+ require_unique_paths(rows, "Rank input")
569
+
570
+ output_dir = Path(args.out_dir).expanduser()
571
+ output_dir.mkdir(parents=True, exist_ok=True)
572
+ existing = sorted((*output_dir.glob(SHARD_INPUT_GLOB), *output_dir.glob(SHARD_OUTPUT_GLOB)))
573
+ if existing:
574
+ raise SystemExit(f"Rank shard directory already contains shard files: {output_dir}")
575
+
576
+ shard_count = 0
577
+ for start in range(0, len(rows), args.max_rows):
578
+ shard_count += 1
579
+ shard_path = output_dir / f"rank-shard-{shard_count:04d}.input.jsonl"
580
+ write_jsonl(shard_path, rows[start : start + args.max_rows])
581
+
582
+ print(f"Wrote {shard_count} rank shards to {output_dir}")
583
+
584
+
585
+ def discover_input_shards(shard_dir: Path) -> list[Path]:
586
+ if not shard_dir.is_dir():
587
+ raise SystemExit(f"Rank shard directory missing: {shard_dir}")
588
+
589
+ numbered_shards: list[tuple[int, Path]] = []
590
+ for path in shard_dir.glob(SHARD_INPUT_GLOB):
591
+ match = SHARD_INPUT_PATTERN.fullmatch(path.name)
592
+ if match is None:
593
+ raise SystemExit(f"Rank input shard has invalid name: {path.name}")
594
+ numbered_shards.append((int(match.group(1)), path))
595
+ numbered_shards.sort(key=lambda item: (item[0], item[1].name))
596
+ input_shards = [path for _, path in numbered_shards]
597
+ expected_names = [
598
+ f"rank-shard-{index:04d}.input.jsonl" for index in range(1, len(input_shards) + 1)
599
+ ]
600
+ actual_names = [path.name for path in input_shards]
601
+ if actual_names != expected_names:
602
+ raise SystemExit(
603
+ "Rank input shards must use contiguous canonical names; "
604
+ f"expected={expected_names}; actual={actual_names}"
605
+ )
606
+ return input_shards
607
+
608
+
609
+ def output_name_for(input_name: str) -> str:
610
+ return input_name.replace(".input.jsonl", ".output.jsonl")
611
+
612
+
613
+ def require_plan_shard_dir(plan_path: Path, shard_dir: Path) -> None:
614
+ expected = plan_path.parent / "rank_shards"
615
+ if shard_dir.resolve() != expected.resolve():
616
+ raise SystemExit(
617
+ "Rank shard directory must be the assignment plan's sibling rank_shards "
618
+ f"directory; expected={expected}; actual={shard_dir}"
619
+ )
620
+
621
+
622
+ def require_no_misplaced_rank_shards(plan_path: Path) -> None:
623
+ misplaced = sorted(
624
+ (
625
+ *plan_path.parent.glob(SHARD_INPUT_GLOB),
626
+ *plan_path.parent.glob(SHARD_OUTPUT_GLOB),
627
+ ),
628
+ key=lambda path: path.name,
629
+ )
630
+ if misplaced:
631
+ raise SystemExit(
632
+ "Rank shard artifacts must be stored in the assignment plan's sibling "
633
+ f"rank_shards directory; misplaced={[path.name for path in misplaced]}"
634
+ )
635
+
636
+
637
+ def make_rank_pool_plan(args: argparse.Namespace) -> None:
638
+ if args.usable_worker_slots < 1:
639
+ raise SystemExit("--usable-worker-slots must be at least 1")
640
+
641
+ shard_dir = Path(args.shard_dir).expanduser()
642
+ output = Path(args.out).expanduser()
643
+ require_plan_shard_dir(output, shard_dir)
644
+ input_shards = discover_input_shards(shard_dir)
645
+ worker_count = min(len(input_shards), args.usable_worker_slots, RANK_POOL_WORKER_CAP)
646
+ workers: list[dict[str, object]] = []
647
+ for worker_index in range(worker_count):
648
+ assigned_inputs = [path.name for path in input_shards[worker_index::worker_count]]
649
+ workers.append(
650
+ {
651
+ "slot": worker_index + 1,
652
+ "input_shards": assigned_inputs,
653
+ "output_shards": [output_name_for(name) for name in assigned_inputs],
654
+ }
655
+ )
656
+
657
+ plan: dict[str, object] = {
658
+ "schema_version": RANK_POOL_PLAN_SCHEMA_VERSION,
659
+ "strategy": RANK_POOL_STRATEGY,
660
+ "shard_count": len(input_shards),
661
+ "ranking_worker_count": worker_count,
662
+ "workers": workers,
663
+ }
664
+ write_json(output, plan)
665
+ print(f"Assigned {len(input_shards)} rank shards to {worker_count} ranking workers in {output}")
666
+
667
+
668
+ def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]:
669
+ if not plan_path.exists():
670
+ raise SystemExit(f"Rank pool plan missing: {plan_path}")
671
+ plan_bytes = plan_path.read_bytes()
672
+ try:
673
+ payload: object = json.loads(plan_bytes)
674
+ except json.JSONDecodeError as exc:
675
+ raise SystemExit(f"{plan_path}: invalid JSON: {exc.msg}") from exc
676
+ if not isinstance(payload, dict):
677
+ raise SystemExit(f"{plan_path}: expected a JSON object")
678
+ return {str(key): value for key, value in payload.items()}, plan_bytes
679
+
680
+
681
+ def require_integer(value: object, label: str, *, minimum: int) -> int:
682
+ if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
683
+ raise SystemExit(f"{label} must be an integer of at least {minimum}")
684
+ return value
685
+
686
+
687
+ def require_string_list(value: object, label: str) -> list[str]:
688
+ if not isinstance(value, list) or not value:
689
+ raise SystemExit(f"{label} must be a non-empty list")
690
+ if any(not isinstance(item, str) or not item for item in value):
691
+ raise SystemExit(f"{label} entries must be non-empty strings")
692
+ return [item for item in value if isinstance(item, str)]
693
+
694
+
695
+ def assignment_differences(
696
+ assigned_names: list[str], expected_names: list[str]
697
+ ) -> tuple[list[str], list[str], list[str]]:
698
+ counts = Counter(assigned_names)
699
+ duplicates = sorted(name for name, count in counts.items() if count > 1)
700
+ assigned = set(assigned_names)
701
+ expected = set(expected_names)
702
+ return sorted(expected - assigned), duplicates, sorted(assigned - expected)
703
+
704
+
705
+ def validate_rank_pool_plan(
706
+ plan_path: Path, shard_dir: Path
707
+ ) -> tuple[list[Path], list[str], list[RankWorkerAssignment], bytes]:
708
+ require_plan_shard_dir(plan_path, shard_dir)
709
+ require_no_misplaced_rank_shards(plan_path)
710
+ input_shards = discover_input_shards(shard_dir)
711
+ input_names = [path.name for path in input_shards]
712
+ output_names = [output_name_for(name) for name in input_names]
713
+ plan, plan_bytes = load_rank_pool_plan(plan_path)
714
+ expected_fields = {
715
+ "schema_version",
716
+ "strategy",
717
+ "shard_count",
718
+ "ranking_worker_count",
719
+ "workers",
720
+ }
721
+ actual_fields = set(plan)
722
+ if actual_fields != expected_fields:
723
+ raise SystemExit(
724
+ f"{plan_path}: rank pool plan fields do not match schema; "
725
+ f"missing={sorted(expected_fields - actual_fields)}; "
726
+ f"unexpected={sorted(actual_fields - expected_fields)}"
727
+ )
728
+ schema_version = require_integer(
729
+ plan["schema_version"], f"{plan_path}: schema_version", minimum=1
730
+ )
731
+ if schema_version != RANK_POOL_PLAN_SCHEMA_VERSION:
732
+ raise SystemExit(f"{plan_path}: schema_version must be {RANK_POOL_PLAN_SCHEMA_VERSION}")
733
+ if plan["strategy"] != RANK_POOL_STRATEGY:
734
+ raise SystemExit(f"{plan_path}: strategy must be {RANK_POOL_STRATEGY}")
735
+
736
+ shard_count = require_integer(plan["shard_count"], f"{plan_path}: shard_count", minimum=0)
737
+ if shard_count != len(input_shards):
738
+ raise SystemExit(
739
+ f"{plan_path}: shard_count does not match input shards; "
740
+ f"plan={shard_count}; actual={len(input_shards)}"
741
+ )
742
+ worker_count = require_integer(
743
+ plan["ranking_worker_count"], f"{plan_path}: ranking_worker_count", minimum=0
744
+ )
745
+ if shard_count > 0 and worker_count == 0:
746
+ raise SystemExit(
747
+ f"{plan_path}: ranking_worker_count must be at least 1 when input shards exist"
748
+ )
749
+ if worker_count > shard_count:
750
+ raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed shard_count")
751
+ if worker_count > RANK_POOL_WORKER_CAP:
752
+ raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed {RANK_POOL_WORKER_CAP}")
753
+
754
+ workers = plan["workers"]
755
+ if not isinstance(workers, list) or len(workers) != worker_count:
756
+ raise SystemExit(
757
+ f"{plan_path}: workers must contain exactly {worker_count} worker assignments"
758
+ )
759
+
760
+ assigned_inputs: list[str] = []
761
+ assigned_outputs: list[str] = []
762
+ parsed_workers: list[RankWorkerAssignment] = []
763
+ worker_fields = {"slot", "input_shards", "output_shards"}
764
+ for worker_index, raw_worker in enumerate(workers):
765
+ label = f"{plan_path}: workers[{worker_index}]"
766
+ if not isinstance(raw_worker, dict):
767
+ raise SystemExit(f"{label} must be a JSON object")
768
+ worker = {str(key): value for key, value in raw_worker.items()}
769
+ if set(worker) != worker_fields:
770
+ raise SystemExit(
771
+ f"{label} fields do not match schema; "
772
+ f"missing={sorted(worker_fields - set(worker))}; "
773
+ f"unexpected={sorted(set(worker) - worker_fields)}"
774
+ )
775
+ slot = require_integer(worker["slot"], f"{label}.slot", minimum=1)
776
+ if slot != worker_index + 1:
777
+ raise SystemExit(f"{label}.slot must be {worker_index + 1}")
778
+ worker_inputs = require_string_list(worker["input_shards"], f"{label}.input_shards")
779
+ worker_outputs = require_string_list(worker["output_shards"], f"{label}.output_shards")
780
+ if len(worker_inputs) != len(worker_outputs):
781
+ raise SystemExit(f"{label} input_shards and output_shards lengths must match")
782
+ expected_worker_outputs = [output_name_for(name) for name in worker_inputs]
783
+ if worker_outputs != expected_worker_outputs:
784
+ raise SystemExit(f"{label}.output_shards do not match its input_shards")
785
+ assigned_inputs.extend(worker_inputs)
786
+ assigned_outputs.extend(worker_outputs)
787
+ parsed_workers.append((slot, worker_inputs, worker_outputs))
788
+
789
+ missing, duplicates, unexpected = assignment_differences(assigned_inputs, input_names)
790
+ if missing or duplicates or unexpected:
791
+ raise SystemExit(
792
+ f"{plan_path}: pool plan must assign each input shard exactly once; "
793
+ f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
794
+ )
795
+ missing, duplicates, unexpected = assignment_differences(assigned_outputs, output_names)
796
+ if missing or duplicates or unexpected:
797
+ raise SystemExit(
798
+ f"{plan_path}: pool plan must assign each output shard exactly once; "
799
+ f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
800
+ )
801
+
802
+ for worker_index, (_, worker_inputs, worker_outputs) in enumerate(parsed_workers):
803
+ expected_inputs = input_names[worker_index::worker_count]
804
+ expected_outputs = output_names[worker_index::worker_count]
805
+ if worker_inputs != expected_inputs or worker_outputs != expected_outputs:
806
+ raise SystemExit(
807
+ f"{plan_path}: worker slot {worker_index + 1} does not match the deterministic "
808
+ f"{RANK_POOL_STRATEGY} assignment"
809
+ )
810
+ return input_shards, output_names, parsed_workers, plan_bytes
811
+
812
+
813
+ def validate_rank_worker_command(args: argparse.Namespace) -> None:
814
+ plan_path = Path(args.plan).expanduser()
815
+ shard_dir = Path(args.shard_dir).expanduser()
816
+ _, _, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir)
817
+
818
+ slot = require_integer(args.slot, "--slot", minimum=1)
819
+ worker_count = len(workers)
820
+ if slot > worker_count:
821
+ raise SystemExit(f"--slot must be at most {worker_count}")
822
+
823
+ assigned_slot, input_names, output_names = workers[slot - 1]
824
+ if assigned_slot != slot:
825
+ raise SystemExit(f"{plan_path}: worker assignment for slot {slot} is inconsistent")
826
+
827
+ row_count = 0
828
+ outputs_digest = hashlib.sha256()
829
+ for input_name, output_name in zip(input_names, output_names, strict=True):
830
+ input_shard = shard_dir / input_name
831
+ output_shard = shard_dir / output_name
832
+ _, output_rows = validate_rank_shard(input_shard, output_shard)
833
+ output_bytes = output_shard.read_bytes()
834
+ row_count += len(output_rows)
835
+ outputs_digest.update(output_name.encode("utf-8"))
836
+ outputs_digest.update(b"\0")
837
+ outputs_digest.update(output_bytes)
838
+ outputs_digest.update(b"\0")
839
+
840
+ receipt: dict[str, object] = {
841
+ "schema_version": 1,
842
+ "plan_sha256": hashlib.sha256(plan_bytes).hexdigest(),
843
+ "slot": slot,
844
+ "ranking_worker_count": worker_count,
845
+ "output_shards": len(output_names),
846
+ "rows": row_count,
847
+ "outputs_sha256": outputs_digest.hexdigest(),
848
+ "status": "complete",
849
+ }
850
+ print("RANK_WORKER_RECEIPT " + json.dumps(receipt, sort_keys=True, separators=(",", ":")))
851
+
852
+
853
+ def validate_rank_pool_command(args: argparse.Namespace) -> None:
854
+ plan_path = Path(args.plan).expanduser()
855
+ shard_dir = Path(args.shard_dir).expanduser()
856
+ input_shards, expected_output_names, workers, _ = validate_rank_pool_plan(plan_path, shard_dir)
857
+
858
+ actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)}
859
+ expected_outputs = set(expected_output_names)
860
+ if actual_output_names != expected_outputs:
861
+ missing = sorted(expected_outputs - actual_output_names)
862
+ unexpected = sorted(actual_output_names - expected_outputs)
863
+ raise SystemExit(
864
+ "Rank pool outputs are incomplete; "
865
+ f"missing output shards={missing}; unexpected output shards={unexpected}"
866
+ )
867
+
868
+ row_count = 0
869
+ for input_shard in input_shards:
870
+ output_shard = input_shard.with_name(output_name_for(input_shard.name))
871
+ _, shard_outputs = validate_rank_shard(input_shard, output_shard)
872
+ row_count += len(shard_outputs)
873
+ print(
874
+ f"Validated {len(workers)} ranking workers, "
875
+ f"{len(input_shards)} shards, and {row_count} ranking rows"
876
+ )
877
+
878
+
879
+ def validate_rank_shard(
880
+ input_shard: Path, output_shard: Path
881
+ ) -> tuple[list[JsonRow], list[JsonRow]]:
882
+ shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row)
883
+ require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
884
+ shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row)
885
+ require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
886
+
887
+ expected_paths = {str(row["path"]) for row in shard_inputs}
888
+ actual_paths = {str(row["path"]) for row in shard_outputs}
889
+ if expected_paths != actual_paths:
890
+ missing = sorted(expected_paths - actual_paths)
891
+ unknown = sorted(actual_paths - expected_paths)
892
+ raise SystemExit(
893
+ f"{output_shard}: paths do not match its input shard; "
894
+ f"missing={missing}; unknown={unknown}"
895
+ )
896
+
897
+ area_by_path = {str(row["path"]): row["area"] for row in shard_inputs}
898
+ for row in shard_outputs:
899
+ row_path = str(row["path"])
900
+ if row["area"] != area_by_path[row_path]:
901
+ raise SystemExit(f"{output_shard}: area does not match rank input for {row_path}")
902
+ return shard_inputs, shard_outputs
903
+
904
+
905
+ def validate_rank_shard_command(args: argparse.Namespace) -> None:
906
+ input_shard = Path(args.input).expanduser()
907
+ output_shard = Path(args.output).expanduser()
908
+ _, output_rows = validate_rank_shard(input_shard, output_shard)
909
+ print(f"Validated {len(output_rows)} ranking rows in {output_shard}")
910
+
911
+
912
+ def merge_rank_outputs(args: argparse.Namespace) -> None:
913
+ rank_input = Path(args.rank_input).expanduser()
914
+ authoritative_rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
915
+ require_unique_paths(authoritative_rows, "Rank input")
916
+
917
+ shard_dir = Path(args.shard_dir).expanduser()
918
+ input_shards = discover_input_shards(shard_dir)
919
+ output_shards = sorted(shard_dir.glob(SHARD_OUTPUT_GLOB))
920
+ expected_output_names = {
921
+ path.name.replace(".input.jsonl", ".output.jsonl") for path in input_shards
922
+ }
923
+ actual_output_names = {path.name for path in output_shards}
924
+ if expected_output_names != actual_output_names:
925
+ missing = sorted(expected_output_names - actual_output_names)
926
+ unexpected = sorted(actual_output_names - expected_output_names)
927
+ details: list[str] = []
928
+ if missing:
929
+ details.append(f"missing output shards {missing}")
930
+ if unexpected:
931
+ details.append(f"unexpected output shards {unexpected}")
932
+ raise SystemExit(f"Rank shard outputs are incomplete: {'; '.join(details)}")
933
+
934
+ sharded_inputs: list[JsonRow] = []
935
+ output_by_path: dict[str, JsonRow] = {}
936
+ for input_shard in input_shards:
937
+ output_shard = input_shard.with_name(
938
+ input_shard.name.replace(".input.jsonl", ".output.jsonl")
939
+ )
940
+ shard_inputs, shard_outputs = validate_rank_shard(input_shard, output_shard)
941
+ sharded_inputs.extend(shard_inputs)
942
+ for row in shard_outputs:
943
+ row_path = str(row["path"])
944
+ if row_path in output_by_path:
945
+ raise SystemExit(f"Rank outputs contain duplicate path: {row_path}")
946
+ output_by_path[row_path] = row
947
+
948
+ if sharded_inputs != authoritative_rows:
949
+ raise SystemExit("Rank input shards do not exactly partition the authoritative rank input")
950
+
951
+ merged = [output_by_path[str(row["path"])] for row in authoritative_rows]
952
+ output = Path(args.out).expanduser()
953
+ write_jsonl(output, merged)
954
+ print(f"Merged {len(merged)} ranking rows into {output}")
955
+
956
+
957
+ def copy_deep_review_input(args: argparse.Namespace) -> None:
958
+ rank_input = Path(args.rank_input).expanduser()
959
+ rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
960
+ require_unique_paths(rows, "Rank input")
961
+ selected = [{"path": row["path"], "area": row["area"]} for row in rows]
962
+
963
+ output = Path(args.out).expanduser()
964
+ write_jsonl(output, selected)
965
+ print(f"Copied {len(selected)} rows into {output}")
966
+
967
+
968
+ def select_deep_review_input(args: argparse.Namespace) -> None:
969
+ rank_output = Path(args.rank_output).expanduser()
970
+ rows = load_jsonl(rank_output, "Rank output", validate_rank_output_row)
971
+ require_unique_paths(rows, "Rank output")
972
+
973
+ included = [row for row in rows if row["include"]]
974
+ base_rows = included if included else rows
975
+ base_rows.sort(key=lambda row: (-int(row["score"]), str(row["path"])))
976
+ keep = max(1, int(len(base_rows) * (args.top_percent / 100.0))) if base_rows else 0
977
+ selected = [{"path": row["path"], "area": row["area"]} for row in base_rows[:keep]]
978
+
979
+ output = Path(args.out).expanduser()
980
+ write_jsonl(output, selected)
981
+ print(f"Selected {len(selected)} of {len(base_rows)} rows into {output}")
982
+
983
+
984
+ def main() -> None:
985
+ args = parse_args()
986
+ if args.command == "make-repo-rank-input":
987
+ make_repo_rank_input(args)
988
+ elif args.command == "bind-repo-scopes":
989
+ bind_repo_scopes(args)
990
+ elif args.command == "make-diff-rank-input":
991
+ make_diff_rank_input(args)
992
+ elif args.command == "make-rank-shards":
993
+ make_rank_shards(args)
994
+ elif args.command == "make-rank-pool-plan":
995
+ make_rank_pool_plan(args)
996
+ elif args.command == "validate-rank-shard":
997
+ validate_rank_shard_command(args)
998
+ elif args.command == "validate-rank-worker":
999
+ validate_rank_worker_command(args)
1000
+ elif args.command == "validate-rank-pool":
1001
+ validate_rank_pool_command(args)
1002
+ elif args.command == "merge-rank-outputs":
1003
+ merge_rank_outputs(args)
1004
+ elif args.command == "copy-deep-review-input":
1005
+ copy_deep_review_input(args)
1006
+ elif args.command == "select-deep-review-input":
1007
+ select_deep_review_input(args)
1008
+ else:
1009
+ raise SystemExit(f"Unknown command: {args.command}")
1010
+
1011
+
1012
+ if __name__ == "__main__":
1013
+ main()