@openai/codex-security 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (484) hide show
  1. package/README.md +437 -2692
  2. package/_bundled_plugin/.app.json +5 -1
  3. package/_bundled_plugin/.codex-plugin/plugin.json +4 -0
  4. package/_bundled_plugin/.mcp.json +2 -0
  5. package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
  6. package/_bundled_plugin/examples/completed-scan/report.md +28 -26
  7. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
  8. package/_bundled_plugin/mcp/helpers.mjs +1 -0
  9. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  10. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +6 -213
  11. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  12. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  13. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +556 -0
  20. package/_bundled_plugin/mcp/server.mjs +1 -0
  21. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  22. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  23. package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
  24. package/_bundled_plugin/references/artifact-storage.md +2 -2
  25. package/_bundled_plugin/references/config-preflight.md +0 -4
  26. package/_bundled_plugin/references/core-scan.md +21 -9
  27. package/_bundled_plugin/references/final-report.md +11 -13
  28. package/_bundled_plugin/references/finding-detail-fields.md +90 -80
  29. package/_bundled_plugin/references/sarif-adapter.md +1 -1
  30. package/_bundled_plugin/references/scan-artifacts.md +1 -2
  31. package/_bundled_plugin/references/scan-contract.md +9 -4
  32. package/_bundled_plugin/references/scan-prologue.md +12 -0
  33. package/_bundled_plugin/references/security-guidance.md +2 -2
  34. package/_bundled_plugin/references/standalone-validation.md +11 -0
  35. package/_bundled_plugin/references/threat-model.md +1 -1
  36. package/_bundled_plugin/schemas/coverage.schema.json +43 -170
  37. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
  38. package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
  39. package/_bundled_plugin/schemas/findings.schema.json +159 -567
  40. package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +106 -119
  41. package/_bundled_plugin/schemas/scan-manifest.schema.json +78 -220
  42. package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
  43. package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
  44. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
  45. package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
  46. package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
  47. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +169 -294
  48. package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
  49. package/_bundled_plugin/scripts/codex_session.mjs +53 -0
  50. package/_bundled_plugin/scripts/config_preflight.py +26 -62
  51. package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
  52. package/_bundled_plugin/scripts/deep_scan_workbench.py +129 -150
  53. package/_bundled_plugin/scripts/finalize_scan_contract.py +321 -237
  54. package/_bundled_plugin/scripts/finding_preview.py +39 -9
  55. package/_bundled_plugin/scripts/generate_in_scope_files.py +27 -77
  56. package/_bundled_plugin/scripts/generate_rank_input.py +115 -926
  57. package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -10
  58. package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
  59. package/_bundled_plugin/scripts/rank_preview.py +42 -918
  60. package/_bundled_plugin/scripts/report_projection.py +150 -183
  61. package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
  62. package/_bundled_plugin/scripts/threat_model_projection.py +4 -7
  63. package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
  64. package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
  65. package/_bundled_plugin/scripts/windows_scan_local_files.py +48 -3
  66. package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
  67. package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
  68. package/_bundled_plugin/scripts/workbench/storage.py +10 -6
  69. package/_bundled_plugin/scripts/workbench_cli.py +102 -184
  70. package/_bundled_plugin/scripts/workbench_db.py +463 -464
  71. package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
  72. package/_bundled_plugin/scripts/workbench_finding_index.py +19 -1
  73. package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
  74. package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
  75. package/_bundled_plugin/scripts/workbench_native_indexes.py +27 -14
  76. package/_bundled_plugin/scripts/workbench_progress.py +14 -58
  77. package/_bundled_plugin/scripts/workbench_publication.py +234 -289
  78. package/_bundled_plugin/scripts/workbench_remediation.py +12 -0
  79. package/_bundled_plugin/scripts/workbench_saved_results.py +172 -49
  80. package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
  81. package/_bundled_plugin/scripts/workbench_scan_start.py +105 -41
  82. package/_bundled_plugin/scripts/workbench_scan_usage.py +194 -42
  83. package/_bundled_plugin/scripts/workbench_schema.py +37 -1151
  84. package/_bundled_plugin/scripts/workbench_source_excerpt.py +11 -1
  85. package/_bundled_plugin/scripts/workbench_target.py +430 -142
  86. package/_bundled_plugin/scripts/workbench_validation.py +31 -28
  87. package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
  88. package/_bundled_plugin/shared/workbench-migrations.json +414 -0
  89. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
  90. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
  91. package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
  92. package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
  93. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
  94. package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
  95. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
  96. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
  97. package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
  98. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  99. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +15 -1
  100. package/_bundled_plugin/skills/threat-model/SKILL.md +1 -1
  101. package/_bundled_plugin/skills/track-findings/SKILL.md +48 -176
  102. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
  103. package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
  104. package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
  105. package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
  106. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
  107. package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
  108. package/bin/codex-security.mjs +9 -2
  109. package/dist/api.d.ts +63 -10
  110. package/dist/api.d.ts.map +1 -1
  111. package/dist/api.js +1015 -383
  112. package/dist/api.js.map +1 -1
  113. package/dist/artifact-export.d.ts +1 -1
  114. package/dist/artifact-export.d.ts.map +1 -1
  115. package/dist/artifact-export.js +28 -13
  116. package/dist/artifact-export.js.map +1 -1
  117. package/dist/auth.d.ts +2 -6
  118. package/dist/auth.d.ts.map +1 -1
  119. package/dist/auth.js +19 -9
  120. package/dist/auth.js.map +1 -1
  121. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  122. package/dist/bulk-scan-discovery.js +1 -0
  123. package/dist/bulk-scan-discovery.js.map +1 -1
  124. package/dist/classify-severity.d.ts +26 -9
  125. package/dist/classify-severity.d.ts.map +1 -1
  126. package/dist/classify-severity.js +13 -30
  127. package/dist/classify-severity.js.map +1 -1
  128. package/dist/cli-help.d.ts.map +1 -1
  129. package/dist/cli-help.js +17 -13
  130. package/dist/cli-help.js.map +1 -1
  131. package/dist/cli-signals.d.ts +8 -0
  132. package/dist/cli-signals.d.ts.map +1 -0
  133. package/dist/cli-signals.js +28 -0
  134. package/dist/cli-signals.js.map +1 -0
  135. package/dist/cli.d.ts +4 -3
  136. package/dist/cli.d.ts.map +1 -1
  137. package/dist/cli.js +1102 -905
  138. package/dist/cli.js.map +1 -1
  139. package/dist/cloud-publish.d.ts.map +1 -1
  140. package/dist/cloud-publish.js +10 -9
  141. package/dist/cloud-publish.js.map +1 -1
  142. package/dist/codeowners.d.ts +19 -0
  143. package/dist/codeowners.d.ts.map +1 -0
  144. package/dist/codeowners.js +138 -0
  145. package/dist/codeowners.js.map +1 -0
  146. package/dist/codex-sdk-environment.d.ts +1 -0
  147. package/dist/codex-sdk-environment.d.ts.map +1 -1
  148. package/dist/codex-sdk-environment.js +40 -2
  149. package/dist/codex-sdk-environment.js.map +1 -1
  150. package/dist/component-plan.d.ts +2 -5
  151. package/dist/component-plan.d.ts.map +1 -1
  152. package/dist/component-plan.js +39 -31
  153. package/dist/component-plan.js.map +1 -1
  154. package/dist/component-scan.d.ts +6 -5
  155. package/dist/component-scan.d.ts.map +1 -1
  156. package/dist/component-scan.js +25 -36
  157. package/dist/component-scan.js.map +1 -1
  158. package/dist/config.d.ts.map +1 -1
  159. package/dist/config.js +64 -74
  160. package/dist/config.js.map +1 -1
  161. package/dist/contract.d.ts +16 -3
  162. package/dist/contract.d.ts.map +1 -1
  163. package/dist/contract.js +91 -255
  164. package/dist/contract.js.map +1 -1
  165. package/dist/cost-model.d.ts +13 -6
  166. package/dist/cost-model.d.ts.map +1 -1
  167. package/dist/cost-model.js +14 -4
  168. package/dist/cost-model.js.map +1 -1
  169. package/dist/cost.d.ts +10 -2
  170. package/dist/cost.d.ts.map +1 -1
  171. package/dist/cost.js +102 -104
  172. package/dist/cost.js.map +1 -1
  173. package/dist/custom-validation-prompt.d.ts.map +1 -1
  174. package/dist/custom-validation-prompt.js +6 -5
  175. package/dist/custom-validation-prompt.js.map +1 -1
  176. package/dist/custom-validation.d.ts +1 -2
  177. package/dist/custom-validation.d.ts.map +1 -1
  178. package/dist/custom-validation.js +9 -3
  179. package/dist/custom-validation.js.map +1 -1
  180. package/dist/deduplication/checkpointed-review.d.ts +4 -2
  181. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  182. package/dist/deduplication/checkpointed-review.js +23 -9
  183. package/dist/deduplication/checkpointed-review.js.map +1 -1
  184. package/dist/deduplication/codex-review.d.ts +8 -6
  185. package/dist/deduplication/codex-review.d.ts.map +1 -1
  186. package/dist/deduplication/codex-review.js +128 -19
  187. package/dist/deduplication/codex-review.js.map +1 -1
  188. package/dist/deduplication/deduplication-prompts.d.ts +5 -2
  189. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  190. package/dist/deduplication/deduplication-prompts.js +18 -4
  191. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  192. package/dist/deduplication/deduplication-reviewer.d.ts +27 -5
  193. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  194. package/dist/deduplication/deduplication-reviewer.js +58 -48
  195. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  196. package/dist/deduplication/deduplication.d.ts.map +1 -1
  197. package/dist/deduplication/deduplication.js +19 -51
  198. package/dist/deduplication/deduplication.js.map +1 -1
  199. package/dist/deduplication/diagnostics.d.ts +17 -0
  200. package/dist/deduplication/diagnostics.d.ts.map +1 -0
  201. package/dist/deduplication/diagnostics.js +5 -0
  202. package/dist/deduplication/diagnostics.js.map +1 -0
  203. package/dist/deduplication/finding-schema.d.ts.map +1 -1
  204. package/dist/deduplication/finding-schema.js +7 -1
  205. package/dist/deduplication/finding-schema.js.map +1 -1
  206. package/dist/deduplication/local.d.ts +31 -0
  207. package/dist/deduplication/local.d.ts.map +1 -0
  208. package/dist/deduplication/local.js +166 -0
  209. package/dist/deduplication/local.js.map +1 -0
  210. package/dist/deduplication/records-protocol.d.ts +3 -8
  211. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  212. package/dist/deduplication/records-protocol.js +11 -184
  213. package/dist/deduplication/records-protocol.js.map +1 -1
  214. package/dist/deduplication/records.d.ts +186 -11
  215. package/dist/deduplication/records.d.ts.map +1 -1
  216. package/dist/deduplication/records.js +10 -18
  217. package/dist/deduplication/records.js.map +1 -1
  218. package/dist/deduplication/scan.d.ts +9 -3
  219. package/dist/deduplication/scan.d.ts.map +1 -1
  220. package/dist/deduplication/scan.js +158 -39
  221. package/dist/deduplication/scan.js.map +1 -1
  222. package/dist/deep-config.d.ts +2 -1
  223. package/dist/deep-config.d.ts.map +1 -1
  224. package/dist/deep-config.js +4 -4
  225. package/dist/deep-config.js.map +1 -1
  226. package/dist/deep-progress.d.ts +17 -1
  227. package/dist/deep-progress.d.ts.map +1 -1
  228. package/dist/deep-progress.js +19 -17
  229. package/dist/deep-progress.js.map +1 -1
  230. package/dist/deep-scan.d.ts +20 -0
  231. package/dist/deep-scan.d.ts.map +1 -0
  232. package/dist/deep-scan.js +113 -0
  233. package/dist/deep-scan.js.map +1 -0
  234. package/dist/errors.d.ts +2 -0
  235. package/dist/errors.d.ts.map +1 -1
  236. package/dist/errors.js +2 -0
  237. package/dist/errors.js.map +1 -1
  238. package/dist/executable-path.d.ts +2 -0
  239. package/dist/executable-path.d.ts.map +1 -0
  240. package/dist/executable-path.js +12 -0
  241. package/dist/executable-path.js.map +1 -0
  242. package/dist/feedback.d.ts +1 -4
  243. package/dist/feedback.d.ts.map +1 -1
  244. package/dist/feedback.js +54 -22
  245. package/dist/feedback.js.map +1 -1
  246. package/dist/finding-catalogue.d.ts.map +1 -1
  247. package/dist/finding-catalogue.js +4 -8
  248. package/dist/finding-catalogue.js.map +1 -1
  249. package/dist/finding-normalization.d.ts +5 -0
  250. package/dist/finding-normalization.d.ts.map +1 -0
  251. package/dist/finding-normalization.js +120 -0
  252. package/dist/finding-normalization.js.map +1 -0
  253. package/dist/finding-retrieval.d.ts +9 -0
  254. package/dist/finding-retrieval.d.ts.map +1 -1
  255. package/dist/finding-retrieval.js.map +1 -1
  256. package/dist/finding-workflow.d.ts +13 -8
  257. package/dist/finding-workflow.d.ts.map +1 -1
  258. package/dist/finding-workflow.js +14 -6
  259. package/dist/finding-workflow.js.map +1 -1
  260. package/dist/findings-client.d.ts +2 -1
  261. package/dist/findings-client.d.ts.map +1 -1
  262. package/dist/findings-client.js +36 -9
  263. package/dist/findings-client.js.map +1 -1
  264. package/dist/findings-errors.d.ts +16 -0
  265. package/dist/findings-errors.d.ts.map +1 -0
  266. package/dist/findings-errors.js +24 -0
  267. package/dist/findings-errors.js.map +1 -0
  268. package/dist/findings-import.d.ts +1 -1
  269. package/dist/findings-import.d.ts.map +1 -1
  270. package/dist/findings-import.js +12 -30
  271. package/dist/findings-import.js.map +1 -1
  272. package/dist/github.d.ts.map +1 -1
  273. package/dist/github.js +7 -1
  274. package/dist/github.js.map +1 -1
  275. package/dist/host-protocol.d.ts +19 -0
  276. package/dist/host-protocol.d.ts.map +1 -0
  277. package/dist/host-protocol.js +270 -0
  278. package/dist/host-protocol.js.map +1 -0
  279. package/dist/hosted-scan-protocol.d.ts +4 -0
  280. package/dist/hosted-scan-protocol.d.ts.map +1 -0
  281. package/dist/hosted-scan-protocol.js +25 -0
  282. package/dist/hosted-scan-protocol.js.map +1 -0
  283. package/dist/hosted-scan.d.ts +117 -0
  284. package/dist/hosted-scan.d.ts.map +1 -0
  285. package/dist/hosted-scan.js +260 -0
  286. package/dist/hosted-scan.js.map +1 -0
  287. package/dist/import-scan.d.ts.map +1 -1
  288. package/dist/import-scan.js +48 -38
  289. package/dist/import-scan.js.map +1 -1
  290. package/dist/index.d.ts +11 -4
  291. package/dist/index.d.ts.map +1 -1
  292. package/dist/index.js +2 -1
  293. package/dist/index.js.map +1 -1
  294. package/dist/knowledge-base.d.ts +7 -1
  295. package/dist/knowledge-base.d.ts.map +1 -1
  296. package/dist/knowledge-base.js +114 -46
  297. package/dist/knowledge-base.js.map +1 -1
  298. package/dist/linear.d.ts.map +1 -1
  299. package/dist/linear.js +2 -7
  300. package/dist/linear.js.map +1 -1
  301. package/dist/mock-scan.d.ts.map +1 -1
  302. package/dist/mock-scan.js +0 -1
  303. package/dist/mock-scan.js.map +1 -1
  304. package/dist/models.d.ts +0 -2
  305. package/dist/models.d.ts.map +1 -1
  306. package/dist/models.js.map +1 -1
  307. package/dist/multiscan.d.ts +9 -3
  308. package/dist/multiscan.d.ts.map +1 -1
  309. package/dist/multiscan.js +240 -135
  310. package/dist/multiscan.js.map +1 -1
  311. package/dist/owner-evidence.d.ts +10 -2
  312. package/dist/owner-evidence.d.ts.map +1 -1
  313. package/dist/owner-evidence.js +67 -8
  314. package/dist/owner-evidence.js.map +1 -1
  315. package/dist/patch-tui.d.ts.map +1 -1
  316. package/dist/patch-tui.js +3 -2
  317. package/dist/patch-tui.js.map +1 -1
  318. package/dist/project-config.js +3 -3
  319. package/dist/project-config.js.map +1 -1
  320. package/dist/provider-profile.d.ts +21 -0
  321. package/dist/provider-profile.d.ts.map +1 -0
  322. package/dist/provider-profile.js +133 -0
  323. package/dist/provider-profile.js.map +1 -0
  324. package/dist/publication-events.d.ts +6 -8
  325. package/dist/publication-events.d.ts.map +1 -1
  326. package/dist/publication-events.js +25 -50
  327. package/dist/publication-events.js.map +1 -1
  328. package/dist/publication-store.d.ts.map +1 -1
  329. package/dist/publication-store.js +95 -109
  330. package/dist/publication-store.js.map +1 -1
  331. package/dist/publication.d.ts +1 -4
  332. package/dist/publication.d.ts.map +1 -1
  333. package/dist/publication.js +2 -13
  334. package/dist/publication.js.map +1 -1
  335. package/dist/publish.d.ts +1 -5
  336. package/dist/publish.d.ts.map +1 -1
  337. package/dist/publish.js +109 -160
  338. package/dist/publish.js.map +1 -1
  339. package/dist/request-metadata.d.ts +9 -0
  340. package/dist/request-metadata.d.ts.map +1 -0
  341. package/dist/request-metadata.js +13 -0
  342. package/dist/request-metadata.js.map +1 -0
  343. package/dist/runtime.d.ts +20 -6
  344. package/dist/runtime.d.ts.map +1 -1
  345. package/dist/runtime.js +407 -148
  346. package/dist/runtime.js.map +1 -1
  347. package/dist/saved-scan-bootstrap.d.ts +14 -0
  348. package/dist/saved-scan-bootstrap.d.ts.map +1 -0
  349. package/dist/saved-scan-bootstrap.js +261 -0
  350. package/dist/saved-scan-bootstrap.js.map +1 -0
  351. package/dist/saved-scan.d.ts +1 -1
  352. package/dist/saved-scan.d.ts.map +1 -1
  353. package/dist/saved-scan.js +6 -12
  354. package/dist/saved-scan.js.map +1 -1
  355. package/dist/scan-activity.js +11 -8
  356. package/dist/scan-activity.js.map +1 -1
  357. package/dist/scan-comparison.d.ts +28 -15
  358. package/dist/scan-comparison.d.ts.map +1 -1
  359. package/dist/scan-comparison.js +276 -281
  360. package/dist/scan-comparison.js.map +1 -1
  361. package/dist/scan-dashboard.d.ts +3 -0
  362. package/dist/scan-dashboard.d.ts.map +1 -1
  363. package/dist/scan-dashboard.js +345 -77
  364. package/dist/scan-dashboard.js.map +1 -1
  365. package/dist/scan-executor.d.ts +104 -0
  366. package/dist/scan-executor.d.ts.map +1 -0
  367. package/dist/scan-executor.js +62 -0
  368. package/dist/scan-executor.js.map +1 -0
  369. package/dist/scan-history-renderer.d.ts.map +1 -1
  370. package/dist/scan-history-renderer.js +54 -13
  371. package/dist/scan-history-renderer.js.map +1 -1
  372. package/dist/scan-inputs.d.ts +6 -0
  373. package/dist/scan-inputs.d.ts.map +1 -0
  374. package/dist/scan-inputs.js +71 -0
  375. package/dist/scan-inputs.js.map +1 -0
  376. package/dist/scan-logs.d.ts +2 -3
  377. package/dist/scan-logs.d.ts.map +1 -1
  378. package/dist/scan-logs.js +12 -6
  379. package/dist/scan-logs.js.map +1 -1
  380. package/dist/scan-sessions.d.ts +4 -0
  381. package/dist/scan-sessions.d.ts.map +1 -1
  382. package/dist/scan-sessions.js +17 -0
  383. package/dist/scan-sessions.js.map +1 -1
  384. package/dist/scan-skill-paths.d.ts +8 -0
  385. package/dist/scan-skill-paths.d.ts.map +1 -0
  386. package/dist/scan-skill-paths.js +47 -0
  387. package/dist/scan-skill-paths.js.map +1 -0
  388. package/dist/security-policy-cli.d.ts.map +1 -1
  389. package/dist/security-policy-cli.js +7 -27
  390. package/dist/security-policy-cli.js.map +1 -1
  391. package/dist/security-policy.d.ts +9 -5
  392. package/dist/security-policy.d.ts.map +1 -1
  393. package/dist/security-policy.js +33 -14
  394. package/dist/security-policy.js.map +1 -1
  395. package/dist/server/api.d.ts +0 -1
  396. package/dist/server/api.d.ts.map +1 -1
  397. package/dist/server/api.js +0 -1
  398. package/dist/server/api.js.map +1 -1
  399. package/dist/server/embeddings.d.ts +3 -1
  400. package/dist/server/embeddings.d.ts.map +1 -1
  401. package/dist/server/embeddings.js +10 -3
  402. package/dist/server/embeddings.js.map +1 -1
  403. package/dist/server/errors.d.ts +5 -2
  404. package/dist/server/errors.d.ts.map +1 -1
  405. package/dist/server/errors.js +7 -0
  406. package/dist/server/errors.js.map +1 -1
  407. package/dist/server/sqlite-store.d.ts +0 -2
  408. package/dist/server/sqlite-store.d.ts.map +1 -1
  409. package/dist/server/sqlite-store.js +54 -44
  410. package/dist/server/sqlite-store.js.map +1 -1
  411. package/dist/server/storage.d.ts +0 -2
  412. package/dist/server/storage.d.ts.map +1 -1
  413. package/dist/server/storage.js.map +1 -1
  414. package/dist/severity-store.js +19 -24
  415. package/dist/severity-store.js.map +1 -1
  416. package/dist/standalone-validation-prompt.d.ts +2 -0
  417. package/dist/standalone-validation-prompt.d.ts.map +1 -0
  418. package/dist/standalone-validation-prompt.js +10 -0
  419. package/dist/standalone-validation-prompt.js.map +1 -0
  420. package/dist/suggest-owners.d.ts +4 -2
  421. package/dist/suggest-owners.d.ts.map +1 -1
  422. package/dist/suggest-owners.js +11 -5
  423. package/dist/suggest-owners.js.map +1 -1
  424. package/dist/targets.d.ts +11 -0
  425. package/dist/targets.d.ts.map +1 -1
  426. package/dist/targets.js +123 -72
  427. package/dist/targets.js.map +1 -1
  428. package/dist/trusted-executable.d.ts +2 -2
  429. package/dist/trusted-executable.d.ts.map +1 -1
  430. package/dist/trusted-executable.js +8 -6
  431. package/dist/trusted-executable.js.map +1 -1
  432. package/dist/value.js +7 -0
  433. package/dist/value.js.map +1 -1
  434. package/dist/version.js.map +1 -1
  435. package/dist/workbench-arguments.d.ts +3 -0
  436. package/dist/workbench-arguments.d.ts.map +1 -0
  437. package/dist/workbench-arguments.js +15 -0
  438. package/dist/workbench-arguments.js.map +1 -0
  439. package/dist/worker-progress.d.ts +44 -11
  440. package/dist/worker-progress.d.ts.map +1 -1
  441. package/dist/worker-progress.js +48 -49
  442. package/dist/worker-progress.js.map +1 -1
  443. package/docs/cli.md +1788 -0
  444. package/docs/findings-service.md +402 -0
  445. package/docs/hosted-scans.md +242 -0
  446. package/package.json +23 -22
  447. package/schemas/hosted-scan-v2.fixture.json +145 -0
  448. package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
  449. package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
  450. package/_bundled_plugin/scripts/workbench_findings.py +0 -221
  451. package/_bundled_plugin/scripts/workbench_severity.py +0 -122
  452. package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
  453. package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
  454. package/dist/server/dashboard/app.css +0 -2
  455. package/dist/server/dashboard/app.js +0 -66
  456. package/dist/server/dashboard/index.html +0 -18
  457. package/dist/server/dashboard-types.d.ts +0 -48
  458. package/dist/server/dashboard-types.d.ts.map +0 -1
  459. package/dist/server/dashboard-types.js +0 -2
  460. package/dist/server/dashboard-types.js.map +0 -1
  461. package/dist/server/dashboard.d.ts +0 -6
  462. package/dist/server/dashboard.d.ts.map +0 -1
  463. package/dist/server/dashboard.js +0 -59
  464. package/dist/server/dashboard.js.map +0 -1
  465. package/dist/server/index.d.ts +0 -2
  466. package/dist/server/index.d.ts.map +0 -1
  467. package/dist/server/index.js +0 -6
  468. package/dist/server/index.js.map +0 -1
  469. package/dist/server/routes.d.ts +0 -7
  470. package/dist/server/routes.d.ts.map +0 -1
  471. package/dist/server/routes.js +0 -98
  472. package/dist/server/routes.js.map +0 -1
  473. package/dist/server/serve.d.ts +0 -2
  474. package/dist/server/serve.d.ts.map +0 -1
  475. package/dist/server/serve.js +0 -28
  476. package/dist/server/serve.js.map +0 -1
  477. package/dist/server/server.d.ts +0 -10
  478. package/dist/server/server.d.ts.map +0 -1
  479. package/dist/server/server.js +0 -16
  480. package/dist/server/server.js.map +0 -1
  481. package/dist/server/validation.d.ts +0 -17
  482. package/dist/server/validation.d.ts.map +0 -1
  483. package/dist/server/validation.js +0 -53
  484. package/dist/server/validation.js.map +0 -1
@@ -3,36 +3,23 @@
3
3
 
4
4
  This script stays deliberately model-free:
5
5
 
6
- - `make-repo-rank-input` creates the deterministic repository or scoped-path
7
- JSONL candidate worklist that ranking subagents consume.
6
+ - `make-repo-scope-input` lists every explicitly scoped repository file.
8
7
  - `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
9
8
  worklist from Git changed paths. It supports committed revision diffs and
10
9
  local working-tree patches.
11
- - `make-rank-shards` partitions the ranking input into deterministic shards.
12
- - `make-rank-pool-plan` assigns those shards to a deterministic bounded worker
13
- pool.
14
- - `validate-rank-worker` validates one worker slot and emits a content-bound
15
- completion receipt.
16
- - `validate-rank-shard` validates one completed worker output before the
17
- coordinator accepts it.
18
- - `validate-rank-pool` validates the pool plan and every assigned shard output.
19
- - `merge-rank-outputs` validates and combines worker-local shard outputs.
20
- - `copy-deep-review-input` copies every candidate into the deep-review worklist
21
- for exhaustive mode.
22
- - `select-deep-review-input` selects the ranked rows for deep review.
10
+
11
+ Candidate selection uses repository scope, native tool ignore rules, and binary detection,
12
+ not filename or directory classifications. Diff inputs use bounded source previews.
23
13
  """
24
14
 
25
15
  from __future__ import annotations
26
16
 
27
17
  import argparse
28
- import hashlib
29
18
  import json
30
19
  import os
31
- import re
32
20
  import subprocess
33
21
  import sys
34
- from collections import Counter
35
- from collections.abc import Callable, KeysView
22
+ from collections.abc import Iterable
36
23
  from pathlib import Path
37
24
 
38
25
  # Some plugin hosts launch Python with safe-path isolation enabled.
@@ -40,125 +27,19 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
40
27
  from generate_in_scope_files import windows_stream_component
41
28
  from rank_preview import (
42
29
  DEFAULT_PREVIEW_BYTES,
43
- TEXT_CODE_EXTENSIONS,
30
+ is_binary_file,
44
31
  preview_for,
45
32
  preview_for_bytes,
46
33
  )
47
- from workbench_target import git_blob_bytes, git_command, git_directory_snapshot_paths
48
-
49
- EXCLUDED_DIRS = {
50
- ".cache",
51
- ".circleci",
52
- ".devcontainer",
53
- ".git",
54
- ".github",
55
- ".idea",
56
- ".mypy_cache",
57
- ".pytest_cache",
58
- ".ruff_cache",
59
- ".tox",
60
- ".venv",
61
- ".vscode",
62
- "__pycache__",
63
- "bench",
64
- "benchmark",
65
- "bintest",
66
- "build",
67
- "build_config",
68
- "build_configs",
69
- "build-tools",
70
- "build_tools",
71
- "ci",
72
- "coverage",
73
- "deps",
74
- "dev",
75
- "dist",
76
- "doc",
77
- "docs",
78
- "example",
79
- "examples",
80
- "external",
81
- "extern",
82
- "fixture",
83
- "fixtures",
84
- "generated",
85
- "node_modules",
86
- "sample",
87
- "samples",
88
- "target",
89
- "test",
90
- "tests",
91
- "testing",
92
- "third-party",
93
- "third_party",
94
- "tmp",
95
- "vendor",
96
- }
34
+ from workbench_target import git_blob_samples, git_command, git_directory_snapshot_paths
97
35
 
98
- EXCLUDED_FILENAMES = {
99
- ".DS_Store",
100
- "CHANGELOG",
101
- "CHANGELOG.md",
102
- "CONTRIBUTING.md",
103
- "Dockerfile",
104
- "Gemfile",
105
- "Gemfile.lock",
106
- "LICENSE",
107
- "LICENSE.md",
108
- "Makefile",
109
- "NEWS",
110
- "NEWS.md",
111
- "NOTICE",
112
- "README",
113
- "README.md",
114
- "README.rst",
115
- "Rakefile",
116
- "SECURITY.md",
117
- "TODO",
118
- "TODO.md",
119
- "docker-compose.yml",
120
- "package-lock.json",
121
- "pnpm-lock.yaml",
122
- "yarn.lock",
123
- }
124
-
125
- SHARD_INPUT_GLOB = "rank-shard-*.input.jsonl"
126
- SHARD_OUTPUT_GLOB = "rank-shard-*.output.jsonl"
127
- SHARD_INPUT_PATTERN = re.compile(r"^rank-shard-([0-9]{4,})\.input\.jsonl$")
128
- RANK_POOL_PLAN_SCHEMA_VERSION = 1
129
- RANK_POOL_STRATEGY = "round_robin"
130
- RANK_POOL_WORKER_CAP = 6
131
36
  JsonRow = dict[str, object]
132
- RowValidator = Callable[[JsonRow, Path, int], None]
133
37
 
134
38
 
135
39
  def parse_args() -> argparse.Namespace:
136
40
  parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
137
41
  subparsers = parser.add_subparsers(dest="command", required=True)
138
42
 
139
- make = subparsers.add_parser(
140
- "make-repo-rank-input",
141
- help="Create rank_input.jsonl for subagent-based file ranking.",
142
- )
143
- make.add_argument("--repo", required=True, help="Repository root.")
144
- make.add_argument(
145
- "--scope",
146
- default=".",
147
- help="Path within the repository to scan. Defaults to the repository root.",
148
- )
149
- make.add_argument(
150
- "--scopes-file",
151
- help="JSON array of repository-relative files and directories to scan together.",
152
- )
153
- make.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
154
- make.add_argument("--area", default="", help="Area label. Defaults to scope.")
155
- make.add_argument(
156
- "--preview-bytes",
157
- type=int,
158
- default=DEFAULT_PREVIEW_BYTES,
159
- help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
160
- )
161
-
162
43
  scoped = subparsers.add_parser(
163
44
  "make-repo-scope-input",
164
45
  help="List every explicitly scoped file without ranking or reading its contents.",
@@ -171,17 +52,9 @@ def parse_args() -> argparse.Namespace:
171
52
  )
172
53
  scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
173
54
 
174
- bind = subparsers.add_parser(
175
- "bind-repo-scopes",
176
- help="Copy SDK scoped-path targets into the unsealed manifest and coverage documents.",
177
- )
178
- bind.add_argument("--scopes-file", required=True, help="JSON array of requested scopes.")
179
- bind.add_argument("--manifest", required=True, help="Unsealed scan-manifest.json path.")
180
- bind.add_argument("--coverage", required=True, help="Unsealed coverage.json path.")
181
-
182
55
  diff = subparsers.add_parser(
183
56
  "make-diff-rank-input",
184
- help="Create rank_input.jsonl from Git changed source-like files.",
57
+ help="Create rank_input.jsonl from Git changed text files.",
185
58
  )
186
59
  diff.add_argument("--repo", required=True, help="Repository root.")
187
60
  diff.add_argument("--base", required=True, help="Git diff base revision.")
@@ -193,7 +66,7 @@ def parse_args() -> argparse.Namespace:
193
66
  )
194
67
  diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
195
68
  diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
196
- diff.add_argument("--area", default="diff", help="Area label for ranking rows.")
69
+ diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")
197
70
  diff.add_argument(
198
71
  "--preview-bytes",
199
72
  type=int,
@@ -201,137 +74,35 @@ def parse_args() -> argparse.Namespace:
201
74
  help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
202
75
  )
203
76
 
204
- shards = subparsers.add_parser(
205
- "make-rank-shards",
206
- help="Partition rank_input.jsonl into deterministic worker input shards.",
207
- )
208
- shards.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
209
- shards.add_argument("--out-dir", required=True, help="Directory for worker input shards.")
210
- shards.add_argument(
211
- "--max-rows",
212
- type=int,
213
- default=150,
214
- help="Maximum rows per shard. Defaults to 150.",
215
- )
216
-
217
- pool_plan = subparsers.add_parser(
218
- "make-rank-pool-plan",
219
- help="Assign rank shards to a deterministic bounded worker pool.",
220
- )
221
- pool_plan.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
222
- pool_plan.add_argument(
223
- "--usable-worker-slots",
224
- required=True,
225
- type=int,
226
- help="Usable ranking-worker slots reported by capability preflight; capped at 6.",
227
- )
228
- pool_plan.add_argument("--out", required=True, help="Output rank_worker_assignments.json path.")
229
-
230
- validate_shard = subparsers.add_parser(
231
- "validate-rank-shard",
232
- help="Validate one worker output against its rank input shard.",
233
- )
234
- validate_shard.add_argument("--input", required=True, help="Worker rank input shard.")
235
- validate_shard.add_argument("--output", required=True, help="Worker rank output shard.")
236
-
237
- validate_worker = subparsers.add_parser(
238
- "validate-rank-worker",
239
- help="Validate one assigned ranking-worker slot and emit its completion receipt.",
240
- )
241
- validate_worker.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
242
- validate_worker.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
243
- validate_worker.add_argument(
244
- "--slot",
245
- required=True,
246
- type=int,
247
- help="One-based ranking-worker slot from the rank pool plan.",
248
- )
249
-
250
- validate_pool = subparsers.add_parser(
251
- "validate-rank-pool",
252
- help="Validate a rank pool plan and every assigned shard output.",
253
- )
254
- validate_pool.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
255
- validate_pool.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
256
-
257
- merge = subparsers.add_parser(
258
- "merge-rank-outputs",
259
- help="Validate worker shard outputs and create rank_output.jsonl.",
260
- )
261
- merge.add_argument("--rank-input", required=True, help="Authoritative rank input JSONL.")
262
- merge.add_argument("--shard-dir", required=True, help="Directory of input and output shards.")
263
- merge.add_argument("--out", required=True, help="Output rank_output.jsonl path.")
264
-
265
- copy = subparsers.add_parser(
266
- "copy-deep-review-input",
267
- help="Create deep_review_input.jsonl directly from rank_input.jsonl.",
268
- )
269
- copy.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
270
- copy.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
271
-
272
- select = subparsers.add_parser(
273
- "select-deep-review-input",
274
- help="Create deep_review_input.jsonl from worker-produced rank_output.jsonl.",
275
- )
276
- select.add_argument("--rank-output", required=True, help="Worker ranking output JSONL.")
277
- select.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
278
- select.add_argument(
279
- "--top-percent",
280
- type=int,
281
- default=100,
282
- help="Percent of included files to keep for deep review.",
283
- )
284
77
  return parser.parse_args()
285
78
 
286
79
 
287
- def path_is_excluded(path: Path) -> bool:
288
- if any(part in EXCLUDED_DIRS for part in path.parts):
289
- return True
290
- if path.name in EXCLUDED_FILENAMES:
291
- return True
292
- return path.name.endswith((".min.js", ".map"))
293
-
294
-
295
- def path_is_diff_excluded(path: Path) -> bool:
296
- """Apply repository exclusions while retaining changed workflow files."""
297
- if path.parts[:2] == (".github", "workflows"):
298
- return False
299
- return path_is_excluded(path)
300
-
301
-
302
- def resolve_scope(
303
- repo: Path,
304
- scope: str,
305
- *,
306
- expand_user: bool = True,
307
- reject_symlinks: bool = False,
308
- ) -> Path:
309
- scope_path = Path(scope).expanduser() if expand_user else Path(scope)
80
+ def resolve_scope(repo: Path, scope: str) -> Path:
81
+ scope_path = Path(scope)
310
82
  stream = windows_stream_component(scope_path)
311
83
  if stream is not None:
312
84
  raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")
313
85
  if not scope_path.is_absolute():
314
86
  scope_path = repo / scope_path
315
- if reject_symlinks:
316
- repository = repo.resolve()
87
+ repository = repo.resolve()
88
+ try:
89
+ relative = scope_path.relative_to(repository)
90
+ except ValueError as exc:
91
+ raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
92
+ ancestor = repository
93
+ for part in relative.parts:
94
+ if part == "..":
95
+ if ancestor == repository:
96
+ raise SystemExit(f"Scope must be inside repo: {scope_path}")
97
+ ancestor = ancestor.parent
98
+ continue
99
+ ancestor /= part
317
100
  try:
318
- relative = scope_path.relative_to(repository)
319
- except ValueError as exc:
320
- raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
321
- ancestor = repository
322
- for part in relative.parts:
323
- if part == "..":
324
- if ancestor == repository:
325
- raise SystemExit(f"Scope must be inside repo: {scope_path}")
326
- ancestor = ancestor.parent
327
- continue
328
- ancestor /= part
329
- try:
330
- metadata = ancestor.stat(follow_symlinks=False)
331
- except OSError as exc:
332
- raise SystemExit(f"Scope path not found: {ancestor}") from exc
333
- if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
334
- raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
101
+ metadata = ancestor.stat(follow_symlinks=False)
102
+ except OSError as exc:
103
+ raise SystemExit(f"Scope path not found: {ancestor}") from exc
104
+ if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
105
+ raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
335
106
  scope_path = scope_path.resolve()
336
107
  repo_resolved = repo.resolve()
337
108
  try:
@@ -351,14 +122,6 @@ def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
351
122
  handle.write("\n")
352
123
 
353
124
 
354
- def write_json(output: Path, payload: dict[str, object]) -> None:
355
- output.parent.mkdir(parents=True, exist_ok=True)
356
- output.write_text(
357
- json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
358
- encoding="utf-8",
359
- )
360
-
361
-
362
125
  def load_scopes_file(scopes_file: Path) -> list[str]:
363
126
  try:
364
127
  loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
@@ -373,135 +136,58 @@ def load_scopes_file(scopes_file: Path) -> list[str]:
373
136
  return loaded
374
137
 
375
138
 
376
- def load_jsonl(path: Path, label: str, validator: RowValidator) -> list[JsonRow]:
377
- if not path.exists():
378
- raise SystemExit(f"{label} missing: {path}")
379
-
380
- rows: list[JsonRow] = []
381
- with path.open(encoding="utf-8") as handle:
382
- for line_number, raw_line in enumerate(handle, start=1):
383
- if not raw_line.strip():
384
- raise SystemExit(f"{path}:{line_number}: blank JSONL rows are not allowed")
385
- try:
386
- row: object = json.loads(raw_line)
387
- except json.JSONDecodeError as exc:
388
- raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc
389
- if not isinstance(row, dict):
390
- raise SystemExit(f"{path}:{line_number}: expected a JSON object")
391
- validator(row, path, line_number)
392
- rows.append(row)
393
- return rows
394
-
395
-
396
- def require_exact_fields(row: JsonRow, expected: set[str], path: Path, line_number: int) -> None:
397
- actual = set(row)
398
- if actual != expected:
399
- missing = sorted(expected - actual)
400
- unexpected = sorted(actual - expected)
401
- details: list[str] = []
402
- if missing:
403
- details.append(f"missing fields {missing}")
404
- if unexpected:
405
- details.append(f"unexpected fields {unexpected}")
406
- raise SystemExit(f"{path}:{line_number}: {'; '.join(details)}")
407
-
408
-
409
- def require_string(
410
- row: JsonRow, field: str, path: Path, line_number: int, *, allow_empty: bool
411
- ) -> None:
412
- value = row[field]
413
- if not isinstance(value, str) or (not allow_empty and not value.strip()):
414
- requirement = "a string" if allow_empty else "a non-empty string"
415
- raise SystemExit(f"{path}:{line_number}: {field} must be {requirement}")
416
-
417
-
418
- def validate_rank_input_row(row: JsonRow, path: Path, line_number: int) -> None:
419
- require_exact_fields(row, {"path", "area", "preview"}, path, line_number)
420
- require_string(row, "path", path, line_number, allow_empty=False)
421
- require_string(row, "area", path, line_number, allow_empty=True)
422
- require_string(row, "preview", path, line_number, allow_empty=True)
423
-
424
-
425
- def validate_rank_output_row(row: JsonRow, path: Path, line_number: int) -> None:
426
- require_exact_fields(row, {"path", "area", "score", "include", "reason"}, path, line_number)
427
- require_string(row, "path", path, line_number, allow_empty=False)
428
- require_string(row, "area", path, line_number, allow_empty=True)
429
- score = row["score"]
430
- if isinstance(score, bool) or not isinstance(score, int):
431
- raise SystemExit(f"{path}:{line_number}: score must be an integer from 1 through 10")
432
- if not 1 <= score <= 10:
433
- raise SystemExit(f"{path}:{line_number}: score must be from 1 through 10")
434
- if not isinstance(row["include"], bool):
435
- raise SystemExit(f"{path}:{line_number}: include must be a boolean")
436
- require_string(row, "reason", path, line_number, allow_empty=False)
437
-
438
-
439
- def require_unique_paths(rows: list[JsonRow], label: str) -> KeysView[str]:
440
- counts = Counter(str(row["path"]) for row in rows)
441
- if len(counts) < len(rows):
442
- duplicates = sorted(path for path, count in counts.items() if count > 1)
443
- raise SystemExit(f"{label} contains duplicate paths: {duplicates}")
444
- return counts.keys()
445
-
446
-
447
- def make_repo_rank_input(args: argparse.Namespace) -> None:
448
- repo = Path(args.repo).expanduser().resolve()
449
- if not repo.is_dir():
450
- raise SystemExit(f"Repo path not found: {repo}")
451
- scopes = [args.scope]
452
- explicit_scopes = args.scopes_file is not None
453
- if explicit_scopes:
454
- scopes = load_scopes_file(Path(args.scopes_file).expanduser())
455
-
456
- resolved_scopes = [
457
- resolve_scope(repo, scope, expand_user=not explicit_scopes) for scope in scopes
139
+ def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
140
+ """Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
141
+
142
+ Git retains tracked files and applies its standard exclusions to untracked files.
143
+ Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
144
+ Explicit file scopes bypass directory ignore rules.
145
+ """
146
+ if scope_path.is_file():
147
+ return (scope_path,)
148
+ git_candidates = git_directory_snapshot_paths(scope_path)
149
+ if git_candidates is not None:
150
+ return git_candidates
151
+
152
+ command = [
153
+ "rg",
154
+ "--files",
155
+ "--hidden",
156
+ "--no-require-git",
157
+ "--null",
158
+ # Also exclude descendants when the scope starts inside .git.
159
+ "--glob",
160
+ "!**/.git",
161
+ "--glob",
162
+ "!**/.git/**",
163
+ "--",
164
+ str(scope_path.relative_to(repo)),
458
165
  ]
459
- directly_requested_files = {
460
- scope_abs for scope_abs in resolved_scopes if explicit_scopes and scope_abs.is_file()
461
- }
462
- rows_by_path: dict[str, JsonRow] = {}
463
- for scope_abs in resolved_scopes:
464
- scope_rel = scope_abs.relative_to(repo)
465
- area = args.area or scope_rel.as_posix()
466
- candidates = (scope_abs,) if scope_abs.is_file() else scope_abs.rglob("*")
467
- for path in candidates:
468
- try:
469
- if path.is_symlink() or not path.is_file():
470
- continue
471
- path.resolve(strict=True).relative_to(repo)
472
- except (OSError, ValueError):
473
- continue
474
- rel = path.relative_to(repo)
475
- directly_requested = path in directly_requested_files
476
- excluded_path = (
477
- path.relative_to(scope_abs if scope_abs.is_dir() else scope_abs.parent)
478
- if explicit_scopes
479
- else rel
480
- )
481
- if not directly_requested and (
482
- path_is_excluded(excluded_path) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS
483
- ):
484
- continue
485
-
486
- if (
487
- directly_requested
488
- and path.suffix.lower() not in TEXT_CODE_EXTENSIONS
489
- and path.name not in EXCLUDED_FILENAMES
490
- ):
491
- preview = ""
492
- else:
493
- preview, is_binary = preview_for(path, args.preview_bytes)
494
- if is_binary and not directly_requested:
495
- continue
496
- rows_by_path.setdefault(
497
- rel.as_posix(),
498
- {"path": rel.as_posix(), "area": area, "preview": preview},
166
+ try:
167
+ result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
168
+ except OSError as exc:
169
+ ignore_names = (".gitignore", ".ignore", ".rgignore")
170
+ ancestors = (scope_path, *scope_path.parents)
171
+ has_ignore_rules = (
172
+ any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
173
+ or any(
174
+ (ancestor / name).is_file()
175
+ for ancestor in ancestors
176
+ if ancestor == repo or repo in ancestor.parents
177
+ for name in ignore_names
499
178
  )
179
+ or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
180
+ )
181
+ if has_ignore_rules:
182
+ raise SystemExit(
183
+ "Could not safely enumerate ignored scoped files without Git or ripgrep."
184
+ ) from exc
185
+ return scope_path.rglob("*")
500
186
 
501
- rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
502
- output = Path(args.out).expanduser()
503
- write_jsonl(output, rows)
504
- print(f"Wrote {len(rows)} rows to {output}")
187
+ if result.returncode not in (0, 1):
188
+ detail = result.stderr.decode("utf-8", errors="replace").strip()
189
+ raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
190
+ return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
505
191
 
506
192
 
507
193
  def make_repo_scope_input(args: argparse.Namespace) -> None:
@@ -512,59 +198,8 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
512
198
  scopes = load_scopes_file(Path(args.scopes_file).expanduser())
513
199
  rows_by_path: dict[str, JsonRow] = {}
514
200
  for scope in scopes:
515
- scope_path = resolve_scope(repo, scope, expand_user=False, reject_symlinks=True)
516
- if scope_path.is_file():
517
- candidates = (scope_path,)
518
- else:
519
- git_candidates = git_directory_snapshot_paths(scope_path)
520
- if git_candidates is not None:
521
- candidates = git_candidates
522
- else:
523
- command = [
524
- "rg",
525
- "--files",
526
- "--hidden",
527
- "--no-require-git",
528
- "--null",
529
- # Also exclude descendants when the scope starts inside .git.
530
- "--glob",
531
- "!**/.git",
532
- "--glob",
533
- "!**/.git/**",
534
- "--",
535
- str(scope_path.relative_to(repo)),
536
- ]
537
- try:
538
- result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
539
- except OSError as exc:
540
- ignore_names = (".gitignore", ".ignore", ".rgignore")
541
- ancestors = (scope_path, *scope_path.parents)
542
- has_ignore_rules = (
543
- any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
544
- or any(
545
- (ancestor / name).is_file()
546
- for ancestor in ancestors
547
- if ancestor == repo or repo in ancestor.parents
548
- for name in ignore_names
549
- )
550
- or any(
551
- path.name in ignore_names
552
- for path in scope_path.rglob("*")
553
- if path.is_file()
554
- )
555
- )
556
- if has_ignore_rules:
557
- raise SystemExit(
558
- "Could not safely enumerate ignored scoped files without Git or ripgrep."
559
- ) from exc
560
- candidates = scope_path.rglob("*")
561
- else:
562
- if result.returncode not in (0, 1):
563
- detail = result.stderr.decode("utf-8", errors="replace").strip()
564
- raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
565
- candidates = (
566
- repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path
567
- )
201
+ scope_path = resolve_scope(repo, scope)
202
+ candidates = scope_candidates(repo, scope_path)
568
203
  for path in candidates:
569
204
  try:
570
205
  if path.is_symlink() or not path.is_file():
@@ -582,41 +217,15 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
582
217
  print(f"Wrote {len(rows)} scoped paths to {output}")
583
218
 
584
219
 
585
- def bind_repo_scopes(args: argparse.Namespace) -> None:
586
- scopes = load_scopes_file(Path(args.scopes_file).expanduser())
587
- manifest_path = Path(args.manifest).expanduser()
588
- coverage_path = Path(args.coverage).expanduser()
589
- try:
590
- manifest: object = json.loads(manifest_path.read_text(encoding="utf-8"))
591
- coverage: object = json.loads(coverage_path.read_text(encoding="utf-8"))
592
- if not isinstance(manifest, dict) or not isinstance(coverage, dict):
593
- raise ValueError("expected JSON objects")
594
- scan = manifest.get("scan")
595
- if not isinstance(scan, dict):
596
- raise ValueError("manifest.scan must be an object")
597
- scope = scan.get("scope")
598
- if not isinstance(scope, dict):
599
- raise ValueError("manifest.scan.scope must be an object")
600
- except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
601
- raise SystemExit("Unable to bind requested scopes into the scan contract") from exc
602
- scope["includePaths"] = scopes
603
- coverage["includePaths"] = scopes
604
- manifest_path.write_text(
605
- json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
606
- )
607
- coverage_path.write_text(
608
- json.dumps(coverage, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
609
- )
610
- print(f"Bound {len(scopes)} requested scopes into the scan contract")
611
-
612
-
613
220
  def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
221
+ """Return changed regular files from the selected side of each change."""
614
222
  result = git_command(
615
223
  repo,
616
224
  "diff",
617
- "--name-status",
225
+ "--ignore-submodules=all",
226
+ "--raw",
618
227
  "-z",
619
- "--diff-filter=ACMRD",
228
+ "--diff-filter=ACMRDT",
620
229
  *diff_args,
621
230
  text=False,
622
231
  )
@@ -628,13 +237,16 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
628
237
  changed: list[tuple[Path, str]] = []
629
238
  index = 0
630
239
  while index < len(fields):
631
- status = chr(fields[index][0])
240
+ metadata = fields[index].split()
241
+ status = chr(metadata[-1][0])
632
242
  index += 1
633
243
  if status in {"C", "R"}:
634
244
  index += 1
635
245
  path = repo / os.fsdecode(fields[index])
636
246
  index += 1
637
- changed.append((path, status))
247
+ selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
248
+ if selected_mode.startswith(b"100"):
249
+ changed.append((path, status))
638
250
  return changed
639
251
 
640
252
 
@@ -658,9 +270,13 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
658
270
  combined.update(
659
271
  (repo / os.fsdecode(relative), "A")
660
272
  for relative in untracked.stdout.split(b"\0")
661
- if relative
273
+ if relative and not relative.endswith(b"/")
274
+ )
275
+ return sorted(
276
+ (path, status)
277
+ for path, status in combined.items()
278
+ if status == "D" or (not path.is_symlink() and path.is_file())
662
279
  )
663
- return sorted(combined.items())
664
280
  raise SystemExit(f"Unknown diff mode: {mode}")
665
281
 
666
282
 
@@ -669,25 +285,16 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
669
285
  if not repo.is_dir():
670
286
  raise SystemExit(f"Repo path not found: {repo}")
671
287
 
672
- changed = [
673
- (path, status)
674
- for path, status in git_changed_paths(repo, args.base, args.head, args.mode)
675
- if not path_is_diff_excluded(path.relative_to(repo))
676
- and path.suffix.lower() in TEXT_CODE_EXTENSIONS
677
- ]
678
- revision_paths = [
679
- path.relative_to(repo)
680
- for path, status in changed
681
- if args.mode == "revisions" and status != "D"
682
- ]
683
- revision_blobs = dict(
684
- zip(
685
- revision_paths,
686
- git_blob_bytes(
687
- repo,
688
- [f"{args.head}:{path.as_posix()}" for path in revision_paths],
689
- ),
288
+ changed = git_changed_paths(repo, args.base, args.head, args.mode)
289
+ revision_refs = {
290
+ path.relative_to(repo): (
291
+ f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
690
292
  )
293
+ for path, status in changed
294
+ if args.mode == "revisions" or status == "D"
295
+ }
296
+ revision_samples = dict(
297
+ zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
691
298
  )
692
299
 
693
300
  rows: list[JsonRow] = []
@@ -695,23 +302,24 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
695
302
  rel = path.relative_to(repo)
696
303
 
697
304
  preview = ""
698
- if status != "D" and args.mode == "revisions":
699
- content = revision_blobs[rel]
700
- if content is None:
701
- raise SystemExit(
702
- f"Unable to read committed diff blob: {args.head}:{rel.as_posix()}"
703
- )
704
- preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
305
+ if args.mode == "revisions" or status == "D":
306
+ sample = revision_samples[rel]
307
+ if sample is None:
308
+ revision = args.base if status == "D" else args.head
309
+ raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
310
+ content, is_binary = sample
705
311
  if is_binary:
706
312
  continue
707
- elif status != "D" and not path.is_symlink() and path.is_file():
313
+ if status != "D":
314
+ preview, _ = preview_for_bytes(content, args.preview_bytes)
315
+ elif not path.is_symlink() and path.is_file():
708
316
  try:
709
317
  path.resolve(strict=True).relative_to(repo)
710
318
  except (OSError, ValueError):
711
319
  preview = ""
712
320
  else:
713
321
  preview, is_binary = preview_for(path, args.preview_bytes)
714
- if is_binary:
322
+ if is_binary or is_binary_file(path):
715
323
  continue
716
324
  rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
717
325
 
@@ -721,431 +329,12 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
721
329
  print(f"Wrote {len(rows)} rows to {output}")
722
330
 
723
331
 
724
- def make_rank_shards(args: argparse.Namespace) -> None:
725
- if args.max_rows < 1:
726
- raise SystemExit("--max-rows must be at least 1")
727
-
728
- rank_input = Path(args.rank_input).expanduser()
729
- rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
730
- require_unique_paths(rows, "Rank input")
731
-
732
- output_dir = Path(args.out_dir).expanduser()
733
- output_dir.mkdir(parents=True, exist_ok=True)
734
- existing = sorted((*output_dir.glob(SHARD_INPUT_GLOB), *output_dir.glob(SHARD_OUTPUT_GLOB)))
735
- if existing:
736
- raise SystemExit(f"Rank shard directory already contains shard files: {output_dir}")
737
-
738
- shard_count = 0
739
- for shard_count, start in enumerate(range(0, len(rows), args.max_rows), start=1):
740
- shard_path = output_dir / f"rank-shard-{shard_count:04d}.input.jsonl"
741
- write_jsonl(shard_path, rows[start : start + args.max_rows])
742
-
743
- print(f"Wrote {shard_count} rank shards to {output_dir}")
744
-
745
-
746
- def discover_input_shards(shard_dir: Path) -> list[Path]:
747
- if not shard_dir.is_dir():
748
- raise SystemExit(f"Rank shard directory missing: {shard_dir}")
749
-
750
- numbered_shards: list[tuple[int, Path]] = []
751
- for path in shard_dir.glob(SHARD_INPUT_GLOB):
752
- match = SHARD_INPUT_PATTERN.fullmatch(path.name)
753
- if match is None:
754
- raise SystemExit(f"Rank input shard has invalid name: {path.name}")
755
- numbered_shards.append((int(match.group(1)), path))
756
- numbered_shards.sort(key=lambda item: (item[0], item[1].name))
757
- input_shards = [path for _, path in numbered_shards]
758
- expected_names = [
759
- f"rank-shard-{index:04d}.input.jsonl" for index in range(1, len(input_shards) + 1)
760
- ]
761
- actual_names = [path.name for path in input_shards]
762
- if actual_names != expected_names:
763
- raise SystemExit(
764
- "Rank input shards must use contiguous canonical names; "
765
- f"expected={expected_names}; actual={actual_names}"
766
- )
767
- return input_shards
768
-
769
-
770
- def output_name_for(input_name: str) -> str:
771
- return input_name.replace(".input.jsonl", ".output.jsonl")
772
-
773
-
774
- def require_plan_shard_dir(plan_path: Path, shard_dir: Path) -> None:
775
- expected = plan_path.parent / "rank_shards"
776
- if shard_dir.resolve() != expected.resolve():
777
- raise SystemExit(
778
- "Rank shard directory must be the assignment plan's sibling rank_shards "
779
- f"directory; expected={expected}; actual={shard_dir}"
780
- )
781
-
782
-
783
- def require_no_misplaced_rank_shards(plan_path: Path) -> None:
784
- misplaced = sorted(
785
- (
786
- *plan_path.parent.glob(SHARD_INPUT_GLOB),
787
- *plan_path.parent.glob(SHARD_OUTPUT_GLOB),
788
- ),
789
- key=lambda path: path.name,
790
- )
791
- if misplaced:
792
- raise SystemExit(
793
- "Rank shard artifacts must be stored in the assignment plan's sibling "
794
- f"rank_shards directory; misplaced={[path.name for path in misplaced]}"
795
- )
796
-
797
-
798
- def make_rank_pool_plan(args: argparse.Namespace) -> None:
799
- if args.usable_worker_slots < 1:
800
- raise SystemExit("--usable-worker-slots must be at least 1")
801
-
802
- shard_dir = Path(args.shard_dir).expanduser()
803
- output = Path(args.out).expanduser()
804
- require_plan_shard_dir(output, shard_dir)
805
- input_shards = discover_input_shards(shard_dir)
806
- worker_count = min(len(input_shards), args.usable_worker_slots, RANK_POOL_WORKER_CAP)
807
- workers: list[dict[str, object]] = []
808
- for worker_index in range(worker_count):
809
- assigned_inputs = [path.name for path in input_shards[worker_index::worker_count]]
810
- workers.append(
811
- {
812
- "slot": worker_index + 1,
813
- "input_shards": assigned_inputs,
814
- "output_shards": [output_name_for(name) for name in assigned_inputs],
815
- }
816
- )
817
-
818
- plan: dict[str, object] = {
819
- "schema_version": RANK_POOL_PLAN_SCHEMA_VERSION,
820
- "strategy": RANK_POOL_STRATEGY,
821
- "shard_count": len(input_shards),
822
- "ranking_worker_count": worker_count,
823
- "workers": workers,
824
- }
825
- write_json(output, plan)
826
- print(f"Assigned {len(input_shards)} rank shards to {worker_count} ranking workers in {output}")
827
-
828
-
829
- def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]:
830
- if not plan_path.exists():
831
- raise SystemExit(f"Rank pool plan missing: {plan_path}")
832
- plan_bytes = plan_path.read_bytes()
833
- try:
834
- payload: object = json.loads(plan_bytes)
835
- except json.JSONDecodeError as exc:
836
- raise SystemExit(f"{plan_path}: invalid JSON: {exc.msg}") from exc
837
- if not isinstance(payload, dict):
838
- raise SystemExit(f"{plan_path}: expected a JSON object")
839
- return payload, plan_bytes
840
-
841
-
842
- def require_integer(value: object, label: str, *, minimum: int) -> int:
843
- if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
844
- raise SystemExit(f"{label} must be an integer of at least {minimum}")
845
- return value
846
-
847
-
848
- def require_string_list(value: object, label: str) -> list[str]:
849
- if not isinstance(value, list) or not value:
850
- raise SystemExit(f"{label} must be a non-empty list")
851
- if any(not isinstance(item, str) or not item for item in value):
852
- raise SystemExit(f"{label} entries must be non-empty strings")
853
- return [item for item in value if isinstance(item, str)]
854
-
855
-
856
- def validate_rank_pool_plan(
857
- plan_path: Path, shard_dir: Path
858
- ) -> tuple[list[Path], list[tuple[list[str], list[str]]], bytes]:
859
- require_plan_shard_dir(plan_path, shard_dir)
860
- require_no_misplaced_rank_shards(plan_path)
861
- input_shards = discover_input_shards(shard_dir)
862
- input_names = [path.name for path in input_shards]
863
- plan, plan_bytes = load_rank_pool_plan(plan_path)
864
- expected_fields = {
865
- "schema_version",
866
- "strategy",
867
- "shard_count",
868
- "ranking_worker_count",
869
- "workers",
870
- }
871
- actual_fields = set(plan)
872
- if actual_fields != expected_fields:
873
- raise SystemExit(
874
- f"{plan_path}: rank pool plan fields do not match schema; "
875
- f"missing={sorted(expected_fields - actual_fields)}; "
876
- f"unexpected={sorted(actual_fields - expected_fields)}"
877
- )
878
- schema_version = require_integer(
879
- plan["schema_version"], f"{plan_path}: schema_version", minimum=1
880
- )
881
- if schema_version != RANK_POOL_PLAN_SCHEMA_VERSION:
882
- raise SystemExit(f"{plan_path}: schema_version must be {RANK_POOL_PLAN_SCHEMA_VERSION}")
883
- if plan["strategy"] != RANK_POOL_STRATEGY:
884
- raise SystemExit(f"{plan_path}: strategy must be {RANK_POOL_STRATEGY}")
885
-
886
- shard_count = require_integer(plan["shard_count"], f"{plan_path}: shard_count", minimum=0)
887
- if shard_count != len(input_shards):
888
- raise SystemExit(
889
- f"{plan_path}: shard_count does not match input shards; "
890
- f"plan={shard_count}; actual={len(input_shards)}"
891
- )
892
- worker_count = require_integer(
893
- plan["ranking_worker_count"], f"{plan_path}: ranking_worker_count", minimum=0
894
- )
895
- if shard_count > 0 and worker_count == 0:
896
- raise SystemExit(
897
- f"{plan_path}: ranking_worker_count must be at least 1 when input shards exist"
898
- )
899
- if worker_count > shard_count:
900
- raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed shard_count")
901
- if worker_count > RANK_POOL_WORKER_CAP:
902
- raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed {RANK_POOL_WORKER_CAP}")
903
-
904
- workers = plan["workers"]
905
- if not isinstance(workers, list) or len(workers) != worker_count:
906
- raise SystemExit(
907
- f"{plan_path}: workers must contain exactly {worker_count} worker assignments"
908
- )
909
-
910
- counts: Counter[str] = Counter()
911
- parsed_workers: list[tuple[list[str], list[str]]] = []
912
- worker_fields = {"slot", "input_shards", "output_shards"}
913
- for worker_index, worker in enumerate(workers):
914
- label = f"{plan_path}: workers[{worker_index}]"
915
- if not isinstance(worker, dict):
916
- raise SystemExit(f"{label} must be a JSON object")
917
- if set(worker) != worker_fields:
918
- raise SystemExit(
919
- f"{label} fields do not match schema; "
920
- f"missing={sorted(worker_fields - set(worker))}; "
921
- f"unexpected={sorted(set(worker) - worker_fields)}"
922
- )
923
- slot = require_integer(worker["slot"], f"{label}.slot", minimum=1)
924
- if slot != worker_index + 1:
925
- raise SystemExit(f"{label}.slot must be {worker_index + 1}")
926
- worker_inputs = require_string_list(worker["input_shards"], f"{label}.input_shards")
927
- worker_outputs = require_string_list(worker["output_shards"], f"{label}.output_shards")
928
- if len(worker_inputs) != len(worker_outputs):
929
- raise SystemExit(f"{label} input_shards and output_shards lengths must match")
930
- expected_worker_outputs = [output_name_for(name) for name in worker_inputs]
931
- if worker_outputs != expected_worker_outputs:
932
- raise SystemExit(f"{label}.output_shards do not match its input_shards")
933
- counts.update(worker_inputs)
934
- parsed_workers.append((worker_inputs, worker_outputs))
935
-
936
- # Canonical input names map one-to-one to the per-worker outputs checked above.
937
- duplicates = sorted(name for name, count in counts.items() if count > 1)
938
- expected = set(input_names)
939
- missing = sorted(expected - counts.keys())
940
- unexpected = sorted(counts.keys() - expected)
941
- if missing or duplicates or unexpected:
942
- raise SystemExit(
943
- f"{plan_path}: pool plan must assign each input shard exactly once; "
944
- f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
945
- )
946
- for worker_index, (worker_inputs, _) in enumerate(parsed_workers):
947
- expected_inputs = input_names[worker_index::worker_count]
948
- if worker_inputs != expected_inputs:
949
- raise SystemExit(
950
- f"{plan_path}: worker slot {worker_index + 1} does not match the deterministic "
951
- f"{RANK_POOL_STRATEGY} assignment"
952
- )
953
- return input_shards, parsed_workers, plan_bytes
954
-
955
-
956
- def validate_rank_worker_command(args: argparse.Namespace) -> None:
957
- plan_path = Path(args.plan).expanduser()
958
- shard_dir = Path(args.shard_dir).expanduser()
959
- _, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir)
960
-
961
- slot = require_integer(args.slot, "--slot", minimum=1)
962
- worker_count = len(workers)
963
- if slot > worker_count:
964
- raise SystemExit(f"--slot must be at most {worker_count}")
965
-
966
- input_names, output_names = workers[slot - 1]
967
-
968
- row_count = 0
969
- outputs_digest = hashlib.sha256()
970
- for input_name, output_name in zip(input_names, output_names, strict=True):
971
- input_shard = shard_dir / input_name
972
- output_shard = shard_dir / output_name
973
- _, output_rows = validate_rank_shard(input_shard, output_shard)
974
- output_bytes = output_shard.read_bytes()
975
- row_count += len(output_rows)
976
- outputs_digest.update(output_name.encode("utf-8"))
977
- outputs_digest.update(b"\0")
978
- outputs_digest.update(output_bytes)
979
- outputs_digest.update(b"\0")
980
-
981
- receipt: dict[str, object] = {
982
- "schema_version": 1,
983
- "plan_sha256": hashlib.sha256(plan_bytes).hexdigest(),
984
- "slot": slot,
985
- "ranking_worker_count": worker_count,
986
- "output_shards": len(output_names),
987
- "rows": row_count,
988
- "outputs_sha256": outputs_digest.hexdigest(),
989
- "status": "complete",
990
- }
991
- print("RANK_WORKER_RECEIPT " + json.dumps(receipt, sort_keys=True, separators=(",", ":")))
992
-
993
-
994
- def validate_rank_pool_command(args: argparse.Namespace) -> None:
995
- plan_path = Path(args.plan).expanduser()
996
- shard_dir = Path(args.shard_dir).expanduser()
997
- input_shards, workers, _ = validate_rank_pool_plan(plan_path, shard_dir)
998
-
999
- actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)}
1000
- expected_outputs = {output_name_for(path.name) for path in input_shards}
1001
- if actual_output_names != expected_outputs:
1002
- missing = sorted(expected_outputs - actual_output_names)
1003
- unexpected = sorted(actual_output_names - expected_outputs)
1004
- raise SystemExit(
1005
- "Rank pool outputs are incomplete; "
1006
- f"missing output shards={missing}; unexpected output shards={unexpected}"
1007
- )
1008
-
1009
- row_count = 0
1010
- for input_shard in input_shards:
1011
- output_shard = input_shard.with_name(output_name_for(input_shard.name))
1012
- _, shard_outputs = validate_rank_shard(input_shard, output_shard)
1013
- row_count += len(shard_outputs)
1014
- print(
1015
- f"Validated {len(workers)} ranking workers, "
1016
- f"{len(input_shards)} shards, and {row_count} ranking rows"
1017
- )
1018
-
1019
-
1020
- def validate_rank_shard(
1021
- input_shard: Path, output_shard: Path
1022
- ) -> tuple[list[JsonRow], list[JsonRow]]:
1023
- shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row)
1024
- expected_paths = require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
1025
- shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row)
1026
- actual_paths = require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
1027
-
1028
- if expected_paths != actual_paths:
1029
- missing = sorted(expected_paths - actual_paths)
1030
- unknown = sorted(actual_paths - expected_paths)
1031
- raise SystemExit(
1032
- f"{output_shard}: paths do not match its input shard; "
1033
- f"missing={missing}; unknown={unknown}"
1034
- )
1035
-
1036
- area_by_path = {str(row["path"]): row["area"] for row in shard_inputs}
1037
- for row in shard_outputs:
1038
- row_path = str(row["path"])
1039
- if row["area"] != area_by_path[row_path]:
1040
- raise SystemExit(f"{output_shard}: area does not match rank input for {row_path}")
1041
- return shard_inputs, shard_outputs
1042
-
1043
-
1044
- def validate_rank_shard_command(args: argparse.Namespace) -> None:
1045
- input_shard = Path(args.input).expanduser()
1046
- output_shard = Path(args.output).expanduser()
1047
- _, output_rows = validate_rank_shard(input_shard, output_shard)
1048
- print(f"Validated {len(output_rows)} ranking rows in {output_shard}")
1049
-
1050
-
1051
- def merge_rank_outputs(args: argparse.Namespace) -> None:
1052
- rank_input = Path(args.rank_input).expanduser()
1053
- authoritative_rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
1054
- require_unique_paths(authoritative_rows, "Rank input")
1055
-
1056
- shard_dir = Path(args.shard_dir).expanduser()
1057
- input_shards = discover_input_shards(shard_dir)
1058
- output_shards = sorted(shard_dir.glob(SHARD_OUTPUT_GLOB))
1059
- expected_output_names = {
1060
- path.name.replace(".input.jsonl", ".output.jsonl") for path in input_shards
1061
- }
1062
- actual_output_names = {path.name for path in output_shards}
1063
- if expected_output_names != actual_output_names:
1064
- missing = sorted(expected_output_names - actual_output_names)
1065
- unexpected = sorted(actual_output_names - expected_output_names)
1066
- details: list[str] = []
1067
- if missing:
1068
- details.append(f"missing output shards {missing}")
1069
- if unexpected:
1070
- details.append(f"unexpected output shards {unexpected}")
1071
- raise SystemExit(f"Rank shard outputs are incomplete: {'; '.join(details)}")
1072
-
1073
- sharded_inputs: list[JsonRow] = []
1074
- output_by_path: dict[str, JsonRow] = {}
1075
- for input_shard in input_shards:
1076
- output_shard = input_shard.with_name(
1077
- input_shard.name.replace(".input.jsonl", ".output.jsonl")
1078
- )
1079
- shard_inputs, shard_outputs = validate_rank_shard(input_shard, output_shard)
1080
- sharded_inputs.extend(shard_inputs)
1081
- for row in shard_outputs:
1082
- row_path = str(row["path"])
1083
- if row_path in output_by_path:
1084
- raise SystemExit(f"Rank outputs contain duplicate path: {row_path}")
1085
- output_by_path[row_path] = row
1086
-
1087
- if sharded_inputs != authoritative_rows:
1088
- raise SystemExit("Rank input shards do not exactly partition the authoritative rank input")
1089
-
1090
- merged = [output_by_path[str(row["path"])] for row in authoritative_rows]
1091
- output = Path(args.out).expanduser()
1092
- write_jsonl(output, merged)
1093
- print(f"Merged {len(merged)} ranking rows into {output}")
1094
-
1095
-
1096
- def copy_deep_review_input(args: argparse.Namespace) -> None:
1097
- rank_input = Path(args.rank_input).expanduser()
1098
- rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
1099
- require_unique_paths(rows, "Rank input")
1100
- selected = [{"path": row["path"], "area": row["area"]} for row in rows]
1101
-
1102
- output = Path(args.out).expanduser()
1103
- write_jsonl(output, selected)
1104
- print(f"Copied {len(selected)} rows into {output}")
1105
-
1106
-
1107
- def select_deep_review_input(args: argparse.Namespace) -> None:
1108
- rank_output = Path(args.rank_output).expanduser()
1109
- rows = load_jsonl(rank_output, "Rank output", validate_rank_output_row)
1110
- require_unique_paths(rows, "Rank output")
1111
-
1112
- included = [row for row in rows if row["include"]]
1113
- base_rows = included if included else rows
1114
- base_rows.sort(key=lambda row: (-int(row["score"]), str(row["path"])))
1115
- keep = max(1, int(len(base_rows) * (args.top_percent / 100.0))) if base_rows else 0
1116
- selected = [{"path": row["path"], "area": row["area"]} for row in base_rows[:keep]]
1117
-
1118
- output = Path(args.out).expanduser()
1119
- write_jsonl(output, selected)
1120
- print(f"Selected {len(selected)} of {len(base_rows)} rows into {output}")
1121
-
1122
-
1123
332
  def main() -> None:
1124
333
  args = parse_args()
1125
- if args.command == "make-repo-rank-input":
1126
- make_repo_rank_input(args)
1127
- elif args.command == "make-repo-scope-input":
334
+ if args.command == "make-repo-scope-input":
1128
335
  make_repo_scope_input(args)
1129
- elif args.command == "bind-repo-scopes":
1130
- bind_repo_scopes(args)
1131
336
  elif args.command == "make-diff-rank-input":
1132
337
  make_diff_rank_input(args)
1133
- elif args.command == "make-rank-shards":
1134
- make_rank_shards(args)
1135
- elif args.command == "make-rank-pool-plan":
1136
- make_rank_pool_plan(args)
1137
- elif args.command == "validate-rank-shard":
1138
- validate_rank_shard_command(args)
1139
- elif args.command == "validate-rank-worker":
1140
- validate_rank_worker_command(args)
1141
- elif args.command == "validate-rank-pool":
1142
- validate_rank_pool_command(args)
1143
- elif args.command == "merge-rank-outputs":
1144
- merge_rank_outputs(args)
1145
- elif args.command == "copy-deep-review-input":
1146
- copy_deep_review_input(args)
1147
- elif args.command == "select-deep-review-input":
1148
- select_deep_review_input(args)
1149
338
  else:
1150
339
  raise SystemExit(f"Unknown command: {args.command}")
1151
340