@openai/codex-security 0.1.31 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (514) hide show
  1. package/README.md +442 -2434
  2. package/_bundled_plugin/.app.json +5 -1
  3. package/_bundled_plugin/.codex-plugin/plugin.json +9 -5
  4. package/_bundled_plugin/.mcp.json +3 -0
  5. package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
  6. package/_bundled_plugin/examples/completed-scan/report.md +28 -26
  7. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
  8. package/_bundled_plugin/mcp/helpers.mjs +1 -0
  9. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  10. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +7 -214
  11. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  12. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  13. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +563 -0
  20. package/_bundled_plugin/mcp/server.mjs +1 -0
  21. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  22. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  23. package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
  24. package/_bundled_plugin/references/artifact-storage.md +5 -5
  25. package/_bundled_plugin/references/config-preflight.md +1 -5
  26. package/_bundled_plugin/references/core-scan.md +34 -10
  27. package/_bundled_plugin/references/final-report.md +13 -13
  28. package/_bundled_plugin/references/finding-detail-fields.md +90 -80
  29. package/_bundled_plugin/references/sarif-adapter.md +1 -1
  30. package/_bundled_plugin/references/scan-artifacts.md +3 -4
  31. package/_bundled_plugin/references/scan-contract.md +11 -6
  32. package/_bundled_plugin/references/scan-prologue.md +12 -0
  33. package/_bundled_plugin/references/security-guidance.md +2 -2
  34. package/_bundled_plugin/references/standalone-validation.md +11 -0
  35. package/_bundled_plugin/references/threat-model.md +2 -2
  36. package/_bundled_plugin/schemas/coverage.schema.json +23 -47
  37. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
  38. package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
  39. package/_bundled_plugin/schemas/findings.schema.json +71 -192
  40. package/_bundled_plugin/schemas/scan-manifest.schema.json +83 -95
  41. package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
  42. package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
  43. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
  44. package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
  45. package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
  46. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +245 -356
  47. package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
  48. package/_bundled_plugin/scripts/codex_session.mjs +53 -0
  49. package/_bundled_plugin/scripts/config_preflight.py +29 -69
  50. package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
  51. package/_bundled_plugin/scripts/deep_scan_workbench.py +208 -363
  52. package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
  53. package/_bundled_plugin/scripts/finalize_scan_contract.py +733 -375
  54. package/_bundled_plugin/scripts/finding_preview.py +81 -91
  55. package/_bundled_plugin/scripts/generate_in_scope_files.py +48 -102
  56. package/_bundled_plugin/scripts/generate_rank_input.py +117 -888
  57. package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -6
  58. package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
  59. package/_bundled_plugin/scripts/rank_preview.py +43 -902
  60. package/_bundled_plugin/scripts/report_projection.py +143 -219
  61. package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
  62. package/_bundled_plugin/scripts/threat_model_projection.py +161 -0
  63. package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
  64. package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
  65. package/_bundled_plugin/scripts/windows_scan_local_files.py +59 -9
  66. package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
  67. package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
  68. package/_bundled_plugin/scripts/workbench/storage.py +21 -6
  69. package/_bundled_plugin/scripts/workbench_cli.py +99 -190
  70. package/_bundled_plugin/scripts/workbench_constants.py +6 -14
  71. package/_bundled_plugin/scripts/workbench_db.py +524 -672
  72. package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
  73. package/_bundled_plugin/scripts/workbench_finding_index.py +20 -7
  74. package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
  75. package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
  76. package/_bundled_plugin/scripts/workbench_native_indexes.py +28 -18
  77. package/_bundled_plugin/scripts/workbench_progress.py +26 -92
  78. package/_bundled_plugin/scripts/workbench_publication.py +77 -116
  79. package/_bundled_plugin/scripts/workbench_remediation.py +14 -20
  80. package/_bundled_plugin/scripts/workbench_saved_results.py +1402 -263
  81. package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
  82. package/_bundled_plugin/scripts/workbench_scan_start.py +109 -52
  83. package/_bundled_plugin/scripts/workbench_scan_usage.py +30 -49
  84. package/_bundled_plugin/scripts/workbench_schema.py +71 -1370
  85. package/_bundled_plugin/scripts/workbench_source_excerpt.py +16 -7
  86. package/_bundled_plugin/scripts/workbench_target.py +526 -150
  87. package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
  88. package/_bundled_plugin/scripts/workbench_validation.py +19 -44
  89. package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
  90. package/_bundled_plugin/shared/workbench-migrations.json +397 -0
  91. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
  92. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
  93. package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
  94. package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
  95. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +6 -6
  96. package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
  97. package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
  98. package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
  99. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
  100. package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
  101. package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
  102. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
  103. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  104. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +13 -1
  105. package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
  106. package/_bundled_plugin/skills/track-findings/SKILL.md +46 -178
  107. package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
  108. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
  109. package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
  110. package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
  111. package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
  112. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
  113. package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
  114. package/bin/codex-security.mjs +9 -2
  115. package/dist/api.d.ts +63 -15
  116. package/dist/api.d.ts.map +1 -1
  117. package/dist/api.js +970 -553
  118. package/dist/api.js.map +1 -1
  119. package/dist/artifact-export.d.ts +70 -0
  120. package/dist/artifact-export.d.ts.map +1 -0
  121. package/dist/artifact-export.js +292 -0
  122. package/dist/artifact-export.js.map +1 -0
  123. package/dist/auth.d.ts +2 -6
  124. package/dist/auth.d.ts.map +1 -1
  125. package/dist/auth.js +27 -12
  126. package/dist/auth.js.map +1 -1
  127. package/dist/bulk-scan-discovery.d.ts +1 -0
  128. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  129. package/dist/bulk-scan-discovery.js +9 -9
  130. package/dist/bulk-scan-discovery.js.map +1 -1
  131. package/dist/classify-scan-severity.d.ts.map +1 -1
  132. package/dist/classify-scan-severity.js +2 -5
  133. package/dist/classify-scan-severity.js.map +1 -1
  134. package/dist/classify-severity.d.ts +28 -17
  135. package/dist/classify-severity.d.ts.map +1 -1
  136. package/dist/classify-severity.js +16 -35
  137. package/dist/classify-severity.js.map +1 -1
  138. package/dist/cli-help.d.ts +3 -0
  139. package/dist/cli-help.d.ts.map +1 -0
  140. package/dist/cli-help.js +285 -0
  141. package/dist/cli-help.js.map +1 -0
  142. package/dist/cli-signals.d.ts +8 -0
  143. package/dist/cli-signals.d.ts.map +1 -0
  144. package/dist/cli-signals.js +28 -0
  145. package/dist/cli-signals.js.map +1 -0
  146. package/dist/cli.d.ts +22 -24
  147. package/dist/cli.d.ts.map +1 -1
  148. package/dist/cli.js +1660 -1556
  149. package/dist/cli.js.map +1 -1
  150. package/dist/cloud-publish.d.ts.map +1 -1
  151. package/dist/cloud-publish.js +11 -17
  152. package/dist/cloud-publish.js.map +1 -1
  153. package/dist/codeowners.d.ts +19 -0
  154. package/dist/codeowners.d.ts.map +1 -0
  155. package/dist/codeowners.js +138 -0
  156. package/dist/codeowners.js.map +1 -0
  157. package/dist/codex-prompt.d.ts.map +1 -1
  158. package/dist/codex-prompt.js +1 -2
  159. package/dist/codex-prompt.js.map +1 -1
  160. package/dist/codex-sdk-environment.d.ts +3 -0
  161. package/dist/codex-sdk-environment.d.ts.map +1 -0
  162. package/dist/codex-sdk-environment.js +68 -0
  163. package/dist/codex-sdk-environment.js.map +1 -0
  164. package/dist/component-plan.d.ts +2 -5
  165. package/dist/component-plan.d.ts.map +1 -1
  166. package/dist/component-plan.js +38 -34
  167. package/dist/component-plan.js.map +1 -1
  168. package/dist/component-scan.d.ts +7 -5
  169. package/dist/component-scan.d.ts.map +1 -1
  170. package/dist/component-scan.js +54 -44
  171. package/dist/component-scan.js.map +1 -1
  172. package/dist/config.d.ts.map +1 -1
  173. package/dist/config.js +93 -104
  174. package/dist/config.js.map +1 -1
  175. package/dist/contract.d.ts +15 -2
  176. package/dist/contract.d.ts.map +1 -1
  177. package/dist/contract.js +124 -348
  178. package/dist/contract.js.map +1 -1
  179. package/dist/cost-model.d.ts +13 -6
  180. package/dist/cost-model.d.ts.map +1 -1
  181. package/dist/cost-model.js +75 -34
  182. package/dist/cost-model.js.map +1 -1
  183. package/dist/cost.d.ts +10 -2
  184. package/dist/cost.d.ts.map +1 -1
  185. package/dist/cost.js +116 -134
  186. package/dist/cost.js.map +1 -1
  187. package/dist/custom-validation-prompt.d.ts.map +1 -1
  188. package/dist/custom-validation-prompt.js +7 -6
  189. package/dist/custom-validation-prompt.js.map +1 -1
  190. package/dist/custom-validation.d.ts +1 -2
  191. package/dist/custom-validation.d.ts.map +1 -1
  192. package/dist/custom-validation.js +23 -16
  193. package/dist/custom-validation.js.map +1 -1
  194. package/dist/deduplication/checkpointed-review.d.ts +4 -2
  195. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  196. package/dist/deduplication/checkpointed-review.js +33 -20
  197. package/dist/deduplication/checkpointed-review.js.map +1 -1
  198. package/dist/deduplication/codex-review.d.ts +9 -1
  199. package/dist/deduplication/codex-review.d.ts.map +1 -1
  200. package/dist/deduplication/codex-review.js +129 -20
  201. package/dist/deduplication/codex-review.js.map +1 -1
  202. package/dist/deduplication/deduplication-prompts.d.ts +5 -2
  203. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  204. package/dist/deduplication/deduplication-prompts.js +18 -4
  205. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  206. package/dist/deduplication/deduplication-reviewer.d.ts +26 -4
  207. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  208. package/dist/deduplication/deduplication-reviewer.js +59 -47
  209. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  210. package/dist/deduplication/deduplication.d.ts.map +1 -1
  211. package/dist/deduplication/deduplication.js +59 -113
  212. package/dist/deduplication/deduplication.js.map +1 -1
  213. package/dist/deduplication/diagnostics.d.ts +17 -0
  214. package/dist/deduplication/diagnostics.d.ts.map +1 -0
  215. package/dist/deduplication/diagnostics.js +5 -0
  216. package/dist/deduplication/diagnostics.js.map +1 -0
  217. package/dist/deduplication/finding-schema.d.ts.map +1 -1
  218. package/dist/deduplication/finding-schema.js +7 -1
  219. package/dist/deduplication/finding-schema.js.map +1 -1
  220. package/dist/deduplication/local.d.ts +31 -0
  221. package/dist/deduplication/local.d.ts.map +1 -0
  222. package/dist/deduplication/local.js +166 -0
  223. package/dist/deduplication/local.js.map +1 -0
  224. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  225. package/dist/deduplication/records-protocol.js +12 -28
  226. package/dist/deduplication/records-protocol.js.map +1 -1
  227. package/dist/deduplication/records.d.ts +188 -12
  228. package/dist/deduplication/records.d.ts.map +1 -1
  229. package/dist/deduplication/records.js +34 -24
  230. package/dist/deduplication/records.js.map +1 -1
  231. package/dist/deduplication/review.d.ts +7 -0
  232. package/dist/deduplication/review.d.ts.map +1 -1
  233. package/dist/deduplication/review.js.map +1 -1
  234. package/dist/deduplication/scan.d.ts +9 -3
  235. package/dist/deduplication/scan.d.ts.map +1 -1
  236. package/dist/deduplication/scan.js +159 -44
  237. package/dist/deduplication/scan.js.map +1 -1
  238. package/dist/deep-config.d.ts +2 -1
  239. package/dist/deep-config.d.ts.map +1 -1
  240. package/dist/deep-config.js +7 -23
  241. package/dist/deep-config.js.map +1 -1
  242. package/dist/deep-progress.d.ts +2 -1
  243. package/dist/deep-progress.d.ts.map +1 -1
  244. package/dist/deep-progress.js +32 -47
  245. package/dist/deep-progress.js.map +1 -1
  246. package/dist/deep-scan.d.ts +20 -0
  247. package/dist/deep-scan.d.ts.map +1 -0
  248. package/dist/deep-scan.js +113 -0
  249. package/dist/deep-scan.js.map +1 -0
  250. package/dist/errors.d.ts +5 -2
  251. package/dist/errors.d.ts.map +1 -1
  252. package/dist/errors.js +7 -7
  253. package/dist/errors.js.map +1 -1
  254. package/dist/feedback.d.ts.map +1 -1
  255. package/dist/feedback.js +53 -21
  256. package/dist/feedback.js.map +1 -1
  257. package/dist/finding-catalogue.d.ts.map +1 -1
  258. package/dist/finding-catalogue.js +26 -30
  259. package/dist/finding-catalogue.js.map +1 -1
  260. package/dist/finding-normalization.d.ts +5 -0
  261. package/dist/finding-normalization.d.ts.map +1 -0
  262. package/dist/finding-normalization.js +120 -0
  263. package/dist/finding-normalization.js.map +1 -0
  264. package/dist/finding-retrieval.d.ts +9 -0
  265. package/dist/finding-retrieval.d.ts.map +1 -1
  266. package/dist/finding-retrieval.js.map +1 -1
  267. package/dist/finding-workflow.d.ts +13 -8
  268. package/dist/finding-workflow.d.ts.map +1 -1
  269. package/dist/finding-workflow.js +18 -13
  270. package/dist/finding-workflow.js.map +1 -1
  271. package/dist/findings-client.d.ts +2 -1
  272. package/dist/findings-client.d.ts.map +1 -1
  273. package/dist/findings-client.js +36 -9
  274. package/dist/findings-client.js.map +1 -1
  275. package/dist/findings-errors.d.ts +16 -0
  276. package/dist/findings-errors.d.ts.map +1 -0
  277. package/dist/findings-errors.js +24 -0
  278. package/dist/findings-errors.js.map +1 -0
  279. package/dist/findings-import.d.ts +1 -1
  280. package/dist/findings-import.d.ts.map +1 -1
  281. package/dist/findings-import.js +16 -44
  282. package/dist/findings-import.js.map +1 -1
  283. package/dist/github.d.ts.map +1 -1
  284. package/dist/github.js +9 -4
  285. package/dist/github.js.map +1 -1
  286. package/dist/import-scan.d.ts.map +1 -1
  287. package/dist/import-scan.js +49 -39
  288. package/dist/import-scan.js.map +1 -1
  289. package/dist/index.d.ts +11 -5
  290. package/dist/index.d.ts.map +1 -1
  291. package/dist/index.js +2 -1
  292. package/dist/index.js.map +1 -1
  293. package/dist/knowledge-base.d.ts +7 -1
  294. package/dist/knowledge-base.d.ts.map +1 -1
  295. package/dist/knowledge-base.js +115 -47
  296. package/dist/knowledge-base.js.map +1 -1
  297. package/dist/linear.d.ts.map +1 -1
  298. package/dist/linear.js +4 -10
  299. package/dist/linear.js.map +1 -1
  300. package/dist/mock-scan.d.ts.map +1 -1
  301. package/dist/mock-scan.js +0 -1
  302. package/dist/mock-scan.js.map +1 -1
  303. package/dist/models.d.ts +24 -2
  304. package/dist/models.d.ts.map +1 -1
  305. package/dist/models.js.map +1 -1
  306. package/dist/multiscan.d.ts +9 -3
  307. package/dist/multiscan.d.ts.map +1 -1
  308. package/dist/multiscan.js +225 -153
  309. package/dist/multiscan.js.map +1 -1
  310. package/dist/owner-evidence.d.ts +10 -2
  311. package/dist/owner-evidence.d.ts.map +1 -1
  312. package/dist/owner-evidence.js +69 -14
  313. package/dist/owner-evidence.js.map +1 -1
  314. package/dist/patch-tui.d.ts.map +1 -1
  315. package/dist/patch-tui.js +5 -6
  316. package/dist/patch-tui.js.map +1 -1
  317. package/dist/project-config-schema.d.ts +10 -0
  318. package/dist/project-config-schema.d.ts.map +1 -1
  319. package/dist/project-config-schema.js +1 -0
  320. package/dist/project-config-schema.js.map +1 -1
  321. package/dist/project-config.d.ts +1 -0
  322. package/dist/project-config.d.ts.map +1 -1
  323. package/dist/project-config.js +5 -3
  324. package/dist/project-config.js.map +1 -1
  325. package/dist/prompt-files.d.ts +0 -1
  326. package/dist/prompt-files.d.ts.map +1 -1
  327. package/dist/prompt-files.js +2 -4
  328. package/dist/prompt-files.js.map +1 -1
  329. package/dist/provider-profile.d.ts +21 -0
  330. package/dist/provider-profile.d.ts.map +1 -0
  331. package/dist/provider-profile.js +133 -0
  332. package/dist/provider-profile.js.map +1 -0
  333. package/dist/publication-events.d.ts +6 -8
  334. package/dist/publication-events.d.ts.map +1 -1
  335. package/dist/publication-events.js +40 -79
  336. package/dist/publication-events.js.map +1 -1
  337. package/dist/publication-store.d.ts.map +1 -1
  338. package/dist/publication-store.js +48 -76
  339. package/dist/publication-store.js.map +1 -1
  340. package/dist/publication.d.ts +1 -2
  341. package/dist/publication.d.ts.map +1 -1
  342. package/dist/publication.js +7 -16
  343. package/dist/publication.js.map +1 -1
  344. package/dist/publish.d.ts +1 -5
  345. package/dist/publish.d.ts.map +1 -1
  346. package/dist/publish.js +138 -199
  347. package/dist/publish.js.map +1 -1
  348. package/dist/record.d.ts +2 -0
  349. package/dist/record.d.ts.map +1 -0
  350. package/dist/record.js +4 -0
  351. package/dist/record.js.map +1 -0
  352. package/dist/request-metadata.d.ts +9 -0
  353. package/dist/request-metadata.d.ts.map +1 -0
  354. package/dist/request-metadata.js +13 -0
  355. package/dist/request-metadata.js.map +1 -0
  356. package/dist/result.d.ts +4 -1
  357. package/dist/result.d.ts.map +1 -1
  358. package/dist/result.js +7 -0
  359. package/dist/result.js.map +1 -1
  360. package/dist/runtime.d.ts +25 -17
  361. package/dist/runtime.d.ts.map +1 -1
  362. package/dist/runtime.js +619 -255
  363. package/dist/runtime.js.map +1 -1
  364. package/dist/saved-scan-bootstrap.d.ts +14 -0
  365. package/dist/saved-scan-bootstrap.d.ts.map +1 -0
  366. package/dist/saved-scan-bootstrap.js +261 -0
  367. package/dist/saved-scan-bootstrap.js.map +1 -0
  368. package/dist/saved-scan.d.ts +1 -1
  369. package/dist/saved-scan.d.ts.map +1 -1
  370. package/dist/saved-scan.js +16 -32
  371. package/dist/saved-scan.js.map +1 -1
  372. package/dist/scan-activity.d.ts.map +1 -1
  373. package/dist/scan-activity.js +19 -30
  374. package/dist/scan-activity.js.map +1 -1
  375. package/dist/scan-comparison.d.ts +31 -18
  376. package/dist/scan-comparison.d.ts.map +1 -1
  377. package/dist/scan-comparison.js +307 -347
  378. package/dist/scan-comparison.js.map +1 -1
  379. package/dist/scan-dashboard.d.ts +3 -1
  380. package/dist/scan-dashboard.d.ts.map +1 -1
  381. package/dist/scan-dashboard.js +351 -92
  382. package/dist/scan-dashboard.js.map +1 -1
  383. package/dist/scan-history-renderer.d.ts.map +1 -1
  384. package/dist/scan-history-renderer.js +55 -15
  385. package/dist/scan-history-renderer.js.map +1 -1
  386. package/dist/scan-inputs.d.ts +6 -0
  387. package/dist/scan-inputs.d.ts.map +1 -0
  388. package/dist/scan-inputs.js +71 -0
  389. package/dist/scan-inputs.js.map +1 -0
  390. package/dist/scan-logs.d.ts +2 -3
  391. package/dist/scan-logs.d.ts.map +1 -1
  392. package/dist/scan-logs.js +14 -12
  393. package/dist/scan-logs.js.map +1 -1
  394. package/dist/scan-sessions.d.ts +4 -0
  395. package/dist/scan-sessions.d.ts.map +1 -1
  396. package/dist/scan-sessions.js +18 -3
  397. package/dist/scan-sessions.js.map +1 -1
  398. package/dist/scan-settings.d.ts +5 -0
  399. package/dist/scan-settings.d.ts.map +1 -1
  400. package/dist/scan-settings.js +4 -0
  401. package/dist/scan-settings.js.map +1 -1
  402. package/dist/scan-skill-paths.d.ts +8 -0
  403. package/dist/scan-skill-paths.d.ts.map +1 -0
  404. package/dist/scan-skill-paths.js +47 -0
  405. package/dist/scan-skill-paths.js.map +1 -0
  406. package/dist/security-policy-cli.d.ts.map +1 -1
  407. package/dist/security-policy-cli.js +15 -33
  408. package/dist/security-policy-cli.js.map +1 -1
  409. package/dist/security-policy.d.ts +17 -9
  410. package/dist/security-policy.d.ts.map +1 -1
  411. package/dist/security-policy.js +131 -98
  412. package/dist/security-policy.js.map +1 -1
  413. package/dist/server/api.d.ts +0 -1
  414. package/dist/server/api.d.ts.map +1 -1
  415. package/dist/server/api.js +0 -1
  416. package/dist/server/api.js.map +1 -1
  417. package/dist/server/embeddings.d.ts +3 -1
  418. package/dist/server/embeddings.d.ts.map +1 -1
  419. package/dist/server/embeddings.js +11 -3
  420. package/dist/server/embeddings.js.map +1 -1
  421. package/dist/server/errors.d.ts +5 -2
  422. package/dist/server/errors.d.ts.map +1 -1
  423. package/dist/server/errors.js +7 -0
  424. package/dist/server/errors.js.map +1 -1
  425. package/dist/server/sqlite-store.d.ts +0 -2
  426. package/dist/server/sqlite-store.d.ts.map +1 -1
  427. package/dist/server/sqlite-store.js +55 -55
  428. package/dist/server/sqlite-store.js.map +1 -1
  429. package/dist/server/storage.d.ts +0 -2
  430. package/dist/server/storage.d.ts.map +1 -1
  431. package/dist/server/storage.js.map +1 -1
  432. package/dist/severity-store.js +20 -32
  433. package/dist/severity-store.js.map +1 -1
  434. package/dist/standalone-validation-prompt.d.ts +2 -0
  435. package/dist/standalone-validation-prompt.d.ts.map +1 -0
  436. package/dist/standalone-validation-prompt.js +10 -0
  437. package/dist/standalone-validation-prompt.js.map +1 -0
  438. package/dist/suggest-owners.d.ts +6 -5
  439. package/dist/suggest-owners.d.ts.map +1 -1
  440. package/dist/suggest-owners.js +15 -10
  441. package/dist/suggest-owners.js.map +1 -1
  442. package/dist/targets.d.ts +12 -0
  443. package/dist/targets.d.ts.map +1 -1
  444. package/dist/targets.js +124 -62
  445. package/dist/targets.js.map +1 -1
  446. package/dist/trusted-executable.d.ts +6 -1
  447. package/dist/trusted-executable.d.ts.map +1 -1
  448. package/dist/trusted-executable.js +28 -9
  449. package/dist/trusted-executable.js.map +1 -1
  450. package/dist/value.d.ts +2 -0
  451. package/dist/value.d.ts.map +1 -0
  452. package/dist/value.js +27 -0
  453. package/dist/value.js.map +1 -0
  454. package/dist/version.d.ts.map +1 -1
  455. package/dist/version.js +3 -6
  456. package/dist/version.js.map +1 -1
  457. package/dist/windows-path.d.ts.map +1 -1
  458. package/dist/windows-path.js +1 -2
  459. package/dist/windows-path.js.map +1 -1
  460. package/dist/workbench-arguments.d.ts +3 -0
  461. package/dist/workbench-arguments.d.ts.map +1 -0
  462. package/dist/workbench-arguments.js +15 -0
  463. package/dist/workbench-arguments.js.map +1 -0
  464. package/dist/worker-progress.d.ts +2 -0
  465. package/dist/worker-progress.d.ts.map +1 -1
  466. package/dist/worker-progress.js +40 -46
  467. package/dist/worker-progress.js.map +1 -1
  468. package/docs/cli.md +1778 -0
  469. package/docs/dedupe-records.md +24 -0
  470. package/docs/findings-service.md +402 -0
  471. package/package.json +23 -23
  472. package/schemas/project-config.schema.json +5 -0
  473. package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
  474. package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
  475. package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
  476. package/_bundled_plugin/scripts/workbench_findings.py +0 -221
  477. package/_bundled_plugin/scripts/workbench_severity.py +0 -119
  478. package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
  479. package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
  480. package/dist/server/dashboard/app.css +0 -2
  481. package/dist/server/dashboard/app.js +0 -66
  482. package/dist/server/dashboard/index.html +0 -18
  483. package/dist/server/dashboard-types.d.ts +0 -48
  484. package/dist/server/dashboard-types.d.ts.map +0 -1
  485. package/dist/server/dashboard-types.js +0 -2
  486. package/dist/server/dashboard-types.js.map +0 -1
  487. package/dist/server/dashboard.d.ts +0 -6
  488. package/dist/server/dashboard.d.ts.map +0 -1
  489. package/dist/server/dashboard.js +0 -59
  490. package/dist/server/dashboard.js.map +0 -1
  491. package/dist/server/findings-service.d.ts +0 -20
  492. package/dist/server/findings-service.d.ts.map +0 -1
  493. package/dist/server/findings-service.js +0 -31
  494. package/dist/server/findings-service.js.map +0 -1
  495. package/dist/server/index.d.ts +0 -2
  496. package/dist/server/index.d.ts.map +0 -1
  497. package/dist/server/index.js +0 -6
  498. package/dist/server/index.js.map +0 -1
  499. package/dist/server/routes.d.ts +0 -6
  500. package/dist/server/routes.d.ts.map +0 -1
  501. package/dist/server/routes.js +0 -86
  502. package/dist/server/routes.js.map +0 -1
  503. package/dist/server/serve.d.ts +0 -2
  504. package/dist/server/serve.d.ts.map +0 -1
  505. package/dist/server/serve.js +0 -28
  506. package/dist/server/serve.js.map +0 -1
  507. package/dist/server/server.d.ts +0 -10
  508. package/dist/server/server.d.ts.map +0 -1
  509. package/dist/server/server.js +0 -17
  510. package/dist/server/server.js.map +0 -1
  511. package/dist/server/validation.d.ts +0 -17
  512. package/dist/server/validation.d.ts.map +0 -1
  513. package/dist/server/validation.js +0 -53
  514. package/dist/server/validation.js.map +0 -1
@@ -8,128 +8,33 @@ This script stays deliberately model-free:
8
8
  - `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
9
9
  worklist from Git changed paths. It supports committed revision diffs and
10
10
  local working-tree patches.
11
- - `make-rank-shards` partitions the ranking input into deterministic shards.
12
- - `make-rank-pool-plan` assigns those shards to a deterministic bounded worker
13
- pool.
14
- - `validate-rank-worker` validates one worker slot and emits a content-bound
15
- completion receipt.
16
- - `validate-rank-shard` validates one completed worker output before the
17
- coordinator accepts it.
18
- - `validate-rank-pool` validates the pool plan and every assigned shard output.
19
- - `merge-rank-outputs` validates and combines worker-local shard outputs.
20
- - `copy-deep-review-input` copies every candidate into the deep-review worklist
21
- for exhaustive mode.
22
- - `select-deep-review-input` selects the ranked rows for deep review.
11
+
12
+ Candidate selection uses repository scope, native tool ignore rules, and binary detection,
13
+ not filename or directory classifications. Ranking inputs use bounded source previews.
23
14
  """
24
15
 
25
16
  from __future__ import annotations
26
17
 
27
18
  import argparse
28
- import hashlib
29
19
  import json
30
20
  import os
31
- import re
32
21
  import subprocess
33
22
  import sys
34
- from collections import Counter
35
- from collections.abc import Callable
23
+ from collections.abc import Iterable
36
24
  from pathlib import Path
37
25
 
38
26
  # Some plugin hosts launch Python with safe-path isolation enabled.
39
27
  sys.path.insert(0, str(Path(__file__).resolve().parent))
28
+ from generate_in_scope_files import windows_stream_component
40
29
  from rank_preview import (
41
30
  DEFAULT_PREVIEW_BYTES,
42
- TEXT_CODE_EXTENSIONS,
31
+ is_binary_file,
43
32
  preview_for,
44
33
  preview_for_bytes,
45
34
  )
46
- from workbench_target import git_blob_bytes, git_directory_snapshot_paths
47
-
48
- EXCLUDED_DIRS = {
49
- ".cache",
50
- ".circleci",
51
- ".devcontainer",
52
- ".git",
53
- ".github",
54
- ".idea",
55
- ".mypy_cache",
56
- ".pytest_cache",
57
- ".ruff_cache",
58
- ".tox",
59
- ".venv",
60
- ".vscode",
61
- "__pycache__",
62
- "bench",
63
- "benchmark",
64
- "bintest",
65
- "build",
66
- "build_config",
67
- "build_configs",
68
- "build-tools",
69
- "build_tools",
70
- "ci",
71
- "coverage",
72
- "deps",
73
- "dev",
74
- "dist",
75
- "doc",
76
- "docs",
77
- "example",
78
- "examples",
79
- "external",
80
- "extern",
81
- "fixture",
82
- "fixtures",
83
- "generated",
84
- "node_modules",
85
- "sample",
86
- "samples",
87
- "target",
88
- "test",
89
- "tests",
90
- "testing",
91
- "third-party",
92
- "third_party",
93
- "tmp",
94
- "vendor",
95
- }
96
-
97
- EXCLUDED_FILENAMES = {
98
- ".DS_Store",
99
- "CHANGELOG",
100
- "CHANGELOG.md",
101
- "CONTRIBUTING.md",
102
- "Dockerfile",
103
- "Gemfile",
104
- "Gemfile.lock",
105
- "LICENSE",
106
- "LICENSE.md",
107
- "Makefile",
108
- "NEWS",
109
- "NEWS.md",
110
- "NOTICE",
111
- "README",
112
- "README.md",
113
- "README.rst",
114
- "Rakefile",
115
- "SECURITY.md",
116
- "TODO",
117
- "TODO.md",
118
- "docker-compose.yml",
119
- "package-lock.json",
120
- "pnpm-lock.yaml",
121
- "yarn.lock",
122
- }
35
+ from workbench_target import git_blob_samples, git_command, git_directory_snapshot_paths
123
36
 
124
- SHARD_INPUT_GLOB = "rank-shard-*.input.jsonl"
125
- SHARD_OUTPUT_GLOB = "rank-shard-*.output.jsonl"
126
- SHARD_INPUT_PATTERN = re.compile(r"^rank-shard-([0-9]{4,})\.input\.jsonl$")
127
- RANK_POOL_PLAN_SCHEMA_VERSION = 1
128
- RANK_POOL_STRATEGY = "round_robin"
129
- RANK_POOL_WORKER_CAP = 6
130
37
  JsonRow = dict[str, object]
131
- RowValidator = Callable[[JsonRow, Path, int], None]
132
- RankWorkerAssignment = tuple[int, list[str], list[str]]
133
38
 
134
39
 
135
40
  def parse_args() -> argparse.Namespace:
@@ -171,17 +76,9 @@ def parse_args() -> argparse.Namespace:
171
76
  )
172
77
  scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
173
78
 
174
- bind = subparsers.add_parser(
175
- "bind-repo-scopes",
176
- help="Copy SDK scoped-path targets into the unsealed manifest and coverage documents.",
177
- )
178
- bind.add_argument("--scopes-file", required=True, help="JSON array of requested scopes.")
179
- bind.add_argument("--manifest", required=True, help="Unsealed scan-manifest.json path.")
180
- bind.add_argument("--coverage", required=True, help="Unsealed coverage.json path.")
181
-
182
79
  diff = subparsers.add_parser(
183
80
  "make-diff-rank-input",
184
- help="Create rank_input.jsonl from Git changed source-like files.",
81
+ help="Create rank_input.jsonl from Git changed text files.",
185
82
  )
186
83
  diff.add_argument("--repo", required=True, help="Repository root.")
187
84
  diff.add_argument("--base", required=True, help="Git diff base revision.")
@@ -201,115 +98,9 @@ def parse_args() -> argparse.Namespace:
201
98
  help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
202
99
  )
203
100
 
204
- shards = subparsers.add_parser(
205
- "make-rank-shards",
206
- help="Partition rank_input.jsonl into deterministic worker input shards.",
207
- )
208
- shards.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
209
- shards.add_argument("--out-dir", required=True, help="Directory for worker input shards.")
210
- shards.add_argument(
211
- "--max-rows",
212
- type=int,
213
- default=150,
214
- help="Maximum rows per shard. Defaults to 150.",
215
- )
216
-
217
- pool_plan = subparsers.add_parser(
218
- "make-rank-pool-plan",
219
- help="Assign rank shards to a deterministic bounded worker pool.",
220
- )
221
- pool_plan.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
222
- pool_plan.add_argument(
223
- "--usable-worker-slots",
224
- required=True,
225
- type=int,
226
- help="Usable ranking-worker slots reported by capability preflight; capped at 6.",
227
- )
228
- pool_plan.add_argument("--out", required=True, help="Output rank_worker_assignments.json path.")
229
-
230
- validate_shard = subparsers.add_parser(
231
- "validate-rank-shard",
232
- help="Validate one worker output against its rank input shard.",
233
- )
234
- validate_shard.add_argument("--input", required=True, help="Worker rank input shard.")
235
- validate_shard.add_argument("--output", required=True, help="Worker rank output shard.")
236
-
237
- validate_worker = subparsers.add_parser(
238
- "validate-rank-worker",
239
- help="Validate one assigned ranking-worker slot and emit its completion receipt.",
240
- )
241
- validate_worker.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
242
- validate_worker.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
243
- validate_worker.add_argument(
244
- "--slot",
245
- required=True,
246
- type=int,
247
- help="One-based ranking-worker slot from the rank pool plan.",
248
- )
249
-
250
- validate_pool = subparsers.add_parser(
251
- "validate-rank-pool",
252
- help="Validate a rank pool plan and every assigned shard output.",
253
- )
254
- validate_pool.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
255
- validate_pool.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
256
-
257
- merge = subparsers.add_parser(
258
- "merge-rank-outputs",
259
- help="Validate worker shard outputs and create rank_output.jsonl.",
260
- )
261
- merge.add_argument("--rank-input", required=True, help="Authoritative rank input JSONL.")
262
- merge.add_argument("--shard-dir", required=True, help="Directory of input and output shards.")
263
- merge.add_argument("--out", required=True, help="Output rank_output.jsonl path.")
264
-
265
- copy = subparsers.add_parser(
266
- "copy-deep-review-input",
267
- help="Create deep_review_input.jsonl directly from rank_input.jsonl.",
268
- )
269
- copy.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
270
- copy.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
271
-
272
- select = subparsers.add_parser(
273
- "select-deep-review-input",
274
- help="Create deep_review_input.jsonl from worker-produced rank_output.jsonl.",
275
- )
276
- select.add_argument("--rank-output", required=True, help="Worker ranking output JSONL.")
277
- select.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
278
- select.add_argument(
279
- "--top-percent",
280
- type=int,
281
- default=100,
282
- help="Percent of included files to keep for deep review.",
283
- )
284
101
  return parser.parse_args()
285
102
 
286
103
 
287
- def path_is_excluded(path: Path) -> bool:
288
- if any(part in EXCLUDED_DIRS for part in path.parts):
289
- return True
290
- if path.name in EXCLUDED_FILENAMES:
291
- return True
292
- return path.name.endswith((".min.js", ".map"))
293
-
294
-
295
- def path_is_diff_excluded(path: Path) -> bool:
296
- """Apply repository exclusions while retaining changed workflow files."""
297
- if path.parts[:2] == (".github", "workflows"):
298
- return False
299
- return path_is_excluded(path)
300
-
301
-
302
- def windows_stream_component(path: Path) -> str | None:
303
- """Return the first NTFS alternate-data-stream component."""
304
-
305
- if os.name != "nt":
306
- return None
307
- return next(
308
- (component for component in path.parts if component != path.anchor and ":" in component),
309
- None,
310
- )
311
-
312
-
313
104
  def resolve_scope(
314
105
  repo: Path,
315
106
  scope: str,
@@ -362,14 +153,6 @@ def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
362
153
  handle.write("\n")
363
154
 
364
155
 
365
- def write_json(output: Path, payload: dict[str, object]) -> None:
366
- output.parent.mkdir(parents=True, exist_ok=True)
367
- output.write_text(
368
- json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
369
- encoding="utf-8",
370
- )
371
-
372
-
373
156
  def load_scopes_file(scopes_file: Path) -> list[str]:
374
157
  try:
375
158
  loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
@@ -384,80 +167,58 @@ def load_scopes_file(scopes_file: Path) -> list[str]:
384
167
  return loaded
385
168
 
386
169
 
387
- def load_jsonl(path: Path, label: str, validator: RowValidator) -> list[JsonRow]:
388
- if not path.exists():
389
- raise SystemExit(f"{label} missing: {path}")
390
-
391
- rows: list[JsonRow] = []
392
- with path.open(encoding="utf-8") as handle:
393
- for line_number, raw_line in enumerate(handle, start=1):
394
- if not raw_line.strip():
395
- raise SystemExit(f"{path}:{line_number}: blank JSONL rows are not allowed")
396
- try:
397
- parsed: object = json.loads(raw_line)
398
- except json.JSONDecodeError as exc:
399
- raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc
400
- if not isinstance(parsed, dict):
401
- raise SystemExit(f"{path}:{line_number}: expected a JSON object")
402
- row = {str(key): value for key, value in parsed.items()}
403
- validator(row, path, line_number)
404
- rows.append(row)
405
- return rows
406
-
407
-
408
- def require_exact_fields(row: JsonRow, expected: set[str], path: Path, line_number: int) -> None:
409
- actual = set(row)
410
- if actual != expected:
411
- missing = sorted(expected - actual)
412
- unexpected = sorted(actual - expected)
413
- details: list[str] = []
414
- if missing:
415
- details.append(f"missing fields {missing}")
416
- if unexpected:
417
- details.append(f"unexpected fields {unexpected}")
418
- raise SystemExit(f"{path}:{line_number}: {'; '.join(details)}")
419
-
420
-
421
- def require_string(
422
- row: JsonRow, field: str, path: Path, line_number: int, *, allow_empty: bool
423
- ) -> None:
424
- value = row[field]
425
- if not isinstance(value, str) or (not allow_empty and not value.strip()):
426
- requirement = "a string" if allow_empty else "a non-empty string"
427
- raise SystemExit(f"{path}:{line_number}: {field} must be {requirement}")
428
-
429
-
430
- def validate_rank_input_row(row: JsonRow, path: Path, line_number: int) -> None:
431
- require_exact_fields(row, {"path", "area", "preview"}, path, line_number)
432
- require_string(row, "path", path, line_number, allow_empty=False)
433
- require_string(row, "area", path, line_number, allow_empty=True)
434
- require_string(row, "preview", path, line_number, allow_empty=True)
435
-
436
-
437
- def validate_rank_output_row(row: JsonRow, path: Path, line_number: int) -> None:
438
- require_exact_fields(row, {"path", "area", "score", "include", "reason"}, path, line_number)
439
- require_string(row, "path", path, line_number, allow_empty=False)
440
- require_string(row, "area", path, line_number, allow_empty=True)
441
- score = row["score"]
442
- if isinstance(score, bool) or not isinstance(score, int):
443
- raise SystemExit(f"{path}:{line_number}: score must be an integer from 1 through 10")
444
- if not 1 <= score <= 10:
445
- raise SystemExit(f"{path}:{line_number}: score must be from 1 through 10")
446
- if not isinstance(row["include"], bool):
447
- raise SystemExit(f"{path}:{line_number}: include must be a boolean")
448
- require_string(row, "reason", path, line_number, allow_empty=False)
449
-
170
+ def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
171
+ """Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
172
+
173
+ Git retains tracked files and applies its standard exclusions to untracked files.
174
+ Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
175
+ Explicit file scopes bypass directory ignore rules.
176
+ """
177
+ if scope_path.is_file():
178
+ return (scope_path,)
179
+ git_candidates = git_directory_snapshot_paths(scope_path)
180
+ if git_candidates is not None:
181
+ return git_candidates
182
+
183
+ command = [
184
+ "rg",
185
+ "--files",
186
+ "--hidden",
187
+ "--no-require-git",
188
+ "--null",
189
+ # Also exclude descendants when the scope starts inside .git.
190
+ "--glob",
191
+ "!**/.git",
192
+ "--glob",
193
+ "!**/.git/**",
194
+ "--",
195
+ str(scope_path.relative_to(repo)),
196
+ ]
197
+ try:
198
+ result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
199
+ except OSError as exc:
200
+ ignore_names = (".gitignore", ".ignore", ".rgignore")
201
+ ancestors = (scope_path, *scope_path.parents)
202
+ has_ignore_rules = (
203
+ any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
204
+ or any(
205
+ (ancestor / name).is_file()
206
+ for ancestor in ancestors
207
+ if ancestor == repo or repo in ancestor.parents
208
+ for name in ignore_names
209
+ )
210
+ or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
211
+ )
212
+ if has_ignore_rules:
213
+ raise SystemExit(
214
+ "Could not safely enumerate ignored scoped files without Git or ripgrep."
215
+ ) from exc
216
+ return scope_path.rglob("*")
450
217
 
451
- def require_unique_paths(rows: list[JsonRow], label: str) -> None:
452
- seen: set[str] = set()
453
- duplicates: set[str] = set()
454
- for row in rows:
455
- path = str(row["path"])
456
- if path in seen:
457
- duplicates.add(path)
458
- seen.add(path)
459
- if duplicates:
460
- raise SystemExit(f"{label} contains duplicate paths: {sorted(duplicates)}")
218
+ if result.returncode not in (0, 1):
219
+ detail = result.stderr.decode("utf-8", errors="replace").strip()
220
+ raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
221
+ return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
461
222
 
462
223
 
463
224
  def make_repo_rank_input(args: argparse.Namespace) -> None:
@@ -479,7 +240,7 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
479
240
  for scope_abs in resolved_scopes:
480
241
  scope_rel = scope_abs.relative_to(repo)
481
242
  area = args.area or scope_rel.as_posix()
482
- candidates = (scope_abs,) if scope_abs.is_file() else scope_abs.rglob("*")
243
+ candidates = scope_candidates(repo, scope_abs)
483
244
  for path in candidates:
484
245
  try:
485
246
  if path.is_symlink() or not path.is_file():
@@ -489,26 +250,14 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
489
250
  continue
490
251
  rel = path.relative_to(repo)
491
252
  directly_requested = path in directly_requested_files
492
- excluded_path = (
493
- path.relative_to(scope_abs if scope_abs.is_dir() else scope_abs.parent)
494
- if explicit_scopes
495
- else rel
496
- )
497
- if not directly_requested and (
498
- path_is_excluded(excluded_path) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS
499
- ):
253
+ if ".git" in rel.parts:
500
254
  continue
501
255
 
502
- if (
503
- directly_requested
504
- and path.suffix.lower() not in TEXT_CODE_EXTENSIONS
505
- and path.name not in EXCLUDED_FILENAMES
506
- ):
507
- preview = ""
508
- else:
509
- preview, is_binary = preview_for(path, args.preview_bytes)
510
- if is_binary and not directly_requested:
256
+ preview, is_binary = preview_for(path, args.preview_bytes)
257
+ if is_binary or is_binary_file(path):
258
+ if not directly_requested:
511
259
  continue
260
+ preview = ""
512
261
  rows_by_path.setdefault(
513
262
  rel.as_posix(),
514
263
  {"path": rel.as_posix(), "area": area, "preview": preview},
@@ -529,55 +278,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
529
278
  rows_by_path: dict[str, JsonRow] = {}
530
279
  for scope in scopes:
531
280
  scope_path = resolve_scope(repo, scope, expand_user=False, reject_symlinks=True)
532
- if scope_path.is_file():
533
- candidates = (scope_path,)
534
- else:
535
- git_candidates = git_directory_snapshot_paths(scope_path)
536
- if git_candidates is not None:
537
- candidates = git_candidates
538
- else:
539
- command = [
540
- "rg",
541
- "--files",
542
- "--hidden",
543
- "--no-require-git",
544
- "--null",
545
- "--glob",
546
- "!.git/**",
547
- "--",
548
- str(scope_path.relative_to(repo)),
549
- ]
550
- try:
551
- result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
552
- except OSError as exc:
553
- ignore_names = (".gitignore", ".ignore", ".rgignore")
554
- ancestors = (scope_path, *scope_path.parents)
555
- has_ignore_rules = (
556
- any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
557
- or any(
558
- (ancestor / name).is_file()
559
- for ancestor in ancestors
560
- if ancestor == repo or repo in ancestor.parents
561
- for name in ignore_names
562
- )
563
- or any(
564
- path.name in ignore_names
565
- for path in scope_path.rglob("*")
566
- if path.is_file()
567
- )
568
- )
569
- if has_ignore_rules:
570
- raise SystemExit(
571
- "Could not safely enumerate ignored scoped files without Git or ripgrep."
572
- ) from exc
573
- candidates = scope_path.rglob("*")
574
- else:
575
- if result.returncode not in (0, 1):
576
- detail = result.stderr.decode("utf-8", errors="replace").strip()
577
- raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
578
- candidates = (
579
- repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path
580
- )
281
+ candidates = scope_candidates(repo, scope_path)
581
282
  for path in candidates:
582
283
  try:
583
284
  if path.is_symlink() or not path.is_file():
@@ -595,49 +296,19 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
595
296
  print(f"Wrote {len(rows)} scoped paths to {output}")
596
297
 
597
298
 
598
- def bind_repo_scopes(args: argparse.Namespace) -> None:
599
- scopes = load_scopes_file(Path(args.scopes_file).expanduser())
600
- manifest_path = Path(args.manifest).expanduser()
601
- coverage_path = Path(args.coverage).expanduser()
602
- try:
603
- manifest: object = json.loads(manifest_path.read_text(encoding="utf-8"))
604
- coverage: object = json.loads(coverage_path.read_text(encoding="utf-8"))
605
- if not isinstance(manifest, dict) or not isinstance(coverage, dict):
606
- raise ValueError("expected JSON objects")
607
- scan = manifest.get("scan")
608
- if not isinstance(scan, dict):
609
- raise ValueError("manifest.scan must be an object")
610
- scope = scan.get("scope")
611
- if not isinstance(scope, dict):
612
- raise ValueError("manifest.scan.scope must be an object")
613
- except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
614
- raise SystemExit("Unable to bind requested scopes into the scan contract") from exc
615
- scope["includePaths"] = scopes
616
- coverage["includePaths"] = scopes
617
- manifest_path.write_text(
618
- json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
619
- )
620
- coverage_path.write_text(
621
- json.dumps(coverage, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
622
- )
623
- print(f"Bound {len(scopes)} requested scopes into the scan contract")
624
-
625
-
626
299
  def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
627
- result = subprocess.run(
628
- [
629
- "git",
630
- "-C",
631
- str(repo),
632
- "diff",
633
- "--name-status",
634
- "-z",
635
- "--diff-filter=ACMRD",
636
- *diff_args,
637
- ],
638
- check=True,
639
- capture_output=True,
640
- )
300
+ """Return changed regular files from the selected side of each change."""
301
+ result = git_command(
302
+ repo,
303
+ "diff",
304
+ "--ignore-submodules=all",
305
+ "--raw",
306
+ "-z",
307
+ "--diff-filter=ACMRDT",
308
+ *diff_args,
309
+ text=False,
310
+ )
311
+ result.check_returncode()
641
312
  fields = result.stdout.split(b"\0")
642
313
  if fields and not fields[-1]:
643
314
  fields.pop()
@@ -645,13 +316,16 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
645
316
  changed: list[tuple[Path, str]] = []
646
317
  index = 0
647
318
  while index < len(fields):
648
- status = chr(fields[index][0])
319
+ metadata = fields[index].split()
320
+ status = chr(metadata[-1][0])
649
321
  index += 1
650
322
  if status in {"C", "R"}:
651
323
  index += 1
652
324
  path = repo / os.fsdecode(fields[index])
653
325
  index += 1
654
- changed.append((path, status))
326
+ selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
327
+ if selected_mode.startswith(b"100"):
328
+ changed.append((path, status))
655
329
  return changed
656
330
 
657
331
 
@@ -661,19 +335,27 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
661
335
  if mode == "local-patch":
662
336
  unstaged = run_git_changed_paths(repo, [base])
663
337
  staged = run_git_changed_paths(repo, ["--cached", base])
664
- untracked = subprocess.run(
665
- ["git", "-C", str(repo), "ls-files", "--others", "--exclude-standard", "-z"],
666
- capture_output=True,
667
- check=True,
338
+ untracked = git_command(
339
+ repo,
340
+ "ls-files",
341
+ "--others",
342
+ "--exclude-standard",
343
+ "-z",
344
+ text=False,
668
345
  )
346
+ untracked.check_returncode()
669
347
  combined = dict(staged)
670
348
  combined.update(unstaged)
671
349
  combined.update(
672
350
  (repo / os.fsdecode(relative), "A")
673
351
  for relative in untracked.stdout.split(b"\0")
674
- if relative
352
+ if relative and not relative.endswith(b"/")
353
+ )
354
+ return sorted(
355
+ (path, status)
356
+ for path, status in combined.items()
357
+ if status == "D" or (not path.is_symlink() and path.is_file())
675
358
  )
676
- return sorted(combined.items())
677
359
  raise SystemExit(f"Unknown diff mode: {mode}")
678
360
 
679
361
 
@@ -682,55 +364,42 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
682
364
  if not repo.is_dir():
683
365
  raise SystemExit(f"Repo path not found: {repo}")
684
366
 
685
- changed = [
686
- (path, status)
687
- for path, status in git_changed_paths(repo, args.base, args.head, args.mode)
688
- if not path_is_diff_excluded(path.relative_to(repo))
689
- and path.suffix.lower() in TEXT_CODE_EXTENSIONS
690
- ]
691
- revision_paths = [
692
- path.relative_to(repo)
693
- for path, status in changed
694
- if args.mode == "revisions" and status != "D"
695
- ]
696
- revision_blobs = dict(
697
- zip(
698
- revision_paths,
699
- git_blob_bytes(
700
- repo,
701
- [f"{args.head}:{path.as_posix()}" for path in revision_paths],
702
- ),
367
+ changed = git_changed_paths(repo, args.base, args.head, args.mode)
368
+ revision_refs = {
369
+ path.relative_to(repo): (
370
+ f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
703
371
  )
372
+ for path, status in changed
373
+ if args.mode == "revisions" or status == "D"
374
+ }
375
+ revision_samples = dict(
376
+ zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
704
377
  )
705
378
 
706
379
  rows: list[JsonRow] = []
707
380
  for path, status in changed:
708
381
  rel = path.relative_to(repo)
709
382
 
710
- if status == "D":
711
- preview = ""
712
- elif args.mode == "revisions":
713
- content = revision_blobs[rel]
714
- if content is None:
715
- raise SystemExit(
716
- f"Unable to read committed diff blob: {args.head}:{rel.as_posix()}"
717
- )
718
- preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
383
+ preview = ""
384
+ if args.mode == "revisions" or status == "D":
385
+ sample = revision_samples[rel]
386
+ if sample is None:
387
+ revision = args.base if status == "D" else args.head
388
+ raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
389
+ content, is_binary = sample
719
390
  if is_binary:
720
391
  continue
721
- elif path.is_symlink():
722
- preview = ""
723
- elif path.is_file():
392
+ if status != "D":
393
+ preview, _ = preview_for_bytes(content, args.preview_bytes)
394
+ elif not path.is_symlink() and path.is_file():
724
395
  try:
725
396
  path.resolve(strict=True).relative_to(repo)
726
397
  except (OSError, ValueError):
727
398
  preview = ""
728
399
  else:
729
400
  preview, is_binary = preview_for(path, args.preview_bytes)
730
- if is_binary:
401
+ if is_binary or is_binary_file(path):
731
402
  continue
732
- else:
733
- preview = ""
734
403
  rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
735
404
 
736
405
  rows.sort(key=lambda row: str(row["path"]))
@@ -739,454 +408,14 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
739
408
  print(f"Wrote {len(rows)} rows to {output}")
740
409
 
741
410
 
742
- def make_rank_shards(args: argparse.Namespace) -> None:
743
- if args.max_rows < 1:
744
- raise SystemExit("--max-rows must be at least 1")
745
-
746
- rank_input = Path(args.rank_input).expanduser()
747
- rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
748
- require_unique_paths(rows, "Rank input")
749
-
750
- output_dir = Path(args.out_dir).expanduser()
751
- output_dir.mkdir(parents=True, exist_ok=True)
752
- existing = sorted((*output_dir.glob(SHARD_INPUT_GLOB), *output_dir.glob(SHARD_OUTPUT_GLOB)))
753
- if existing:
754
- raise SystemExit(f"Rank shard directory already contains shard files: {output_dir}")
755
-
756
- shard_count = 0
757
- for start in range(0, len(rows), args.max_rows):
758
- shard_count += 1
759
- shard_path = output_dir / f"rank-shard-{shard_count:04d}.input.jsonl"
760
- write_jsonl(shard_path, rows[start : start + args.max_rows])
761
-
762
- print(f"Wrote {shard_count} rank shards to {output_dir}")
763
-
764
-
765
- def discover_input_shards(shard_dir: Path) -> list[Path]:
766
- if not shard_dir.is_dir():
767
- raise SystemExit(f"Rank shard directory missing: {shard_dir}")
768
-
769
- numbered_shards: list[tuple[int, Path]] = []
770
- for path in shard_dir.glob(SHARD_INPUT_GLOB):
771
- match = SHARD_INPUT_PATTERN.fullmatch(path.name)
772
- if match is None:
773
- raise SystemExit(f"Rank input shard has invalid name: {path.name}")
774
- numbered_shards.append((int(match.group(1)), path))
775
- numbered_shards.sort(key=lambda item: (item[0], item[1].name))
776
- input_shards = [path for _, path in numbered_shards]
777
- expected_names = [
778
- f"rank-shard-{index:04d}.input.jsonl" for index in range(1, len(input_shards) + 1)
779
- ]
780
- actual_names = [path.name for path in input_shards]
781
- if actual_names != expected_names:
782
- raise SystemExit(
783
- "Rank input shards must use contiguous canonical names; "
784
- f"expected={expected_names}; actual={actual_names}"
785
- )
786
- return input_shards
787
-
788
-
789
- def output_name_for(input_name: str) -> str:
790
- return input_name.replace(".input.jsonl", ".output.jsonl")
791
-
792
-
793
- def require_plan_shard_dir(plan_path: Path, shard_dir: Path) -> None:
794
- expected = plan_path.parent / "rank_shards"
795
- if shard_dir.resolve() != expected.resolve():
796
- raise SystemExit(
797
- "Rank shard directory must be the assignment plan's sibling rank_shards "
798
- f"directory; expected={expected}; actual={shard_dir}"
799
- )
800
-
801
-
802
- def require_no_misplaced_rank_shards(plan_path: Path) -> None:
803
- misplaced = sorted(
804
- (
805
- *plan_path.parent.glob(SHARD_INPUT_GLOB),
806
- *plan_path.parent.glob(SHARD_OUTPUT_GLOB),
807
- ),
808
- key=lambda path: path.name,
809
- )
810
- if misplaced:
811
- raise SystemExit(
812
- "Rank shard artifacts must be stored in the assignment plan's sibling "
813
- f"rank_shards directory; misplaced={[path.name for path in misplaced]}"
814
- )
815
-
816
-
817
- def make_rank_pool_plan(args: argparse.Namespace) -> None:
818
- if args.usable_worker_slots < 1:
819
- raise SystemExit("--usable-worker-slots must be at least 1")
820
-
821
- shard_dir = Path(args.shard_dir).expanduser()
822
- output = Path(args.out).expanduser()
823
- require_plan_shard_dir(output, shard_dir)
824
- input_shards = discover_input_shards(shard_dir)
825
- worker_count = min(len(input_shards), args.usable_worker_slots, RANK_POOL_WORKER_CAP)
826
- workers: list[dict[str, object]] = []
827
- for worker_index in range(worker_count):
828
- assigned_inputs = [path.name for path in input_shards[worker_index::worker_count]]
829
- workers.append(
830
- {
831
- "slot": worker_index + 1,
832
- "input_shards": assigned_inputs,
833
- "output_shards": [output_name_for(name) for name in assigned_inputs],
834
- }
835
- )
836
-
837
- plan: dict[str, object] = {
838
- "schema_version": RANK_POOL_PLAN_SCHEMA_VERSION,
839
- "strategy": RANK_POOL_STRATEGY,
840
- "shard_count": len(input_shards),
841
- "ranking_worker_count": worker_count,
842
- "workers": workers,
843
- }
844
- write_json(output, plan)
845
- print(f"Assigned {len(input_shards)} rank shards to {worker_count} ranking workers in {output}")
846
-
847
-
848
- def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]:
849
- if not plan_path.exists():
850
- raise SystemExit(f"Rank pool plan missing: {plan_path}")
851
- plan_bytes = plan_path.read_bytes()
852
- try:
853
- payload: object = json.loads(plan_bytes)
854
- except json.JSONDecodeError as exc:
855
- raise SystemExit(f"{plan_path}: invalid JSON: {exc.msg}") from exc
856
- if not isinstance(payload, dict):
857
- raise SystemExit(f"{plan_path}: expected a JSON object")
858
- return {str(key): value for key, value in payload.items()}, plan_bytes
859
-
860
-
861
- def require_integer(value: object, label: str, *, minimum: int) -> int:
862
- if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
863
- raise SystemExit(f"{label} must be an integer of at least {minimum}")
864
- return value
865
-
866
-
867
- def require_string_list(value: object, label: str) -> list[str]:
868
- if not isinstance(value, list) or not value:
869
- raise SystemExit(f"{label} must be a non-empty list")
870
- if any(not isinstance(item, str) or not item for item in value):
871
- raise SystemExit(f"{label} entries must be non-empty strings")
872
- return [item for item in value if isinstance(item, str)]
873
-
874
-
875
- def assignment_differences(
876
- assigned_names: list[str], expected_names: list[str]
877
- ) -> tuple[list[str], list[str], list[str]]:
878
- counts = Counter(assigned_names)
879
- duplicates = sorted(name for name, count in counts.items() if count > 1)
880
- assigned = set(assigned_names)
881
- expected = set(expected_names)
882
- return sorted(expected - assigned), duplicates, sorted(assigned - expected)
883
-
884
-
885
- def validate_rank_pool_plan(
886
- plan_path: Path, shard_dir: Path
887
- ) -> tuple[list[Path], list[str], list[RankWorkerAssignment], bytes]:
888
- require_plan_shard_dir(plan_path, shard_dir)
889
- require_no_misplaced_rank_shards(plan_path)
890
- input_shards = discover_input_shards(shard_dir)
891
- input_names = [path.name for path in input_shards]
892
- output_names = [output_name_for(name) for name in input_names]
893
- plan, plan_bytes = load_rank_pool_plan(plan_path)
894
- expected_fields = {
895
- "schema_version",
896
- "strategy",
897
- "shard_count",
898
- "ranking_worker_count",
899
- "workers",
900
- }
901
- actual_fields = set(plan)
902
- if actual_fields != expected_fields:
903
- raise SystemExit(
904
- f"{plan_path}: rank pool plan fields do not match schema; "
905
- f"missing={sorted(expected_fields - actual_fields)}; "
906
- f"unexpected={sorted(actual_fields - expected_fields)}"
907
- )
908
- schema_version = require_integer(
909
- plan["schema_version"], f"{plan_path}: schema_version", minimum=1
910
- )
911
- if schema_version != RANK_POOL_PLAN_SCHEMA_VERSION:
912
- raise SystemExit(f"{plan_path}: schema_version must be {RANK_POOL_PLAN_SCHEMA_VERSION}")
913
- if plan["strategy"] != RANK_POOL_STRATEGY:
914
- raise SystemExit(f"{plan_path}: strategy must be {RANK_POOL_STRATEGY}")
915
-
916
- shard_count = require_integer(plan["shard_count"], f"{plan_path}: shard_count", minimum=0)
917
- if shard_count != len(input_shards):
918
- raise SystemExit(
919
- f"{plan_path}: shard_count does not match input shards; "
920
- f"plan={shard_count}; actual={len(input_shards)}"
921
- )
922
- worker_count = require_integer(
923
- plan["ranking_worker_count"], f"{plan_path}: ranking_worker_count", minimum=0
924
- )
925
- if shard_count > 0 and worker_count == 0:
926
- raise SystemExit(
927
- f"{plan_path}: ranking_worker_count must be at least 1 when input shards exist"
928
- )
929
- if worker_count > shard_count:
930
- raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed shard_count")
931
- if worker_count > RANK_POOL_WORKER_CAP:
932
- raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed {RANK_POOL_WORKER_CAP}")
933
-
934
- workers = plan["workers"]
935
- if not isinstance(workers, list) or len(workers) != worker_count:
936
- raise SystemExit(
937
- f"{plan_path}: workers must contain exactly {worker_count} worker assignments"
938
- )
939
-
940
- assigned_inputs: list[str] = []
941
- assigned_outputs: list[str] = []
942
- parsed_workers: list[RankWorkerAssignment] = []
943
- worker_fields = {"slot", "input_shards", "output_shards"}
944
- for worker_index, raw_worker in enumerate(workers):
945
- label = f"{plan_path}: workers[{worker_index}]"
946
- if not isinstance(raw_worker, dict):
947
- raise SystemExit(f"{label} must be a JSON object")
948
- worker = {str(key): value for key, value in raw_worker.items()}
949
- if set(worker) != worker_fields:
950
- raise SystemExit(
951
- f"{label} fields do not match schema; "
952
- f"missing={sorted(worker_fields - set(worker))}; "
953
- f"unexpected={sorted(set(worker) - worker_fields)}"
954
- )
955
- slot = require_integer(worker["slot"], f"{label}.slot", minimum=1)
956
- if slot != worker_index + 1:
957
- raise SystemExit(f"{label}.slot must be {worker_index + 1}")
958
- worker_inputs = require_string_list(worker["input_shards"], f"{label}.input_shards")
959
- worker_outputs = require_string_list(worker["output_shards"], f"{label}.output_shards")
960
- if len(worker_inputs) != len(worker_outputs):
961
- raise SystemExit(f"{label} input_shards and output_shards lengths must match")
962
- expected_worker_outputs = [output_name_for(name) for name in worker_inputs]
963
- if worker_outputs != expected_worker_outputs:
964
- raise SystemExit(f"{label}.output_shards do not match its input_shards")
965
- assigned_inputs.extend(worker_inputs)
966
- assigned_outputs.extend(worker_outputs)
967
- parsed_workers.append((slot, worker_inputs, worker_outputs))
968
-
969
- missing, duplicates, unexpected = assignment_differences(assigned_inputs, input_names)
970
- if missing or duplicates or unexpected:
971
- raise SystemExit(
972
- f"{plan_path}: pool plan must assign each input shard exactly once; "
973
- f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
974
- )
975
- missing, duplicates, unexpected = assignment_differences(assigned_outputs, output_names)
976
- if missing or duplicates or unexpected:
977
- raise SystemExit(
978
- f"{plan_path}: pool plan must assign each output shard exactly once; "
979
- f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
980
- )
981
-
982
- for worker_index, (_, worker_inputs, worker_outputs) in enumerate(parsed_workers):
983
- expected_inputs = input_names[worker_index::worker_count]
984
- expected_outputs = output_names[worker_index::worker_count]
985
- if worker_inputs != expected_inputs or worker_outputs != expected_outputs:
986
- raise SystemExit(
987
- f"{plan_path}: worker slot {worker_index + 1} does not match the deterministic "
988
- f"{RANK_POOL_STRATEGY} assignment"
989
- )
990
- return input_shards, output_names, parsed_workers, plan_bytes
991
-
992
-
993
- def validate_rank_worker_command(args: argparse.Namespace) -> None:
994
- plan_path = Path(args.plan).expanduser()
995
- shard_dir = Path(args.shard_dir).expanduser()
996
- _, _, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir)
997
-
998
- slot = require_integer(args.slot, "--slot", minimum=1)
999
- worker_count = len(workers)
1000
- if slot > worker_count:
1001
- raise SystemExit(f"--slot must be at most {worker_count}")
1002
-
1003
- assigned_slot, input_names, output_names = workers[slot - 1]
1004
- if assigned_slot != slot:
1005
- raise SystemExit(f"{plan_path}: worker assignment for slot {slot} is inconsistent")
1006
-
1007
- row_count = 0
1008
- outputs_digest = hashlib.sha256()
1009
- for input_name, output_name in zip(input_names, output_names, strict=True):
1010
- input_shard = shard_dir / input_name
1011
- output_shard = shard_dir / output_name
1012
- _, output_rows = validate_rank_shard(input_shard, output_shard)
1013
- output_bytes = output_shard.read_bytes()
1014
- row_count += len(output_rows)
1015
- outputs_digest.update(output_name.encode("utf-8"))
1016
- outputs_digest.update(b"\0")
1017
- outputs_digest.update(output_bytes)
1018
- outputs_digest.update(b"\0")
1019
-
1020
- receipt: dict[str, object] = {
1021
- "schema_version": 1,
1022
- "plan_sha256": hashlib.sha256(plan_bytes).hexdigest(),
1023
- "slot": slot,
1024
- "ranking_worker_count": worker_count,
1025
- "output_shards": len(output_names),
1026
- "rows": row_count,
1027
- "outputs_sha256": outputs_digest.hexdigest(),
1028
- "status": "complete",
1029
- }
1030
- print("RANK_WORKER_RECEIPT " + json.dumps(receipt, sort_keys=True, separators=(",", ":")))
1031
-
1032
-
1033
- def validate_rank_pool_command(args: argparse.Namespace) -> None:
1034
- plan_path = Path(args.plan).expanduser()
1035
- shard_dir = Path(args.shard_dir).expanduser()
1036
- input_shards, expected_output_names, workers, _ = validate_rank_pool_plan(plan_path, shard_dir)
1037
-
1038
- actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)}
1039
- expected_outputs = set(expected_output_names)
1040
- if actual_output_names != expected_outputs:
1041
- missing = sorted(expected_outputs - actual_output_names)
1042
- unexpected = sorted(actual_output_names - expected_outputs)
1043
- raise SystemExit(
1044
- "Rank pool outputs are incomplete; "
1045
- f"missing output shards={missing}; unexpected output shards={unexpected}"
1046
- )
1047
-
1048
- row_count = 0
1049
- for input_shard in input_shards:
1050
- output_shard = input_shard.with_name(output_name_for(input_shard.name))
1051
- _, shard_outputs = validate_rank_shard(input_shard, output_shard)
1052
- row_count += len(shard_outputs)
1053
- print(
1054
- f"Validated {len(workers)} ranking workers, "
1055
- f"{len(input_shards)} shards, and {row_count} ranking rows"
1056
- )
1057
-
1058
-
1059
- def validate_rank_shard(
1060
- input_shard: Path, output_shard: Path
1061
- ) -> tuple[list[JsonRow], list[JsonRow]]:
1062
- shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row)
1063
- require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
1064
- shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row)
1065
- require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
1066
-
1067
- expected_paths = {str(row["path"]) for row in shard_inputs}
1068
- actual_paths = {str(row["path"]) for row in shard_outputs}
1069
- if expected_paths != actual_paths:
1070
- missing = sorted(expected_paths - actual_paths)
1071
- unknown = sorted(actual_paths - expected_paths)
1072
- raise SystemExit(
1073
- f"{output_shard}: paths do not match its input shard; "
1074
- f"missing={missing}; unknown={unknown}"
1075
- )
1076
-
1077
- area_by_path = {str(row["path"]): row["area"] for row in shard_inputs}
1078
- for row in shard_outputs:
1079
- row_path = str(row["path"])
1080
- if row["area"] != area_by_path[row_path]:
1081
- raise SystemExit(f"{output_shard}: area does not match rank input for {row_path}")
1082
- return shard_inputs, shard_outputs
1083
-
1084
-
1085
- def validate_rank_shard_command(args: argparse.Namespace) -> None:
1086
- input_shard = Path(args.input).expanduser()
1087
- output_shard = Path(args.output).expanduser()
1088
- _, output_rows = validate_rank_shard(input_shard, output_shard)
1089
- print(f"Validated {len(output_rows)} ranking rows in {output_shard}")
1090
-
1091
-
1092
- def merge_rank_outputs(args: argparse.Namespace) -> None:
1093
- rank_input = Path(args.rank_input).expanduser()
1094
- authoritative_rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
1095
- require_unique_paths(authoritative_rows, "Rank input")
1096
-
1097
- shard_dir = Path(args.shard_dir).expanduser()
1098
- input_shards = discover_input_shards(shard_dir)
1099
- output_shards = sorted(shard_dir.glob(SHARD_OUTPUT_GLOB))
1100
- expected_output_names = {
1101
- path.name.replace(".input.jsonl", ".output.jsonl") for path in input_shards
1102
- }
1103
- actual_output_names = {path.name for path in output_shards}
1104
- if expected_output_names != actual_output_names:
1105
- missing = sorted(expected_output_names - actual_output_names)
1106
- unexpected = sorted(actual_output_names - expected_output_names)
1107
- details: list[str] = []
1108
- if missing:
1109
- details.append(f"missing output shards {missing}")
1110
- if unexpected:
1111
- details.append(f"unexpected output shards {unexpected}")
1112
- raise SystemExit(f"Rank shard outputs are incomplete: {'; '.join(details)}")
1113
-
1114
- sharded_inputs: list[JsonRow] = []
1115
- output_by_path: dict[str, JsonRow] = {}
1116
- for input_shard in input_shards:
1117
- output_shard = input_shard.with_name(
1118
- input_shard.name.replace(".input.jsonl", ".output.jsonl")
1119
- )
1120
- shard_inputs, shard_outputs = validate_rank_shard(input_shard, output_shard)
1121
- sharded_inputs.extend(shard_inputs)
1122
- for row in shard_outputs:
1123
- row_path = str(row["path"])
1124
- if row_path in output_by_path:
1125
- raise SystemExit(f"Rank outputs contain duplicate path: {row_path}")
1126
- output_by_path[row_path] = row
1127
-
1128
- if sharded_inputs != authoritative_rows:
1129
- raise SystemExit("Rank input shards do not exactly partition the authoritative rank input")
1130
-
1131
- merged = [output_by_path[str(row["path"])] for row in authoritative_rows]
1132
- output = Path(args.out).expanduser()
1133
- write_jsonl(output, merged)
1134
- print(f"Merged {len(merged)} ranking rows into {output}")
1135
-
1136
-
1137
- def copy_deep_review_input(args: argparse.Namespace) -> None:
1138
- rank_input = Path(args.rank_input).expanduser()
1139
- rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
1140
- require_unique_paths(rows, "Rank input")
1141
- selected = [{"path": row["path"], "area": row["area"]} for row in rows]
1142
-
1143
- output = Path(args.out).expanduser()
1144
- write_jsonl(output, selected)
1145
- print(f"Copied {len(selected)} rows into {output}")
1146
-
1147
-
1148
- def select_deep_review_input(args: argparse.Namespace) -> None:
1149
- rank_output = Path(args.rank_output).expanduser()
1150
- rows = load_jsonl(rank_output, "Rank output", validate_rank_output_row)
1151
- require_unique_paths(rows, "Rank output")
1152
-
1153
- included = [row for row in rows if row["include"]]
1154
- base_rows = included if included else rows
1155
- base_rows.sort(key=lambda row: (-int(row["score"]), str(row["path"])))
1156
- keep = max(1, int(len(base_rows) * (args.top_percent / 100.0))) if base_rows else 0
1157
- selected = [{"path": row["path"], "area": row["area"]} for row in base_rows[:keep]]
1158
-
1159
- output = Path(args.out).expanduser()
1160
- write_jsonl(output, selected)
1161
- print(f"Selected {len(selected)} of {len(base_rows)} rows into {output}")
1162
-
1163
-
1164
411
  def main() -> None:
1165
412
  args = parse_args()
1166
413
  if args.command == "make-repo-rank-input":
1167
414
  make_repo_rank_input(args)
1168
415
  elif args.command == "make-repo-scope-input":
1169
416
  make_repo_scope_input(args)
1170
- elif args.command == "bind-repo-scopes":
1171
- bind_repo_scopes(args)
1172
417
  elif args.command == "make-diff-rank-input":
1173
418
  make_diff_rank_input(args)
1174
- elif args.command == "make-rank-shards":
1175
- make_rank_shards(args)
1176
- elif args.command == "make-rank-pool-plan":
1177
- make_rank_pool_plan(args)
1178
- elif args.command == "validate-rank-shard":
1179
- validate_rank_shard_command(args)
1180
- elif args.command == "validate-rank-worker":
1181
- validate_rank_worker_command(args)
1182
- elif args.command == "validate-rank-pool":
1183
- validate_rank_pool_command(args)
1184
- elif args.command == "merge-rank-outputs":
1185
- merge_rank_outputs(args)
1186
- elif args.command == "copy-deep-review-input":
1187
- copy_deep_review_input(args)
1188
- elif args.command == "select-deep-review-input":
1189
- select_deep_review_input(args)
1190
419
  else:
1191
420
  raise SystemExit(f"Unknown command: {args.command}")
1192
421