@openai/codex-security 0.8.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (649) hide show
  1. package/README.md +201 -178
  2. package/_bundled_plugin/.app.json +4 -0
  3. package/_bundled_plugin/.mcp.json +6 -1
  4. package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
  5. package/_bundled_plugin/examples/completed-scan/report.md +11 -8
  6. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
  7. package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
  8. package/_bundled_plugin/mcp/helpers.mjs +2 -11
  9. package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
  10. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
  11. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  12. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  13. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/package.json +1 -0
  20. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +51 -14
  21. package/_bundled_plugin/mcp/server.mjs +2 -11
  22. package/_bundled_plugin/mcp/server.mjs.br +0 -0
  23. package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
  24. package/_bundled_plugin/references/artifact-storage.md +1 -1
  25. package/_bundled_plugin/references/core-scan.md +14 -5
  26. package/_bundled_plugin/references/default-severity-rubric.md +33 -0
  27. package/_bundled_plugin/references/final-report.md +5 -3
  28. package/_bundled_plugin/references/finding-detail-fields.md +19 -18
  29. package/_bundled_plugin/references/sarif-adapter.md +8 -2
  30. package/_bundled_plugin/references/scan-artifacts.md +21 -12
  31. package/_bundled_plugin/references/scan-contract.md +27 -33
  32. package/_bundled_plugin/references/threat-model.md +1 -0
  33. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
  34. package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
  35. package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
  36. package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
  37. package/_bundled_plugin/scripts/deep_scan_workbench.py +188 -454
  38. package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
  39. package/_bundled_plugin/scripts/finalize_scan_contract.py +317 -377
  40. package/_bundled_plugin/scripts/finding_preview.py +25 -29
  41. package/_bundled_plugin/scripts/launch_codex_security_mcp +1 -0
  42. package/_bundled_plugin/scripts/legacy_findings.py +148 -0
  43. package/_bundled_plugin/scripts/posix_scan_local_files.py +198 -0
  44. package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
  45. package/_bundled_plugin/scripts/report_projection.py +167 -178
  46. package/_bundled_plugin/scripts/review_state.py +8 -2
  47. package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
  48. package/_bundled_plugin/scripts/snapshot_sqlite.py +1 -0
  49. package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
  50. package/_bundled_plugin/scripts/validate_scan_contract.py +9 -7
  51. package/_bundled_plugin/scripts/windows_scan_local_files.py +600 -72
  52. package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
  53. package/_bundled_plugin/scripts/workbench/handoff.py +47 -0
  54. package/_bundled_plugin/scripts/workbench/retained_coverage.py +38 -75
  55. package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
  56. package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
  57. package/_bundled_plugin/scripts/workbench/saved_coverage.py +152 -248
  58. package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
  59. package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
  60. package/_bundled_plugin/scripts/workbench/storage.py +71 -7
  61. package/_bundled_plugin/scripts/workbench_cli.py +121 -14
  62. package/_bundled_plugin/scripts/workbench_constants.py +19 -0
  63. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
  64. package/_bundled_plugin/scripts/workbench_db.py +550 -492
  65. package/_bundled_plugin/scripts/workbench_feedback.py +8 -7
  66. package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
  67. package/_bundled_plugin/scripts/workbench_finding_workflows.py +90 -4
  68. package/_bundled_plugin/scripts/workbench_native_indexes.py +29 -33
  69. package/_bundled_plugin/scripts/workbench_progress.py +10 -1
  70. package/_bundled_plugin/scripts/workbench_publication.py +14 -10
  71. package/_bundled_plugin/scripts/workbench_remediation.py +508 -11
  72. package/_bundled_plugin/scripts/workbench_saved_results.py +132 -77
  73. package/_bundled_plugin/scripts/workbench_scan_history.py +139 -80
  74. package/_bundled_plugin/scripts/workbench_scan_start.py +84 -25
  75. package/_bundled_plugin/scripts/workbench_scan_usage.py +666 -117
  76. package/_bundled_plugin/scripts/workbench_schema.py +44 -55
  77. package/_bundled_plugin/scripts/workbench_setup.py +251 -0
  78. package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
  79. package/_bundled_plugin/scripts/workbench_source_scopes.py +33 -23
  80. package/_bundled_plugin/scripts/workbench_target.py +330 -66
  81. package/_bundled_plugin/scripts/workbench_target_state.py +3 -0
  82. package/_bundled_plugin/scripts/workbench_validation.py +44 -12
  83. package/_bundled_plugin/shared/workbench-history-repairs.json +19 -16
  84. package/_bundled_plugin/shared/workbench-migrations.json +53 -0
  85. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -1
  86. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +4 -68
  87. package/_bundled_plugin/skills/check-skills/SKILL.md +43 -0
  88. package/_bundled_plugin/skills/check-skills/agents/openai.yaml +4 -0
  89. package/_bundled_plugin/skills/cloud-findings/SKILL.md +6 -4
  90. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
  91. package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -2
  92. package/_bundled_plugin/skills/publish-findings/SKILL.md +81 -45
  93. package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
  94. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +5 -24
  95. package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +464 -0
  96. package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +36 -9
  97. package/_bundled_plugin/skills/track-findings/SKILL.md +5 -3
  98. package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +22 -68
  99. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +12 -6
  100. package/dist/api.d.ts +45 -5
  101. package/dist/api.d.ts.map +1 -1
  102. package/dist/api.js +1144 -411
  103. package/dist/api.js.map +1 -1
  104. package/dist/auth.d.ts.map +1 -1
  105. package/dist/auth.js +10 -7
  106. package/dist/auth.js.map +1 -1
  107. package/dist/bedrock-oss-pricing.d.ts +6 -0
  108. package/dist/bedrock-oss-pricing.d.ts.map +1 -0
  109. package/dist/bedrock-oss-pricing.js +58 -0
  110. package/dist/bedrock-oss-pricing.js.map +1 -0
  111. package/dist/bedrock-pricing.d.ts +15 -0
  112. package/dist/bedrock-pricing.d.ts.map +1 -0
  113. package/dist/bedrock-pricing.js +117 -0
  114. package/dist/bedrock-pricing.js.map +1 -0
  115. package/dist/blackduck-findings.d.ts +8 -0
  116. package/dist/blackduck-findings.d.ts.map +1 -0
  117. package/dist/blackduck-findings.js +320 -0
  118. package/dist/blackduck-findings.js.map +1 -0
  119. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  120. package/dist/bulk-scan-discovery.js +51 -36
  121. package/dist/bulk-scan-discovery.js.map +1 -1
  122. package/dist/checkmarx-evidence.d.ts +19 -0
  123. package/dist/checkmarx-evidence.d.ts.map +1 -0
  124. package/dist/checkmarx-evidence.js +36 -0
  125. package/dist/checkmarx-evidence.js.map +1 -0
  126. package/dist/checkmarx-findings.d.ts +4 -0
  127. package/dist/checkmarx-findings.d.ts.map +1 -0
  128. package/dist/checkmarx-findings.js +27 -0
  129. package/dist/checkmarx-findings.js.map +1 -0
  130. package/dist/checkmarx-one.d.ts +4 -0
  131. package/dist/checkmarx-one.d.ts.map +1 -0
  132. package/dist/checkmarx-one.js +53 -0
  133. package/dist/checkmarx-one.js.map +1 -0
  134. package/dist/checkmarx-repository.d.ts +12 -0
  135. package/dist/checkmarx-repository.d.ts.map +1 -0
  136. package/dist/checkmarx-repository.js +181 -0
  137. package/dist/checkmarx-repository.js.map +1 -0
  138. package/dist/checkmarx-xml.d.ts +12 -0
  139. package/dist/checkmarx-xml.d.ts.map +1 -0
  140. package/dist/checkmarx-xml.js +196 -0
  141. package/dist/checkmarx-xml.js.map +1 -0
  142. package/dist/classify-severity-records.d.ts +6 -9
  143. package/dist/classify-severity-records.d.ts.map +1 -1
  144. package/dist/classify-severity-records.js +9 -21
  145. package/dist/classify-severity-records.js.map +1 -1
  146. package/dist/classify-severity.d.ts +4 -1
  147. package/dist/classify-severity.d.ts.map +1 -1
  148. package/dist/classify-severity.js +30 -13
  149. package/dist/classify-severity.js.map +1 -1
  150. package/dist/cli.d.ts +14 -5
  151. package/dist/cli.d.ts.map +1 -1
  152. package/dist/cli.js +2112 -1266
  153. package/dist/cli.js.map +1 -1
  154. package/dist/cloud-publish.js +2 -2
  155. package/dist/cloud-publish.js.map +1 -1
  156. package/dist/codex-prompt.d.ts +2 -0
  157. package/dist/codex-prompt.d.ts.map +1 -1
  158. package/dist/codex-prompt.js +6 -0
  159. package/dist/codex-prompt.js.map +1 -1
  160. package/dist/component-scan.d.ts +16 -3
  161. package/dist/component-scan.d.ts.map +1 -1
  162. package/dist/component-scan.js +32 -6
  163. package/dist/component-scan.js.map +1 -1
  164. package/dist/config.d.ts +2 -1
  165. package/dist/config.d.ts.map +1 -1
  166. package/dist/config.js +221 -65
  167. package/dist/config.js.map +1 -1
  168. package/dist/continuum-findings.d.ts +7 -0
  169. package/dist/continuum-findings.d.ts.map +1 -0
  170. package/dist/continuum-findings.js +225 -0
  171. package/dist/continuum-findings.js.map +1 -0
  172. package/dist/contract-values.d.ts +30 -0
  173. package/dist/contract-values.d.ts.map +1 -0
  174. package/dist/contract-values.js +78 -0
  175. package/dist/contract-values.js.map +1 -0
  176. package/dist/contract.d.ts +2 -20
  177. package/dist/contract.d.ts.map +1 -1
  178. package/dist/contract.js +58 -669
  179. package/dist/contract.js.map +1 -1
  180. package/dist/cost-model.d.ts +13 -1
  181. package/dist/cost-model.d.ts.map +1 -1
  182. package/dist/cost-model.js +66 -115
  183. package/dist/cost-model.js.map +1 -1
  184. package/dist/cost.d.ts +61 -4
  185. package/dist/cost.d.ts.map +1 -1
  186. package/dist/cost.js +2560 -100
  187. package/dist/cost.js.map +1 -1
  188. package/dist/custom-publish.d.ts +4 -0
  189. package/dist/custom-publish.d.ts.map +1 -1
  190. package/dist/custom-publish.js +40 -3
  191. package/dist/custom-publish.js.map +1 -1
  192. package/dist/custom-validation-prompt.d.ts +1 -1
  193. package/dist/custom-validation-prompt.d.ts.map +1 -1
  194. package/dist/custom-validation-prompt.js +5 -0
  195. package/dist/custom-validation-prompt.js.map +1 -1
  196. package/dist/custom-validation.d.ts +2 -0
  197. package/dist/custom-validation.d.ts.map +1 -1
  198. package/dist/custom-validation.js +4 -0
  199. package/dist/custom-validation.js.map +1 -1
  200. package/dist/deduplication/deduplication-prompts.d.ts +4 -4
  201. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  202. package/dist/deduplication/deduplication-prompts.js +2 -2
  203. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  204. package/dist/deduplication/deduplication-reviewer.d.ts +14 -11
  205. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  206. package/dist/deduplication/deduplication-reviewer.js +11 -4
  207. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  208. package/dist/deduplication/deduplication.js.map +1 -1
  209. package/dist/deduplication/finding-schema.d.ts +19 -1
  210. package/dist/deduplication/finding-schema.d.ts.map +1 -1
  211. package/dist/deduplication/finding-schema.js +25 -0
  212. package/dist/deduplication/finding-schema.js.map +1 -1
  213. package/dist/deduplication/local.d.ts +2 -2
  214. package/dist/deduplication/local.d.ts.map +1 -1
  215. package/dist/deduplication/local.js +5 -10
  216. package/dist/deduplication/local.js.map +1 -1
  217. package/dist/deduplication/records.d.ts +8 -2
  218. package/dist/deduplication/records.d.ts.map +1 -1
  219. package/dist/deduplication/records.js +36 -8
  220. package/dist/deduplication/records.js.map +1 -1
  221. package/dist/deduplication/refusal.d.ts.map +1 -1
  222. package/dist/deduplication/refusal.js +25 -5
  223. package/dist/deduplication/refusal.js.map +1 -1
  224. package/dist/deduplication/scan.d.ts +3 -0
  225. package/dist/deduplication/scan.d.ts.map +1 -1
  226. package/dist/deduplication/scan.js +1 -1
  227. package/dist/deduplication/scan.js.map +1 -1
  228. package/dist/deep-config.d.ts +0 -1
  229. package/dist/deep-config.d.ts.map +1 -1
  230. package/dist/deep-config.js +22 -33
  231. package/dist/deep-config.js.map +1 -1
  232. package/dist/deep-scan-defaults.d.ts +2 -0
  233. package/dist/deep-scan-defaults.d.ts.map +1 -1
  234. package/dist/deep-scan-defaults.js +28 -1
  235. package/dist/deep-scan-defaults.js.map +1 -1
  236. package/dist/deep-scan.d.ts +13 -0
  237. package/dist/deep-scan.d.ts.map +1 -1
  238. package/dist/deep-scan.js +262 -7
  239. package/dist/deep-scan.js.map +1 -1
  240. package/dist/default-severity-rubric.d.ts +1 -2
  241. package/dist/default-severity-rubric.d.ts.map +1 -1
  242. package/dist/default-severity-rubric.js +2 -66
  243. package/dist/default-severity-rubric.js.map +1 -1
  244. package/dist/endor-findings.d.ts +3 -1
  245. package/dist/endor-findings.d.ts.map +1 -1
  246. package/dist/endor-findings.js +108 -80
  247. package/dist/endor-findings.js.map +1 -1
  248. package/dist/errors.d.ts +1 -1
  249. package/dist/errors.d.ts.map +1 -1
  250. package/dist/errors.js +9 -2
  251. package/dist/errors.js.map +1 -1
  252. package/dist/external-findings-import.d.ts +5 -3
  253. package/dist/external-findings-import.d.ts.map +1 -1
  254. package/dist/external-findings-import.js +11 -16
  255. package/dist/external-findings-import.js.map +1 -1
  256. package/dist/external-findings-local.d.ts +15 -4
  257. package/dist/external-findings-local.d.ts.map +1 -1
  258. package/dist/external-findings-local.js +619 -225
  259. package/dist/external-findings-local.js.map +1 -1
  260. package/dist/external-findings-publish.d.ts +20 -6
  261. package/dist/external-findings-publish.d.ts.map +1 -1
  262. package/dist/external-findings-publish.js +2356 -308
  263. package/dist/external-findings-publish.js.map +1 -1
  264. package/dist/external-findings-read.d.ts +28 -2
  265. package/dist/external-findings-read.d.ts.map +1 -1
  266. package/dist/external-findings-read.js +174 -81
  267. package/dist/external-findings-read.js.map +1 -1
  268. package/dist/external-findings.d.ts +17 -3
  269. package/dist/external-findings.d.ts.map +1 -1
  270. package/dist/external-findings.js +153 -1
  271. package/dist/external-findings.js.map +1 -1
  272. package/dist/external-import-contract.d.ts +33 -1
  273. package/dist/external-import-contract.d.ts.map +1 -1
  274. package/dist/external-import-contract.js +335 -51
  275. package/dist/external-import-contract.js.map +1 -1
  276. package/dist/external-import-models.d.ts +1 -1
  277. package/dist/external-import-models.d.ts.map +1 -1
  278. package/dist/external-import-models.js.map +1 -1
  279. package/dist/feedback.d.ts.map +1 -1
  280. package/dist/feedback.js +5 -4
  281. package/dist/feedback.js.map +1 -1
  282. package/dist/finding-prompt.d.ts +5 -0
  283. package/dist/finding-prompt.d.ts.map +1 -0
  284. package/dist/finding-prompt.js +32 -0
  285. package/dist/finding-prompt.js.map +1 -0
  286. package/dist/finding-retrieval.d.ts +3 -3
  287. package/dist/finding-retrieval.d.ts.map +1 -1
  288. package/dist/finding-retrieval.js.map +1 -1
  289. package/dist/finding-workflow.d.ts +6 -1
  290. package/dist/finding-workflow.d.ts.map +1 -1
  291. package/dist/finding-workflow.js +2 -2
  292. package/dist/finding-workflow.js.map +1 -1
  293. package/dist/findings-client.d.ts +1 -0
  294. package/dist/findings-client.d.ts.map +1 -1
  295. package/dist/findings-client.js +27 -3
  296. package/dist/findings-client.js.map +1 -1
  297. package/dist/git-remote.d.ts +4 -0
  298. package/dist/git-remote.d.ts.map +1 -1
  299. package/dist/git-remote.js +20 -0
  300. package/dist/git-remote.js.map +1 -1
  301. package/dist/github-advisories.d.ts +3 -12
  302. package/dist/github-advisories.d.ts.map +1 -1
  303. package/dist/github-advisories.js +9 -31
  304. package/dist/github-advisories.js.map +1 -1
  305. package/dist/github-dependabot-findings.d.ts.map +1 -1
  306. package/dist/github-dependabot-findings.js +22 -20
  307. package/dist/github-dependabot-findings.js.map +1 -1
  308. package/dist/github-findings.d.ts +4 -1
  309. package/dist/github-findings.d.ts.map +1 -1
  310. package/dist/github-findings.js +392 -79
  311. package/dist/github-findings.js.map +1 -1
  312. package/dist/github.d.ts +17 -7
  313. package/dist/github.d.ts.map +1 -1
  314. package/dist/github.js +43 -30
  315. package/dist/github.js.map +1 -1
  316. package/dist/host-protocol.d.ts +2 -2
  317. package/dist/host-protocol.d.ts.map +1 -1
  318. package/dist/host-protocol.js.map +1 -1
  319. package/dist/host-records-protocol.d.ts +10 -11
  320. package/dist/host-records-protocol.d.ts.map +1 -1
  321. package/dist/host-records-protocol.js +69 -9
  322. package/dist/host-records-protocol.js.map +1 -1
  323. package/dist/hosted-git-remotes.d.ts +5 -0
  324. package/dist/hosted-git-remotes.d.ts.map +1 -0
  325. package/dist/hosted-git-remotes.js +285 -0
  326. package/dist/hosted-git-remotes.js.map +1 -0
  327. package/dist/hosted-git-repository.d.ts +28 -0
  328. package/dist/hosted-git-repository.d.ts.map +1 -0
  329. package/dist/hosted-git-repository.js +222 -0
  330. package/dist/hosted-git-repository.js.map +1 -0
  331. package/dist/hosted-scan-protocol.js +2 -2
  332. package/dist/hosted-scan-protocol.js.map +1 -1
  333. package/dist/hosted-scan.d.ts +15 -0
  334. package/dist/hosted-scan.d.ts.map +1 -1
  335. package/dist/hosted-scan.js +71 -44
  336. package/dist/hosted-scan.js.map +1 -1
  337. package/dist/hosted-scanner-http.d.ts +21 -0
  338. package/dist/hosted-scanner-http.d.ts.map +1 -0
  339. package/dist/hosted-scanner-http.js +204 -0
  340. package/dist/hosted-scanner-http.js.map +1 -0
  341. package/dist/hosted-scanner-v1.d.ts +6 -0
  342. package/dist/hosted-scanner-v1.d.ts.map +1 -0
  343. package/dist/hosted-scanner-v1.js +60 -0
  344. package/dist/hosted-scanner-v1.js.map +1 -0
  345. package/dist/import-scan.d.ts.map +1 -1
  346. package/dist/import-scan.js +3 -0
  347. package/dist/import-scan.js.map +1 -1
  348. package/dist/index.d.ts +24 -6
  349. package/dist/index.d.ts.map +1 -1
  350. package/dist/index.js +15 -1
  351. package/dist/index.js.map +1 -1
  352. package/dist/inspector-findings.d.ts.map +1 -1
  353. package/dist/inspector-findings.js +58 -81
  354. package/dist/inspector-findings.js.map +1 -1
  355. package/dist/knowledge-base.d.ts.map +1 -1
  356. package/dist/knowledge-base.js +86 -10
  357. package/dist/knowledge-base.js.map +1 -1
  358. package/dist/local-git-metadata.d.ts +5 -0
  359. package/dist/local-git-metadata.d.ts.map +1 -0
  360. package/dist/local-git-metadata.js +130 -0
  361. package/dist/local-git-metadata.js.map +1 -0
  362. package/dist/multiscan.d.ts +6 -4
  363. package/dist/multiscan.d.ts.map +1 -1
  364. package/dist/multiscan.js +160 -77
  365. package/dist/multiscan.js.map +1 -1
  366. package/dist/native-json-findings.d.ts +2 -1
  367. package/dist/native-json-findings.d.ts.map +1 -1
  368. package/dist/native-json-findings.js +12 -2
  369. package/dist/native-json-findings.js.map +1 -1
  370. package/dist/opengrep-findings.d.ts +1 -0
  371. package/dist/opengrep-findings.d.ts.map +1 -1
  372. package/dist/opengrep-findings.js +49 -31
  373. package/dist/opengrep-findings.js.map +1 -1
  374. package/dist/operation-telemetry.d.ts +104 -0
  375. package/dist/operation-telemetry.d.ts.map +1 -0
  376. package/dist/operation-telemetry.js +293 -0
  377. package/dist/operation-telemetry.js.map +1 -0
  378. package/dist/patch-tui.d.ts.map +1 -1
  379. package/dist/patch-tui.js +90 -35
  380. package/dist/patch-tui.js.map +1 -1
  381. package/dist/plugin-python-probe.d.ts +4 -0
  382. package/dist/plugin-python-probe.d.ts.map +1 -0
  383. package/dist/plugin-python-probe.js +10 -0
  384. package/dist/plugin-python-probe.js.map +1 -0
  385. package/dist/private-finding-import.d.ts +341 -0
  386. package/dist/private-finding-import.d.ts.map +1 -0
  387. package/dist/private-finding-import.js +411 -0
  388. package/dist/private-finding-import.js.map +1 -0
  389. package/dist/project-config.d.ts.map +1 -1
  390. package/dist/project-config.js +2 -2
  391. package/dist/project-config.js.map +1 -1
  392. package/dist/provider-profile.d.ts +11 -3
  393. package/dist/provider-profile.d.ts.map +1 -1
  394. package/dist/provider-profile.js +80 -11
  395. package/dist/provider-profile.js.map +1 -1
  396. package/dist/publication-telemetry.d.ts +45 -0
  397. package/dist/publication-telemetry.d.ts.map +1 -0
  398. package/dist/publication-telemetry.js +50 -0
  399. package/dist/publication-telemetry.js.map +1 -0
  400. package/dist/publication.js +7 -1
  401. package/dist/publication.js.map +1 -1
  402. package/dist/publish.d.ts +4 -0
  403. package/dist/publish.d.ts.map +1 -1
  404. package/dist/publish.js +42 -2
  405. package/dist/publish.js.map +1 -1
  406. package/dist/reader-selection.d.ts +15 -0
  407. package/dist/reader-selection.d.ts.map +1 -0
  408. package/dist/reader-selection.js +89 -0
  409. package/dist/reader-selection.js.map +1 -0
  410. package/dist/record.d.ts +6 -1
  411. package/dist/record.d.ts.map +1 -1
  412. package/dist/record.js +53 -2
  413. package/dist/record.js.map +1 -1
  414. package/dist/remediation.d.ts +3 -1
  415. package/dist/remediation.d.ts.map +1 -1
  416. package/dist/remediation.js +28 -2
  417. package/dist/remediation.js.map +1 -1
  418. package/dist/request-metadata.d.ts +3 -0
  419. package/dist/request-metadata.d.ts.map +1 -1
  420. package/dist/request-metadata.js +9 -0
  421. package/dist/request-metadata.js.map +1 -1
  422. package/dist/result.d.ts +6 -0
  423. package/dist/result.d.ts.map +1 -1
  424. package/dist/result.js +3 -1
  425. package/dist/result.js.map +1 -1
  426. package/dist/retry-windows-rename.d.ts +8 -0
  427. package/dist/retry-windows-rename.d.ts.map +1 -0
  428. package/dist/retry-windows-rename.js +24 -0
  429. package/dist/retry-windows-rename.js.map +1 -0
  430. package/dist/runtime.d.ts +22 -1
  431. package/dist/runtime.d.ts.map +1 -1
  432. package/dist/runtime.js +408 -162
  433. package/dist/runtime.js.map +1 -1
  434. package/dist/saved-scan-bootstrap.d.ts +1 -0
  435. package/dist/saved-scan-bootstrap.d.ts.map +1 -1
  436. package/dist/saved-scan-bootstrap.js +18 -15
  437. package/dist/saved-scan-bootstrap.js.map +1 -1
  438. package/dist/scan-activity.d.ts.map +1 -1
  439. package/dist/scan-activity.js +172 -64
  440. package/dist/scan-activity.js.map +1 -1
  441. package/dist/scan-comparison.d.ts +10 -5
  442. package/dist/scan-comparison.d.ts.map +1 -1
  443. package/dist/scan-comparison.js +68 -25
  444. package/dist/scan-comparison.js.map +1 -1
  445. package/dist/scan-dashboard.d.ts +7 -6
  446. package/dist/scan-dashboard.d.ts.map +1 -1
  447. package/dist/scan-dashboard.js +91 -79
  448. package/dist/scan-dashboard.js.map +1 -1
  449. package/dist/scan-executor.d.ts +12 -1
  450. package/dist/scan-executor.d.ts.map +1 -1
  451. package/dist/scan-executor.js +22 -2
  452. package/dist/scan-executor.js.map +1 -1
  453. package/dist/scan-history-renderer.d.ts.map +1 -1
  454. package/dist/scan-history-renderer.js +31 -28
  455. package/dist/scan-history-renderer.js.map +1 -1
  456. package/dist/scan-logs.d.ts +12 -0
  457. package/dist/scan-logs.d.ts.map +1 -1
  458. package/dist/scan-logs.js +106 -77
  459. package/dist/scan-logs.js.map +1 -1
  460. package/dist/scan-measurement.d.ts +26 -0
  461. package/dist/scan-measurement.d.ts.map +1 -0
  462. package/dist/scan-measurement.js +77 -0
  463. package/dist/scan-measurement.js.map +1 -0
  464. package/dist/scan-progress.d.ts +1 -0
  465. package/dist/scan-progress.d.ts.map +1 -1
  466. package/dist/scan-progress.js +3 -0
  467. package/dist/scan-progress.js.map +1 -1
  468. package/dist/scan-sessions.d.ts.map +1 -1
  469. package/dist/scan-sessions.js +4 -1
  470. package/dist/scan-sessions.js.map +1 -1
  471. package/dist/scan-settings.d.ts +1 -1
  472. package/dist/scan-settings.d.ts.map +1 -1
  473. package/dist/scan-settings.js +1 -24
  474. package/dist/scan-settings.js.map +1 -1
  475. package/dist/scanner-import.d.ts +64 -6
  476. package/dist/scanner-import.d.ts.map +1 -1
  477. package/dist/scanner-import.js +406 -48
  478. package/dist/scanner-import.js.map +1 -1
  479. package/dist/scanner-repository-url.d.ts +17 -0
  480. package/dist/scanner-repository-url.d.ts.map +1 -0
  481. package/dist/scanner-repository-url.js +71 -0
  482. package/dist/scanner-repository-url.js.map +1 -0
  483. package/dist/security-policy.d.ts +1 -1
  484. package/dist/security-policy.d.ts.map +1 -1
  485. package/dist/security-policy.js.map +1 -1
  486. package/dist/semgrep-findings.d.ts +6 -3
  487. package/dist/semgrep-findings.d.ts.map +1 -1
  488. package/dist/semgrep-findings.js +124 -88
  489. package/dist/semgrep-findings.js.map +1 -1
  490. package/dist/semgrep-guardian-findings.d.ts +38 -0
  491. package/dist/semgrep-guardian-findings.d.ts.map +1 -0
  492. package/dist/semgrep-guardian-findings.js +165 -0
  493. package/dist/semgrep-guardian-findings.js.map +1 -0
  494. package/dist/semgrep-platform-collector.d.ts +26 -0
  495. package/dist/semgrep-platform-collector.d.ts.map +1 -0
  496. package/dist/semgrep-platform-collector.js +108 -0
  497. package/dist/semgrep-platform-collector.js.map +1 -0
  498. package/dist/semgrep-platform-findings.d.ts +213 -0
  499. package/dist/semgrep-platform-findings.d.ts.map +1 -0
  500. package/dist/semgrep-platform-findings.js +1078 -0
  501. package/dist/semgrep-platform-findings.js.map +1 -0
  502. package/dist/semgrep-platform-v1.d.ts +6 -0
  503. package/dist/semgrep-platform-v1.d.ts.map +1 -0
  504. package/dist/semgrep-platform-v1.js +83 -0
  505. package/dist/semgrep-platform-v1.js.map +1 -0
  506. package/dist/server/api.d.ts +1 -1
  507. package/dist/server/api.d.ts.map +1 -1
  508. package/dist/server/api.js.map +1 -1
  509. package/dist/server/embedding-tokenizer.cdx.json +33 -0
  510. package/dist/server/embedding-tokenizer.d.ts +2 -0
  511. package/dist/server/embedding-tokenizer.d.ts.map +1 -0
  512. package/dist/server/embedding-tokenizer.js +322 -0
  513. package/dist/server/embedding-tokenizer.js.map +6 -0
  514. package/dist/server/embeddings.d.ts +16 -2
  515. package/dist/server/embeddings.d.ts.map +1 -1
  516. package/dist/server/embeddings.js +35 -5
  517. package/dist/server/embeddings.js.map +1 -1
  518. package/dist/session-lines.d.ts +5 -0
  519. package/dist/session-lines.d.ts.map +1 -0
  520. package/dist/session-lines.js +60 -0
  521. package/dist/session-lines.js.map +1 -0
  522. package/dist/severity-review.d.ts +3 -1
  523. package/dist/severity-review.d.ts.map +1 -1
  524. package/dist/severity-review.js +3 -2
  525. package/dist/severity-review.js.map +1 -1
  526. package/dist/severity-rubric.d.ts +20 -0
  527. package/dist/severity-rubric.d.ts.map +1 -0
  528. package/dist/severity-rubric.js +73 -0
  529. package/dist/severity-rubric.js.map +1 -0
  530. package/dist/severity-store.js +15 -5
  531. package/dist/severity-store.js.map +1 -1
  532. package/dist/skill-authentication.d.ts +1 -0
  533. package/dist/skill-authentication.d.ts.map +1 -1
  534. package/dist/skill-authentication.js +1 -0
  535. package/dist/skill-authentication.js.map +1 -1
  536. package/dist/snyk-findings.d.ts +7 -2
  537. package/dist/snyk-findings.d.ts.map +1 -1
  538. package/dist/snyk-findings.js +108 -84
  539. package/dist/snyk-findings.js.map +1 -1
  540. package/dist/snyk-rest-findings.d.ts +76 -0
  541. package/dist/snyk-rest-findings.d.ts.map +1 -0
  542. package/dist/snyk-rest-findings.js +1051 -0
  543. package/dist/snyk-rest-findings.js.map +1 -0
  544. package/dist/snyk-rest-v1.d.ts +28 -0
  545. package/dist/snyk-rest-v1.d.ts.map +1 -0
  546. package/dist/snyk-rest-v1.js +132 -0
  547. package/dist/snyk-rest-v1.js.map +1 -0
  548. package/dist/snyk-token-origin.d.ts +4 -0
  549. package/dist/snyk-token-origin.d.ts.map +1 -0
  550. package/dist/snyk-token-origin.js +82 -0
  551. package/dist/snyk-token-origin.js.map +1 -0
  552. package/dist/sonar-collector.d.ts +16 -0
  553. package/dist/sonar-collector.d.ts.map +1 -0
  554. package/dist/sonar-collector.js +380 -0
  555. package/dist/sonar-collector.js.map +1 -0
  556. package/dist/sonar-findings.d.ts +49 -0
  557. package/dist/sonar-findings.d.ts.map +1 -0
  558. package/dist/sonar-findings.js +288 -0
  559. package/dist/sonar-findings.js.map +1 -0
  560. package/dist/sonar-import.d.ts +111 -0
  561. package/dist/sonar-import.d.ts.map +1 -0
  562. package/dist/sonar-import.js +277 -0
  563. package/dist/sonar-import.js.map +1 -0
  564. package/dist/suggest-owners.d.ts.map +1 -1
  565. package/dist/suggest-owners.js +2 -1
  566. package/dist/suggest-owners.js.map +1 -1
  567. package/dist/target-drift.d.ts +9 -0
  568. package/dist/target-drift.d.ts.map +1 -0
  569. package/dist/target-drift.js +27 -0
  570. package/dist/target-drift.js.map +1 -0
  571. package/dist/targets.d.ts +5 -0
  572. package/dist/targets.d.ts.map +1 -1
  573. package/dist/targets.js +73 -31
  574. package/dist/targets.js.map +1 -1
  575. package/dist/tenable-findings.d.ts +9 -0
  576. package/dist/tenable-findings.d.ts.map +1 -0
  577. package/dist/tenable-findings.js +122 -0
  578. package/dist/tenable-findings.js.map +1 -0
  579. package/dist/terminal-columns.d.ts +4 -0
  580. package/dist/terminal-columns.d.ts.map +1 -0
  581. package/dist/terminal-columns.js +19 -0
  582. package/dist/terminal-columns.js.map +1 -0
  583. package/dist/terminal-stream.d.ts +13 -0
  584. package/dist/terminal-stream.d.ts.map +1 -0
  585. package/dist/terminal-stream.js +35 -0
  586. package/dist/terminal-stream.js.map +1 -0
  587. package/dist/trivy-findings.d.ts +19 -0
  588. package/dist/trivy-findings.d.ts.map +1 -0
  589. package/dist/trivy-findings.js +435 -0
  590. package/dist/trivy-findings.js.map +1 -0
  591. package/dist/trivy-import.d.ts +19 -0
  592. package/dist/trivy-import.d.ts.map +1 -0
  593. package/dist/trivy-import.js +151 -0
  594. package/dist/trivy-import.js.map +1 -0
  595. package/dist/validation-records.d.ts +0 -3
  596. package/dist/validation-records.d.ts.map +1 -1
  597. package/dist/validation-records.js +1 -17
  598. package/dist/validation-records.js.map +1 -1
  599. package/dist/veracode-findings.d.ts +9 -0
  600. package/dist/veracode-findings.d.ts.map +1 -0
  601. package/dist/veracode-findings.js +409 -0
  602. package/dist/veracode-findings.js.map +1 -0
  603. package/dist/vertex-pricing.d.ts +8 -0
  604. package/dist/vertex-pricing.d.ts.map +1 -0
  605. package/dist/vertex-pricing.js +20 -0
  606. package/dist/vertex-pricing.js.map +1 -0
  607. package/dist/windows-path.d.ts +1 -0
  608. package/dist/windows-path.d.ts.map +1 -1
  609. package/dist/windows-path.js +27 -1
  610. package/dist/windows-path.js.map +1 -1
  611. package/dist/wiz-csv.d.ts +2 -2
  612. package/dist/wiz-csv.d.ts.map +1 -1
  613. package/dist/wiz-csv.js +1 -3
  614. package/dist/wiz-csv.js.map +1 -1
  615. package/dist/wiz-findings.d.ts +14 -2
  616. package/dist/wiz-findings.d.ts.map +1 -1
  617. package/dist/wiz-findings.js +143 -50
  618. package/dist/wiz-findings.js.map +1 -1
  619. package/docs/classify-severity-records.md +28 -13
  620. package/docs/cli.md +623 -843
  621. package/docs/dedupe-records.md +54 -24
  622. package/docs/findings-service.md +87 -47
  623. package/docs/hosted-remediation.md +14 -7
  624. package/docs/hosted-scans.md +64 -41
  625. package/docs/telemetry.md +236 -0
  626. package/docs/validate-records.md +78 -44
  627. package/package.json +19 -17
  628. package/schemas/external-findings.schema.json +14 -1
  629. package/schemas/hosted-scan-v2.fixture.json +16 -1
  630. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  631. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  632. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  633. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  634. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  635. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
  636. package/_bundled_plugin/scripts/validate_tracking_source.py +0 -82
  637. package/_bundled_plugin/scripts/workbench_local_dedupe.py +0 -136
  638. package/dist/deduplication/records-protocol.d.ts +0 -4
  639. package/dist/deduplication/records-protocol.d.ts.map +0 -1
  640. package/dist/deduplication/records-protocol.js +0 -49
  641. package/dist/deduplication/records-protocol.js.map +0 -1
  642. package/dist/external-imports-feature.d.ts +0 -6
  643. package/dist/external-imports-feature.d.ts.map +0 -1
  644. package/dist/external-imports-feature.js +0 -51
  645. package/dist/external-imports-feature.js.map +0 -1
  646. package/dist/finding-normalization.d.ts +0 -5
  647. package/dist/finding-normalization.d.ts.map +0 -1
  648. package/dist/finding-normalization.js +0 -120
  649. package/dist/finding-normalization.js.map +0 -1
@@ -8,14 +8,20 @@ import math
8
8
  import os
9
9
  import re
10
10
  import sqlite3
11
+ import subprocess
11
12
  import sys
13
+ import tempfile
12
14
  import uuid
15
+ from contextlib import contextmanager
13
16
  from dataclasses import dataclass
14
17
  from datetime import datetime, timezone
15
18
  from pathlib import Path
16
19
  from typing import Any, Mapping
17
20
  from urllib.parse import quote
18
21
 
22
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
23
+ from workbench.storage import worker_usage_directory
24
+
19
25
  TOKEN_FIELDS = {
20
26
  "input_tokens": "inputTokens",
21
27
  "cached_input_tokens": "cachedInputTokens",
@@ -30,6 +36,8 @@ STATE_DATABASE_TIMEOUT_SECONDS = 1.0
30
36
 
31
37
  @dataclass(frozen=True)
32
38
  class RolloutSession:
39
+ """Associate a thread and its optional parent with the path of the rollout used for usage accounting."""
40
+
33
41
  thread_id: str
34
42
  parent_thread_id: str | None
35
43
  path: Path
@@ -37,6 +45,8 @@ class RolloutSession:
37
45
 
38
46
  @dataclass
39
47
  class RolloutUsage:
48
+ """Collect token counts, their update time, warnings, and account rate-limit observations for a rollout."""
49
+
40
50
  tokens: dict[str, int]
41
51
  warnings: set[str]
42
52
  tokens_updated_at: datetime | None
@@ -66,13 +76,27 @@ def reconcile_completed_scan_cost(
66
76
  ) -> None:
67
77
  """Persist authoritative SDK cost without discarding measured worker usage."""
68
78
 
69
- existing = json.loads(scan["cost_json"]) if scan["cost_json"] is not None else {}
70
- if isinstance(existing, dict) and "usage" in existing:
71
- cost_json = json.dumps(
72
- {**existing, "cost": json.loads(cost_json)},
73
- separators=(",", ":"),
74
- allow_nan=False,
75
- )
79
+ stored = stored_scan_cost_fields(scan["cost_json"])
80
+ completion = stored.get("usage", {}).get("completion")
81
+ native = collect_scan_usage_safely(
82
+ connection,
83
+ scan,
84
+ thread_id=completion["threadId"] if completion else None,
85
+ completed_turn_id=completion["turnId"] if completion and completion["finalized"] else None,
86
+ )
87
+ fields = merge_scan_usage_fields(stored, stored_scan_cost_fields(cost_json), {"usage": native})
88
+ if completion:
89
+ usage = {**fields["usage"], "completion": completion}
90
+ if not completion["finalized"]:
91
+ if usage["coverage"] == "complete":
92
+ usage["coverage"] = "partial"
93
+ usage["warnings"] = sorted({*usage.get("warnings", []), "turn_in_progress"})
94
+ fields["usage"] = usage
95
+ cost_json = json.dumps(
96
+ fields if "usage" in fields else fields["cost"],
97
+ separators=(",", ":"),
98
+ allow_nan=False,
99
+ )
76
100
  connection.execute("BEGIN IMMEDIATE")
77
101
  with connection:
78
102
  connection.execute(
@@ -81,53 +105,219 @@ def reconcile_completed_scan_cost(
81
105
  )
82
106
 
83
107
 
108
+ def merge_scan_usage_fields(*observations: Mapping[str, Any]) -> dict[str, Any]:
109
+ """Reconcile overlapping measurements without summing the same execution twice."""
110
+
111
+ merged: dict[str, Any] = {}
112
+ latest_coverage: dict[str, Any] | None = None
113
+ rate_limits: dict[str, dict[str, Any]] = {}
114
+ for observation in observations:
115
+ if isinstance(observation.get("cost"), dict):
116
+ merged["cost"] = observation["cost"]
117
+ usage = observation.get("usage")
118
+ if not isinstance(usage, dict):
119
+ continue
120
+ for snapshot in usage.get("accountRateLimits", []):
121
+ key = snapshot["limitId"]
122
+ if key not in rate_limits or snapshot["observedAt"] >= rate_limits[key]["observedAt"]:
123
+ rate_limits[key] = snapshot
124
+ previous = merged.get("usage")
125
+ if previous is None or _usage_quality(usage) >= _usage_quality(previous):
126
+ merged["usage"] = usage
127
+ if usage.get("coverage") not in ("complete", "partial"):
128
+ continue
129
+ latest_coverage = (
130
+ usage if latest_coverage is None else _merge_usage_coverage(latest_coverage, usage)
131
+ )
132
+ cost = merged.get("cost")
133
+ usage = merged.get("usage")
134
+ if usage is not None and latest_coverage is not None and latest_coverage is not usage:
135
+ usage = {**usage}
136
+ for field in (
137
+ "coverage",
138
+ "coverageScope",
139
+ "threadCount",
140
+ "missingThreadCount",
141
+ "warnings",
142
+ "observedAt",
143
+ "completion",
144
+ "byRole",
145
+ ):
146
+ if field in latest_coverage:
147
+ usage[field] = latest_coverage[field]
148
+ else:
149
+ usage.pop(field, None)
150
+ # Counts describe the coverage observation, even when its scope narrows.
151
+ # Its whole role breakdown must still sum to the retained token subtotal.
152
+ if "byRole" in usage and any(
153
+ usage.get(field) != latest_coverage.get(field) for field in TOKEN_FIELDS.values()
154
+ ):
155
+ usage.pop("byRole")
156
+ merged["usage"] = usage
157
+ cost_warning = "cost_estimate_covers_earlier_usage"
158
+ if (
159
+ cost is not None
160
+ and usage is not None
161
+ and usage.get("coverage") in ("complete", "partial")
162
+ and type(usage.get("totalTokens")) is int
163
+ ):
164
+ warnings = set(usage.get("warnings", []))
165
+ warnings.discard(cost_warning)
166
+ if usage["totalTokens"] > cost["inputTokens"] + cost["outputTokens"]:
167
+ warnings.add(cost_warning)
168
+ if warnings != set(usage.get("warnings", [])):
169
+ updated = {**usage}
170
+ updated["coverage"] = (
171
+ "partial"
172
+ if warnings - {cost_warning} or usage.get("missingThreadCount", 0)
173
+ else "complete"
174
+ )
175
+ if warnings:
176
+ updated["warnings"] = sorted(warnings)
177
+ else:
178
+ updated.pop("warnings", None)
179
+ merged["usage"] = updated
180
+ if rate_limits:
181
+ merged["usage"] = {
182
+ **merged["usage"],
183
+ "accountRateLimits": list(rate_limits.values()),
184
+ }
185
+ return merged
186
+
187
+
188
+ def _merge_usage_coverage(previous: dict[str, Any], observed: dict[str, Any]) -> dict[str, Any]:
189
+ if previous == observed:
190
+ return observed
191
+ previous_at = _timestamp(previous.get("observedAt"))
192
+ observed_at = _timestamp(observed.get("observedAt"))
193
+ if previous_at is not None and observed_at is not None and previous_at > observed_at:
194
+ previous, observed = observed, previous
195
+ previous_scope = previous.get("coverageScope")
196
+ observed_scope = observed.get("coverageScope")
197
+ covers_previous = (
198
+ previous_scope is not None
199
+ and observed_scope is not None
200
+ and set(previous_scope).issubset(observed_scope)
201
+ )
202
+ scope_gap = previous_scope is not None and observed_scope is not None and not covers_previous
203
+ unresolved_previous = (previous["coverage"] == "partial" and not covers_previous) or scope_gap
204
+ merged = {**observed}
205
+ if unresolved_previous:
206
+ merged["coverage"] = "partial"
207
+ merged["missingThreadCount"] = max(
208
+ previous.get("missingThreadCount", 0), observed.get("missingThreadCount", 0)
209
+ )
210
+ warnings = set(previous.get("warnings", [])) | set(observed.get("warnings", []))
211
+ if scope_gap:
212
+ # A newer subset can grow while still staying below the older
213
+ # aggregate. Neither subtotal proves complete usage for the union.
214
+ warnings.add("usage_scope_incomplete")
215
+ if warnings:
216
+ merged["warnings"] = sorted(warnings)
217
+ if previous.get("completion", {}).get("finalized") is False:
218
+ merged["completion"] = previous["completion"]
219
+ if previous_scope is not None and observed_scope is not None:
220
+ merged["coverageScope"] = sorted(set(previous_scope) | set(observed_scope))
221
+ elif previous_scope is not None and observed["coverage"] == "complete":
222
+ merged["coverageScope"] = previous_scope
223
+ elif unresolved_previous:
224
+ # An older subtotal without scope cannot prove which gaps a later
225
+ # collector has resolved. Retain that uncertainty across future merges.
226
+ merged.pop("coverageScope", None)
227
+ return merged
228
+
229
+
230
+ def _usage_quality(
231
+ usage: Mapping[str, Any], *, observed_at: datetime | None = None
232
+ ) -> tuple[int, bool, datetime]:
233
+ return (
234
+ usage.get("totalTokens", -1),
235
+ usage.get("cacheWriteInputTokensReported") is not False,
236
+ observed_at
237
+ or _timestamp(usage.get("observedAt"))
238
+ or datetime.min.replace(tzinfo=timezone.utc),
239
+ )
240
+
241
+
242
+ def collect_scan_usage_safely(
243
+ connection: sqlite3.Connection, scan: sqlite3.Row, **options: Any
244
+ ) -> dict[str, Any]:
245
+ try:
246
+ return collect_scan_usage(connection, scan, **options)
247
+ except (Exception, SystemExit) as exc:
248
+ try:
249
+ print(f"Could not collect scan token usage: {exc}", file=sys.stderr)
250
+ except OSError:
251
+ # A closed diagnostic stream must not replace the unavailable-usage result.
252
+ pass
253
+ return _unavailable_usage("usage_collection_failed")
254
+
255
+
84
256
  def collect_scan_usage(
85
257
  connection: sqlite3.Connection,
86
258
  scan: sqlite3.Row,
87
259
  *,
88
260
  thread_id: str | None = None,
261
+ validation_thread_id: str | None = None,
89
262
  completed_at: str | None = None,
90
263
  completed_turn_id: str | None = None,
91
264
  ) -> dict[str, Any]:
92
- """Count attributable receipts and flag observable gaps inside the scan window."""
265
+ """Count attributable session receipts during the scan window and report missing coverage.
266
+
267
+ The optional thread and completion arguments identify the finishing turn when
268
+ the scan record alone is insufficient. Return an unavailable usage record when
269
+ the window, Codex state, or relevant threads cannot be established.
270
+ """
93
271
 
94
272
  roots = _scan_root_thread_ids(connection, scan, thread_id)
95
273
  parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
96
- worker_directories = _scan_worker_directories(connection, scan)
97
- if not roots and not worker_directories:
98
- return _unavailable_usage("scan_thread_unavailable")
99
-
100
- state_database = _codex_state_database()
101
- if state_database is None:
102
- return _unavailable_usage("codex_state_unavailable")
103
-
274
+ if validation_thread_id:
275
+ roots.append(validation_thread_id)
276
+ parent_ids.add(validation_thread_id)
277
+ warnings: set[str] = set()
278
+ worker_directories = _scan_worker_directories(connection, scan, warnings)
104
279
  started_at = _timestamp(scan["started_at"])
105
280
  observed_at = datetime.now(timezone.utc)
106
281
  stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
107
282
  if started_at is None:
108
283
  return _unavailable_usage("scan_window_unavailable")
109
284
 
110
- warnings: set[str] = set()
111
- try:
112
- sessions, missing_thread_ids = _discover_rollout_sessions(
113
- state_database,
114
- roots,
115
- warnings,
116
- worker_directories=worker_directories,
117
- started_at=started_at,
118
- completed_at=stopped_at,
119
- )
120
- except (OSError, sqlite3.Error, ValueError):
121
- return _unavailable_usage("codex_state_unavailable")
285
+ receipts, missing_receipts, receipt_scope = _worker_usage_receipts(
286
+ scan, started_at, stopped_at, warnings
287
+ )
288
+ state_database = _codex_state_database()
289
+ sessions: list[RolloutSession] = []
290
+ missing_thread_ids: set[str] = set()
291
+ if state_database is None:
292
+ if not receipts:
293
+ return _unavailable_usage("codex_state_unavailable", warnings=warnings)
294
+ warnings.add("codex_state_unavailable")
295
+ missing_thread_ids.update(roots)
296
+ else:
297
+ try:
298
+ sessions, missing_thread_ids = _discover_rollout_sessions(
299
+ state_database,
300
+ roots,
301
+ warnings,
302
+ worker_directories=worker_directories,
303
+ scan_directory=Path(scan["scan_dir"]) if worker_directories else None,
304
+ started_at=started_at,
305
+ completed_at=stopped_at,
306
+ )
307
+ except (OSError, sqlite3.Error, ValueError):
308
+ if not receipts:
309
+ return _unavailable_usage("codex_state_unavailable", warnings=warnings)
310
+ warnings.add("codex_state_unavailable")
311
+ missing_thread_ids.update(roots)
122
312
 
123
- if not sessions:
313
+ if not sessions and not receipts:
124
314
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
125
315
 
126
- by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
127
316
  rate_limits: dict[str, dict[str, Any]] = {}
128
317
  tokens_updated_at = None
129
318
  accepted_thread_ids: set[str] = set()
130
- measured_thread_ids: set[str] = set()
319
+ measured_threads: dict[str, dict[str, int]] = {}
320
+ complete_native_thread_ids: set[str] = set()
131
321
  excluded_thread_ids: set[str] = set()
132
322
  for session in sessions:
133
323
  if session.parent_thread_id in excluded_thread_ids:
@@ -168,25 +358,60 @@ def collect_scan_usage(
168
358
  for key, snapshot in usage.rate_limits.items():
169
359
  if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
170
360
  rate_limits[key] = snapshot
171
- if "token_usage_unavailable" in usage.warnings:
361
+ if usage.warnings & {"token_usage_unavailable", "token_breakdown_unavailable"}:
172
362
  missing_thread_ids.add(session.thread_id)
173
363
  continue
174
- measured_thread_ids.add(session.thread_id)
175
- role = by_role["parent" if session.thread_id in parent_ids else "workers"]
176
- _add_token_usage(role, usage.tokens)
177
- role["threadCount"] += 1
364
+ measured_threads[session.thread_id] = usage.tokens
365
+ if not usage.warnings:
366
+ complete_native_thread_ids.add(session.thread_id)
178
367
  if usage.tokens_updated_at is not None:
179
368
  tokens_updated_at = max(
180
369
  tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
181
370
  )
182
371
 
372
+ runtime_receipts_used = False
373
+ for receipt_thread_id, receipt in receipts.items():
374
+ native = measured_threads.get(receipt_thread_id)
375
+ extends_native = native is None or receipt.tokens["totalTokens"] > native["totalTokens"]
376
+ fills_cache_write_usage = (
377
+ native is not None
378
+ and receipt.tokens["totalTokens"] == native["totalTokens"]
379
+ and native.get("cacheWriteInputTokensReported") is False
380
+ and receipt.tokens.get("cacheWriteInputTokensReported") is not False
381
+ )
382
+ if extends_native or fills_cache_write_usage:
383
+ measured_threads[receipt_thread_id] = receipt.tokens
384
+ accepted_thread_ids.add(receipt_thread_id)
385
+ runtime_receipts_used = True
386
+ if extends_native:
387
+ complete_native_thread_ids.discard(receipt_thread_id)
388
+ # Numeric SDK subtotals cannot establish that descendants and
389
+ # compaction requests are present when their rollouts are absent.
390
+ missing_thread_ids.add(receipt_thread_id)
391
+ warnings.add("worker_rollout_usage_incomplete")
392
+ if receipt.tokens_updated_at is not None:
393
+ tokens_updated_at = max(
394
+ tokens_updated_at or receipt.tokens_updated_at, receipt.tokens_updated_at
395
+ )
396
+ unresolved_receipts = missing_receipts - complete_native_thread_ids
397
+ if unresolved_receipts:
398
+ missing_thread_ids.update(unresolved_receipts)
399
+ warnings.add("worker_usage_unavailable")
183
400
  if not accepted_thread_ids:
184
401
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
185
- if not measured_thread_ids:
186
- return _unavailable_usage(
187
- "token_usage_unavailable", warnings=warnings, rate_limits=rate_limits
402
+ if not measured_threads:
403
+ reason = (
404
+ "token_breakdown_unavailable"
405
+ if "token_breakdown_unavailable" in warnings
406
+ else "token_usage_unavailable"
188
407
  )
408
+ return _unavailable_usage(reason, warnings=warnings, rate_limits=rate_limits)
189
409
 
410
+ by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
411
+ for measured_id, tokens in measured_threads.items():
412
+ role = by_role["parent" if measured_id in parent_ids else "workers"]
413
+ _add_token_usage(role, tokens)
414
+ role["threadCount"] += 1
190
415
  if any(
191
416
  usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
192
417
  or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
@@ -196,18 +421,28 @@ def collect_scan_usage(
196
421
  "token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
197
422
  )
198
423
 
199
- if completed_turn_id is not None and tokens_updated_at is not None:
424
+ if (completed_turn_id is not None or runtime_receipts_used) and tokens_updated_at is not None:
200
425
  stopped_at = max(stopped_at, tokens_updated_at)
201
426
  result: dict[str, Any] = {
202
427
  "coverage": "partial" if missing_thread_ids or warnings else "complete",
203
- "source": "codex_rollout",
428
+ "source": "codex_runtime" if runtime_receipts_used else "codex_rollout",
204
429
  **{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
205
- "threadCount": len(measured_thread_ids),
430
+ "threadCount": len(measured_threads),
431
+ "coverageScope": sorted(
432
+ receipt_scope
433
+ | {f"thread:{identity}" for identity in measured_threads}
434
+ | {
435
+ identity if identity.startswith("worker-attempt:") else f"thread:{identity}"
436
+ for identity in missing_thread_ids
437
+ }
438
+ ),
206
439
  "byRole": by_role,
207
440
  "window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
208
441
  "observedAt": observed_at.isoformat(),
209
442
  "accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
210
443
  }
444
+ if any(role.get("cacheWriteInputTokensReported") is False for role in by_role.values()):
445
+ result["cacheWriteInputTokensReported"] = False
211
446
  if tokens_updated_at is not None:
212
447
  result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
213
448
  if missing_thread_ids:
@@ -217,6 +452,79 @@ def collect_scan_usage(
217
452
  return result
218
453
 
219
454
 
455
+ def _worker_usage_receipts(
456
+ scan: Mapping[str, Any],
457
+ started_at: datetime,
458
+ completed_at: datetime,
459
+ warnings: set[str],
460
+ ) -> tuple[dict[str, RolloutUsage], set[str], set[str]]:
461
+ if scan["mode"] != "deep":
462
+ return {}, set(), set()
463
+ directory = worker_usage_directory(scan["id"])
464
+ try:
465
+ if directory.resolve(strict=True) != directory:
466
+ raise ValueError("Worker usage directory is not a canonical protected directory.")
467
+ except FileNotFoundError:
468
+ return {}, set(), set()
469
+ except (OSError, RuntimeError, ValueError):
470
+ warnings.add("worker_usage_receipt_unavailable")
471
+ return {}, {"worker-receipts"}, {"worker-receipts"}
472
+ receipts: dict[str, RolloutUsage] = {}
473
+ missing: set[str] = set()
474
+ attempts: set[str] = set()
475
+ scope: set[str] = set()
476
+ for path in sorted(directory.glob("*.json")):
477
+ try:
478
+ if _rollout_path(str(path)) is None:
479
+ raise ValueError("Worker usage receipt path is unavailable.")
480
+ receipt = json.loads(path.read_text(encoding="utf-8"))
481
+ if (
482
+ not isinstance(receipt, dict)
483
+ or receipt.get("schemaVersion") != 1
484
+ or receipt.get("source") != "codex_sdk"
485
+ or receipt.get("usageScope") != "thread_cumulative"
486
+ or receipt.get("scanId") != scan["id"]
487
+ or receipt.get("kind") not in ("discovery", "dedup", "compare")
488
+ or receipt.get("attemptId") != path.stem
489
+ ):
490
+ raise ValueError("Worker usage receipt identity is invalid.")
491
+ attempt_id = receipt["attemptId"]
492
+ begin = _timestamp(receipt.get("startedAt"))
493
+ observed = _timestamp(receipt.get("observedAt"))
494
+ if begin is None or observed is None:
495
+ raise ValueError("Worker usage receipt timestamps are invalid.")
496
+ if begin < started_at or begin > completed_at or attempt_id in attempts:
497
+ continue
498
+ attempts.add(attempt_id)
499
+ scope.add(f"worker-attempt:{attempt_id}")
500
+ thread_id = receipt.get("threadId")
501
+ if isinstance(thread_id, str) and thread_id:
502
+ scope.add(f"thread:{thread_id}")
503
+ tokens = _usage_counts(receipt.get("usage"))
504
+ if not isinstance(thread_id, str) or not thread_id or tokens is None:
505
+ missing.add(
506
+ thread_id
507
+ if isinstance(thread_id, str) and thread_id
508
+ else f"worker-attempt:{attempt_id}"
509
+ )
510
+ continue
511
+ previous = receipts.get(thread_id)
512
+ if previous is None or _usage_quality(tokens, observed_at=observed) > _usage_quality(
513
+ previous.tokens, observed_at=previous.tokens_updated_at
514
+ ):
515
+ receipts[thread_id] = RolloutUsage(tokens, set(), observed, {})
516
+ except (OSError, UnicodeError, ValueError) as exc:
517
+ warnings.add("worker_usage_receipt_unavailable")
518
+ missing.add(f"worker-attempt:{path.stem}")
519
+ scope.add(f"worker-attempt:{path.stem}")
520
+ try:
521
+ print(f"Could not read worker token usage: {exc}", file=sys.stderr)
522
+ except OSError:
523
+ # The receipt warning and missing-attempt scope are already retained.
524
+ pass
525
+ return receipts, missing, scope
526
+
527
+
220
528
  def _scan_root_thread_ids(
221
529
  connection: sqlite3.Connection,
222
530
  scan: sqlite3.Row,
@@ -267,19 +575,35 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
267
575
  )
268
576
 
269
577
 
270
- def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]:
578
+ def _scan_worker_directories(
579
+ connection: sqlite3.Connection, scan: sqlite3.Row, warnings: set[str]
580
+ ) -> dict[str, bool]:
271
581
  if scan["mode"] != "deep":
272
- return set()
273
- return {
274
- row["artifact_dir"]
275
- if row["kind"] == "discovery"
276
- else str(Path(scan["scan_dir"]) / "artifacts")
277
- for row in connection.execute(
278
- "SELECT kind, artifact_dir FROM deep_scan_workers "
279
- "WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
280
- (scan["id"],),
281
- )
282
- }
582
+ return {}
583
+ if connection.execute(
584
+ "SELECT 1 FROM deep_scan_runs WHERE scan_id = ? AND worker_usage_gap = 1", (scan["id"],)
585
+ ).fetchone():
586
+ warnings.add("worker_thread_unavailable")
587
+ artifacts = Path(scan["scan_dir"]) / "artifacts"
588
+ # Comparison turns run in this directory without a discovery worker row.
589
+ directories = {str(artifacts / "deep_discovery"): False}
590
+ for row in connection.execute(
591
+ "SELECT kind, artifact_dir, started_at, sdk_thread_id, attempt FROM deep_scan_workers "
592
+ "WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
593
+ (scan["id"],),
594
+ ):
595
+ if row["kind"] == "discovery":
596
+ if row["sdk_thread_id"] and row["attempt"] == 1:
597
+ continue # The first attempt already has an exact thread identity.
598
+ # An attempt that started without a saved ID must be found by its assigned cwd.
599
+ missing_thread = bool(row["started_at"] and not row["sdk_thread_id"])
600
+ directories[row["artifact_dir"]] = missing_thread
601
+ else:
602
+ directories[str(artifacts)] = False
603
+ if row["attempt"] > 1 and row["started_at"] and not row["sdk_thread_id"]:
604
+ # Retries share a cwd. A rollout from an older attempt cannot prove the new one.
605
+ warnings.add("worker_thread_unavailable")
606
+ return directories
283
607
 
284
608
 
285
609
  def _codex_state_database() -> Path | None:
@@ -324,7 +648,8 @@ def _discover_rollout_sessions(
324
648
  roots: list[str],
325
649
  warnings: set[str],
326
650
  *,
327
- worker_directories: set[str],
651
+ worker_directories: dict[str, bool],
652
+ scan_directory: Path | None = None,
328
653
  started_at: datetime,
329
654
  completed_at: datetime | None,
330
655
  ) -> tuple[list[RolloutSession], set[str]]:
@@ -343,9 +668,12 @@ def _discover_rollout_sessions(
343
668
  columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
344
669
  if not required.issubset(columns):
345
670
  raise ValueError("Codex state graph does not expose the required thread columns.")
671
+ if table == "threads" and worker_directories and not {"cwd", "created_at"} <= columns:
672
+ worker_directories = {}
673
+ warnings.add("worker_directories_unavailable")
346
674
  sessions: dict[str, RolloutSession] = {}
347
675
  missing_thread_ids: set[str] = set()
348
- root_ids = dict.fromkeys(roots)
676
+ root_ids: dict[str, str | None] = dict.fromkeys(roots)
349
677
  started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
350
678
  completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
351
679
  directory_expression = "cwd"
@@ -356,16 +684,52 @@ def _discover_rollout_sessions(
356
684
  "codex_security_path_key", 1, _normalized_windows_path, deterministic=True
357
685
  )
358
686
  directory_expression = "codex_security_path_key(cwd)"
359
- worker_directories = {_normalized_windows_path(value) for value in worker_directories}
360
- for directory in sorted(worker_directories):
687
+ worker_directories = {
688
+ _normalized_windows_path(path): required
689
+ for path, required in worker_directories.items()
690
+ }
691
+ if scan_directory is not None:
692
+ scan_directory = Path(_normalized_windows_path(scan_directory))
693
+ unmatched = {path for path, required in worker_directories.items() if required}
694
+ directory_sources = {path: {path} for path in worker_directories}
695
+ if worker_directories and scan_directory and ".previous-" in scan_directory.name:
696
+ # Archived files move, but session working directories describe where execution ran.
697
+ # Confirm the original output with an already-bound session before including peers.
698
+ bound_directories = {
699
+ Path(row[0])
700
+ for row in database.execute(
701
+ f"SELECT DISTINCT {directory_expression} FROM threads WHERE cwd IS NOT NULL "
702
+ "AND id IN (SELECT value FROM json_each(?))",
703
+ (json.dumps(roots),),
704
+ )
705
+ }
706
+ relatives = {
707
+ Path(path).relative_to(scan_directory): path
708
+ for path in worker_directories
709
+ if Path(path).is_relative_to(scan_directory)
710
+ }
711
+ for marker in re.finditer(r"\.previous-", scan_directory.name):
712
+ if marker.start() == 0:
713
+ continue
714
+ original = scan_directory.with_name(scan_directory.name[: marker.start()])
715
+ candidates = {original / relative: path for relative, path in relatives.items()}
716
+ # Older archives kept absolute discovery paths in their original directory.
717
+ if (
718
+ original in bound_directories
719
+ or candidates.keys() & bound_directories
720
+ or any(Path(path).is_relative_to(original) for path in worker_directories)
721
+ ):
722
+ for candidate, path in candidates.items():
723
+ directory_sources.setdefault(str(candidate), set()).add(path)
724
+ for directory in sorted(directory_sources):
361
725
  for row in database.execute(
362
726
  "SELECT id FROM threads WHERE cwd IS NOT NULL "
363
727
  f"AND {directory_expression} = ? AND created_at >= ? "
364
728
  "AND (? IS NULL OR created_at <= ?) ORDER BY id",
365
729
  (directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
366
730
  ):
367
- root_ids[row["id"]] = None
368
- for root in root_ids:
731
+ root_ids[row["id"]] = directory
732
+ for root, directory in root_ids.items():
369
733
  row = database.execute(
370
734
  "SELECT id, rollout_path FROM threads WHERE id = ?",
371
735
  (root,),
@@ -381,8 +745,8 @@ def _discover_rollout_sessions(
381
745
  continue
382
746
  if root not in roots:
383
747
  try:
384
- with path.open("rb") as source:
385
- metadata = json.loads(source.readline())
748
+ with _rollout_lines(path) as source:
749
+ metadata = json.loads(next(source, b""))
386
750
  except (OSError, ValueError):
387
751
  missing_thread_ids.add(root)
388
752
  warnings.add("rollout_unavailable")
@@ -393,9 +757,15 @@ def _discover_rollout_sessions(
393
757
  and isinstance(payload := metadata.get("payload"), dict)
394
758
  and (payload.get("id") or payload.get("session_id")) == root
395
759
  and (created_at := _timestamp(payload.get("timestamp"))) is not None
396
- and created_at < started_millisecond
760
+ and (
761
+ created_at < started_millisecond
762
+ or completed_at is not None
763
+ and created_at > completed_at
764
+ )
397
765
  ):
398
766
  continue
767
+ if directory is not None:
768
+ unmatched.difference_update(directory_sources[directory])
399
769
  sessions[root] = RolloutSession(root, None, path)
400
770
  descendants = database.execute(
401
771
  """
@@ -453,6 +823,8 @@ def _discover_rollout_sessions(
453
823
  warnings.add("rollout_unavailable")
454
824
  continue
455
825
  sessions[child_id] = RolloutSession(child_id, parent_id, path)
826
+ if unmatched:
827
+ warnings.add("worker_thread_unavailable")
456
828
  return list(sessions.values()), missing_thread_ids
457
829
  finally:
458
830
  database.close()
@@ -465,6 +837,13 @@ def _rollout_path(value: object) -> Path | None:
465
837
  if not candidate.is_absolute():
466
838
  return None
467
839
  try:
840
+ try:
841
+ candidate.lstat()
842
+ except FileNotFoundError:
843
+ # Native cold storage may compress an indexed JSONL file in place.
844
+ if candidate.suffix != ".jsonl":
845
+ return None
846
+ candidate = candidate.with_suffix(".jsonl.zst")
468
847
  resolved = candidate.resolve(strict=True)
469
848
  if not resolved.is_file():
470
849
  return None
@@ -481,6 +860,63 @@ def _rollout_path(value: object) -> Path | None:
481
860
  return None
482
861
 
483
862
 
863
+ @contextmanager
864
+ def _rollout_lines(path: Path, *, verify_end: bool = False):
865
+ if path.suffix != ".zst":
866
+ with path.open("rb") as source:
867
+ yield source
868
+ return
869
+ try:
870
+ from compression import zstd # Python 3.14+
871
+ except ImportError:
872
+ try:
873
+ from backports import zstd
874
+ except ImportError:
875
+ zstd = None
876
+ if zstd is not None:
877
+ try:
878
+ with zstd.open(path, "rb") as source:
879
+ yield source
880
+ # A requested completion turn may stop parsing before the frame checksum.
881
+ if verify_end:
882
+ while source.read(64 * 1024):
883
+ pass
884
+ except (zstd.ZstdError, EOFError) as error:
885
+ raise ValueError("Invalid compressed Codex rollout.") from error
886
+ return
887
+ with tempfile.TemporaryFile() as errors:
888
+ with subprocess.Popen(
889
+ [
890
+ os.environ.get("CODEX_MCP_NODE_PATH") or "node",
891
+ str(Path(__file__).with_name("read_codex_session.mjs")),
892
+ str(path),
893
+ ],
894
+ stdout=subprocess.PIPE,
895
+ stderr=errors,
896
+ ) as child:
897
+ exhausted = False
898
+
899
+ def lines():
900
+ nonlocal exhausted
901
+ yield from child.stdout
902
+ exhausted = True
903
+
904
+ try:
905
+ iterator = lines()
906
+ yield iterator
907
+ if verify_end:
908
+ for _ in iterator:
909
+ pass
910
+ finally:
911
+ if not exhausted and child.poll() is None:
912
+ child.terminate()
913
+ child.stdout.close()
914
+ status = child.wait()
915
+ if exhausted and status != 0:
916
+ errors.seek(0)
917
+ raise OSError(errors.read().decode("utf-8", errors="replace"))
918
+
919
+
484
920
  def _read_rollout_usage(
485
921
  session: RolloutSession,
486
922
  *,
@@ -493,8 +929,13 @@ def _read_rollout_usage(
493
929
  total = result.tokens
494
930
  warnings = result.warnings
495
931
  previous = _empty_token_usage()
932
+ previous_cache_writes_reported = True
933
+ prefix_unclassified = 0
934
+ prefix_missing_cache: set[str] = set()
935
+ ambiguous_legacy_cache = False
496
936
  boundary_reached = False
497
937
  legacy_usage = _empty_token_usage()
938
+ completed_legacy_usage = _empty_token_usage()
498
939
  response_usage: dict[str, int] | None = None
499
940
  observed_response_usage = _empty_token_usage()
500
941
  response_ids: set[str] = set()
@@ -506,7 +947,15 @@ def _read_rollout_usage(
506
947
  closed_turns: set[str] = set()
507
948
  measured_turns: set[str] = set()
508
949
 
509
- with session.path.open("rb") as source:
950
+ def past_scan_end(timestamp: datetime, turn_id: object) -> bool:
951
+ # The requested completion turn may finish after the scan was marked complete.
952
+ return (
953
+ completed_at is not None
954
+ and timestamp > completed_at
955
+ and (completed_turn_id is None or turn_id != completed_turn_id)
956
+ )
957
+
958
+ with _rollout_lines(session.path, verify_end=True) as source:
510
959
  for line_number, raw_line in enumerate(source, start=1):
511
960
  if not raw_line.endswith(b"\n"):
512
961
  warnings.add("rollout_record_incomplete")
@@ -559,10 +1008,8 @@ def _read_rollout_usage(
559
1008
  if task_started_at is None:
560
1009
  warnings.add("thread_ownership_unavailable")
561
1010
  return result
562
- if task_started_at < started_at or (
563
- completed_at is not None
564
- and task_started_at > completed_at
565
- and payload.get("turn_id") != completed_turn_id
1011
+ if task_started_at < started_at or past_scan_end(
1012
+ task_started_at, payload.get("turn_id")
566
1013
  ):
567
1014
  warnings.add("thread_outside_scan_window")
568
1015
  return result
@@ -570,22 +1017,21 @@ def _read_rollout_usage(
570
1017
  elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
571
1018
  inherited_usage = _token_snapshot(payload)
572
1019
  if inherited_usage is not None:
573
- previous = inherited_usage
1020
+ missing_cache = _missing_cache_fields(payload)
1021
+ baseline = _retain_cache_baseline(previous, inherited_usage, missing_cache)
1022
+ if baseline is None:
1023
+ warnings.add("token_breakdown_unavailable")
1024
+ else:
1025
+ previous = baseline
1026
+ prefix_unclassified = _uncached_input(baseline)
1027
+ prefix_missing_cache = missing_cache
574
1028
  if not boundary_reached:
575
1029
  continue
576
1030
  if event.get("type") == "event_msg" and payload.get("type") == "task_started":
577
1031
  # The same owner can resume as a model-backed coordinator. Only
578
1032
  # a provably post-scan turn leaves the original control zero intact.
579
- task_started_at = _timestamp(event.get("timestamp"))
580
- if (
581
- completed_at is None
582
- or task_started_at is None
583
- or task_started_at <= completed_at
584
- or (
585
- completed_turn_id is not None
586
- and payload.get("turn_id") == completed_turn_id
587
- )
588
- ):
1033
+ timestamp = _timestamp(event.get("timestamp"))
1034
+ if timestamp is None or not past_scan_end(timestamp, payload.get("turn_id")):
589
1035
  direct_control_session = False
590
1036
  if (
591
1037
  completed_turn_id is not None
@@ -596,19 +1042,19 @@ def _read_rollout_usage(
596
1042
  _add_token_usage(
597
1043
  total, response_usage if response_usage is not None else legacy_usage
598
1044
  )
1045
+ if response_usage is None:
1046
+ _add_token_usage(completed_legacy_usage, legacy_usage)
1047
+ if ambiguous_legacy_cache:
1048
+ warnings.add("token_breakdown_unavailable")
1049
+ ambiguous_legacy_cache = False
599
1050
  legacy_usage = _empty_token_usage()
600
1051
  response_usage = None
601
1052
  if payload.get("turn_id") != current_turn_id:
602
1053
  observed_response_usage = _empty_token_usage()
603
1054
  current_turn_id = payload.get("turn_id")
604
- timestamp = _timestamp(event.get("timestamp"))
605
1055
  if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
606
1056
  warnings.add("turn_record_invalid")
607
- elif (
608
- completed_at is None
609
- or timestamp <= completed_at
610
- or current_turn_id == completed_turn_id
611
- ):
1057
+ elif not past_scan_end(timestamp, current_turn_id):
612
1058
  started_turns.add(current_turn_id)
613
1059
  open_turns.add(current_turn_id)
614
1060
  continue
@@ -624,11 +1070,7 @@ def _read_rollout_usage(
624
1070
  or timestamp is None
625
1071
  ):
626
1072
  warnings.add("turn_record_invalid")
627
- elif (
628
- completed_at is None
629
- or timestamp <= completed_at
630
- or terminal_turn_id == completed_turn_id
631
- ):
1073
+ elif not past_scan_end(timestamp, terminal_turn_id):
632
1074
  open_turns.discard(terminal_turn_id)
633
1075
  if timestamp >= started_at:
634
1076
  if terminal_turn_id in started_turns:
@@ -648,11 +1090,7 @@ def _read_rollout_usage(
648
1090
  if (
649
1091
  timestamp is not None
650
1092
  and timestamp >= started_at
651
- and (
652
- completed_at is None
653
- or timestamp <= completed_at
654
- or (completed_turn_id is not None and current_turn_id == completed_turn_id)
655
- )
1093
+ and not past_scan_end(timestamp, current_turn_id)
656
1094
  and "compaction_response_id" not in payload
657
1095
  and response_usage is None
658
1096
  ):
@@ -675,15 +1113,13 @@ def _read_rollout_usage(
675
1113
  ):
676
1114
  warnings.add("token_record_invalid")
677
1115
  continue
678
- is_completion_turn = (
679
- completed_turn_id is not None and payload.get("turn_id") == completed_turn_id
680
- )
681
- if completed_at is not None and timestamp > completed_at and not is_completion_turn:
1116
+ receipt_turn_id = payload.get("turn_id")
1117
+ if past_scan_end(timestamp, receipt_turn_id):
682
1118
  continue
683
1119
  if response_usage is None:
684
1120
  response_usage = _empty_token_usage()
685
1121
  if current_turn_id is None:
686
- current_turn_id = payload.get("turn_id")
1122
+ current_turn_id = receipt_turn_id
687
1123
  if isinstance(current_turn_id, str):
688
1124
  open_turns.add(current_turn_id)
689
1125
  if response_id in response_ids:
@@ -691,14 +1127,13 @@ def _read_rollout_usage(
691
1127
  response_ids.add(response_id)
692
1128
  _add_token_usage(observed_response_usage, usage)
693
1129
  turn_usage = _usage_counts(payload.get("turn_token_usage"))
694
- receipt_turn_id = payload.get("turn_id")
695
1130
  if isinstance(receipt_turn_id, str):
696
1131
  measured_turns.add(receipt_turn_id)
697
1132
  if timestamp < started_at:
698
1133
  if turn_usage is not None:
699
1134
  observed_response_usage = turn_usage
700
1135
  continue
701
- if is_completion_turn:
1136
+ if completed_turn_id is not None and receipt_turn_id == completed_turn_id:
702
1137
  completed_turn_measured = True
703
1138
  if turn_usage is not None and any(
704
1139
  turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
@@ -713,11 +1148,7 @@ def _read_rollout_usage(
713
1148
  in_window = (
714
1149
  timestamp is not None
715
1150
  and timestamp >= started_at
716
- and (
717
- completed_at is None
718
- or timestamp <= completed_at
719
- or (completed_turn_id is not None and current_turn_id == completed_turn_id)
720
- )
1151
+ and not past_scan_end(timestamp, current_turn_id)
721
1152
  )
722
1153
  if in_window:
723
1154
  rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
@@ -730,33 +1161,102 @@ def _read_rollout_usage(
730
1161
  result.rate_limits[key] = rate_limits
731
1162
  if "info" in payload and payload["info"] is None:
732
1163
  continue
733
- timestamp = _timestamp(event.get("timestamp"))
734
1164
  snapshot = _token_snapshot(payload)
735
1165
  if timestamp is None or snapshot is None:
736
1166
  warnings.add("token_record_invalid")
737
1167
  continue
1168
+ reset = snapshot["totalTokens"] < previous["totalTokens"]
1169
+ if reset:
1170
+ prefix_unclassified = 0
1171
+ missing_cache = _missing_cache_fields(payload)
1172
+ cache_writes_reported = "cacheWriteInputTokens" not in missing_cache
1173
+ baseline = _retain_cache_baseline(previous, snapshot, missing_cache)
1174
+ if baseline is None:
1175
+ if not past_scan_end(timestamp, current_turn_id):
1176
+ warnings.add("token_breakdown_unavailable")
1177
+ continue
1178
+ if baseline is previous:
1179
+ cache_writes_reported = previous_cache_writes_reported and cache_writes_reported
1180
+ refines_classification = (
1181
+ baseline["inputTokens"] >= previous["inputTokens"]
1182
+ and baseline["outputTokens"] >= previous["outputTokens"]
1183
+ and baseline["cachedInputTokens"] >= previous["cachedInputTokens"]
1184
+ and baseline["cacheWriteInputTokens"] >= previous["cacheWriteInputTokens"]
1185
+ and (
1186
+ (not previous_cache_writes_reported and cache_writes_reported)
1187
+ or baseline["cachedInputTokens"] > previous["cachedInputTokens"]
1188
+ or baseline["cacheWriteInputTokens"] > previous["cacheWriteInputTokens"]
1189
+ )
1190
+ )
1191
+ previous_cache_writes_reported = cache_writes_reported
738
1192
  delta = {
739
- key: value - previous[key] if value >= previous[key] else value
740
- for key, value in snapshot.items()
1193
+ key: snapshot[key] - previous[key]
1194
+ if not reset and snapshot[key] >= previous[key]
1195
+ else snapshot[key]
1196
+ for key in TOKEN_FIELDS.values()
741
1197
  }
742
- previous = snapshot
743
- is_completion_turn = (
744
- completed_turn_id is not None and current_turn_id == completed_turn_id
745
- )
746
- if completed_at is not None and timestamp > completed_at and not is_completion_turn:
1198
+ if (
1199
+ snapshot.get("cacheWriteInputTokensReported") is False
1200
+ or previous.get("cacheWriteInputTokensReported") is False
1201
+ ):
1202
+ delta["cacheWriteInputTokensReported"] = False
1203
+ previous = baseline
1204
+ if past_scan_end(timestamp, current_turn_id):
747
1205
  continue
748
1206
  if isinstance(current_turn_id, str):
749
1207
  measured_turns.add(current_turn_id)
750
1208
  if timestamp < started_at:
1209
+ prefix_unclassified = _uncached_input(baseline)
1210
+ prefix_missing_cache = missing_cache
751
1211
  continue
752
- if is_completion_turn:
1212
+ if completed_turn_id is not None and current_turn_id == completed_turn_id:
753
1213
  completed_turn_measured = True
754
1214
  result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
755
- if delta["totalTokens"] <= 0:
1215
+ delta["totalTokens"] = delta["inputTokens"] + delta["outputTokens"]
1216
+ if delta["totalTokens"] <= 0 and not refines_classification:
756
1217
  continue
1218
+ cache_write_capacity = (
1219
+ _uncached_input(legacy_usage) + delta["inputTokens"]
1220
+ if refines_classification
1221
+ else delta["inputTokens"]
1222
+ )
1223
+ # Late cumulative cache details can classify an earlier legacy turn.
1224
+ completed_capacity = (
1225
+ _uncached_input(completed_legacy_usage)
1226
+ if refines_classification and response_usage is None
1227
+ else 0
1228
+ )
1229
+ if (
1230
+ prefix_unclassified > 0
1231
+ and refines_classification
1232
+ and (
1233
+ any(delta[key] > 0 for key in prefix_missing_cache)
1234
+ # Even explicit counters can revise earlier input when the
1235
+ # newly classified tokens exceed all in-scan capacity.
1236
+ or delta["cachedInputTokens"] + delta["cacheWriteInputTokens"]
1237
+ > cache_write_capacity + completed_capacity
1238
+ )
1239
+ ):
1240
+ # The newly classified tokens could have occurred before this scan.
1241
+ ambiguous_legacy_cache = True
1242
+ for key in ("cacheWriteInputTokens", "cachedInputTokens"):
1243
+ current = min(delta[key], cache_write_capacity)
1244
+ completed = min(delta[key] - current, completed_capacity)
1245
+ if completed and ambiguous_legacy_cache:
1246
+ warnings.add("token_breakdown_unavailable")
1247
+ delta[key] = current
1248
+ cache_write_capacity -= current
1249
+ completed_capacity -= completed
1250
+ completed_legacy_usage[key] += completed
1251
+ total[key] += completed
1252
+ delta["reasoningOutputTokens"] = min(
1253
+ delta["reasoningOutputTokens"], delta["outputTokens"]
1254
+ )
757
1255
  _add_token_usage(legacy_usage, delta)
758
1256
 
759
1257
  _add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
1258
+ if response_usage is None and ambiguous_legacy_cache:
1259
+ warnings.add("token_breakdown_unavailable")
760
1260
  if not boundary_reached:
761
1261
  warnings.add("thread_ownership_unavailable")
762
1262
  elif result.tokens_updated_at is None and not direct_control_session:
@@ -827,6 +1327,47 @@ def valid_account_rate_limit(value: object) -> bool:
827
1327
  return True
828
1328
 
829
1329
 
1330
+ def _missing_cache_fields(payload: Mapping[str, Any]) -> set[str]:
1331
+ usage = payload["info"]["total_token_usage"]
1332
+ missing: set[str] = set()
1333
+ if "cached_input_tokens" not in usage:
1334
+ missing.add("cachedInputTokens")
1335
+ if "cache_write_input_tokens" not in usage and "cache_write_tokens" not in usage:
1336
+ missing.add("cacheWriteInputTokens")
1337
+ return missing
1338
+
1339
+
1340
+ def _uncached_input(usage: dict[str, int]) -> int:
1341
+ return usage["inputTokens"] - usage["cachedInputTokens"] - usage["cacheWriteInputTokens"]
1342
+
1343
+
1344
+ def _retain_cache_baseline(
1345
+ previous: dict[str, int], snapshot: dict[str, int], missing: set[str]
1346
+ ) -> dict[str, int] | None:
1347
+ baseline = snapshot
1348
+ if (
1349
+ missing
1350
+ and snapshot["inputTokens"] >= previous["inputTokens"]
1351
+ and snapshot["outputTokens"] >= previous["outputTokens"]
1352
+ ):
1353
+ baseline = {**snapshot, **{key: previous[key] for key in missing}}
1354
+ if (
1355
+ baseline["cachedInputTokens"] + baseline["cacheWriteInputTokens"]
1356
+ > baseline["inputTokens"]
1357
+ ):
1358
+ return None
1359
+ if (
1360
+ baseline["inputTokens"] == previous["inputTokens"]
1361
+ and baseline["outputTokens"] == previous["outputTokens"]
1362
+ and (
1363
+ baseline["cachedInputTokens"] < previous["cachedInputTokens"]
1364
+ or baseline["cacheWriteInputTokens"] < previous["cacheWriteInputTokens"]
1365
+ )
1366
+ ):
1367
+ return previous
1368
+ return baseline
1369
+
1370
+
830
1371
  def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
831
1372
  source = payload.get("source")
832
1373
  if isinstance(source, dict):
@@ -909,6 +1450,12 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
909
1450
  if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
910
1451
  return None
911
1452
  result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
1453
+ if (
1454
+ usage.get("cache_write_input_tokens_reported") is False
1455
+ or usage.get("cache_write_input_tokens") is None
1456
+ and usage.get("cache_write_tokens") is None
1457
+ ):
1458
+ result["cacheWriteInputTokensReported"] = False
912
1459
  return result
913
1460
 
914
1461
 
@@ -919,6 +1466,8 @@ def _empty_token_usage() -> dict[str, int]:
919
1466
  def _add_token_usage(target: dict[str, int], addition: Mapping[str, int]) -> None:
920
1467
  for key in TOKEN_FIELDS.values():
921
1468
  target[key] += addition[key]
1469
+ if addition.get("cacheWriteInputTokensReported") is False:
1470
+ target["cacheWriteInputTokensReported"] = False
922
1471
 
923
1472
 
924
1473
  def _timestamp(value: object) -> datetime | None: