@openai/codex-security 0.8.3 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/README.md +36 -746
  2. package/_bundled_plugin/.codex-plugin/plugin.json +2 -0
  3. package/_bundled_plugin/.mcp.json +7 -1
  4. package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
  5. package/_bundled_plugin/examples/completed-scan/report.md +11 -8
  6. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
  7. package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
  8. package/_bundled_plugin/mcp/helpers.mjs +2 -11
  9. package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
  10. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
  11. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  12. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  13. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  14. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  15. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  16. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  17. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  18. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  19. package/_bundled_plugin/mcp/package.json +1 -0
  20. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +5 -2
  21. package/_bundled_plugin/mcp/server.mjs +2 -11
  22. package/_bundled_plugin/mcp/server.mjs.br +0 -0
  23. package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
  24. package/_bundled_plugin/references/artifact-storage.md +1 -1
  25. package/_bundled_plugin/references/baseline-auditor.md +18 -0
  26. package/_bundled_plugin/references/core-scan.md +5 -42
  27. package/_bundled_plugin/references/default-severity-rubric.md +9 -4
  28. package/_bundled_plugin/references/final-report.md +5 -3
  29. package/_bundled_plugin/references/finding-detail-fields.md +18 -18
  30. package/_bundled_plugin/references/focused-investigator.md +13 -0
  31. package/_bundled_plugin/references/sarif-adapter.md +8 -2
  32. package/_bundled_plugin/references/scan-artifacts.md +21 -12
  33. package/_bundled_plugin/references/scan-contract.md +27 -33
  34. package/_bundled_plugin/references/scan-prologue.md +2 -2
  35. package/_bundled_plugin/references/scan-runtime.md +4 -2
  36. package/_bundled_plugin/references/security-guidance.md +14 -1
  37. package/_bundled_plugin/references/validate-scan-findings.md +3 -0
  38. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
  39. package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
  40. package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
  41. package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
  42. package/_bundled_plugin/scripts/deep_scan_workbench.py +192 -352
  43. package/_bundled_plugin/scripts/dispatch_inventory_tool +7 -0
  44. package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
  45. package/_bundled_plugin/scripts/finalize_scan_contract.py +182 -230
  46. package/_bundled_plugin/scripts/finding_preview.py +20 -0
  47. package/_bundled_plugin/scripts/launch_codex_security_mcp +55 -16
  48. package/_bundled_plugin/scripts/legacy_findings.py +148 -0
  49. package/_bundled_plugin/scripts/posix_scan_local_files.py +37 -1
  50. package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
  51. package/_bundled_plugin/scripts/report_projection.py +81 -43
  52. package/_bundled_plugin/scripts/review_state.py +8 -2
  53. package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
  54. package/_bundled_plugin/scripts/snapshot_sqlite.py +9 -2
  55. package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
  56. package/_bundled_plugin/scripts/validate_scan_contract.py +7 -1
  57. package/_bundled_plugin/scripts/windows_scan_local_files.py +582 -20
  58. package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
  59. package/_bundled_plugin/scripts/workbench/handoff.py +100 -23
  60. package/_bundled_plugin/scripts/workbench/retained_coverage.py +30 -51
  61. package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
  62. package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
  63. package/_bundled_plugin/scripts/workbench/saved_coverage.py +9 -17
  64. package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
  65. package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
  66. package/_bundled_plugin/scripts/workbench/storage.py +91 -7
  67. package/_bundled_plugin/scripts/workbench_cli.py +54 -14
  68. package/_bundled_plugin/scripts/workbench_constants.py +12 -0
  69. package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
  70. package/_bundled_plugin/scripts/workbench_db.py +562 -303
  71. package/_bundled_plugin/scripts/workbench_feedback.py +57 -11
  72. package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
  73. package/_bundled_plugin/scripts/workbench_finding_workflows.py +4 -0
  74. package/_bundled_plugin/scripts/workbench_native_indexes.py +238 -41
  75. package/_bundled_plugin/scripts/workbench_progress.py +169 -4
  76. package/_bundled_plugin/scripts/workbench_publication.py +16 -13
  77. package/_bundled_plugin/scripts/workbench_remediation.py +155 -2
  78. package/_bundled_plugin/scripts/workbench_saved_results.py +176 -81
  79. package/_bundled_plugin/scripts/workbench_scan_history.py +398 -174
  80. package/_bundled_plugin/scripts/workbench_scan_start.py +224 -31
  81. package/_bundled_plugin/scripts/workbench_scan_usage.py +675 -73
  82. package/_bundled_plugin/scripts/workbench_schema.py +258 -63
  83. package/_bundled_plugin/scripts/workbench_setup.py +14 -2
  84. package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
  85. package/_bundled_plugin/scripts/workbench_source_scopes.py +18 -0
  86. package/_bundled_plugin/scripts/workbench_target.py +594 -136
  87. package/_bundled_plugin/scripts/workbench_target_state.py +1110 -21
  88. package/_bundled_plugin/scripts/workbench_validation.py +43 -3
  89. package/_bundled_plugin/shared/workbench-history-repairs.json +77 -35
  90. package/_bundled_plugin/shared/workbench-migrations.json +100 -0
  91. package/_bundled_plugin/skills/cloud-findings/SKILL.md +5 -5
  92. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
  93. package/_bundled_plugin/skills/fix-finding/SKILL.md +2 -1
  94. package/_bundled_plugin/skills/publish-findings/SKILL.md +24 -153
  95. package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +2 -2
  96. package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
  97. package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +3 -3
  98. package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +409 -0
  99. package/_bundled_plugin/skills/publish-findings/scripts/collect_continuum.py +224 -0
  100. package/_bundled_plugin/skills/publish-findings/scripts/collector_paths.py +63 -0
  101. package/_bundled_plugin/skills/security-scan/SKILL.md +9 -46
  102. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +2 -2
  103. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -21
  104. package/_bundled_plugin/skills/security-scan/references/managed-scan.md +11 -0
  105. package/_bundled_plugin/skills/security-scan/references/scan-results.md +15 -0
  106. package/_bundled_plugin/skills/security-scan/references/scoped-paths.md +23 -0
  107. package/_bundled_plugin/skills/security-scan/references/sdk-files.md +24 -0
  108. package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +22 -1
  109. package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -2
  110. package/bin/codex-security.mjs +2 -2
  111. package/dist/api.d.ts +45 -5
  112. package/dist/api.d.ts.map +1 -1
  113. package/dist/api.js +1192 -427
  114. package/dist/api.js.map +1 -1
  115. package/dist/armorcode-findings.d.ts +9 -0
  116. package/dist/armorcode-findings.d.ts.map +1 -0
  117. package/dist/armorcode-findings.js +145 -0
  118. package/dist/armorcode-findings.js.map +1 -0
  119. package/dist/artifact-export.d.ts +2 -0
  120. package/dist/artifact-export.d.ts.map +1 -1
  121. package/dist/artifact-export.js +29 -13
  122. package/dist/artifact-export.js.map +1 -1
  123. package/dist/auth.d.ts +6 -2
  124. package/dist/auth.d.ts.map +1 -1
  125. package/dist/auth.js +37 -18
  126. package/dist/auth.js.map +1 -1
  127. package/dist/batch-cost.d.ts +18 -0
  128. package/dist/batch-cost.d.ts.map +1 -0
  129. package/dist/batch-cost.js +41 -0
  130. package/dist/batch-cost.js.map +1 -0
  131. package/dist/bedrock-oss-pricing.d.ts +6 -0
  132. package/dist/bedrock-oss-pricing.d.ts.map +1 -0
  133. package/dist/bedrock-oss-pricing.js +58 -0
  134. package/dist/bedrock-oss-pricing.js.map +1 -0
  135. package/dist/bedrock-pricing.d.ts +15 -0
  136. package/dist/bedrock-pricing.d.ts.map +1 -0
  137. package/dist/bedrock-pricing.js +117 -0
  138. package/dist/bedrock-pricing.js.map +1 -0
  139. package/dist/blackduck-findings.d.ts +1 -1
  140. package/dist/blackduck-findings.d.ts.map +1 -1
  141. package/dist/blackduck-findings.js +24 -22
  142. package/dist/blackduck-findings.js.map +1 -1
  143. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  144. package/dist/bulk-scan-discovery.js +4 -4
  145. package/dist/bulk-scan-discovery.js.map +1 -1
  146. package/dist/bundled-plugin.d.ts +4 -0
  147. package/dist/bundled-plugin.d.ts.map +1 -0
  148. package/dist/bundled-plugin.js +23 -0
  149. package/dist/bundled-plugin.js.map +1 -0
  150. package/dist/checkmarx-evidence.d.ts +19 -0
  151. package/dist/checkmarx-evidence.d.ts.map +1 -0
  152. package/dist/checkmarx-evidence.js +36 -0
  153. package/dist/checkmarx-evidence.js.map +1 -0
  154. package/dist/checkmarx-findings.d.ts +4 -0
  155. package/dist/checkmarx-findings.d.ts.map +1 -0
  156. package/dist/checkmarx-findings.js +27 -0
  157. package/dist/checkmarx-findings.js.map +1 -0
  158. package/dist/checkmarx-one.d.ts +4 -0
  159. package/dist/checkmarx-one.d.ts.map +1 -0
  160. package/dist/checkmarx-one.js +53 -0
  161. package/dist/checkmarx-one.js.map +1 -0
  162. package/dist/checkmarx-repository.d.ts +12 -0
  163. package/dist/checkmarx-repository.d.ts.map +1 -0
  164. package/dist/checkmarx-repository.js +181 -0
  165. package/dist/checkmarx-repository.js.map +1 -0
  166. package/dist/checkmarx-xml.d.ts +12 -0
  167. package/dist/checkmarx-xml.d.ts.map +1 -0
  168. package/dist/checkmarx-xml.js +196 -0
  169. package/dist/checkmarx-xml.js.map +1 -0
  170. package/dist/classify-scan-severity.d.ts +1 -1
  171. package/dist/classify-scan-severity.d.ts.map +1 -1
  172. package/dist/classify-scan-severity.js +11 -3
  173. package/dist/classify-scan-severity.js.map +1 -1
  174. package/dist/classify-severity-records.d.ts +1 -9
  175. package/dist/classify-severity-records.d.ts.map +1 -1
  176. package/dist/classify-severity-records.js +3 -13
  177. package/dist/classify-severity-records.js.map +1 -1
  178. package/dist/classify-severity.d.ts.map +1 -1
  179. package/dist/classify-severity.js +11 -10
  180. package/dist/classify-severity.js.map +1 -1
  181. package/dist/cli-argument.d.ts +4 -0
  182. package/dist/cli-argument.d.ts.map +1 -0
  183. package/dist/cli-argument.js +17 -0
  184. package/dist/cli-argument.js.map +1 -0
  185. package/dist/cli-help.d.ts +1 -1
  186. package/dist/cli-help.d.ts.map +1 -1
  187. package/dist/cli-help.js +71 -29
  188. package/dist/cli-help.js.map +1 -1
  189. package/dist/cli-hook.d.ts +3 -0
  190. package/dist/cli-hook.d.ts.map +1 -0
  191. package/dist/cli-hook.js +87 -0
  192. package/dist/cli-hook.js.map +1 -0
  193. package/dist/cli-integrations.d.ts +13 -0
  194. package/dist/cli-integrations.d.ts.map +1 -0
  195. package/dist/cli-integrations.js +333 -0
  196. package/dist/cli-integrations.js.map +1 -0
  197. package/dist/cli-mcp.d.ts +15 -0
  198. package/dist/cli-mcp.d.ts.map +1 -0
  199. package/dist/cli-mcp.js +178 -0
  200. package/dist/cli-mcp.js.map +1 -0
  201. package/dist/cli-token-view.d.ts +10 -0
  202. package/dist/cli-token-view.d.ts.map +1 -0
  203. package/dist/cli-token-view.js +41 -0
  204. package/dist/cli-token-view.js.map +1 -0
  205. package/dist/cli.d.ts +16 -8
  206. package/dist/cli.d.ts.map +1 -1
  207. package/dist/cli.js +3382 -2017
  208. package/dist/cli.js.map +1 -1
  209. package/dist/cloud-publish.d.ts.map +1 -1
  210. package/dist/cloud-publish.js +35 -33
  211. package/dist/cloud-publish.js.map +1 -1
  212. package/dist/codex-sdk-environment.d.ts +6 -0
  213. package/dist/codex-sdk-environment.d.ts.map +1 -1
  214. package/dist/codex-sdk-environment.js +19 -0
  215. package/dist/codex-sdk-environment.js.map +1 -1
  216. package/dist/command-words.d.ts +3 -0
  217. package/dist/command-words.d.ts.map +1 -0
  218. package/dist/command-words.js +46 -0
  219. package/dist/command-words.js.map +1 -0
  220. package/dist/component-scan.d.ts +19 -4
  221. package/dist/component-scan.d.ts.map +1 -1
  222. package/dist/component-scan.js +67 -9
  223. package/dist/component-scan.js.map +1 -1
  224. package/dist/config.d.ts +1 -1
  225. package/dist/config.d.ts.map +1 -1
  226. package/dist/config.js +130 -67
  227. package/dist/config.js.map +1 -1
  228. package/dist/continuum-findings.d.ts +1 -1
  229. package/dist/continuum-findings.d.ts.map +1 -1
  230. package/dist/continuum-findings.js +149 -19
  231. package/dist/continuum-findings.js.map +1 -1
  232. package/dist/contract-values.d.ts +2 -0
  233. package/dist/contract-values.d.ts.map +1 -1
  234. package/dist/contract-values.js +12 -9
  235. package/dist/contract-values.js.map +1 -1
  236. package/dist/cost-model.d.ts +13 -1
  237. package/dist/cost-model.d.ts.map +1 -1
  238. package/dist/cost-model.js +66 -112
  239. package/dist/cost-model.js.map +1 -1
  240. package/dist/cost.d.ts +42 -3
  241. package/dist/cost.d.ts.map +1 -1
  242. package/dist/cost.js +484 -149
  243. package/dist/cost.js.map +1 -1
  244. package/dist/custom-publish.d.ts +4 -0
  245. package/dist/custom-publish.d.ts.map +1 -1
  246. package/dist/custom-publish.js +40 -3
  247. package/dist/custom-publish.js.map +1 -1
  248. package/dist/custom-validation-prompt.d.ts +1 -1
  249. package/dist/custom-validation-prompt.d.ts.map +1 -1
  250. package/dist/custom-validation-prompt.js +29 -19
  251. package/dist/custom-validation-prompt.js.map +1 -1
  252. package/dist/deduplication/local.d.ts +4 -2
  253. package/dist/deduplication/local.d.ts.map +1 -1
  254. package/dist/deduplication/local.js +9 -2
  255. package/dist/deduplication/local.js.map +1 -1
  256. package/dist/deduplication/refusal.d.ts.map +1 -1
  257. package/dist/deduplication/refusal.js +26 -5
  258. package/dist/deduplication/refusal.js.map +1 -1
  259. package/dist/deduplication/scan.d.ts +5 -0
  260. package/dist/deduplication/scan.d.ts.map +1 -1
  261. package/dist/deduplication/scan.js +24 -7
  262. package/dist/deduplication/scan.js.map +1 -1
  263. package/dist/deep-config.d.ts +0 -1
  264. package/dist/deep-config.d.ts.map +1 -1
  265. package/dist/deep-config.js +22 -33
  266. package/dist/deep-config.js.map +1 -1
  267. package/dist/deep-scan-defaults.d.ts +2 -0
  268. package/dist/deep-scan-defaults.d.ts.map +1 -1
  269. package/dist/deep-scan-defaults.js +28 -1
  270. package/dist/deep-scan-defaults.js.map +1 -1
  271. package/dist/deep-scan.d.ts +2 -0
  272. package/dist/deep-scan.d.ts.map +1 -1
  273. package/dist/deep-scan.js +44 -19
  274. package/dist/deep-scan.js.map +1 -1
  275. package/dist/default-severity-rubric.d.ts +1 -1
  276. package/dist/default-severity-rubric.d.ts.map +1 -1
  277. package/dist/default-severity-rubric.js +1 -1
  278. package/dist/default-severity-rubric.js.map +1 -1
  279. package/dist/endor-findings.d.ts +3 -1
  280. package/dist/endor-findings.d.ts.map +1 -1
  281. package/dist/endor-findings.js +86 -45
  282. package/dist/endor-findings.js.map +1 -1
  283. package/dist/errors.d.ts +4 -2
  284. package/dist/errors.d.ts.map +1 -1
  285. package/dist/errors.js +15 -4
  286. package/dist/errors.js.map +1 -1
  287. package/dist/executable-path.d.ts +3 -0
  288. package/dist/executable-path.d.ts.map +1 -1
  289. package/dist/executable-path.js +26 -0
  290. package/dist/executable-path.js.map +1 -1
  291. package/dist/external-findings-import.d.ts +5 -3
  292. package/dist/external-findings-import.d.ts.map +1 -1
  293. package/dist/external-findings-import.js +19 -7
  294. package/dist/external-findings-import.js.map +1 -1
  295. package/dist/external-findings-local.d.ts +69 -5
  296. package/dist/external-findings-local.d.ts.map +1 -1
  297. package/dist/external-findings-local.js +537 -108
  298. package/dist/external-findings-local.js.map +1 -1
  299. package/dist/external-findings-publish.d.ts +13 -6
  300. package/dist/external-findings-publish.d.ts.map +1 -1
  301. package/dist/external-findings-publish.js +895 -1142
  302. package/dist/external-findings-publish.js.map +1 -1
  303. package/dist/external-findings-read.d.ts +21 -2
  304. package/dist/external-findings-read.d.ts.map +1 -1
  305. package/dist/external-findings-read.js +130 -68
  306. package/dist/external-findings-read.js.map +1 -1
  307. package/dist/external-findings.d.ts +15 -3
  308. package/dist/external-findings.d.ts.map +1 -1
  309. package/dist/external-findings.js +54 -2
  310. package/dist/external-findings.js.map +1 -1
  311. package/dist/external-import-contract.d.ts +8 -12
  312. package/dist/external-import-contract.d.ts.map +1 -1
  313. package/dist/external-import-contract.js +51 -184
  314. package/dist/external-import-contract.js.map +1 -1
  315. package/dist/external-import-models.d.ts +1 -1
  316. package/dist/external-import-models.d.ts.map +1 -1
  317. package/dist/external-import-models.js.map +1 -1
  318. package/dist/feedback.d.ts.map +1 -1
  319. package/dist/feedback.js +5 -4
  320. package/dist/feedback.js.map +1 -1
  321. package/dist/finding-prompt.d.ts +5 -0
  322. package/dist/finding-prompt.d.ts.map +1 -0
  323. package/dist/finding-prompt.js +32 -0
  324. package/dist/finding-prompt.js.map +1 -0
  325. package/dist/findings-client.d.ts +1 -0
  326. package/dist/findings-client.d.ts.map +1 -1
  327. package/dist/findings-client.js +27 -3
  328. package/dist/findings-client.js.map +1 -1
  329. package/dist/github-advisories.d.ts.map +1 -1
  330. package/dist/github-advisories.js +0 -2
  331. package/dist/github-advisories.js.map +1 -1
  332. package/dist/github-dependabot-findings.d.ts.map +1 -1
  333. package/dist/github-dependabot-findings.js +18 -8
  334. package/dist/github-dependabot-findings.js.map +1 -1
  335. package/dist/github-findings.d.ts +4 -1
  336. package/dist/github-findings.d.ts.map +1 -1
  337. package/dist/github-findings.js +73 -316
  338. package/dist/github-findings.js.map +1 -1
  339. package/dist/hosted-git-remotes.d.ts +5 -0
  340. package/dist/hosted-git-remotes.d.ts.map +1 -0
  341. package/dist/hosted-git-remotes.js +285 -0
  342. package/dist/hosted-git-remotes.js.map +1 -0
  343. package/dist/hosted-git-repository.d.ts +28 -0
  344. package/dist/hosted-git-repository.d.ts.map +1 -0
  345. package/dist/hosted-git-repository.js +222 -0
  346. package/dist/hosted-git-repository.js.map +1 -0
  347. package/dist/hosted-remediation.d.ts +1 -0
  348. package/dist/hosted-remediation.d.ts.map +1 -1
  349. package/dist/hosted-remediation.js +23 -3
  350. package/dist/hosted-remediation.js.map +1 -1
  351. package/dist/hosted-scan-protocol.js +2 -2
  352. package/dist/hosted-scan-protocol.js.map +1 -1
  353. package/dist/hosted-scan.d.ts.map +1 -1
  354. package/dist/hosted-scan.js +16 -11
  355. package/dist/hosted-scan.js.map +1 -1
  356. package/dist/hosted-scanner-http.d.ts +21 -0
  357. package/dist/hosted-scanner-http.d.ts.map +1 -0
  358. package/dist/hosted-scanner-http.js +204 -0
  359. package/dist/hosted-scanner-http.js.map +1 -0
  360. package/dist/import-scan.d.ts.map +1 -1
  361. package/dist/import-scan.js +3 -0
  362. package/dist/import-scan.js.map +1 -1
  363. package/dist/index.d.ts +21 -9
  364. package/dist/index.d.ts.map +1 -1
  365. package/dist/index.js +12 -3
  366. package/dist/index.js.map +1 -1
  367. package/dist/inspector-findings.d.ts.map +1 -1
  368. package/dist/inspector-findings.js +52 -86
  369. package/dist/inspector-findings.js.map +1 -1
  370. package/dist/knowledge-base.d.ts +1 -1
  371. package/dist/knowledge-base.d.ts.map +1 -1
  372. package/dist/knowledge-base.js +102 -32
  373. package/dist/knowledge-base.js.map +1 -1
  374. package/dist/linear.d.ts +11 -0
  375. package/dist/linear.d.ts.map +1 -1
  376. package/dist/linear.js.map +1 -1
  377. package/dist/local-git-metadata.d.ts +5 -2
  378. package/dist/local-git-metadata.d.ts.map +1 -1
  379. package/dist/local-git-metadata.js +8 -5
  380. package/dist/local-git-metadata.js.map +1 -1
  381. package/dist/multiscan.d.ts +10 -4
  382. package/dist/multiscan.d.ts.map +1 -1
  383. package/dist/multiscan.js +217 -102
  384. package/dist/multiscan.js.map +1 -1
  385. package/dist/native-json-findings.d.ts +2 -1
  386. package/dist/native-json-findings.d.ts.map +1 -1
  387. package/dist/native-json-findings.js +12 -2
  388. package/dist/native-json-findings.js.map +1 -1
  389. package/dist/opengrep-findings.d.ts +5 -2
  390. package/dist/opengrep-findings.d.ts.map +1 -1
  391. package/dist/opengrep-findings.js +54 -33
  392. package/dist/opengrep-findings.js.map +1 -1
  393. package/dist/operation-telemetry.d.ts +104 -0
  394. package/dist/operation-telemetry.d.ts.map +1 -0
  395. package/dist/operation-telemetry.js +293 -0
  396. package/dist/operation-telemetry.js.map +1 -0
  397. package/dist/plugin-python-probe.d.ts +4 -0
  398. package/dist/plugin-python-probe.d.ts.map +1 -0
  399. package/dist/plugin-python-probe.js +10 -0
  400. package/dist/plugin-python-probe.js.map +1 -0
  401. package/dist/private-finding-import.d.ts +99 -9
  402. package/dist/private-finding-import.d.ts.map +1 -1
  403. package/dist/private-finding-import.js +116 -152
  404. package/dist/private-finding-import.js.map +1 -1
  405. package/dist/project-config.d.ts.map +1 -1
  406. package/dist/project-config.js +53 -13
  407. package/dist/project-config.js.map +1 -1
  408. package/dist/prompt-files.d.ts +7 -2
  409. package/dist/prompt-files.d.ts.map +1 -1
  410. package/dist/prompt-files.js +42 -20
  411. package/dist/prompt-files.js.map +1 -1
  412. package/dist/provider-profile.d.ts +1 -1
  413. package/dist/provider-profile.d.ts.map +1 -1
  414. package/dist/provider-profile.js +12 -10
  415. package/dist/provider-profile.js.map +1 -1
  416. package/dist/publication-telemetry.d.ts +45 -0
  417. package/dist/publication-telemetry.d.ts.map +1 -0
  418. package/dist/publication-telemetry.js +50 -0
  419. package/dist/publication-telemetry.js.map +1 -0
  420. package/dist/publication.d.ts +1 -1
  421. package/dist/publication.d.ts.map +1 -1
  422. package/dist/publication.js.map +1 -1
  423. package/dist/publish.d.ts +4 -0
  424. package/dist/publish.d.ts.map +1 -1
  425. package/dist/publish.js +42 -2
  426. package/dist/publish.js.map +1 -1
  427. package/dist/record.d.ts +1 -5
  428. package/dist/record.d.ts.map +1 -1
  429. package/dist/record.js +3 -51
  430. package/dist/record.js.map +1 -1
  431. package/dist/remediation.d.ts +1 -4
  432. package/dist/remediation.d.ts.map +1 -1
  433. package/dist/remediation.js +6 -8
  434. package/dist/remediation.js.map +1 -1
  435. package/dist/request-metadata.d.ts +3 -0
  436. package/dist/request-metadata.d.ts.map +1 -1
  437. package/dist/request-metadata.js +9 -0
  438. package/dist/request-metadata.js.map +1 -1
  439. package/dist/result.d.ts +4 -0
  440. package/dist/result.d.ts.map +1 -1
  441. package/dist/result.js +1 -1
  442. package/dist/result.js.map +1 -1
  443. package/dist/retry-windows-rename.d.ts +8 -0
  444. package/dist/retry-windows-rename.d.ts.map +1 -0
  445. package/dist/retry-windows-rename.js +24 -0
  446. package/dist/retry-windows-rename.js.map +1 -0
  447. package/dist/runtime.d.ts +54 -6
  448. package/dist/runtime.d.ts.map +1 -1
  449. package/dist/runtime.js +1084 -256
  450. package/dist/runtime.js.map +1 -1
  451. package/dist/saved-scan-bootstrap.d.ts +7 -1
  452. package/dist/saved-scan-bootstrap.d.ts.map +1 -1
  453. package/dist/saved-scan-bootstrap.js +269 -114
  454. package/dist/saved-scan-bootstrap.js.map +1 -1
  455. package/dist/scan-activity.d.ts.map +1 -1
  456. package/dist/scan-activity.js +141 -56
  457. package/dist/scan-activity.js.map +1 -1
  458. package/dist/scan-archive-path.d.ts +2 -0
  459. package/dist/scan-archive-path.d.ts.map +1 -0
  460. package/dist/scan-archive-path.js +5 -0
  461. package/dist/scan-archive-path.js.map +1 -0
  462. package/dist/scan-comparison.d.ts +10 -5
  463. package/dist/scan-comparison.d.ts.map +1 -1
  464. package/dist/scan-comparison.js +66 -17
  465. package/dist/scan-comparison.js.map +1 -1
  466. package/dist/scan-dashboard.d.ts +7 -6
  467. package/dist/scan-dashboard.d.ts.map +1 -1
  468. package/dist/scan-dashboard.js +92 -43
  469. package/dist/scan-dashboard.js.map +1 -1
  470. package/dist/scan-executor.d.ts +10 -1
  471. package/dist/scan-executor.d.ts.map +1 -1
  472. package/dist/scan-executor.js +24 -2
  473. package/dist/scan-executor.js.map +1 -1
  474. package/dist/scan-history-renderer.d.ts +1 -0
  475. package/dist/scan-history-renderer.d.ts.map +1 -1
  476. package/dist/scan-history-renderer.js +34 -28
  477. package/dist/scan-history-renderer.js.map +1 -1
  478. package/dist/scan-logs.d.ts +12 -0
  479. package/dist/scan-logs.d.ts.map +1 -1
  480. package/dist/scan-logs.js +131 -78
  481. package/dist/scan-logs.js.map +1 -1
  482. package/dist/scan-measurement.d.ts +26 -0
  483. package/dist/scan-measurement.d.ts.map +1 -0
  484. package/dist/scan-measurement.js +77 -0
  485. package/dist/scan-measurement.js.map +1 -0
  486. package/dist/scan-progress.d.ts +1 -0
  487. package/dist/scan-progress.d.ts.map +1 -1
  488. package/dist/scan-progress.js +3 -0
  489. package/dist/scan-progress.js.map +1 -1
  490. package/dist/scan-sessions.d.ts.map +1 -1
  491. package/dist/scan-sessions.js +4 -1
  492. package/dist/scan-sessions.js.map +1 -1
  493. package/dist/scan-settings.d.ts +1 -1
  494. package/dist/scan-settings.d.ts.map +1 -1
  495. package/dist/scan-settings.js +1 -24
  496. package/dist/scan-settings.js.map +1 -1
  497. package/dist/scanner-import.d.ts +25 -13
  498. package/dist/scanner-import.d.ts.map +1 -1
  499. package/dist/scanner-import.js +216 -97
  500. package/dist/scanner-import.js.map +1 -1
  501. package/dist/scanner-repository-url.d.ts +4 -1
  502. package/dist/scanner-repository-url.d.ts.map +1 -1
  503. package/dist/scanner-repository-url.js +10 -6
  504. package/dist/scanner-repository-url.js.map +1 -1
  505. package/dist/security-policy.d.ts +1 -1
  506. package/dist/security-policy.d.ts.map +1 -1
  507. package/dist/security-policy.js +26 -6
  508. package/dist/security-policy.js.map +1 -1
  509. package/dist/semgrep-findings.d.ts +6 -3
  510. package/dist/semgrep-findings.d.ts.map +1 -1
  511. package/dist/semgrep-findings.js +72 -42
  512. package/dist/semgrep-findings.js.map +1 -1
  513. package/dist/semgrep-guardian-findings.d.ts +38 -0
  514. package/dist/semgrep-guardian-findings.d.ts.map +1 -0
  515. package/dist/semgrep-guardian-findings.js +165 -0
  516. package/dist/semgrep-guardian-findings.js.map +1 -0
  517. package/dist/semgrep-platform-collector.d.ts +26 -0
  518. package/dist/semgrep-platform-collector.d.ts.map +1 -0
  519. package/dist/semgrep-platform-collector.js +108 -0
  520. package/dist/semgrep-platform-collector.js.map +1 -0
  521. package/dist/semgrep-platform-findings.d.ts +157 -6
  522. package/dist/semgrep-platform-findings.d.ts.map +1 -1
  523. package/dist/semgrep-platform-findings.js +766 -103
  524. package/dist/semgrep-platform-findings.js.map +1 -1
  525. package/dist/server/api.d.ts +1 -1
  526. package/dist/server/api.d.ts.map +1 -1
  527. package/dist/server/api.js.map +1 -1
  528. package/dist/server/embedding-tokenizer.cdx.json +33 -0
  529. package/dist/server/embedding-tokenizer.d.ts +2 -0
  530. package/dist/server/embedding-tokenizer.d.ts.map +1 -0
  531. package/dist/server/embedding-tokenizer.js +322 -0
  532. package/dist/server/embedding-tokenizer.js.map +6 -0
  533. package/dist/server/embeddings.d.ts +16 -2
  534. package/dist/server/embeddings.d.ts.map +1 -1
  535. package/dist/server/embeddings.js +35 -5
  536. package/dist/server/embeddings.js.map +1 -1
  537. package/dist/session-lines.d.ts +5 -0
  538. package/dist/session-lines.d.ts.map +1 -0
  539. package/dist/session-lines.js +60 -0
  540. package/dist/session-lines.js.map +1 -0
  541. package/dist/severity-review.d.ts +3 -1
  542. package/dist/severity-review.d.ts.map +1 -1
  543. package/dist/severity-review.js +3 -2
  544. package/dist/severity-review.js.map +1 -1
  545. package/dist/skill-authentication.d.ts +1 -0
  546. package/dist/skill-authentication.d.ts.map +1 -1
  547. package/dist/skill-authentication.js +1 -0
  548. package/dist/skill-authentication.js.map +1 -1
  549. package/dist/snyk-findings.d.ts +9 -2
  550. package/dist/snyk-findings.d.ts.map +1 -1
  551. package/dist/snyk-findings.js +90 -70
  552. package/dist/snyk-findings.js.map +1 -1
  553. package/dist/snyk-rest-findings.d.ts +59 -0
  554. package/dist/snyk-rest-findings.d.ts.map +1 -0
  555. package/dist/snyk-rest-findings.js +879 -0
  556. package/dist/snyk-rest-findings.js.map +1 -0
  557. package/dist/snyk-token-origin.d.ts +4 -0
  558. package/dist/snyk-token-origin.d.ts.map +1 -0
  559. package/dist/snyk-token-origin.js +82 -0
  560. package/dist/snyk-token-origin.js.map +1 -0
  561. package/dist/sonar-collector.d.ts.map +1 -1
  562. package/dist/sonar-collector.js +0 -3
  563. package/dist/sonar-collector.js.map +1 -1
  564. package/dist/sonar-findings.d.ts +1 -1
  565. package/dist/sonar-findings.d.ts.map +1 -1
  566. package/dist/sonar-findings.js +2 -4
  567. package/dist/sonar-findings.js.map +1 -1
  568. package/dist/sonar-import.d.ts +3 -0
  569. package/dist/sonar-import.d.ts.map +1 -1
  570. package/dist/sonar-import.js +30 -34
  571. package/dist/sonar-import.js.map +1 -1
  572. package/dist/stored-finding-projection.d.ts +6 -0
  573. package/dist/stored-finding-projection.d.ts.map +1 -0
  574. package/dist/stored-finding-projection.js +104 -0
  575. package/dist/stored-finding-projection.js.map +1 -0
  576. package/dist/stored-findings-publication.d.ts +23 -0
  577. package/dist/stored-findings-publication.d.ts.map +1 -0
  578. package/dist/stored-findings-publication.js +169 -0
  579. package/dist/stored-findings-publication.js.map +1 -0
  580. package/dist/target-drift.d.ts +9 -0
  581. package/dist/target-drift.d.ts.map +1 -0
  582. package/dist/target-drift.js +27 -0
  583. package/dist/target-drift.js.map +1 -0
  584. package/dist/targets.d.ts +8 -1
  585. package/dist/targets.d.ts.map +1 -1
  586. package/dist/targets.js +131 -42
  587. package/dist/targets.js.map +1 -1
  588. package/dist/tenable-findings.d.ts +6 -0
  589. package/dist/tenable-findings.d.ts.map +1 -0
  590. package/dist/tenable-findings.js +122 -0
  591. package/dist/tenable-findings.js.map +1 -0
  592. package/dist/terminal-stream.d.ts +13 -0
  593. package/dist/terminal-stream.d.ts.map +1 -0
  594. package/dist/terminal-stream.js +35 -0
  595. package/dist/terminal-stream.js.map +1 -0
  596. package/dist/trivy-findings.d.ts +19 -0
  597. package/dist/trivy-findings.d.ts.map +1 -0
  598. package/dist/trivy-findings.js +424 -0
  599. package/dist/trivy-findings.js.map +1 -0
  600. package/dist/trivy-import.d.ts +19 -0
  601. package/dist/trivy-import.d.ts.map +1 -0
  602. package/dist/trivy-import.js +151 -0
  603. package/dist/trivy-import.js.map +1 -0
  604. package/dist/trusted-executable.d.ts +6 -0
  605. package/dist/trusted-executable.d.ts.map +1 -1
  606. package/dist/trusted-executable.js +140 -37
  607. package/dist/trusted-executable.js.map +1 -1
  608. package/dist/validation-records.js +1 -1
  609. package/dist/validation-records.js.map +1 -1
  610. package/dist/veracode-findings.d.ts +1 -1
  611. package/dist/veracode-findings.d.ts.map +1 -1
  612. package/dist/veracode-findings.js +19 -6
  613. package/dist/veracode-findings.js.map +1 -1
  614. package/dist/version.d.ts +3 -1
  615. package/dist/version.d.ts.map +1 -1
  616. package/dist/version.js +184 -32
  617. package/dist/version.js.map +1 -1
  618. package/dist/vertex-pricing.d.ts +8 -0
  619. package/dist/vertex-pricing.d.ts.map +1 -0
  620. package/dist/vertex-pricing.js +20 -0
  621. package/dist/vertex-pricing.js.map +1 -0
  622. package/dist/windows-path.d.ts +1 -0
  623. package/dist/windows-path.d.ts.map +1 -1
  624. package/dist/windows-path.js +26 -1
  625. package/dist/windows-path.js.map +1 -1
  626. package/dist/wiz-findings.d.ts +6 -2
  627. package/dist/wiz-findings.d.ts.map +1 -1
  628. package/dist/wiz-findings.js +65 -89
  629. package/dist/wiz-findings.js.map +1 -1
  630. package/docs/classify-severity-records.md +27 -12
  631. package/docs/cli.md +853 -639
  632. package/docs/dedupe-records.md +44 -24
  633. package/docs/findings-service.md +84 -44
  634. package/docs/hosted-remediation.md +27 -11
  635. package/docs/hosted-scans.md +50 -42
  636. package/docs/sdk.md +903 -0
  637. package/docs/telemetry.md +236 -0
  638. package/docs/validate-records.md +78 -44
  639. package/package.json +23 -16
  640. package/schemas/external-findings.schema.json +4 -1
  641. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  642. package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
  643. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  644. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  645. package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
  646. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
  647. package/dist/external-imports-feature.d.ts +0 -6
  648. package/dist/external-imports-feature.d.ts.map +0 -1
  649. package/dist/external-imports-feature.js +0 -51
  650. package/dist/external-imports-feature.js.map +0 -1
  651. package/dist/finding-normalization.d.ts +0 -5
  652. package/dist/finding-normalization.d.ts.map +0 -1
  653. package/dist/finding-normalization.js +0 -155
  654. package/dist/finding-normalization.js.map +0 -1
  655. package/dist/reader-selection.d.ts +0 -13
  656. package/dist/reader-selection.d.ts.map +0 -1
  657. package/dist/reader-selection.js +0 -81
  658. package/dist/reader-selection.js.map +0 -1
@@ -8,13 +8,19 @@ import math
8
8
  import os
9
9
  import re
10
10
  import sqlite3
11
+ import subprocess
11
12
  import sys
13
+ import tempfile
12
14
  import uuid
15
+ from contextlib import contextmanager
13
16
  from dataclasses import dataclass
14
17
  from datetime import datetime, timezone
15
18
  from pathlib import Path
16
19
  from typing import Any, Mapping
17
- from urllib.parse import quote
20
+
21
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
22
+ from snapshot_sqlite import readonly_sqlite_uri
23
+ from workbench.storage import archive_source_directory, worker_usage_directory
18
24
 
19
25
  TOKEN_FIELDS = {
20
26
  "input_tokens": "inputTokens",
@@ -30,6 +36,8 @@ STATE_DATABASE_TIMEOUT_SECONDS = 1.0
30
36
 
31
37
  @dataclass(frozen=True)
32
38
  class RolloutSession:
39
+ """Associate a thread and its optional parent with the path of the rollout used for usage accounting."""
40
+
33
41
  thread_id: str
34
42
  parent_thread_id: str | None
35
43
  path: Path
@@ -37,6 +45,8 @@ class RolloutSession:
37
45
 
38
46
  @dataclass
39
47
  class RolloutUsage:
48
+ """Collect token counts, their update time, warnings, and account rate-limit observations for a rollout."""
49
+
40
50
  tokens: dict[str, int]
41
51
  warnings: set[str]
42
52
  tokens_updated_at: datetime | None
@@ -66,13 +76,27 @@ def reconcile_completed_scan_cost(
66
76
  ) -> None:
67
77
  """Persist authoritative SDK cost without discarding measured worker usage."""
68
78
 
69
- existing = json.loads(scan["cost_json"]) if scan["cost_json"] is not None else {}
70
- if isinstance(existing, dict) and "usage" in existing:
71
- cost_json = json.dumps(
72
- {**existing, "cost": json.loads(cost_json)},
73
- separators=(",", ":"),
74
- allow_nan=False,
75
- )
79
+ stored = stored_scan_cost_fields(scan["cost_json"])
80
+ completion = stored.get("usage", {}).get("completion")
81
+ native = collect_scan_usage_safely(
82
+ connection,
83
+ scan,
84
+ thread_id=completion["threadId"] if completion else None,
85
+ completed_turn_id=completion["turnId"] if completion and completion["finalized"] else None,
86
+ )
87
+ fields = merge_scan_usage_fields(stored, stored_scan_cost_fields(cost_json), {"usage": native})
88
+ if completion:
89
+ usage = {**fields["usage"], "completion": completion}
90
+ if not completion["finalized"]:
91
+ if usage["coverage"] == "complete":
92
+ usage["coverage"] = "partial"
93
+ usage["warnings"] = sorted({*usage.get("warnings", []), "turn_in_progress"})
94
+ fields["usage"] = usage
95
+ cost_json = json.dumps(
96
+ fields if "usage" in fields else fields["cost"],
97
+ separators=(",", ":"),
98
+ allow_nan=False,
99
+ )
76
100
  connection.execute("BEGIN IMMEDIATE")
77
101
  with connection:
78
102
  connection.execute(
@@ -81,52 +105,219 @@ def reconcile_completed_scan_cost(
81
105
  )
82
106
 
83
107
 
108
+ def merge_scan_usage_fields(*observations: Mapping[str, Any]) -> dict[str, Any]:
109
+ """Reconcile overlapping measurements without summing the same execution twice."""
110
+
111
+ merged: dict[str, Any] = {}
112
+ latest_coverage: dict[str, Any] | None = None
113
+ rate_limits: dict[str, dict[str, Any]] = {}
114
+ for observation in observations:
115
+ if isinstance(observation.get("cost"), dict):
116
+ merged["cost"] = observation["cost"]
117
+ usage = observation.get("usage")
118
+ if not isinstance(usage, dict):
119
+ continue
120
+ for snapshot in usage.get("accountRateLimits", []):
121
+ key = snapshot["limitId"]
122
+ if key not in rate_limits or snapshot["observedAt"] >= rate_limits[key]["observedAt"]:
123
+ rate_limits[key] = snapshot
124
+ previous = merged.get("usage")
125
+ if previous is None or _usage_quality(usage) >= _usage_quality(previous):
126
+ merged["usage"] = usage
127
+ if usage.get("coverage") not in ("complete", "partial"):
128
+ continue
129
+ latest_coverage = (
130
+ usage if latest_coverage is None else _merge_usage_coverage(latest_coverage, usage)
131
+ )
132
+ cost = merged.get("cost")
133
+ usage = merged.get("usage")
134
+ if usage is not None and latest_coverage is not None and latest_coverage is not usage:
135
+ usage = {**usage}
136
+ for field in (
137
+ "coverage",
138
+ "coverageScope",
139
+ "threadCount",
140
+ "missingThreadCount",
141
+ "warnings",
142
+ "observedAt",
143
+ "completion",
144
+ "byRole",
145
+ ):
146
+ if field in latest_coverage:
147
+ usage[field] = latest_coverage[field]
148
+ else:
149
+ usage.pop(field, None)
150
+ # Counts describe the coverage observation, even when its scope narrows.
151
+ # Its whole role breakdown must still sum to the retained token subtotal.
152
+ if "byRole" in usage and any(
153
+ usage.get(field) != latest_coverage.get(field) for field in TOKEN_FIELDS.values()
154
+ ):
155
+ usage.pop("byRole")
156
+ merged["usage"] = usage
157
+ cost_warning = "cost_estimate_covers_earlier_usage"
158
+ if (
159
+ cost is not None
160
+ and usage is not None
161
+ and usage.get("coverage") in ("complete", "partial")
162
+ and type(usage.get("totalTokens")) is int
163
+ ):
164
+ warnings = set(usage.get("warnings", []))
165
+ warnings.discard(cost_warning)
166
+ if usage["totalTokens"] > cost["inputTokens"] + cost["outputTokens"]:
167
+ warnings.add(cost_warning)
168
+ if warnings != set(usage.get("warnings", [])):
169
+ updated = {**usage}
170
+ updated["coverage"] = (
171
+ "partial"
172
+ if warnings - {cost_warning} or usage.get("missingThreadCount", 0)
173
+ else "complete"
174
+ )
175
+ if warnings:
176
+ updated["warnings"] = sorted(warnings)
177
+ else:
178
+ updated.pop("warnings", None)
179
+ merged["usage"] = updated
180
+ if rate_limits:
181
+ merged["usage"] = {
182
+ **merged["usage"],
183
+ "accountRateLimits": list(rate_limits.values()),
184
+ }
185
+ return merged
186
+
187
+
188
+ def _merge_usage_coverage(previous: dict[str, Any], observed: dict[str, Any]) -> dict[str, Any]:
189
+ if previous == observed:
190
+ return observed
191
+ previous_at = _timestamp(previous.get("observedAt"))
192
+ observed_at = _timestamp(observed.get("observedAt"))
193
+ if previous_at is not None and observed_at is not None and previous_at > observed_at:
194
+ previous, observed = observed, previous
195
+ previous_scope = previous.get("coverageScope")
196
+ observed_scope = observed.get("coverageScope")
197
+ covers_previous = (
198
+ previous_scope is not None
199
+ and observed_scope is not None
200
+ and set(previous_scope).issubset(observed_scope)
201
+ )
202
+ scope_gap = previous_scope is not None and observed_scope is not None and not covers_previous
203
+ unresolved_previous = (previous["coverage"] == "partial" and not covers_previous) or scope_gap
204
+ merged = {**observed}
205
+ if unresolved_previous:
206
+ merged["coverage"] = "partial"
207
+ merged["missingThreadCount"] = max(
208
+ previous.get("missingThreadCount", 0), observed.get("missingThreadCount", 0)
209
+ )
210
+ warnings = set(previous.get("warnings", [])) | set(observed.get("warnings", []))
211
+ if scope_gap:
212
+ # A newer subset can grow while still staying below the older
213
+ # aggregate. Neither subtotal proves complete usage for the union.
214
+ warnings.add("usage_scope_incomplete")
215
+ if warnings:
216
+ merged["warnings"] = sorted(warnings)
217
+ if previous.get("completion", {}).get("finalized") is False:
218
+ merged["completion"] = previous["completion"]
219
+ if previous_scope is not None and observed_scope is not None:
220
+ merged["coverageScope"] = sorted(set(previous_scope) | set(observed_scope))
221
+ elif previous_scope is not None and observed["coverage"] == "complete":
222
+ merged["coverageScope"] = previous_scope
223
+ elif unresolved_previous:
224
+ # An older subtotal without scope cannot prove which gaps a later
225
+ # collector has resolved. Retain that uncertainty across future merges.
226
+ merged.pop("coverageScope", None)
227
+ return merged
228
+
229
+
230
+ def _usage_quality(
231
+ usage: Mapping[str, Any], *, observed_at: datetime | None = None
232
+ ) -> tuple[int, bool, datetime]:
233
+ return (
234
+ usage.get("totalTokens", -1),
235
+ usage.get("cacheWriteInputTokensReported") is not False,
236
+ observed_at
237
+ or _timestamp(usage.get("observedAt"))
238
+ or datetime.min.replace(tzinfo=timezone.utc),
239
+ )
240
+
241
+
242
+ def collect_scan_usage_safely(
243
+ connection: sqlite3.Connection, scan: sqlite3.Row, **options: Any
244
+ ) -> dict[str, Any]:
245
+ try:
246
+ return collect_scan_usage(connection, scan, **options)
247
+ except (Exception, SystemExit) as exc:
248
+ try:
249
+ print(f"Could not collect scan token usage: {exc}", file=sys.stderr)
250
+ except OSError:
251
+ # A closed diagnostic stream must not replace the unavailable-usage result.
252
+ pass
253
+ return _unavailable_usage("usage_collection_failed")
254
+
255
+
84
256
  def collect_scan_usage(
85
257
  connection: sqlite3.Connection,
86
258
  scan: sqlite3.Row,
87
259
  *,
88
260
  thread_id: str | None = None,
261
+ validation_thread_id: str | None = None,
89
262
  completed_at: str | None = None,
90
263
  completed_turn_id: str | None = None,
91
264
  ) -> dict[str, Any]:
92
- """Count attributable receipts and flag observable gaps inside the scan window."""
265
+ """Count attributable session receipts during the scan window and report missing coverage.
266
+
267
+ The optional thread and completion arguments identify the finishing turn when
268
+ the scan record alone is insufficient. Return an unavailable usage record when
269
+ the window, Codex state, or relevant threads cannot be established.
270
+ """
93
271
 
94
272
  roots = _scan_root_thread_ids(connection, scan, thread_id)
95
273
  parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
96
- worker_directories = _scan_worker_directories(connection, scan)
97
- if not roots and not worker_directories:
98
- return _unavailable_usage("scan_thread_unavailable")
99
-
100
- state_database = _codex_state_database()
101
- if state_database is None:
102
- return _unavailable_usage("codex_state_unavailable")
103
-
274
+ if validation_thread_id:
275
+ roots.append(validation_thread_id)
276
+ parent_ids.add(validation_thread_id)
277
+ warnings: set[str] = set()
278
+ worker_directories = _scan_worker_directories(connection, scan, warnings)
104
279
  started_at = _timestamp(scan["started_at"])
105
280
  observed_at = datetime.now(timezone.utc)
106
281
  stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
107
282
  if started_at is None:
108
283
  return _unavailable_usage("scan_window_unavailable")
109
284
 
110
- warnings: set[str] = set()
111
- try:
112
- sessions, missing_thread_ids = _discover_rollout_sessions(
113
- state_database,
114
- roots,
115
- warnings,
116
- worker_directories=worker_directories,
117
- started_at=started_at,
118
- completed_at=stopped_at,
119
- )
120
- except (OSError, sqlite3.Error, ValueError):
121
- return _unavailable_usage("codex_state_unavailable")
285
+ receipts, missing_receipts, receipt_scope = _worker_usage_receipts(
286
+ scan, started_at, stopped_at, warnings
287
+ )
288
+ state_database = _codex_state_database()
289
+ sessions: list[RolloutSession] = []
290
+ missing_thread_ids: set[str] = set()
291
+ if state_database is None:
292
+ if not receipts:
293
+ return _unavailable_usage("codex_state_unavailable", warnings=warnings)
294
+ warnings.add("codex_state_unavailable")
295
+ missing_thread_ids.update(roots)
296
+ else:
297
+ try:
298
+ sessions, missing_thread_ids = _discover_rollout_sessions(
299
+ state_database,
300
+ roots,
301
+ warnings,
302
+ worker_directories=worker_directories,
303
+ scan_directory=Path(scan["scan_dir"]) if worker_directories else None,
304
+ started_at=started_at,
305
+ completed_at=stopped_at,
306
+ )
307
+ except (OSError, sqlite3.Error, ValueError):
308
+ if not receipts:
309
+ return _unavailable_usage("codex_state_unavailable", warnings=warnings)
310
+ warnings.add("codex_state_unavailable")
311
+ missing_thread_ids.update(roots)
122
312
 
123
- if not sessions:
313
+ if not sessions and not receipts:
124
314
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
125
315
 
126
- by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
127
316
  rate_limits: dict[str, dict[str, Any]] = {}
128
317
  tokens_updated_at = None
129
318
  accepted_thread_ids: set[str] = set()
319
+ measured_threads: dict[str, dict[str, int]] = {}
320
+ complete_native_thread_ids: set[str] = set()
130
321
  excluded_thread_ids: set[str] = set()
131
322
  for session in sessions:
132
323
  if session.parent_thread_id in excluded_thread_ids:
@@ -145,6 +336,12 @@ def collect_scan_usage(
145
336
  started_at=started_at,
146
337
  completed_at=stopped_at,
147
338
  completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
339
+ resume_owner=(
340
+ scan["mode"] == "standard"
341
+ and "recipe_json" in scan.keys()
342
+ and scan["recipe_json"] is not None
343
+ and session.thread_id == scan["continuation_thread_id"]
344
+ ),
148
345
  is_scan_owner=(
149
346
  scan["mode"] == "deep"
150
347
  and session.thread_id == scan["deep_scan_owner_thread_id"]
@@ -167,25 +364,60 @@ def collect_scan_usage(
167
364
  for key, snapshot in usage.rate_limits.items():
168
365
  if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
169
366
  rate_limits[key] = snapshot
170
- if "token_usage_unavailable" in usage.warnings:
367
+ if usage.warnings & {"token_usage_unavailable", "token_breakdown_unavailable"}:
171
368
  missing_thread_ids.add(session.thread_id)
172
369
  continue
173
- role = by_role["parent" if session.thread_id in parent_ids else "workers"]
174
- _add_token_usage(role, usage.tokens)
175
- role["threadCount"] += 1
370
+ measured_threads[session.thread_id] = usage.tokens
371
+ if not usage.warnings:
372
+ complete_native_thread_ids.add(session.thread_id)
176
373
  if usage.tokens_updated_at is not None:
177
374
  tokens_updated_at = max(
178
375
  tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
179
376
  )
180
377
 
378
+ runtime_receipts_used = False
379
+ for receipt_thread_id, receipt in receipts.items():
380
+ native = measured_threads.get(receipt_thread_id)
381
+ extends_native = native is None or receipt.tokens["totalTokens"] > native["totalTokens"]
382
+ fills_cache_write_usage = (
383
+ native is not None
384
+ and receipt.tokens["totalTokens"] == native["totalTokens"]
385
+ and native.get("cacheWriteInputTokensReported") is False
386
+ and receipt.tokens.get("cacheWriteInputTokensReported") is not False
387
+ )
388
+ if extends_native or fills_cache_write_usage:
389
+ measured_threads[receipt_thread_id] = receipt.tokens
390
+ accepted_thread_ids.add(receipt_thread_id)
391
+ runtime_receipts_used = True
392
+ if extends_native:
393
+ complete_native_thread_ids.discard(receipt_thread_id)
394
+ # Numeric SDK subtotals cannot establish that descendants and
395
+ # compaction requests are present when their rollouts are absent.
396
+ missing_thread_ids.add(receipt_thread_id)
397
+ warnings.add("worker_rollout_usage_incomplete")
398
+ if receipt.tokens_updated_at is not None:
399
+ tokens_updated_at = max(
400
+ tokens_updated_at or receipt.tokens_updated_at, receipt.tokens_updated_at
401
+ )
402
+ unresolved_receipts = missing_receipts - complete_native_thread_ids
403
+ if unresolved_receipts:
404
+ missing_thread_ids.update(unresolved_receipts)
405
+ warnings.add("worker_usage_unavailable")
181
406
  if not accepted_thread_ids:
182
407
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
183
- thread_count = sum(role["threadCount"] for role in by_role.values())
184
- if not thread_count:
185
- return _unavailable_usage(
186
- "token_usage_unavailable", warnings=warnings, rate_limits=rate_limits
408
+ if not measured_threads:
409
+ reason = (
410
+ "token_breakdown_unavailable"
411
+ if "token_breakdown_unavailable" in warnings
412
+ else "token_usage_unavailable"
187
413
  )
414
+ return _unavailable_usage(reason, warnings=warnings, rate_limits=rate_limits)
188
415
 
416
+ by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
417
+ for measured_id, tokens in measured_threads.items():
418
+ role = by_role["parent" if measured_id in parent_ids else "workers"]
419
+ _add_token_usage(role, tokens)
420
+ role["threadCount"] += 1
189
421
  if any(
190
422
  usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
191
423
  or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
@@ -195,18 +427,28 @@ def collect_scan_usage(
195
427
  "token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
196
428
  )
197
429
 
198
- if completed_turn_id is not None and tokens_updated_at is not None:
430
+ if (completed_turn_id is not None or runtime_receipts_used) and tokens_updated_at is not None:
199
431
  stopped_at = max(stopped_at, tokens_updated_at)
200
432
  result: dict[str, Any] = {
201
433
  "coverage": "partial" if missing_thread_ids or warnings else "complete",
202
- "source": "codex_rollout",
434
+ "source": "codex_runtime" if runtime_receipts_used else "codex_rollout",
203
435
  **{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
204
- "threadCount": thread_count,
436
+ "threadCount": len(measured_threads),
437
+ "coverageScope": sorted(
438
+ receipt_scope
439
+ | {f"thread:{identity}" for identity in measured_threads}
440
+ | {
441
+ identity if identity.startswith("worker-attempt:") else f"thread:{identity}"
442
+ for identity in missing_thread_ids
443
+ }
444
+ ),
205
445
  "byRole": by_role,
206
446
  "window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
207
447
  "observedAt": observed_at.isoformat(),
208
448
  "accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
209
449
  }
450
+ if any(role.get("cacheWriteInputTokensReported") is False for role in by_role.values()):
451
+ result["cacheWriteInputTokensReported"] = False
210
452
  if tokens_updated_at is not None:
211
453
  result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
212
454
  if missing_thread_ids:
@@ -216,6 +458,79 @@ def collect_scan_usage(
216
458
  return result
217
459
 
218
460
 
461
+ def _worker_usage_receipts(
462
+ scan: Mapping[str, Any],
463
+ started_at: datetime,
464
+ completed_at: datetime,
465
+ warnings: set[str],
466
+ ) -> tuple[dict[str, RolloutUsage], set[str], set[str]]:
467
+ if scan["mode"] != "deep":
468
+ return {}, set(), set()
469
+ directory = worker_usage_directory(scan["id"])
470
+ try:
471
+ if directory.resolve(strict=True) != directory:
472
+ raise ValueError("Worker usage directory is not a canonical protected directory.")
473
+ except FileNotFoundError:
474
+ return {}, set(), set()
475
+ except (OSError, RuntimeError, ValueError):
476
+ warnings.add("worker_usage_receipt_unavailable")
477
+ return {}, {"worker-receipts"}, {"worker-receipts"}
478
+ receipts: dict[str, RolloutUsage] = {}
479
+ missing: set[str] = set()
480
+ attempts: set[str] = set()
481
+ scope: set[str] = set()
482
+ for path in sorted(directory.glob("*.json")):
483
+ try:
484
+ if _rollout_path(str(path)) is None:
485
+ raise ValueError("Worker usage receipt path is unavailable.")
486
+ receipt = json.loads(path.read_text(encoding="utf-8"))
487
+ if (
488
+ not isinstance(receipt, dict)
489
+ or receipt.get("schemaVersion") != 1
490
+ or receipt.get("source") != "codex_sdk"
491
+ or receipt.get("usageScope") != "thread_cumulative"
492
+ or receipt.get("scanId") != scan["id"]
493
+ or receipt.get("kind") not in ("discovery", "dedup", "compare")
494
+ or receipt.get("attemptId") != path.stem
495
+ ):
496
+ raise ValueError("Worker usage receipt identity is invalid.")
497
+ attempt_id = receipt["attemptId"]
498
+ begin = _timestamp(receipt.get("startedAt"))
499
+ observed = _timestamp(receipt.get("observedAt"))
500
+ if begin is None or observed is None:
501
+ raise ValueError("Worker usage receipt timestamps are invalid.")
502
+ if begin < started_at or begin > completed_at or attempt_id in attempts:
503
+ continue
504
+ attempts.add(attempt_id)
505
+ scope.add(f"worker-attempt:{attempt_id}")
506
+ thread_id = receipt.get("threadId")
507
+ if isinstance(thread_id, str) and thread_id:
508
+ scope.add(f"thread:{thread_id}")
509
+ tokens = _usage_counts(receipt.get("usage"))
510
+ if not isinstance(thread_id, str) or not thread_id or tokens is None:
511
+ missing.add(
512
+ thread_id
513
+ if isinstance(thread_id, str) and thread_id
514
+ else f"worker-attempt:{attempt_id}"
515
+ )
516
+ continue
517
+ previous = receipts.get(thread_id)
518
+ if previous is None or _usage_quality(tokens, observed_at=observed) > _usage_quality(
519
+ previous.tokens, observed_at=previous.tokens_updated_at
520
+ ):
521
+ receipts[thread_id] = RolloutUsage(tokens, set(), observed, {})
522
+ except (OSError, UnicodeError, ValueError) as exc:
523
+ warnings.add("worker_usage_receipt_unavailable")
524
+ missing.add(f"worker-attempt:{path.stem}")
525
+ scope.add(f"worker-attempt:{path.stem}")
526
+ try:
527
+ print(f"Could not read worker token usage: {exc}", file=sys.stderr)
528
+ except OSError:
529
+ # The receipt warning and missing-attempt scope are already retained.
530
+ pass
531
+ return receipts, missing, scope
532
+
533
+
219
534
  def _scan_root_thread_ids(
220
535
  connection: sqlite3.Connection,
221
536
  scan: sqlite3.Row,
@@ -266,25 +581,41 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
266
581
  )
267
582
 
268
583
 
269
- def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]:
584
+ def _scan_worker_directories(
585
+ connection: sqlite3.Connection, scan: sqlite3.Row, warnings: set[str]
586
+ ) -> dict[str, bool]:
270
587
  if scan["mode"] != "deep":
271
- return set()
272
- return {
273
- row["artifact_dir"]
274
- if row["kind"] == "discovery"
275
- else str(Path(scan["scan_dir"]) / "artifacts")
276
- for row in connection.execute(
277
- "SELECT kind, artifact_dir FROM deep_scan_workers "
278
- "WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
279
- (scan["id"],),
280
- )
281
- }
588
+ return {}
589
+ if connection.execute(
590
+ "SELECT 1 FROM deep_scan_runs WHERE scan_id = ? AND worker_usage_gap = 1", (scan["id"],)
591
+ ).fetchone():
592
+ warnings.add("worker_thread_unavailable")
593
+ artifacts = Path(scan["scan_dir"]) / "artifacts"
594
+ # Comparison turns run in this directory without a discovery worker row.
595
+ directories = {str(artifacts / "deep_discovery"): False}
596
+ for row in connection.execute(
597
+ "SELECT kind, artifact_dir, started_at, sdk_thread_id, attempt FROM deep_scan_workers "
598
+ "WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
599
+ (scan["id"],),
600
+ ):
601
+ if row["kind"] == "discovery":
602
+ if row["sdk_thread_id"] and row["attempt"] == 1:
603
+ continue # The first attempt already has an exact thread identity.
604
+ # An attempt that started without a saved ID must be found by its assigned cwd.
605
+ missing_thread = bool(row["started_at"] and not row["sdk_thread_id"])
606
+ directories[row["artifact_dir"]] = missing_thread
607
+ else:
608
+ directories[str(artifacts)] = False
609
+ if row["attempt"] > 1 and row["started_at"] and not row["sdk_thread_id"]:
610
+ # Retries share a cwd. A rollout from an older attempt cannot prove the new one.
611
+ warnings.add("worker_thread_unavailable")
612
+ return directories
282
613
 
283
614
 
284
615
  def _codex_state_database() -> Path | None:
285
- configured_database = os.environ.get("CODEX_STATE_DB", "").strip()
286
- if configured_database:
287
- path = Path(configured_database).expanduser()
616
+ configured_database = os.environ.get("CODEX_STATE_DB", "")
617
+ if configured_database.strip():
618
+ path = Path(os.path.expanduser(configured_database))
288
619
  return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
289
620
 
290
621
  configured_home = os.environ.get("CODEX_HOME", "")
@@ -323,12 +654,13 @@ def _discover_rollout_sessions(
323
654
  roots: list[str],
324
655
  warnings: set[str],
325
656
  *,
326
- worker_directories: set[str],
657
+ worker_directories: dict[str, bool],
658
+ scan_directory: Path | None = None,
327
659
  started_at: datetime,
328
660
  completed_at: datetime | None,
329
661
  ) -> tuple[list[RolloutSession], set[str]]:
330
662
  database = sqlite3.connect(
331
- f"file:{quote(str(state_database), safe='')}?mode=ro",
663
+ readonly_sqlite_uri(state_database),
332
664
  uri=True,
333
665
  timeout=STATE_DATABASE_TIMEOUT_SECONDS,
334
666
  )
@@ -342,9 +674,12 @@ def _discover_rollout_sessions(
342
674
  columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
343
675
  if not required.issubset(columns):
344
676
  raise ValueError("Codex state graph does not expose the required thread columns.")
677
+ if table == "threads" and worker_directories and not {"cwd", "created_at"} <= columns:
678
+ worker_directories = {}
679
+ warnings.add("worker_directories_unavailable")
345
680
  sessions: dict[str, RolloutSession] = {}
346
681
  missing_thread_ids: set[str] = set()
347
- root_ids = dict.fromkeys(roots)
682
+ root_ids: dict[str, str | None] = dict.fromkeys(roots)
348
683
  started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
349
684
  completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
350
685
  directory_expression = "cwd"
@@ -355,16 +690,54 @@ def _discover_rollout_sessions(
355
690
  "codex_security_path_key", 1, _normalized_windows_path, deterministic=True
356
691
  )
357
692
  directory_expression = "codex_security_path_key(cwd)"
358
- worker_directories = {_normalized_windows_path(value) for value in worker_directories}
359
- for directory in sorted(worker_directories):
693
+ worker_directories = {
694
+ _normalized_windows_path(path): required
695
+ for path, required in worker_directories.items()
696
+ }
697
+ if scan_directory is not None:
698
+ scan_directory = Path(_normalized_windows_path(scan_directory))
699
+ unmatched = {path for path, required in worker_directories.items() if required}
700
+ directory_sources = {path: {path} for path in worker_directories}
701
+ if worker_directories and scan_directory and ".previous-" in scan_directory.name:
702
+ # Archived files move, but session working directories describe where execution ran.
703
+ # Confirm the original output with an already-bound session before including peers.
704
+ bound_directories = {
705
+ Path(row[0])
706
+ for row in database.execute(
707
+ f"SELECT DISTINCT {directory_expression} FROM threads WHERE cwd IS NOT NULL "
708
+ "AND id IN (SELECT value FROM json_each(?))",
709
+ (json.dumps(roots),),
710
+ )
711
+ }
712
+ relatives = {
713
+ Path(path).relative_to(scan_directory): path
714
+ for path in worker_directories
715
+ if Path(path).is_relative_to(scan_directory)
716
+ }
717
+ originals = {
718
+ original
719
+ for path in bound_directories | {Path(path) for path in worker_directories}
720
+ if (original := archive_source_directory(scan_directory, path)) is not None
721
+ }
722
+ for original in originals:
723
+ candidates = {original / relative: path for relative, path in relatives.items()}
724
+ # Older archives kept absolute discovery paths in their original directory.
725
+ if (
726
+ original in bound_directories
727
+ or candidates.keys() & bound_directories
728
+ or any(Path(path).is_relative_to(original) for path in worker_directories)
729
+ ):
730
+ for candidate, path in candidates.items():
731
+ directory_sources.setdefault(str(candidate), set()).add(path)
732
+ for directory in sorted(directory_sources):
360
733
  for row in database.execute(
361
734
  "SELECT id FROM threads WHERE cwd IS NOT NULL "
362
735
  f"AND {directory_expression} = ? AND created_at >= ? "
363
736
  "AND (? IS NULL OR created_at <= ?) ORDER BY id",
364
737
  (directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
365
738
  ):
366
- root_ids[row["id"]] = None
367
- for root in root_ids:
739
+ root_ids[row["id"]] = directory
740
+ for root, directory in root_ids.items():
368
741
  row = database.execute(
369
742
  "SELECT id, rollout_path FROM threads WHERE id = ?",
370
743
  (root,),
@@ -380,8 +753,8 @@ def _discover_rollout_sessions(
380
753
  continue
381
754
  if root not in roots:
382
755
  try:
383
- with path.open("rb") as source:
384
- metadata = json.loads(source.readline())
756
+ with _rollout_lines(path) as source:
757
+ metadata = json.loads(next(source, b""))
385
758
  except (OSError, ValueError):
386
759
  missing_thread_ids.add(root)
387
760
  warnings.add("rollout_unavailable")
@@ -392,9 +765,15 @@ def _discover_rollout_sessions(
392
765
  and isinstance(payload := metadata.get("payload"), dict)
393
766
  and (payload.get("id") or payload.get("session_id")) == root
394
767
  and (created_at := _timestamp(payload.get("timestamp"))) is not None
395
- and created_at < started_millisecond
768
+ and (
769
+ created_at < started_millisecond
770
+ or completed_at is not None
771
+ and created_at > completed_at
772
+ )
396
773
  ):
397
774
  continue
775
+ if directory is not None:
776
+ unmatched.difference_update(directory_sources[directory])
398
777
  sessions[root] = RolloutSession(root, None, path)
399
778
  descendants = database.execute(
400
779
  """
@@ -452,6 +831,8 @@ def _discover_rollout_sessions(
452
831
  warnings.add("rollout_unavailable")
453
832
  continue
454
833
  sessions[child_id] = RolloutSession(child_id, parent_id, path)
834
+ if unmatched:
835
+ warnings.add("worker_thread_unavailable")
455
836
  return list(sessions.values()), missing_thread_ids
456
837
  finally:
457
838
  database.close()
@@ -464,6 +845,13 @@ def _rollout_path(value: object) -> Path | None:
464
845
  if not candidate.is_absolute():
465
846
  return None
466
847
  try:
848
+ try:
849
+ candidate.lstat()
850
+ except FileNotFoundError:
851
+ # Native cold storage may compress an indexed JSONL file in place.
852
+ if candidate.suffix != ".jsonl":
853
+ return None
854
+ candidate = candidate.with_suffix(".jsonl.zst")
467
855
  resolved = candidate.resolve(strict=True)
468
856
  if not resolved.is_file():
469
857
  return None
@@ -480,6 +868,63 @@ def _rollout_path(value: object) -> Path | None:
480
868
  return None
481
869
 
482
870
 
871
+ @contextmanager
872
+ def _rollout_lines(path: Path, *, verify_end: bool = False):
873
+ if path.suffix != ".zst":
874
+ with path.open("rb") as source:
875
+ yield source
876
+ return
877
+ try:
878
+ from compression import zstd # Python 3.14+
879
+ except ImportError:
880
+ try:
881
+ from backports import zstd
882
+ except ImportError:
883
+ zstd = None
884
+ if zstd is not None:
885
+ try:
886
+ with zstd.open(path, "rb") as source:
887
+ yield source
888
+ # A requested completion turn may stop parsing before the frame checksum.
889
+ if verify_end:
890
+ while source.read(64 * 1024):
891
+ pass
892
+ except (zstd.ZstdError, EOFError) as error:
893
+ raise ValueError("Invalid compressed Codex rollout.") from error
894
+ return
895
+ with tempfile.TemporaryFile() as errors:
896
+ with subprocess.Popen(
897
+ [
898
+ os.environ.get("CODEX_MCP_NODE_PATH") or "node",
899
+ str(Path(__file__).with_name("read_codex_session.mjs")),
900
+ str(path),
901
+ ],
902
+ stdout=subprocess.PIPE,
903
+ stderr=errors,
904
+ ) as child:
905
+ exhausted = False
906
+
907
+ def lines():
908
+ nonlocal exhausted
909
+ yield from child.stdout
910
+ exhausted = True
911
+
912
+ try:
913
+ iterator = lines()
914
+ yield iterator
915
+ if verify_end:
916
+ for _ in iterator:
917
+ pass
918
+ finally:
919
+ if not exhausted and child.poll() is None:
920
+ child.terminate()
921
+ child.stdout.close()
922
+ status = child.wait()
923
+ if exhausted and status != 0:
924
+ errors.seek(0)
925
+ raise OSError(errors.read().decode("utf-8", errors="replace"))
926
+
927
+
483
928
  def _read_rollout_usage(
484
929
  session: RolloutSession,
485
930
  *,
@@ -487,13 +932,19 @@ def _read_rollout_usage(
487
932
  completed_at: datetime | None,
488
933
  completed_turn_id: str | None = None,
489
934
  is_scan_owner: bool = False,
935
+ resume_owner: bool = False,
490
936
  ) -> RolloutUsage:
491
937
  result = RolloutUsage(_empty_token_usage(), set(), None, {})
492
938
  total = result.tokens
493
939
  warnings = result.warnings
494
940
  previous = _empty_token_usage()
941
+ previous_cache_writes_reported = True
942
+ prefix_unclassified = 0
943
+ prefix_missing_cache: set[str] = set()
944
+ ambiguous_legacy_cache = False
495
945
  boundary_reached = False
496
946
  legacy_usage = _empty_token_usage()
947
+ completed_legacy_usage = _empty_token_usage()
497
948
  response_usage: dict[str, int] | None = None
498
949
  observed_response_usage = _empty_token_usage()
499
950
  response_ids: set[str] = set()
@@ -503,6 +954,7 @@ def _read_rollout_usage(
503
954
  started_turns: set[str] = set()
504
955
  open_turns: set[str] = set()
505
956
  closed_turns: set[str] = set()
957
+ failed_turns: set[str] = set()
506
958
  measured_turns: set[str] = set()
507
959
 
508
960
  def past_scan_end(timestamp: datetime, turn_id: object) -> bool:
@@ -513,7 +965,7 @@ def _read_rollout_usage(
513
965
  and (completed_turn_id is None or turn_id != completed_turn_id)
514
966
  )
515
967
 
516
- with session.path.open("rb") as source:
968
+ with _rollout_lines(session.path, verify_end=True) as source:
517
969
  for line_number, raw_line in enumerate(source, start=1):
518
970
  if not raw_line.endswith(b"\n"):
519
971
  warnings.add("rollout_record_incomplete")
@@ -575,7 +1027,14 @@ def _read_rollout_usage(
575
1027
  elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
576
1028
  inherited_usage = _token_snapshot(payload)
577
1029
  if inherited_usage is not None:
578
- previous = inherited_usage
1030
+ missing_cache = _missing_cache_fields(payload)
1031
+ baseline = _retain_cache_baseline(previous, inherited_usage, missing_cache)
1032
+ if baseline is None:
1033
+ warnings.add("token_breakdown_unavailable")
1034
+ else:
1035
+ previous = baseline
1036
+ prefix_unclassified = _uncached_input(baseline)
1037
+ prefix_missing_cache = missing_cache
579
1038
  if not boundary_reached:
580
1039
  continue
581
1040
  if event.get("type") == "event_msg" and payload.get("type") == "task_started":
@@ -593,6 +1052,11 @@ def _read_rollout_usage(
593
1052
  _add_token_usage(
594
1053
  total, response_usage if response_usage is not None else legacy_usage
595
1054
  )
1055
+ if response_usage is None:
1056
+ _add_token_usage(completed_legacy_usage, legacy_usage)
1057
+ if ambiguous_legacy_cache:
1058
+ warnings.add("token_breakdown_unavailable")
1059
+ ambiguous_legacy_cache = False
596
1060
  legacy_usage = _empty_token_usage()
597
1061
  response_usage = None
598
1062
  if payload.get("turn_id") != current_turn_id:
@@ -621,6 +1085,8 @@ def _read_rollout_usage(
621
1085
  if timestamp >= started_at:
622
1086
  if terminal_turn_id in started_turns:
623
1087
  closed_turns.add(terminal_turn_id)
1088
+ if payload.get("type") == "task_complete" and payload.get("error"):
1089
+ failed_turns.add(terminal_turn_id)
624
1090
  if payload.get("type") == "turn_aborted":
625
1091
  # Later successful work does not recover interrupted request usage.
626
1092
  warnings.add("turn_interrupted")
@@ -687,6 +1153,20 @@ def _read_rollout_usage(
687
1153
  warnings.add("token_record_incomplete")
688
1154
  result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
689
1155
  _add_token_usage(response_usage, usage)
1156
+ if (
1157
+ resume_owner
1158
+ and receipt_turn_id == current_turn_id
1159
+ and current_turn_id in started_turns
1160
+ and (open_turns - {current_turn_id} or failed_turns)
1161
+ ):
1162
+ # Matching thread totals account for turns left open or failed before resume.
1163
+ cumulative = _usage_counts(payload.get("thread_token_usage"))
1164
+ if cumulative is not None and all(
1165
+ cumulative[key] == total[key] + response_usage[key]
1166
+ for key in TOKEN_FIELDS.values()
1167
+ ):
1168
+ open_turns.intersection_update({current_turn_id})
1169
+ closed_turns.difference_update(failed_turns - {current_turn_id})
690
1170
  continue
691
1171
  if event.get("type") != "event_msg" or payload.get("type") != "token_count":
692
1172
  continue
@@ -711,25 +1191,98 @@ def _read_rollout_usage(
711
1191
  if timestamp is None or snapshot is None:
712
1192
  warnings.add("token_record_invalid")
713
1193
  continue
1194
+ reset = snapshot["totalTokens"] < previous["totalTokens"]
1195
+ if reset:
1196
+ prefix_unclassified = 0
1197
+ missing_cache = _missing_cache_fields(payload)
1198
+ cache_writes_reported = "cacheWriteInputTokens" not in missing_cache
1199
+ baseline = _retain_cache_baseline(previous, snapshot, missing_cache)
1200
+ if baseline is None:
1201
+ if not past_scan_end(timestamp, current_turn_id):
1202
+ warnings.add("token_breakdown_unavailable")
1203
+ continue
1204
+ if baseline is previous:
1205
+ cache_writes_reported = previous_cache_writes_reported and cache_writes_reported
1206
+ refines_classification = (
1207
+ baseline["inputTokens"] >= previous["inputTokens"]
1208
+ and baseline["outputTokens"] >= previous["outputTokens"]
1209
+ and baseline["cachedInputTokens"] >= previous["cachedInputTokens"]
1210
+ and baseline["cacheWriteInputTokens"] >= previous["cacheWriteInputTokens"]
1211
+ and (
1212
+ (not previous_cache_writes_reported and cache_writes_reported)
1213
+ or baseline["cachedInputTokens"] > previous["cachedInputTokens"]
1214
+ or baseline["cacheWriteInputTokens"] > previous["cacheWriteInputTokens"]
1215
+ )
1216
+ )
1217
+ previous_cache_writes_reported = cache_writes_reported
714
1218
  delta = {
715
- key: value - previous[key] if value >= previous[key] else value
716
- for key, value in snapshot.items()
1219
+ key: snapshot[key] - previous[key]
1220
+ if not reset and snapshot[key] >= previous[key]
1221
+ else snapshot[key]
1222
+ for key in TOKEN_FIELDS.values()
717
1223
  }
718
- previous = snapshot
1224
+ if (
1225
+ snapshot.get("cacheWriteInputTokensReported") is False
1226
+ or previous.get("cacheWriteInputTokensReported") is False
1227
+ ):
1228
+ delta["cacheWriteInputTokensReported"] = False
1229
+ previous = baseline
719
1230
  if past_scan_end(timestamp, current_turn_id):
720
1231
  continue
721
1232
  if isinstance(current_turn_id, str):
722
1233
  measured_turns.add(current_turn_id)
723
1234
  if timestamp < started_at:
1235
+ prefix_unclassified = _uncached_input(baseline)
1236
+ prefix_missing_cache = missing_cache
724
1237
  continue
725
1238
  if completed_turn_id is not None and current_turn_id == completed_turn_id:
726
1239
  completed_turn_measured = True
727
1240
  result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
728
- if delta["totalTokens"] <= 0:
1241
+ delta["totalTokens"] = delta["inputTokens"] + delta["outputTokens"]
1242
+ if delta["totalTokens"] <= 0 and not refines_classification:
729
1243
  continue
1244
+ cache_write_capacity = (
1245
+ _uncached_input(legacy_usage) + delta["inputTokens"]
1246
+ if refines_classification
1247
+ else delta["inputTokens"]
1248
+ )
1249
+ # Late cumulative cache details can classify an earlier legacy turn.
1250
+ completed_capacity = (
1251
+ _uncached_input(completed_legacy_usage)
1252
+ if refines_classification and response_usage is None
1253
+ else 0
1254
+ )
1255
+ if (
1256
+ prefix_unclassified > 0
1257
+ and refines_classification
1258
+ and (
1259
+ any(delta[key] > 0 for key in prefix_missing_cache)
1260
+ # Even explicit counters can revise earlier input when the
1261
+ # newly classified tokens exceed all in-scan capacity.
1262
+ or delta["cachedInputTokens"] + delta["cacheWriteInputTokens"]
1263
+ > cache_write_capacity + completed_capacity
1264
+ )
1265
+ ):
1266
+ # The newly classified tokens could have occurred before this scan.
1267
+ ambiguous_legacy_cache = True
1268
+ for key in ("cacheWriteInputTokens", "cachedInputTokens"):
1269
+ current = min(delta[key], cache_write_capacity)
1270
+ completed = min(delta[key] - current, completed_capacity)
1271
+ if completed and ambiguous_legacy_cache:
1272
+ warnings.add("token_breakdown_unavailable")
1273
+ delta[key] = current
1274
+ cache_write_capacity -= current
1275
+ completed_capacity -= completed
1276
+ completed_legacy_usage[key] += completed
1277
+ total[key] += completed
1278
+ delta["reasoningOutputTokens"] = min(
1279
+ delta["reasoningOutputTokens"], delta["outputTokens"]
1280
+ )
730
1281
  _add_token_usage(legacy_usage, delta)
731
1282
 
732
1283
  _add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
1284
+ if response_usage is None and ambiguous_legacy_cache:
1285
+ warnings.add("token_breakdown_unavailable")
733
1286
  if not boundary_reached:
734
1287
  warnings.add("thread_ownership_unavailable")
735
1288
  elif result.tokens_updated_at is None and not direct_control_session:
@@ -800,6 +1353,47 @@ def valid_account_rate_limit(value: object) -> bool:
800
1353
  return True
801
1354
 
802
1355
 
1356
+ def _missing_cache_fields(payload: Mapping[str, Any]) -> set[str]:
1357
+ usage = payload["info"]["total_token_usage"]
1358
+ missing: set[str] = set()
1359
+ if "cached_input_tokens" not in usage:
1360
+ missing.add("cachedInputTokens")
1361
+ if "cache_write_input_tokens" not in usage and "cache_write_tokens" not in usage:
1362
+ missing.add("cacheWriteInputTokens")
1363
+ return missing
1364
+
1365
+
1366
+ def _uncached_input(usage: dict[str, int]) -> int:
1367
+ return usage["inputTokens"] - usage["cachedInputTokens"] - usage["cacheWriteInputTokens"]
1368
+
1369
+
1370
+ def _retain_cache_baseline(
1371
+ previous: dict[str, int], snapshot: dict[str, int], missing: set[str]
1372
+ ) -> dict[str, int] | None:
1373
+ baseline = snapshot
1374
+ if (
1375
+ missing
1376
+ and snapshot["inputTokens"] >= previous["inputTokens"]
1377
+ and snapshot["outputTokens"] >= previous["outputTokens"]
1378
+ ):
1379
+ baseline = {**snapshot, **{key: previous[key] for key in missing}}
1380
+ if (
1381
+ baseline["cachedInputTokens"] + baseline["cacheWriteInputTokens"]
1382
+ > baseline["inputTokens"]
1383
+ ):
1384
+ return None
1385
+ if (
1386
+ baseline["inputTokens"] == previous["inputTokens"]
1387
+ and baseline["outputTokens"] == previous["outputTokens"]
1388
+ and (
1389
+ baseline["cachedInputTokens"] < previous["cachedInputTokens"]
1390
+ or baseline["cacheWriteInputTokens"] < previous["cacheWriteInputTokens"]
1391
+ )
1392
+ ):
1393
+ return previous
1394
+ return baseline
1395
+
1396
+
803
1397
  def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
804
1398
  source = payload.get("source")
805
1399
  if isinstance(source, dict):
@@ -882,6 +1476,12 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
882
1476
  if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
883
1477
  return None
884
1478
  result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
1479
+ if (
1480
+ usage.get("cache_write_input_tokens_reported") is False
1481
+ or usage.get("cache_write_input_tokens") is None
1482
+ and usage.get("cache_write_tokens") is None
1483
+ ):
1484
+ result["cacheWriteInputTokensReported"] = False
885
1485
  return result
886
1486
 
887
1487
 
@@ -892,6 +1492,8 @@ def _empty_token_usage() -> dict[str, int]:
892
1492
  def _add_token_usage(target: dict[str, int], addition: Mapping[str, int]) -> None:
893
1493
  for key in TOKEN_FIELDS.values():
894
1494
  target[key] += addition[key]
1495
+ if addition.get("cacheWriteInputTokensReported") is False:
1496
+ target["cacheWriteInputTokensReported"] = False
895
1497
 
896
1498
 
897
1499
  def _timestamp(value: object) -> datetime | None: