@openai/codex-security 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/README.md +45 -9
  2. package/_bundled_plugin/.codex-plugin/plugin.json +5 -5
  3. package/_bundled_plugin/.mcp.json +1 -0
  4. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  5. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  6. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  7. package/_bundled_plugin/mcp/permission-profile-preflight.mjs +13 -20
  8. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  9. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  10. package/_bundled_plugin/references/core-scan.md +2 -0
  11. package/_bundled_plugin/references/final-report.md +5 -1
  12. package/_bundled_plugin/references/investigation-time-budget.md +11 -0
  13. package/_bundled_plugin/references/sarif-adapter.md +1 -0
  14. package/_bundled_plugin/references/scan-prologue.md +2 -6
  15. package/_bundled_plugin/references/scan-runtime.md +15 -0
  16. package/_bundled_plugin/references/threat-model.md +2 -0
  17. package/_bundled_plugin/schemas/coverage.schema.json +45 -126
  18. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +5 -1
  19. package/_bundled_plugin/schemas/findings.schema.json +140 -427
  20. package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +106 -119
  21. package/_bundled_plugin/schemas/scan-manifest.schema.json +73 -188
  22. package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
  23. package/_bundled_plugin/scripts/finalize_scan_contract.py +87 -49
  24. package/_bundled_plugin/scripts/finding_preview.py +5 -3
  25. package/_bundled_plugin/scripts/generate_rank_input.py +26 -106
  26. package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
  27. package/_bundled_plugin/scripts/report_projection.py +35 -13
  28. package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
  29. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
  30. package/_bundled_plugin/scripts/workbench_cli.py +9 -9
  31. package/_bundled_plugin/scripts/workbench_constants.py +4 -0
  32. package/_bundled_plugin/scripts/workbench_db.py +103 -45
  33. package/_bundled_plugin/scripts/workbench_progress.py +0 -50
  34. package/_bundled_plugin/scripts/workbench_publication.py +216 -247
  35. package/_bundled_plugin/scripts/workbench_scan_history.py +3 -2
  36. package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
  37. package/_bundled_plugin/scripts/workbench_scan_usage.py +360 -37
  38. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  39. package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
  40. package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
  41. package/_bundled_plugin/scripts/workbench_target.py +7 -5
  42. package/_bundled_plugin/scripts/workbench_validation.py +52 -2
  43. package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
  44. package/_bundled_plugin/shared/workbench-migrations.json +22 -0
  45. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
  46. package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
  47. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
  48. package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
  49. package/_bundled_plugin/skills/security-scan/SKILL.md +4 -2
  50. package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
  51. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -3
  52. package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +2 -0
  53. package/_bundled_plugin/skills/track-findings/SKILL.md +8 -4
  54. package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
  55. package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +1 -1
  56. package/dist/api.d.ts +6 -2
  57. package/dist/api.d.ts.map +1 -1
  58. package/dist/api.js +266 -39
  59. package/dist/api.js.map +1 -1
  60. package/dist/artifact-export.d.ts +2 -1
  61. package/dist/artifact-export.d.ts.map +1 -1
  62. package/dist/artifact-export.js +38 -23
  63. package/dist/artifact-export.js.map +1 -1
  64. package/dist/classify-severity-records.d.ts +76 -0
  65. package/dist/classify-severity-records.d.ts.map +1 -0
  66. package/dist/classify-severity-records.js +141 -0
  67. package/dist/classify-severity-records.js.map +1 -0
  68. package/dist/classify-severity.d.ts +1 -23
  69. package/dist/classify-severity.d.ts.map +1 -1
  70. package/dist/classify-severity.js +3 -24
  71. package/dist/classify-severity.js.map +1 -1
  72. package/dist/cli-help.d.ts.map +1 -1
  73. package/dist/cli-help.js +2 -0
  74. package/dist/cli-help.js.map +1 -1
  75. package/dist/cli.d.ts +4 -2
  76. package/dist/cli.d.ts.map +1 -1
  77. package/dist/cli.js +148 -149
  78. package/dist/cli.js.map +1 -1
  79. package/dist/cloud-publish.js +1 -1
  80. package/dist/cloud-publish.js.map +1 -1
  81. package/dist/codex-sdk-environment.js +1 -1
  82. package/dist/codex-sdk-environment.js.map +1 -1
  83. package/dist/component-plan.js +1 -1
  84. package/dist/component-plan.js.map +1 -1
  85. package/dist/component-scan.d.ts.map +1 -1
  86. package/dist/component-scan.js +5 -4
  87. package/dist/component-scan.js.map +1 -1
  88. package/dist/config.d.ts +12 -0
  89. package/dist/config.d.ts.map +1 -1
  90. package/dist/config.js +10 -4
  91. package/dist/config.js.map +1 -1
  92. package/dist/contract.d.ts +6 -0
  93. package/dist/contract.d.ts.map +1 -1
  94. package/dist/contract.js +69 -20
  95. package/dist/contract.js.map +1 -1
  96. package/dist/cost-model.d.ts.map +1 -1
  97. package/dist/cost-model.js +7 -2
  98. package/dist/cost-model.js.map +1 -1
  99. package/dist/custom-validation-prompt.js +3 -3
  100. package/dist/custom-validation-prompt.js.map +1 -1
  101. package/dist/deduplication/codex-review.d.ts +1 -5
  102. package/dist/deduplication/codex-review.d.ts.map +1 -1
  103. package/dist/deduplication/codex-review.js +2 -2
  104. package/dist/deduplication/codex-review.js.map +1 -1
  105. package/dist/deduplication/deduplication-prompts.d.ts +1 -1
  106. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  107. package/dist/deduplication/deduplication-prompts.js +2 -2
  108. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  109. package/dist/deduplication/deduplication-reviewer.d.ts +1 -1
  110. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  111. package/dist/deduplication/deduplication-reviewer.js +1 -1
  112. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  113. package/dist/deduplication/records-protocol.d.ts +3 -8
  114. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  115. package/dist/deduplication/records-protocol.js +11 -171
  116. package/dist/deduplication/records-protocol.js.map +1 -1
  117. package/dist/deduplication/records.js +1 -1
  118. package/dist/deduplication/records.js.map +1 -1
  119. package/dist/deduplication/scan.d.ts.map +1 -1
  120. package/dist/deduplication/scan.js +10 -9
  121. package/dist/deduplication/scan.js.map +1 -1
  122. package/dist/deep-config.d.ts +4 -1
  123. package/dist/deep-config.d.ts.map +1 -1
  124. package/dist/deep-config.js +21 -1
  125. package/dist/deep-config.js.map +1 -1
  126. package/dist/deep-progress.d.ts +15 -1
  127. package/dist/deep-progress.d.ts.map +1 -1
  128. package/dist/deep-progress.js +15 -16
  129. package/dist/deep-progress.js.map +1 -1
  130. package/dist/executable-path.d.ts +2 -0
  131. package/dist/executable-path.d.ts.map +1 -0
  132. package/dist/executable-path.js +12 -0
  133. package/dist/executable-path.js.map +1 -0
  134. package/dist/feedback.d.ts +1 -4
  135. package/dist/feedback.d.ts.map +1 -1
  136. package/dist/feedback.js +1 -1
  137. package/dist/feedback.js.map +1 -1
  138. package/dist/findings-import.d.ts +1 -1
  139. package/dist/findings-import.d.ts.map +1 -1
  140. package/dist/findings-import.js +3 -3
  141. package/dist/findings-import.js.map +1 -1
  142. package/dist/git-remote.d.ts +10 -0
  143. package/dist/git-remote.d.ts.map +1 -0
  144. package/dist/git-remote.js +24 -0
  145. package/dist/git-remote.js.map +1 -0
  146. package/dist/github.d.ts.map +1 -1
  147. package/dist/github.js +5 -7
  148. package/dist/github.js.map +1 -1
  149. package/dist/host-protocol.d.ts +19 -0
  150. package/dist/host-protocol.d.ts.map +1 -0
  151. package/dist/host-protocol.js +270 -0
  152. package/dist/host-protocol.js.map +1 -0
  153. package/dist/host-records-protocol.d.ts +15 -0
  154. package/dist/host-records-protocol.d.ts.map +1 -0
  155. package/dist/host-records-protocol.js +15 -0
  156. package/dist/host-records-protocol.js.map +1 -0
  157. package/dist/hosted-scan-protocol.d.ts +4 -0
  158. package/dist/hosted-scan-protocol.d.ts.map +1 -0
  159. package/dist/hosted-scan-protocol.js +25 -0
  160. package/dist/hosted-scan-protocol.js.map +1 -0
  161. package/dist/hosted-scan.d.ts +112 -0
  162. package/dist/hosted-scan.d.ts.map +1 -0
  163. package/dist/hosted-scan.js +260 -0
  164. package/dist/hosted-scan.js.map +1 -0
  165. package/dist/index.d.ts +7 -0
  166. package/dist/index.d.ts.map +1 -1
  167. package/dist/index.js +3 -0
  168. package/dist/index.js.map +1 -1
  169. package/dist/knowledge-base.d.ts.map +1 -1
  170. package/dist/knowledge-base.js +87 -7
  171. package/dist/knowledge-base.js.map +1 -1
  172. package/dist/multiscan.d.ts.map +1 -1
  173. package/dist/multiscan.js +108 -36
  174. package/dist/multiscan.js.map +1 -1
  175. package/dist/patch-publication.d.ts +26 -0
  176. package/dist/patch-publication.d.ts.map +1 -0
  177. package/dist/patch-publication.js +105 -0
  178. package/dist/patch-publication.js.map +1 -0
  179. package/dist/patch-tui.d.ts.map +1 -1
  180. package/dist/patch-tui.js +4 -1
  181. package/dist/patch-tui.js.map +1 -1
  182. package/dist/project-config-schema.d.ts +2 -0
  183. package/dist/project-config-schema.d.ts.map +1 -1
  184. package/dist/project-config-schema.js +1 -0
  185. package/dist/project-config-schema.js.map +1 -1
  186. package/dist/project-config.d.ts +1 -0
  187. package/dist/project-config.d.ts.map +1 -1
  188. package/dist/project-config.js +16 -6
  189. package/dist/project-config.js.map +1 -1
  190. package/dist/publication-store.d.ts.map +1 -1
  191. package/dist/publication-store.js +73 -63
  192. package/dist/publication-store.js.map +1 -1
  193. package/dist/publication.d.ts +0 -2
  194. package/dist/publication.d.ts.map +1 -1
  195. package/dist/publication.js +135 -24
  196. package/dist/publication.js.map +1 -1
  197. package/dist/publish.d.ts.map +1 -1
  198. package/dist/publish.js +39 -25
  199. package/dist/publish.js.map +1 -1
  200. package/dist/runtime.d.ts +5 -2
  201. package/dist/runtime.d.ts.map +1 -1
  202. package/dist/runtime.js +86 -58
  203. package/dist/runtime.js.map +1 -1
  204. package/dist/scan-executor.d.ts +104 -0
  205. package/dist/scan-executor.d.ts.map +1 -0
  206. package/dist/scan-executor.js +62 -0
  207. package/dist/scan-executor.js.map +1 -0
  208. package/dist/scan-settings.d.ts +1 -0
  209. package/dist/scan-settings.d.ts.map +1 -1
  210. package/dist/scan-settings.js +7 -1
  211. package/dist/scan-settings.js.map +1 -1
  212. package/dist/scan-time-budget.d.ts +9 -0
  213. package/dist/scan-time-budget.d.ts.map +1 -0
  214. package/dist/scan-time-budget.js +16 -0
  215. package/dist/scan-time-budget.js.map +1 -0
  216. package/dist/severity-review.d.ts +29 -0
  217. package/dist/severity-review.d.ts.map +1 -0
  218. package/dist/severity-review.js +47 -0
  219. package/dist/severity-review.js.map +1 -0
  220. package/dist/targets.d.ts.map +1 -1
  221. package/dist/targets.js +17 -25
  222. package/dist/targets.js.map +1 -1
  223. package/dist/validation-records.d.ts +38 -0
  224. package/dist/validation-records.d.ts.map +1 -0
  225. package/dist/validation-records.js +76 -0
  226. package/dist/validation-records.js.map +1 -0
  227. package/dist/version.d.ts.map +1 -1
  228. package/dist/version.js +4 -2
  229. package/dist/version.js.map +1 -1
  230. package/dist/worker-progress.d.ts +43 -15
  231. package/dist/worker-progress.d.ts.map +1 -1
  232. package/dist/worker-progress.js +22 -80
  233. package/dist/worker-progress.js.map +1 -1
  234. package/docs/classify-severity-records.md +89 -0
  235. package/docs/cli.md +281 -35
  236. package/docs/hosted-scans.md +242 -0
  237. package/docs/validate-records.md +69 -0
  238. package/package.json +18 -14
  239. package/schemas/hosted-scan-v2.fixture.json +145 -0
  240. package/schemas/project-config.schema.json +7 -1
  241. package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
  242. package/_bundled_plugin/references/config-preflight.md +0 -93
  243. package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
  244. package/_bundled_plugin/scripts/config_preflight.py +0 -911
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import argparse
6
6
  import json
7
+ import math
7
8
  import os
8
9
  import re
9
10
  import sqlite3
@@ -34,6 +35,14 @@ class RolloutSession:
34
35
  path: Path
35
36
 
36
37
 
38
+ @dataclass
39
+ class RolloutUsage:
40
+ tokens: dict[str, int]
41
+ warnings: set[str]
42
+ tokens_updated_at: datetime | None
43
+ rate_limits: dict[str, dict[str, Any]]
44
+
45
+
37
46
  def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
38
47
  """Project measured usage without changing the existing legacy cost contract."""
39
48
 
@@ -50,12 +59,6 @@ def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
50
59
  }
51
60
 
52
61
 
53
- def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
54
- """Keep usage in the already-migrated scans.cost_json column."""
55
-
56
- return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
57
-
58
-
59
62
  def reconcile_completed_scan_cost(
60
63
  connection: sqlite3.Connection,
61
64
  scan: sqlite3.Row,
@@ -84,11 +87,14 @@ def collect_scan_usage(
84
87
  *,
85
88
  thread_id: str | None = None,
86
89
  completed_at: str | None = None,
90
+ completed_turn_id: str | None = None,
87
91
  ) -> dict[str, Any]:
88
- """Count only complete, attributable rollout events inside this scan's window."""
92
+ """Count attributable receipts and flag observable gaps inside the scan window."""
89
93
 
90
94
  roots = _scan_root_thread_ids(connection, scan, thread_id)
91
- if not roots:
95
+ parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
96
+ worker_directories = _scan_worker_directories(connection, scan)
97
+ if not roots and not worker_directories:
92
98
  return _unavailable_usage("scan_thread_unavailable")
93
99
 
94
100
  state_database = _codex_state_database()
@@ -96,7 +102,8 @@ def collect_scan_usage(
96
102
  return _unavailable_usage("codex_state_unavailable")
97
103
 
98
104
  started_at = _timestamp(scan["started_at"])
99
- stopped_at = _timestamp(completed_at or scan["completed_at"])
105
+ observed_at = datetime.now(timezone.utc)
106
+ stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
100
107
  if started_at is None:
101
108
  return _unavailable_usage("scan_window_unavailable")
102
109
 
@@ -106,6 +113,9 @@ def collect_scan_usage(
106
113
  state_database,
107
114
  roots,
108
115
  warnings,
116
+ worker_directories=worker_directories,
117
+ started_at=started_at,
118
+ completed_at=stopped_at,
109
119
  )
110
120
  except (OSError, sqlite3.Error, ValueError):
111
121
  return _unavailable_usage("codex_state_unavailable")
@@ -113,7 +123,9 @@ def collect_scan_usage(
113
123
  if not sessions:
114
124
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
115
125
 
116
- total = _empty_token_usage()
126
+ by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
127
+ rate_limits: dict[str, dict[str, Any]] = {}
128
+ tokens_updated_at = None
117
129
  accepted_thread_ids: set[str] = set()
118
130
  excluded_thread_ids: set[str] = set()
119
131
  for session in sessions:
@@ -128,36 +140,61 @@ def collect_scan_usage(
128
140
  warnings.add("thread_lineage_incomplete")
129
141
  continue
130
142
  try:
131
- session_usage, session_warnings = _read_rollout_usage(
143
+ usage = _read_rollout_usage(
132
144
  session,
133
145
  started_at=started_at,
134
146
  completed_at=stopped_at,
147
+ completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
135
148
  )
136
149
  except (OSError, UnicodeError, ValueError):
137
150
  missing_thread_ids.add(session.thread_id)
138
151
  warnings.add("rollout_unavailable")
139
152
  continue
140
- if "thread_outside_scan_window" in session_warnings:
153
+ if "thread_outside_scan_window" in usage.warnings:
141
154
  excluded_thread_ids.add(session.thread_id)
142
155
  continue
143
- warnings.update(session_warnings)
144
- if "thread_identity_mismatch" in session_warnings or (
145
- "thread_ownership_unavailable" in session_warnings
156
+ warnings.update(usage.warnings)
157
+ if "thread_identity_mismatch" in usage.warnings or (
158
+ "thread_ownership_unavailable" in usage.warnings
146
159
  ):
147
160
  missing_thread_ids.add(session.thread_id)
148
161
  continue
149
162
  accepted_thread_ids.add(session.thread_id)
150
- _add_token_usage(total, session_usage)
163
+ role = by_role["parent" if session.thread_id in parent_ids else "workers"]
164
+ _add_token_usage(role, usage.tokens)
165
+ role["threadCount"] += 1
166
+ if usage.tokens_updated_at is not None:
167
+ tokens_updated_at = max(
168
+ tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
169
+ )
170
+ for key, snapshot in usage.rate_limits.items():
171
+ if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
172
+ rate_limits[key] = snapshot
151
173
 
152
174
  if not accepted_thread_ids:
153
175
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
154
176
 
177
+ if any(
178
+ usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
179
+ or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
180
+ for usage in by_role.values()
181
+ ):
182
+ return _unavailable_usage("token_breakdown_unavailable", warnings=warnings)
183
+
184
+ if completed_turn_id is not None and tokens_updated_at is not None:
185
+ stopped_at = max(stopped_at, tokens_updated_at)
155
186
  result: dict[str, Any] = {
156
187
  "coverage": "partial" if missing_thread_ids or warnings else "complete",
157
188
  "source": "codex_rollout",
158
- **total,
189
+ **{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
159
190
  "threadCount": len(accepted_thread_ids),
191
+ "byRole": by_role,
192
+ "window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
193
+ "observedAt": observed_at.isoformat(),
194
+ "accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
160
195
  }
196
+ if tokens_updated_at is not None:
197
+ result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
161
198
  if missing_thread_ids:
162
199
  result["missingThreadCount"] = len(missing_thread_ids)
163
200
  if warnings:
@@ -171,6 +208,7 @@ def _scan_root_thread_ids(
171
208
  supplied_thread_id: str | None,
172
209
  *,
173
210
  include_owner_threads: bool = True,
211
+ include_workers: bool = True,
174
212
  ) -> list[str]:
175
213
  candidates: list[str | None] = [supplied_thread_id]
176
214
  if include_owner_threads:
@@ -184,7 +222,7 @@ def _scan_root_thread_ids(
184
222
  ).fetchone()
185
223
  if workspace is not None:
186
224
  candidates.append(workspace["thread_id"])
187
- if scan["mode"] == "deep":
225
+ if include_workers and scan["mode"] == "deep":
188
226
  candidates.extend(
189
227
  row["sdk_thread_id"]
190
228
  for row in connection.execute(
@@ -214,6 +252,21 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
214
252
  )
215
253
 
216
254
 
255
+ def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]:
256
+ if scan["mode"] != "deep":
257
+ return set()
258
+ return {
259
+ row["artifact_dir"]
260
+ if row["kind"] == "discovery"
261
+ else str(Path(scan["scan_dir"]) / "artifacts")
262
+ for row in connection.execute(
263
+ "SELECT kind, artifact_dir FROM deep_scan_workers "
264
+ "WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
265
+ (scan["id"],),
266
+ )
267
+ }
268
+
269
+
217
270
  def _codex_state_database() -> Path | None:
218
271
  configured_database = os.environ.get("CODEX_STATE_DB", "").strip()
219
272
  if configured_database:
@@ -255,6 +308,10 @@ def _discover_rollout_sessions(
255
308
  state_database: Path,
256
309
  roots: list[str],
257
310
  warnings: set[str],
311
+ *,
312
+ worker_directories: set[str],
313
+ started_at: datetime,
314
+ completed_at: datetime | None,
258
315
  ) -> tuple[list[RolloutSession], set[str]]:
259
316
  database = sqlite3.connect(
260
317
  f"file:{quote(str(state_database), safe='')}?mode=ro",
@@ -273,7 +330,27 @@ def _discover_rollout_sessions(
273
330
  raise ValueError("Codex state graph does not expose the required thread columns.")
274
331
  sessions: dict[str, RolloutSession] = {}
275
332
  missing_thread_ids: set[str] = set()
276
- for root in roots:
333
+ root_ids = dict.fromkeys(roots)
334
+ started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
335
+ completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
336
+ directory_expression = "cwd"
337
+ if worker_directories and os.name == "nt":
338
+ from windows_scan_local_files import _normalized_windows_path
339
+
340
+ database.create_function(
341
+ "codex_security_path_key", 1, _normalized_windows_path, deterministic=True
342
+ )
343
+ directory_expression = "codex_security_path_key(cwd)"
344
+ worker_directories = {_normalized_windows_path(value) for value in worker_directories}
345
+ for directory in sorted(worker_directories):
346
+ for row in database.execute(
347
+ "SELECT id FROM threads WHERE cwd IS NOT NULL "
348
+ f"AND {directory_expression} = ? AND created_at >= ? "
349
+ "AND (? IS NULL OR created_at <= ?) ORDER BY id",
350
+ (directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
351
+ ):
352
+ root_ids[row["id"]] = None
353
+ for root in root_ids:
277
354
  row = database.execute(
278
355
  "SELECT id, rollout_path FROM threads WHERE id = ?",
279
356
  (root,),
@@ -282,13 +359,29 @@ def _discover_rollout_sessions(
282
359
  missing_thread_ids.add(root)
283
360
  warnings.add("scan_root_unavailable")
284
361
  continue
285
- if root not in sessions:
286
- path = _rollout_path(row["rollout_path"])
287
- if path is None:
362
+ path = _rollout_path(row["rollout_path"])
363
+ if path is None:
364
+ missing_thread_ids.add(root)
365
+ warnings.add("rollout_unavailable")
366
+ continue
367
+ if root not in roots:
368
+ try:
369
+ with path.open("rb") as source:
370
+ metadata = json.loads(source.readline())
371
+ except (OSError, ValueError):
288
372
  missing_thread_ids.add(root)
289
373
  warnings.add("rollout_unavailable")
290
374
  continue
291
- sessions[root] = RolloutSession(root, None, path)
375
+ if (
376
+ isinstance(metadata, dict)
377
+ and metadata.get("type") == "session_meta"
378
+ and isinstance(payload := metadata.get("payload"), dict)
379
+ and (payload.get("id") or payload.get("session_id")) == root
380
+ and (created_at := _timestamp(payload.get("timestamp"))) is not None
381
+ and created_at < started_millisecond
382
+ ):
383
+ continue
384
+ sessions[root] = RolloutSession(root, None, path)
292
385
  descendants = database.execute(
293
386
  """
294
387
  WITH RECURSIVE descendants(
@@ -378,11 +471,23 @@ def _read_rollout_usage(
378
471
  *,
379
472
  started_at: datetime,
380
473
  completed_at: datetime | None,
381
- ) -> tuple[dict[str, int], set[str]]:
382
- total = _empty_token_usage()
383
- warnings: set[str] = set()
474
+ completed_turn_id: str | None = None,
475
+ ) -> RolloutUsage:
476
+ result = RolloutUsage(_empty_token_usage(), set(), None, {})
477
+ total = result.tokens
478
+ warnings = result.warnings
384
479
  previous = _empty_token_usage()
385
480
  boundary_reached = False
481
+ legacy_usage = _empty_token_usage()
482
+ response_usage: dict[str, int] | None = None
483
+ observed_response_usage = _empty_token_usage()
484
+ response_ids: set[str] = set()
485
+ current_turn_id: str | None = None
486
+ completed_turn_measured = False
487
+ started_turns: set[str] = set()
488
+ open_turns: set[str] = set()
489
+ closed_turns: set[str] = set()
490
+ measured_turns: set[str] = set()
386
491
 
387
492
  with session.path.open("rb") as source:
388
493
  for line_number, raw_line in enumerate(source, start=1):
@@ -405,16 +510,16 @@ def _read_rollout_usage(
405
510
  if line_number == 1:
406
511
  if event.get("type") != "session_meta" or not isinstance(payload, dict):
407
512
  warnings.add("thread_identity_mismatch")
408
- return total, warnings
513
+ return result
409
514
  recorded_id = payload.get("id") or payload.get("session_id")
410
515
  if recorded_id != session.thread_id:
411
516
  warnings.add("thread_identity_mismatch")
412
- return total, warnings
517
+ return result
413
518
  recorded_parent = _session_parent_thread_id(payload)
414
519
  if session.parent_thread_id is not None:
415
520
  if recorded_parent != session.parent_thread_id:
416
521
  warnings.add("thread_identity_mismatch")
417
- return total, warnings
522
+ return result
418
523
  boundary_reached = (
419
524
  session.parent_thread_id is None
420
525
  and not recorded_parent
@@ -429,20 +534,163 @@ def _read_rollout_usage(
429
534
  task_started_at = _timestamp(event.get("timestamp"))
430
535
  if task_started_at is None:
431
536
  warnings.add("thread_ownership_unavailable")
432
- return total, warnings
537
+ return result
433
538
  if task_started_at < started_at or (
434
- completed_at is not None and task_started_at > completed_at
539
+ completed_at is not None
540
+ and task_started_at > completed_at
541
+ and payload.get("turn_id") != completed_turn_id
435
542
  ):
436
543
  warnings.add("thread_outside_scan_window")
437
- return total, warnings
544
+ return result
438
545
  boundary_reached = True
439
546
  elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
440
547
  inherited_usage = _token_snapshot(payload)
441
548
  if inherited_usage is not None:
442
549
  previous = inherited_usage
550
+ if not boundary_reached:
551
+ continue
552
+ if event.get("type") == "event_msg" and payload.get("type") == "task_started":
553
+ if (
554
+ completed_turn_id is not None
555
+ and current_turn_id == completed_turn_id
556
+ and payload.get("turn_id") != completed_turn_id
557
+ ):
558
+ break
559
+ _add_token_usage(
560
+ total, response_usage if response_usage is not None else legacy_usage
561
+ )
562
+ legacy_usage = _empty_token_usage()
563
+ response_usage = None
564
+ if payload.get("turn_id") != current_turn_id:
565
+ observed_response_usage = _empty_token_usage()
566
+ current_turn_id = payload.get("turn_id")
567
+ timestamp = _timestamp(event.get("timestamp"))
568
+ if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
569
+ warnings.add("turn_record_invalid")
570
+ elif (
571
+ completed_at is None
572
+ or timestamp <= completed_at
573
+ or current_turn_id == completed_turn_id
574
+ ):
575
+ started_turns.add(current_turn_id)
576
+ open_turns.add(current_turn_id)
577
+ continue
578
+ if event.get("type") == "event_msg" and payload.get("type") in {
579
+ "task_complete",
580
+ "turn_aborted",
581
+ }:
582
+ timestamp = _timestamp(event.get("timestamp"))
583
+ terminal_turn_id = payload.get("turn_id") or current_turn_id
584
+ if (
585
+ not isinstance(terminal_turn_id, str)
586
+ or not terminal_turn_id
587
+ or timestamp is None
588
+ ):
589
+ warnings.add("turn_record_invalid")
590
+ elif (
591
+ completed_at is None
592
+ or timestamp <= completed_at
593
+ or terminal_turn_id == completed_turn_id
594
+ ):
595
+ open_turns.discard(terminal_turn_id)
596
+ if timestamp >= started_at:
597
+ if terminal_turn_id in started_turns:
598
+ closed_turns.add(terminal_turn_id)
599
+ if payload.get("type") == "turn_aborted":
600
+ # Later successful work does not recover interrupted request usage.
601
+ warnings.add("turn_interrupted")
602
+ if (
603
+ completed_turn_id is not None
604
+ and event.get("type") == "event_msg"
605
+ and payload.get("type") == "task_complete"
606
+ and payload.get("turn_id") == completed_turn_id
607
+ ):
608
+ break
609
+ if event.get("type") == "compacted":
610
+ timestamp = _timestamp(event.get("timestamp"))
611
+ if (
612
+ timestamp is not None
613
+ and timestamp >= started_at
614
+ and (
615
+ completed_at is None
616
+ or timestamp <= completed_at
617
+ or (completed_turn_id is not None and current_turn_id == completed_turn_id)
618
+ )
619
+ and "compaction_response_id" not in payload
620
+ and response_usage is None
621
+ ):
622
+ warnings.add("compaction_usage_unavailable")
623
+ continue
624
+ if event.get("type") == "token_usage_record":
625
+ if not isinstance(payload.get("thread_id"), str):
626
+ warnings.add("token_record_invalid")
627
+ continue
628
+ if payload.get("thread_id") != session.thread_id:
629
+ continue
630
+ timestamp = _timestamp(event.get("timestamp"))
631
+ usage = _usage_counts(payload.get("usage"))
632
+ response_id = payload.get("response_id")
633
+ if (
634
+ timestamp is None
635
+ or usage is None
636
+ or not isinstance(response_id, str)
637
+ or not response_id
638
+ ):
639
+ warnings.add("token_record_invalid")
640
+ continue
641
+ is_completion_turn = (
642
+ completed_turn_id is not None and payload.get("turn_id") == completed_turn_id
643
+ )
644
+ if completed_at is not None and timestamp > completed_at and not is_completion_turn:
645
+ continue
646
+ if response_usage is None:
647
+ response_usage = _empty_token_usage()
648
+ if current_turn_id is None:
649
+ current_turn_id = payload.get("turn_id")
650
+ if isinstance(current_turn_id, str):
651
+ open_turns.add(current_turn_id)
652
+ if response_id in response_ids:
653
+ continue
654
+ response_ids.add(response_id)
655
+ _add_token_usage(observed_response_usage, usage)
656
+ turn_usage = _usage_counts(payload.get("turn_token_usage"))
657
+ receipt_turn_id = payload.get("turn_id")
658
+ if isinstance(receipt_turn_id, str):
659
+ measured_turns.add(receipt_turn_id)
660
+ if timestamp < started_at:
661
+ if turn_usage is not None:
662
+ observed_response_usage = turn_usage
663
+ continue
664
+ if is_completion_turn:
665
+ completed_turn_measured = True
666
+ if turn_usage is not None and any(
667
+ turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
668
+ ):
669
+ warnings.add("token_record_incomplete")
670
+ result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
671
+ _add_token_usage(response_usage, usage)
443
672
  continue
444
673
  if event.get("type") != "event_msg" or payload.get("type") != "token_count":
445
674
  continue
675
+ timestamp = _timestamp(event.get("timestamp"))
676
+ in_window = (
677
+ timestamp is not None
678
+ and timestamp >= started_at
679
+ and (
680
+ completed_at is None
681
+ or timestamp <= completed_at
682
+ or (completed_turn_id is not None and current_turn_id == completed_turn_id)
683
+ )
684
+ )
685
+ if in_window:
686
+ rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
687
+ if rate_limits is not None:
688
+ key = rate_limits["limitId"]
689
+ if (
690
+ key not in result.rate_limits
691
+ or rate_limits["observedAt"] >= result.rate_limits[key]["observedAt"]
692
+ ):
693
+ result.rate_limits[key] = rate_limits
446
694
  if "info" in payload and payload["info"] is None:
447
695
  continue
448
696
  timestamp = _timestamp(event.get("timestamp"))
@@ -455,17 +703,89 @@ def _read_rollout_usage(
455
703
  for key, value in snapshot.items()
456
704
  }
457
705
  previous = snapshot
458
- if timestamp < started_at:
706
+ is_completion_turn = (
707
+ completed_turn_id is not None and current_turn_id == completed_turn_id
708
+ )
709
+ if completed_at is not None and timestamp > completed_at and not is_completion_turn:
459
710
  continue
460
- if completed_at is not None and timestamp > completed_at:
711
+ if isinstance(current_turn_id, str):
712
+ measured_turns.add(current_turn_id)
713
+ if timestamp < started_at:
461
714
  continue
715
+ if is_completion_turn:
716
+ completed_turn_measured = True
717
+ result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
462
718
  if delta["totalTokens"] <= 0:
463
719
  continue
464
- _add_token_usage(total, delta)
720
+ _add_token_usage(legacy_usage, delta)
465
721
 
722
+ _add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
466
723
  if not boundary_reached:
467
724
  warnings.add("thread_ownership_unavailable")
468
- return total, warnings
725
+ if completed_turn_id is not None:
726
+ open_turns.discard(completed_turn_id)
727
+ if open_turns:
728
+ warnings.add("turn_in_progress")
729
+ if boundary_reached and result.tokens_updated_at is None:
730
+ warnings.add("token_record_missing")
731
+ if closed_turns - measured_turns:
732
+ warnings.add("token_record_missing")
733
+ if completed_turn_id is not None and not completed_turn_measured:
734
+ warnings.add("completion_turn_unavailable")
735
+ return result
736
+
737
+
738
+ def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None:
739
+ if not isinstance(value, dict):
740
+ return None
741
+ result: dict[str, Any] = {
742
+ "limitId": value.get("limit_id") or "codex",
743
+ "observedAt": timestamp.isoformat(),
744
+ }
745
+ for name in ("primary", "secondary"):
746
+ window = value.get(name)
747
+ if window is None:
748
+ result[name] = None
749
+ continue
750
+ if not isinstance(window, dict):
751
+ return None
752
+ result[name] = {
753
+ "usedPercent": window.get("used_percent"),
754
+ "windowMinutes": window.get("window_minutes"),
755
+ "resetsAt": window.get("resets_at"),
756
+ }
757
+ return result if valid_account_rate_limit(result) else None
758
+
759
+
760
+ def valid_account_rate_limit(value: object) -> bool:
761
+ if not isinstance(value, dict) or set(value) != {
762
+ "limitId",
763
+ "observedAt",
764
+ "primary",
765
+ "secondary",
766
+ }:
767
+ return False
768
+ if not isinstance(value["limitId"], str) or _timestamp(value["observedAt"]) is None:
769
+ return False
770
+ for name in ("primary", "secondary"):
771
+ window = value[name]
772
+ if window is None:
773
+ continue
774
+ if not isinstance(window, dict) or set(window) != {
775
+ "usedPercent",
776
+ "windowMinutes",
777
+ "resetsAt",
778
+ }:
779
+ return False
780
+ percent = window["usedPercent"]
781
+ if type(percent) not in (int, float) or not math.isfinite(percent):
782
+ return False
783
+ if any(
784
+ window[key] is not None and type(window[key]) is not int
785
+ for key in ("windowMinutes", "resetsAt")
786
+ ):
787
+ return False
788
+ return True
469
789
 
470
790
 
471
791
  def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
@@ -515,7 +835,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
515
835
  info = payload.get("info")
516
836
  if not isinstance(info, dict):
517
837
  return None
518
- usage = info.get("total_token_usage")
838
+ return _usage_counts(info.get("total_token_usage"))
839
+
840
+
841
+ def _usage_counts(usage: object) -> dict[str, int] | None:
519
842
  if not isinstance(usage, dict):
520
843
  return None
521
844
  cache_write = usage.get("cache_write_input_tokens", usage.get("cache_write_tokens", 0))
@@ -75,7 +75,7 @@ def apply_migrations(
75
75
  elif version == 16:
76
76
  should_backfill_targets = repair_stable_targets_migration(connection)
77
77
  elif version in applied:
78
- if version in (2, 12, 13, 26, 28, 31, 32, 47):
78
+ if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50):
79
79
  repair_additive_migration(connection, version)
80
80
  elif version == 11:
81
81
  repair_deep_scan_migration(connection)