@openai/codex-security 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +45 -9
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  5. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  6. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  7. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  8. package/_bundled_plugin/references/core-scan.md +2 -0
  9. package/_bundled_plugin/references/final-report.md +5 -1
  10. package/_bundled_plugin/references/investigation-time-budget.md +11 -0
  11. package/_bundled_plugin/references/sarif-adapter.md +1 -0
  12. package/_bundled_plugin/references/scan-prologue.md +2 -6
  13. package/_bundled_plugin/references/scan-runtime.md +15 -0
  14. package/_bundled_plugin/references/threat-model.md +2 -0
  15. package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +5 -1
  16. package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
  17. package/_bundled_plugin/scripts/finalize_scan_contract.py +80 -43
  18. package/_bundled_plugin/scripts/finding_preview.py +5 -3
  19. package/_bundled_plugin/scripts/generate_rank_input.py +1 -0
  20. package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
  21. package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
  22. package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
  23. package/_bundled_plugin/scripts/workbench_cli.py +0 -3
  24. package/_bundled_plugin/scripts/workbench_constants.py +4 -0
  25. package/_bundled_plugin/scripts/workbench_db.py +56 -33
  26. package/_bundled_plugin/scripts/workbench_progress.py +0 -50
  27. package/_bundled_plugin/scripts/workbench_scan_history.py +3 -2
  28. package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
  29. package/_bundled_plugin/scripts/workbench_scan_usage.py +190 -30
  30. package/_bundled_plugin/scripts/workbench_schema.py +1 -1
  31. package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
  32. package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
  33. package/_bundled_plugin/scripts/workbench_target.py +7 -5
  34. package/_bundled_plugin/scripts/workbench_validation.py +37 -1
  35. package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
  36. package/_bundled_plugin/shared/workbench-migrations.json +5 -0
  37. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
  38. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
  39. package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
  40. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  41. package/dist/api.d.ts +3 -2
  42. package/dist/api.d.ts.map +1 -1
  43. package/dist/api.js +54 -36
  44. package/dist/api.js.map +1 -1
  45. package/dist/artifact-export.d.ts +2 -1
  46. package/dist/artifact-export.d.ts.map +1 -1
  47. package/dist/artifact-export.js +38 -23
  48. package/dist/artifact-export.js.map +1 -1
  49. package/dist/classify-severity-records.d.ts +76 -0
  50. package/dist/classify-severity-records.d.ts.map +1 -0
  51. package/dist/classify-severity-records.js +141 -0
  52. package/dist/classify-severity-records.js.map +1 -0
  53. package/dist/classify-severity.d.ts +1 -23
  54. package/dist/classify-severity.d.ts.map +1 -1
  55. package/dist/classify-severity.js +3 -24
  56. package/dist/classify-severity.js.map +1 -1
  57. package/dist/cli-help.d.ts.map +1 -1
  58. package/dist/cli-help.js +2 -0
  59. package/dist/cli-help.js.map +1 -1
  60. package/dist/cli.d.ts +4 -2
  61. package/dist/cli.d.ts.map +1 -1
  62. package/dist/cli.js +136 -147
  63. package/dist/cli.js.map +1 -1
  64. package/dist/cloud-publish.js +1 -1
  65. package/dist/cloud-publish.js.map +1 -1
  66. package/dist/codex-sdk-environment.js +1 -1
  67. package/dist/codex-sdk-environment.js.map +1 -1
  68. package/dist/config.d.ts +12 -0
  69. package/dist/config.d.ts.map +1 -1
  70. package/dist/config.js +10 -4
  71. package/dist/config.js.map +1 -1
  72. package/dist/contract.d.ts +5 -0
  73. package/dist/contract.d.ts.map +1 -1
  74. package/dist/contract.js +65 -17
  75. package/dist/contract.js.map +1 -1
  76. package/dist/cost-model.d.ts.map +1 -1
  77. package/dist/cost-model.js +7 -2
  78. package/dist/cost-model.js.map +1 -1
  79. package/dist/custom-validation-prompt.js +3 -3
  80. package/dist/custom-validation-prompt.js.map +1 -1
  81. package/dist/deduplication/deduplication-prompts.d.ts +1 -1
  82. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  83. package/dist/deduplication/deduplication-prompts.js +2 -2
  84. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  85. package/dist/deduplication/records.js +1 -1
  86. package/dist/deduplication/records.js.map +1 -1
  87. package/dist/deduplication/scan.d.ts.map +1 -1
  88. package/dist/deduplication/scan.js +10 -9
  89. package/dist/deduplication/scan.js.map +1 -1
  90. package/dist/deep-config.d.ts +4 -1
  91. package/dist/deep-config.d.ts.map +1 -1
  92. package/dist/deep-config.js +21 -1
  93. package/dist/deep-config.js.map +1 -1
  94. package/dist/findings-import.d.ts +1 -1
  95. package/dist/findings-import.d.ts.map +1 -1
  96. package/dist/findings-import.js +3 -3
  97. package/dist/findings-import.js.map +1 -1
  98. package/dist/git-remote.d.ts +10 -0
  99. package/dist/git-remote.d.ts.map +1 -0
  100. package/dist/git-remote.js +24 -0
  101. package/dist/git-remote.js.map +1 -0
  102. package/dist/github.d.ts.map +1 -1
  103. package/dist/github.js +5 -7
  104. package/dist/github.js.map +1 -1
  105. package/dist/host-protocol.js +1 -1
  106. package/dist/host-protocol.js.map +1 -1
  107. package/dist/host-records-protocol.d.ts +15 -0
  108. package/dist/host-records-protocol.d.ts.map +1 -0
  109. package/dist/host-records-protocol.js +15 -0
  110. package/dist/host-records-protocol.js.map +1 -0
  111. package/dist/hosted-scan.d.ts +1 -6
  112. package/dist/hosted-scan.d.ts.map +1 -1
  113. package/dist/hosted-scan.js +2 -2
  114. package/dist/hosted-scan.js.map +1 -1
  115. package/dist/index.d.ts +4 -0
  116. package/dist/index.d.ts.map +1 -1
  117. package/dist/index.js +2 -0
  118. package/dist/index.js.map +1 -1
  119. package/dist/knowledge-base.d.ts.map +1 -1
  120. package/dist/knowledge-base.js +87 -7
  121. package/dist/knowledge-base.js.map +1 -1
  122. package/dist/multiscan.d.ts.map +1 -1
  123. package/dist/multiscan.js +62 -28
  124. package/dist/multiscan.js.map +1 -1
  125. package/dist/patch-publication.d.ts +26 -0
  126. package/dist/patch-publication.d.ts.map +1 -0
  127. package/dist/patch-publication.js +105 -0
  128. package/dist/patch-publication.js.map +1 -0
  129. package/dist/patch-tui.d.ts.map +1 -1
  130. package/dist/patch-tui.js +4 -1
  131. package/dist/patch-tui.js.map +1 -1
  132. package/dist/project-config-schema.d.ts +2 -0
  133. package/dist/project-config-schema.d.ts.map +1 -1
  134. package/dist/project-config-schema.js +1 -0
  135. package/dist/project-config-schema.js.map +1 -1
  136. package/dist/project-config.d.ts +1 -0
  137. package/dist/project-config.d.ts.map +1 -1
  138. package/dist/project-config.js +16 -6
  139. package/dist/project-config.js.map +1 -1
  140. package/dist/publication.d.ts.map +1 -1
  141. package/dist/publication.js +135 -19
  142. package/dist/publication.js.map +1 -1
  143. package/dist/publish.d.ts.map +1 -1
  144. package/dist/publish.js +39 -25
  145. package/dist/publish.js.map +1 -1
  146. package/dist/runtime.d.ts.map +1 -1
  147. package/dist/runtime.js +83 -22
  148. package/dist/runtime.js.map +1 -1
  149. package/dist/scan-settings.d.ts +1 -0
  150. package/dist/scan-settings.d.ts.map +1 -1
  151. package/dist/scan-settings.js +7 -1
  152. package/dist/scan-settings.js.map +1 -1
  153. package/dist/scan-time-budget.d.ts +9 -0
  154. package/dist/scan-time-budget.d.ts.map +1 -0
  155. package/dist/scan-time-budget.js +16 -0
  156. package/dist/scan-time-budget.js.map +1 -0
  157. package/dist/severity-review.d.ts +29 -0
  158. package/dist/severity-review.d.ts.map +1 -0
  159. package/dist/severity-review.js +47 -0
  160. package/dist/severity-review.js.map +1 -0
  161. package/dist/validation-records.d.ts +38 -0
  162. package/dist/validation-records.d.ts.map +1 -0
  163. package/dist/validation-records.js +76 -0
  164. package/dist/validation-records.js.map +1 -0
  165. package/dist/version.d.ts.map +1 -1
  166. package/dist/version.js +4 -2
  167. package/dist/version.js.map +1 -1
  168. package/dist/worker-progress.d.ts +2 -6
  169. package/dist/worker-progress.d.ts.map +1 -1
  170. package/dist/worker-progress.js +1 -43
  171. package/dist/worker-progress.js.map +1 -1
  172. package/docs/classify-severity-records.md +89 -0
  173. package/docs/cli.md +269 -33
  174. package/docs/validate-records.md +69 -0
  175. package/package.json +12 -10
  176. package/schemas/project-config.schema.json +7 -1
  177. package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
  178. package/_bundled_plugin/references/config-preflight.md +0 -93
  179. package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
  180. package/_bundled_plugin/scripts/config_preflight.py +0 -911
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import argparse
6
6
  import json
7
+ import math
7
8
  import os
8
9
  import re
9
10
  import sqlite3
@@ -34,6 +35,14 @@ class RolloutSession:
34
35
  path: Path
35
36
 
36
37
 
38
+ @dataclass
39
+ class RolloutUsage:
40
+ tokens: dict[str, int]
41
+ warnings: set[str]
42
+ tokens_updated_at: datetime | None
43
+ rate_limits: dict[str, dict[str, Any]]
44
+
45
+
37
46
  def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
38
47
  """Project measured usage without changing the existing legacy cost contract."""
39
48
 
@@ -50,12 +59,6 @@ def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
50
59
  }
51
60
 
52
61
 
53
- def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
54
- """Keep usage in the already-migrated scans.cost_json column."""
55
-
56
- return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
57
-
58
-
59
62
  def reconcile_completed_scan_cost(
60
63
  connection: sqlite3.Connection,
61
64
  scan: sqlite3.Row,
@@ -86,9 +89,10 @@ def collect_scan_usage(
86
89
  completed_at: str | None = None,
87
90
  completed_turn_id: str | None = None,
88
91
  ) -> dict[str, Any]:
89
- """Count only complete, attributable rollout events inside this scan's window."""
92
+ """Count attributable receipts and flag observable gaps inside the scan window."""
90
93
 
91
94
  roots = _scan_root_thread_ids(connection, scan, thread_id)
95
+ parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
92
96
  worker_directories = _scan_worker_directories(connection, scan)
93
97
  if not roots and not worker_directories:
94
98
  return _unavailable_usage("scan_thread_unavailable")
@@ -98,7 +102,8 @@ def collect_scan_usage(
98
102
  return _unavailable_usage("codex_state_unavailable")
99
103
 
100
104
  started_at = _timestamp(scan["started_at"])
101
- stopped_at = _timestamp(completed_at or scan["completed_at"])
105
+ observed_at = datetime.now(timezone.utc)
106
+ stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
102
107
  if started_at is None:
103
108
  return _unavailable_usage("scan_window_unavailable")
104
109
 
@@ -118,7 +123,9 @@ def collect_scan_usage(
118
123
  if not sessions:
119
124
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
120
125
 
121
- total = _empty_token_usage()
126
+ by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
127
+ rate_limits: dict[str, dict[str, Any]] = {}
128
+ tokens_updated_at = None
122
129
  accepted_thread_ids: set[str] = set()
123
130
  excluded_thread_ids: set[str] = set()
124
131
  for session in sessions:
@@ -133,7 +140,7 @@ def collect_scan_usage(
133
140
  warnings.add("thread_lineage_incomplete")
134
141
  continue
135
142
  try:
136
- session_usage, session_warnings = _read_rollout_usage(
143
+ usage = _read_rollout_usage(
137
144
  session,
138
145
  started_at=started_at,
139
146
  completed_at=stopped_at,
@@ -143,27 +150,51 @@ def collect_scan_usage(
143
150
  missing_thread_ids.add(session.thread_id)
144
151
  warnings.add("rollout_unavailable")
145
152
  continue
146
- if "thread_outside_scan_window" in session_warnings:
153
+ if "thread_outside_scan_window" in usage.warnings:
147
154
  excluded_thread_ids.add(session.thread_id)
148
155
  continue
149
- warnings.update(session_warnings)
150
- if "thread_identity_mismatch" in session_warnings or (
151
- "thread_ownership_unavailable" in session_warnings
156
+ warnings.update(usage.warnings)
157
+ if "thread_identity_mismatch" in usage.warnings or (
158
+ "thread_ownership_unavailable" in usage.warnings
152
159
  ):
153
160
  missing_thread_ids.add(session.thread_id)
154
161
  continue
155
162
  accepted_thread_ids.add(session.thread_id)
156
- _add_token_usage(total, session_usage)
163
+ role = by_role["parent" if session.thread_id in parent_ids else "workers"]
164
+ _add_token_usage(role, usage.tokens)
165
+ role["threadCount"] += 1
166
+ if usage.tokens_updated_at is not None:
167
+ tokens_updated_at = max(
168
+ tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
169
+ )
170
+ for key, snapshot in usage.rate_limits.items():
171
+ if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
172
+ rate_limits[key] = snapshot
157
173
 
158
174
  if not accepted_thread_ids:
159
175
  return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
160
176
 
177
+ if any(
178
+ usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
179
+ or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
180
+ for usage in by_role.values()
181
+ ):
182
+ return _unavailable_usage("token_breakdown_unavailable", warnings=warnings)
183
+
184
+ if completed_turn_id is not None and tokens_updated_at is not None:
185
+ stopped_at = max(stopped_at, tokens_updated_at)
161
186
  result: dict[str, Any] = {
162
187
  "coverage": "partial" if missing_thread_ids or warnings else "complete",
163
188
  "source": "codex_rollout",
164
- **total,
189
+ **{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
165
190
  "threadCount": len(accepted_thread_ids),
191
+ "byRole": by_role,
192
+ "window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
193
+ "observedAt": observed_at.isoformat(),
194
+ "accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
166
195
  }
196
+ if tokens_updated_at is not None:
197
+ result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
167
198
  if missing_thread_ids:
168
199
  result["missingThreadCount"] = len(missing_thread_ids)
169
200
  if warnings:
@@ -177,6 +208,7 @@ def _scan_root_thread_ids(
177
208
  supplied_thread_id: str | None,
178
209
  *,
179
210
  include_owner_threads: bool = True,
211
+ include_workers: bool = True,
180
212
  ) -> list[str]:
181
213
  candidates: list[str | None] = [supplied_thread_id]
182
214
  if include_owner_threads:
@@ -190,7 +222,7 @@ def _scan_root_thread_ids(
190
222
  ).fetchone()
191
223
  if workspace is not None:
192
224
  candidates.append(workspace["thread_id"])
193
- if scan["mode"] == "deep":
225
+ if include_workers and scan["mode"] == "deep":
194
226
  candidates.extend(
195
227
  row["sdk_thread_id"]
196
228
  for row in connection.execute(
@@ -440,9 +472,10 @@ def _read_rollout_usage(
440
472
  started_at: datetime,
441
473
  completed_at: datetime | None,
442
474
  completed_turn_id: str | None = None,
443
- ) -> tuple[dict[str, int], set[str]]:
444
- total = _empty_token_usage()
445
- warnings: set[str] = set()
475
+ ) -> RolloutUsage:
476
+ result = RolloutUsage(_empty_token_usage(), set(), None, {})
477
+ total = result.tokens
478
+ warnings = result.warnings
446
479
  previous = _empty_token_usage()
447
480
  boundary_reached = False
448
481
  legacy_usage = _empty_token_usage()
@@ -451,6 +484,10 @@ def _read_rollout_usage(
451
484
  response_ids: set[str] = set()
452
485
  current_turn_id: str | None = None
453
486
  completed_turn_measured = False
487
+ started_turns: set[str] = set()
488
+ open_turns: set[str] = set()
489
+ closed_turns: set[str] = set()
490
+ measured_turns: set[str] = set()
454
491
 
455
492
  with session.path.open("rb") as source:
456
493
  for line_number, raw_line in enumerate(source, start=1):
@@ -473,16 +510,16 @@ def _read_rollout_usage(
473
510
  if line_number == 1:
474
511
  if event.get("type") != "session_meta" or not isinstance(payload, dict):
475
512
  warnings.add("thread_identity_mismatch")
476
- return total, warnings
513
+ return result
477
514
  recorded_id = payload.get("id") or payload.get("session_id")
478
515
  if recorded_id != session.thread_id:
479
516
  warnings.add("thread_identity_mismatch")
480
- return total, warnings
517
+ return result
481
518
  recorded_parent = _session_parent_thread_id(payload)
482
519
  if session.parent_thread_id is not None:
483
520
  if recorded_parent != session.parent_thread_id:
484
521
  warnings.add("thread_identity_mismatch")
485
- return total, warnings
522
+ return result
486
523
  boundary_reached = (
487
524
  session.parent_thread_id is None
488
525
  and not recorded_parent
@@ -497,21 +534,21 @@ def _read_rollout_usage(
497
534
  task_started_at = _timestamp(event.get("timestamp"))
498
535
  if task_started_at is None:
499
536
  warnings.add("thread_ownership_unavailable")
500
- return total, warnings
537
+ return result
501
538
  if task_started_at < started_at or (
502
539
  completed_at is not None
503
540
  and task_started_at > completed_at
504
541
  and payload.get("turn_id") != completed_turn_id
505
542
  ):
506
543
  warnings.add("thread_outside_scan_window")
507
- return total, warnings
544
+ return result
508
545
  boundary_reached = True
509
- current_turn_id = payload.get("turn_id")
510
546
  elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
511
547
  inherited_usage = _token_snapshot(payload)
512
548
  if inherited_usage is not None:
513
549
  previous = inherited_usage
514
- continue
550
+ if not boundary_reached:
551
+ continue
515
552
  if event.get("type") == "event_msg" and payload.get("type") == "task_started":
516
553
  if (
517
554
  completed_turn_id is not None
@@ -527,7 +564,41 @@ def _read_rollout_usage(
527
564
  if payload.get("turn_id") != current_turn_id:
528
565
  observed_response_usage = _empty_token_usage()
529
566
  current_turn_id = payload.get("turn_id")
567
+ timestamp = _timestamp(event.get("timestamp"))
568
+ if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
569
+ warnings.add("turn_record_invalid")
570
+ elif (
571
+ completed_at is None
572
+ or timestamp <= completed_at
573
+ or current_turn_id == completed_turn_id
574
+ ):
575
+ started_turns.add(current_turn_id)
576
+ open_turns.add(current_turn_id)
530
577
  continue
578
+ if event.get("type") == "event_msg" and payload.get("type") in {
579
+ "task_complete",
580
+ "turn_aborted",
581
+ }:
582
+ timestamp = _timestamp(event.get("timestamp"))
583
+ terminal_turn_id = payload.get("turn_id") or current_turn_id
584
+ if (
585
+ not isinstance(terminal_turn_id, str)
586
+ or not terminal_turn_id
587
+ or timestamp is None
588
+ ):
589
+ warnings.add("turn_record_invalid")
590
+ elif (
591
+ completed_at is None
592
+ or timestamp <= completed_at
593
+ or terminal_turn_id == completed_turn_id
594
+ ):
595
+ open_turns.discard(terminal_turn_id)
596
+ if timestamp >= started_at:
597
+ if terminal_turn_id in started_turns:
598
+ closed_turns.add(terminal_turn_id)
599
+ if payload.get("type") == "turn_aborted":
600
+ # Later successful work does not recover interrupted request usage.
601
+ warnings.add("turn_interrupted")
531
602
  if (
532
603
  completed_turn_id is not None
533
604
  and event.get("type") == "event_msg"
@@ -576,11 +647,16 @@ def _read_rollout_usage(
576
647
  response_usage = _empty_token_usage()
577
648
  if current_turn_id is None:
578
649
  current_turn_id = payload.get("turn_id")
650
+ if isinstance(current_turn_id, str):
651
+ open_turns.add(current_turn_id)
579
652
  if response_id in response_ids:
580
653
  continue
581
654
  response_ids.add(response_id)
582
655
  _add_token_usage(observed_response_usage, usage)
583
656
  turn_usage = _usage_counts(payload.get("turn_token_usage"))
657
+ receipt_turn_id = payload.get("turn_id")
658
+ if isinstance(receipt_turn_id, str):
659
+ measured_turns.add(receipt_turn_id)
584
660
  if timestamp < started_at:
585
661
  if turn_usage is not None:
586
662
  observed_response_usage = turn_usage
@@ -591,10 +667,30 @@ def _read_rollout_usage(
591
667
  turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
592
668
  ):
593
669
  warnings.add("token_record_incomplete")
670
+ result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
594
671
  _add_token_usage(response_usage, usage)
595
672
  continue
596
673
  if event.get("type") != "event_msg" or payload.get("type") != "token_count":
597
674
  continue
675
+ timestamp = _timestamp(event.get("timestamp"))
676
+ in_window = (
677
+ timestamp is not None
678
+ and timestamp >= started_at
679
+ and (
680
+ completed_at is None
681
+ or timestamp <= completed_at
682
+ or (completed_turn_id is not None and current_turn_id == completed_turn_id)
683
+ )
684
+ )
685
+ if in_window:
686
+ rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
687
+ if rate_limits is not None:
688
+ key = rate_limits["limitId"]
689
+ if (
690
+ key not in result.rate_limits
691
+ or rate_limits["observedAt"] >= result.rate_limits[key]["observedAt"]
692
+ ):
693
+ result.rate_limits[key] = rate_limits
598
694
  if "info" in payload and payload["info"] is None:
599
695
  continue
600
696
  timestamp = _timestamp(event.get("timestamp"))
@@ -607,15 +703,18 @@ def _read_rollout_usage(
607
703
  for key, value in snapshot.items()
608
704
  }
609
705
  previous = snapshot
610
- if timestamp < started_at:
611
- continue
612
706
  is_completion_turn = (
613
707
  completed_turn_id is not None and current_turn_id == completed_turn_id
614
708
  )
615
709
  if completed_at is not None and timestamp > completed_at and not is_completion_turn:
616
710
  continue
711
+ if isinstance(current_turn_id, str):
712
+ measured_turns.add(current_turn_id)
713
+ if timestamp < started_at:
714
+ continue
617
715
  if is_completion_turn:
618
716
  completed_turn_measured = True
717
+ result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
619
718
  if delta["totalTokens"] <= 0:
620
719
  continue
621
720
  _add_token_usage(legacy_usage, delta)
@@ -623,9 +722,70 @@ def _read_rollout_usage(
623
722
  _add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
624
723
  if not boundary_reached:
625
724
  warnings.add("thread_ownership_unavailable")
725
+ if completed_turn_id is not None:
726
+ open_turns.discard(completed_turn_id)
727
+ if open_turns:
728
+ warnings.add("turn_in_progress")
729
+ if boundary_reached and result.tokens_updated_at is None:
730
+ warnings.add("token_record_missing")
731
+ if closed_turns - measured_turns:
732
+ warnings.add("token_record_missing")
626
733
  if completed_turn_id is not None and not completed_turn_measured:
627
734
  warnings.add("completion_turn_unavailable")
628
- return total, warnings
735
+ return result
736
+
737
+
738
+ def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None:
739
+ if not isinstance(value, dict):
740
+ return None
741
+ result: dict[str, Any] = {
742
+ "limitId": value.get("limit_id") or "codex",
743
+ "observedAt": timestamp.isoformat(),
744
+ }
745
+ for name in ("primary", "secondary"):
746
+ window = value.get(name)
747
+ if window is None:
748
+ result[name] = None
749
+ continue
750
+ if not isinstance(window, dict):
751
+ return None
752
+ result[name] = {
753
+ "usedPercent": window.get("used_percent"),
754
+ "windowMinutes": window.get("window_minutes"),
755
+ "resetsAt": window.get("resets_at"),
756
+ }
757
+ return result if valid_account_rate_limit(result) else None
758
+
759
+
760
+ def valid_account_rate_limit(value: object) -> bool:
761
+ if not isinstance(value, dict) or set(value) != {
762
+ "limitId",
763
+ "observedAt",
764
+ "primary",
765
+ "secondary",
766
+ }:
767
+ return False
768
+ if not isinstance(value["limitId"], str) or _timestamp(value["observedAt"]) is None:
769
+ return False
770
+ for name in ("primary", "secondary"):
771
+ window = value[name]
772
+ if window is None:
773
+ continue
774
+ if not isinstance(window, dict) or set(window) != {
775
+ "usedPercent",
776
+ "windowMinutes",
777
+ "resetsAt",
778
+ }:
779
+ return False
780
+ percent = window["usedPercent"]
781
+ if type(percent) not in (int, float) or not math.isfinite(percent):
782
+ return False
783
+ if any(
784
+ window[key] is not None and type(window[key]) is not int
785
+ for key in ("windowMinutes", "resetsAt")
786
+ ):
787
+ return False
788
+ return True
629
789
 
630
790
 
631
791
  def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
@@ -75,7 +75,7 @@ def apply_migrations(
75
75
  elif version == 16:
76
76
  should_backfill_targets = repair_stable_targets_migration(connection)
77
77
  elif version in applied:
78
- if version in (2, 12, 13, 26, 28, 31, 32, 47):
78
+ if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50):
79
79
  repair_additive_migration(connection, version)
80
80
  elif version == 11:
81
81
  repair_deep_scan_migration(connection)
@@ -8,9 +8,16 @@ import sys
8
8
  from pathlib import Path, PurePosixPath
9
9
  from typing import Any
10
10
 
11
- # Some plugin hosts launch Python with safe-path isolation enabled.
12
11
  sys.path.insert(0, str(Path(__file__).resolve().parent))
13
- from workbench_target import clean_worktree_content_digest, git_bytes
12
+ from workbench_source_scopes import (
13
+ load_source_scopes,
14
+ normalized_path_component,
15
+ offline_git_bytes,
16
+ relative_path,
17
+ safe_source_path,
18
+ source_object_for_path,
19
+ )
20
+ from workbench_target import clean_worktree_content_digest
14
21
 
15
22
  CONTEXT_LINES = 3
16
23
  MAX_BYTES = 16_000
@@ -21,23 +28,70 @@ def finding_source_excerpt(
21
28
  scan: sqlite3.Row,
22
29
  target: Path | None,
23
30
  locations: list[dict[str, Any]],
31
+ scopes: list[str],
24
32
  ) -> str | None:
25
- if target is None or not locations:
33
+ if target is None or not locations or scan["target_revision"] == "unversioned":
26
34
  return None
27
- location = next(
28
- (
29
- candidate
30
- for candidate in locations
31
- if "root_control" in str(candidate.get("role") or "").lower()
32
- ),
33
- locations[0],
35
+ snapshot = scan["target_snapshot_digest"]
36
+ if snapshot is not None and snapshot != clean_worktree_content_digest():
37
+ return None
38
+ locations = [
39
+ location
40
+ for location in locations
41
+ if isinstance(path := location.get("path"), str)
42
+ and isinstance(location.get("startLine"), int)
43
+ and safe_source_path(target, path) is not None
44
+ ]
45
+ if not locations:
46
+ return None
47
+ try:
48
+ context = load_source_scopes(scan, target, scopes)
49
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
50
+ return None
51
+ if context is None:
52
+ return None
53
+ repository, tree, records = context
54
+ indexed: dict[tuple[str, ...], list[dict[str, str]]] = {}
55
+ for record in records:
56
+ for name in ("path", "canonicalPath"):
57
+ parts = tuple(
58
+ normalized_path_component(part) for part in PurePosixPath(record[name]).parts
59
+ )
60
+ indexed.setdefault(parts, []).append(record)
61
+ lengths = sorted({len(parts) for parts in indexed})
62
+
63
+ def source_object(path: str) -> str | None:
64
+ parsed = relative_path(path)
65
+ if parsed is None:
66
+ return None
67
+ parts = tuple(normalized_path_component(part) for part in parsed.parts)
68
+ try:
69
+ for length in lengths:
70
+ if length > len(parts):
71
+ break
72
+ for record in indexed.get(parts[:length], []):
73
+ object_id = source_object_for_path(repository, tree, target, path, record)
74
+ if object_id is not None:
75
+ return object_id
76
+ except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
77
+ # An unavailable source object leaves the optional excerpt absent.
78
+ pass
79
+ return None
80
+
81
+ locations.sort(
82
+ key=lambda location: (
83
+ location.get("role") != "root_control",
84
+ "root_control" not in str(location.get("role") or "").lower(),
85
+ )
34
86
  )
35
- path = location.get("path")
36
- start_line = location.get("startLine")
37
- end_line = location.get("endLine")
38
- if not isinstance(path, str) or not isinstance(start_line, int):
87
+ for location in locations:
88
+ object_id = source_object(location["path"])
89
+ if object_id is not None:
90
+ break
91
+ else:
39
92
  return None
40
- source = scanned_source_text(scan, target, path)
93
+ start_line, end_line = location["startLine"], location.get("endLine")
94
+ source = scanned_source_text(repository, object_id)
41
95
  if not source or "\0" in source:
42
96
  return None
43
97
  # Match location validation and SARIF hashing: CR, LF, and CRLF are line breaks.
@@ -58,46 +112,15 @@ def finding_source_excerpt(
58
112
  f"{line_number:>{width}} {lines[line_number - 1]}"
59
113
  for line_number in range(excerpt_start, excerpt_end + 1)
60
114
  )
61
- encoded = excerpt.encode("utf-8")[:MAX_BYTES]
62
- return encoded.decode("utf-8", errors="ignore")
115
+ return excerpt.encode("utf-8")[:MAX_BYTES].decode("utf-8", errors="ignore")
63
116
 
64
117
 
65
- def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | None:
66
- if safe_source_path(target, path) is None:
67
- return None
68
- revision = scan["target_revision"]
69
- if revision == "unversioned":
70
- return None
71
- snapshot_digest = scan["target_snapshot_digest"]
72
- if (
73
- scan["diff_target_kind"] == "working_tree"
74
- and scan["diff_content_digest"] != clean_worktree_content_digest()
75
- ):
76
- return None
77
- if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
78
- return None
79
- if (
80
- scan["diff_target_kind"] == "working_tree"
81
- and scan["diff_content_digest"] != clean_worktree_content_digest()
82
- ):
83
- return None
84
- object_name = f"{revision}:./{path}"
85
- content = git_bytes(target, "cat-file", "blob", object_name)
86
- return content.decode("utf-8", errors="replace") if content is not None else None
87
-
88
-
89
- def safe_source_path(target: Path, relative_path: str) -> Path | None:
90
- if "\\" in relative_path:
91
- return None
92
- parsed = PurePosixPath(relative_path)
93
- if parsed.is_absolute() or ".." in parsed.parts:
94
- return None
118
+ def scanned_source_text(repository: Path, object_id: str) -> str | None:
95
119
  try:
96
- path = (target / parsed.as_posix()).resolve()
97
- path.relative_to(target)
98
- except (OSError, RuntimeError, ValueError):
120
+ content = offline_git_bytes(repository, "cat-file", "blob", object_id)
121
+ except (OSError, RuntimeError, SystemExit):
99
122
  return None
100
- return path
123
+ return content.decode("utf-8", errors="replace") if content is not None else None
101
124
 
102
125
 
103
126
  if __name__ == "__main__":