@openai/codex-security 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -9
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/core-scan.md +2 -0
- package/_bundled_plugin/references/final-report.md +5 -1
- package/_bundled_plugin/references/investigation-time-budget.md +11 -0
- package/_bundled_plugin/references/sarif-adapter.md +1 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -6
- package/_bundled_plugin/references/scan-runtime.md +15 -0
- package/_bundled_plugin/references/threat-model.md +2 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +5 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +80 -43
- package/_bundled_plugin/scripts/finding_preview.py +5 -3
- package/_bundled_plugin/scripts/generate_rank_input.py +1 -0
- package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
- package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
- package/_bundled_plugin/scripts/workbench_cli.py +0 -3
- package/_bundled_plugin/scripts/workbench_constants.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +56 -33
- package/_bundled_plugin/scripts/workbench_progress.py +0 -50
- package/_bundled_plugin/scripts/workbench_scan_history.py +3 -2
- package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
- package/_bundled_plugin/scripts/workbench_scan_usage.py +190 -30
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
- package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
- package/_bundled_plugin/scripts/workbench_target.py +7 -5
- package/_bundled_plugin/scripts/workbench_validation.py +37 -1
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
- package/_bundled_plugin/shared/workbench-migrations.json +5 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/dist/api.d.ts +3 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +54 -36
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +2 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +38 -23
- package/dist/artifact-export.js.map +1 -1
- package/dist/classify-severity-records.d.ts +76 -0
- package/dist/classify-severity-records.d.ts.map +1 -0
- package/dist/classify-severity-records.js +141 -0
- package/dist/classify-severity-records.js.map +1 -0
- package/dist/classify-severity.d.ts +1 -23
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -24
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +2 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +4 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +136 -147
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +1 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-sdk-environment.js +1 -1
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -4
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +5 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +65 -17
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +7 -2
- package/dist/cost-model.js.map +1 -1
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/records.js +1 -1
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +10 -9
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +4 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +21 -1
- package/dist/deep-config.js.map +1 -1
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +3 -3
- package/dist/findings-import.js.map +1 -1
- package/dist/git-remote.d.ts +10 -0
- package/dist/git-remote.d.ts.map +1 -0
- package/dist/git-remote.js +24 -0
- package/dist/git-remote.js.map +1 -0
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +5 -7
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.js +1 -1
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +15 -0
- package/dist/host-records-protocol.d.ts.map +1 -0
- package/dist/host-records-protocol.js +15 -0
- package/dist/host-records-protocol.js.map +1 -0
- package/dist/hosted-scan.d.ts +1 -6
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +2 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +87 -7
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +62 -28
- package/dist/multiscan.js.map +1 -1
- package/dist/patch-publication.d.ts +26 -0
- package/dist/patch-publication.d.ts.map +1 -0
- package/dist/patch-publication.js +105 -0
- package/dist/patch-publication.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +4 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +16 -6
- package/dist/project-config.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +135 -19
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +39 -25
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +83 -22
- package/dist/runtime.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +7 -1
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-time-budget.d.ts +9 -0
- package/dist/scan-time-budget.d.ts.map +1 -0
- package/dist/scan-time-budget.js +16 -0
- package/dist/scan-time-budget.js.map +1 -0
- package/dist/severity-review.d.ts +29 -0
- package/dist/severity-review.d.ts.map +1 -0
- package/dist/severity-review.js +47 -0
- package/dist/severity-review.js.map +1 -0
- package/dist/validation-records.d.ts +38 -0
- package/dist/validation-records.d.ts.map +1 -0
- package/dist/validation-records.js +76 -0
- package/dist/validation-records.js.map +1 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/dist/worker-progress.d.ts +2 -6
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +1 -43
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +89 -0
- package/docs/cli.md +269 -33
- package/docs/validate-records.md +69 -0
- package/package.json +12 -10
- package/schemas/project-config.schema.json +7 -1
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
- package/_bundled_plugin/references/config-preflight.md +0 -93
- package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
- package/_bundled_plugin/scripts/config_preflight.py +0 -911
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
|
+
import math
|
|
7
8
|
import os
|
|
8
9
|
import re
|
|
9
10
|
import sqlite3
|
|
@@ -34,6 +35,14 @@ class RolloutSession:
|
|
|
34
35
|
path: Path
|
|
35
36
|
|
|
36
37
|
|
|
38
|
+
@dataclass
|
|
39
|
+
class RolloutUsage:
|
|
40
|
+
tokens: dict[str, int]
|
|
41
|
+
warnings: set[str]
|
|
42
|
+
tokens_updated_at: datetime | None
|
|
43
|
+
rate_limits: dict[str, dict[str, Any]]
|
|
44
|
+
|
|
45
|
+
|
|
37
46
|
def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
38
47
|
"""Project measured usage without changing the existing legacy cost contract."""
|
|
39
48
|
|
|
@@ -50,12 +59,6 @@ def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
|
50
59
|
}
|
|
51
60
|
|
|
52
61
|
|
|
53
|
-
def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
|
|
54
|
-
"""Keep usage in the already-migrated scans.cost_json column."""
|
|
55
|
-
|
|
56
|
-
return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
|
|
57
|
-
|
|
58
|
-
|
|
59
62
|
def reconcile_completed_scan_cost(
|
|
60
63
|
connection: sqlite3.Connection,
|
|
61
64
|
scan: sqlite3.Row,
|
|
@@ -86,9 +89,10 @@ def collect_scan_usage(
|
|
|
86
89
|
completed_at: str | None = None,
|
|
87
90
|
completed_turn_id: str | None = None,
|
|
88
91
|
) -> dict[str, Any]:
|
|
89
|
-
"""Count
|
|
92
|
+
"""Count attributable receipts and flag observable gaps inside the scan window."""
|
|
90
93
|
|
|
91
94
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
95
|
+
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
92
96
|
worker_directories = _scan_worker_directories(connection, scan)
|
|
93
97
|
if not roots and not worker_directories:
|
|
94
98
|
return _unavailable_usage("scan_thread_unavailable")
|
|
@@ -98,7 +102,8 @@ def collect_scan_usage(
|
|
|
98
102
|
return _unavailable_usage("codex_state_unavailable")
|
|
99
103
|
|
|
100
104
|
started_at = _timestamp(scan["started_at"])
|
|
101
|
-
|
|
105
|
+
observed_at = datetime.now(timezone.utc)
|
|
106
|
+
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
102
107
|
if started_at is None:
|
|
103
108
|
return _unavailable_usage("scan_window_unavailable")
|
|
104
109
|
|
|
@@ -118,7 +123,9 @@ def collect_scan_usage(
|
|
|
118
123
|
if not sessions:
|
|
119
124
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
120
125
|
|
|
121
|
-
|
|
126
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
|
+
tokens_updated_at = None
|
|
122
129
|
accepted_thread_ids: set[str] = set()
|
|
123
130
|
excluded_thread_ids: set[str] = set()
|
|
124
131
|
for session in sessions:
|
|
@@ -133,7 +140,7 @@ def collect_scan_usage(
|
|
|
133
140
|
warnings.add("thread_lineage_incomplete")
|
|
134
141
|
continue
|
|
135
142
|
try:
|
|
136
|
-
|
|
143
|
+
usage = _read_rollout_usage(
|
|
137
144
|
session,
|
|
138
145
|
started_at=started_at,
|
|
139
146
|
completed_at=stopped_at,
|
|
@@ -143,27 +150,51 @@ def collect_scan_usage(
|
|
|
143
150
|
missing_thread_ids.add(session.thread_id)
|
|
144
151
|
warnings.add("rollout_unavailable")
|
|
145
152
|
continue
|
|
146
|
-
if "thread_outside_scan_window" in
|
|
153
|
+
if "thread_outside_scan_window" in usage.warnings:
|
|
147
154
|
excluded_thread_ids.add(session.thread_id)
|
|
148
155
|
continue
|
|
149
|
-
warnings.update(
|
|
150
|
-
if "thread_identity_mismatch" in
|
|
151
|
-
"thread_ownership_unavailable" in
|
|
156
|
+
warnings.update(usage.warnings)
|
|
157
|
+
if "thread_identity_mismatch" in usage.warnings or (
|
|
158
|
+
"thread_ownership_unavailable" in usage.warnings
|
|
152
159
|
):
|
|
153
160
|
missing_thread_ids.add(session.thread_id)
|
|
154
161
|
continue
|
|
155
162
|
accepted_thread_ids.add(session.thread_id)
|
|
156
|
-
|
|
163
|
+
role = by_role["parent" if session.thread_id in parent_ids else "workers"]
|
|
164
|
+
_add_token_usage(role, usage.tokens)
|
|
165
|
+
role["threadCount"] += 1
|
|
166
|
+
if usage.tokens_updated_at is not None:
|
|
167
|
+
tokens_updated_at = max(
|
|
168
|
+
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
169
|
+
)
|
|
170
|
+
for key, snapshot in usage.rate_limits.items():
|
|
171
|
+
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
172
|
+
rate_limits[key] = snapshot
|
|
157
173
|
|
|
158
174
|
if not accepted_thread_ids:
|
|
159
175
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
160
176
|
|
|
177
|
+
if any(
|
|
178
|
+
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
179
|
+
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
180
|
+
for usage in by_role.values()
|
|
181
|
+
):
|
|
182
|
+
return _unavailable_usage("token_breakdown_unavailable", warnings=warnings)
|
|
183
|
+
|
|
184
|
+
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
185
|
+
stopped_at = max(stopped_at, tokens_updated_at)
|
|
161
186
|
result: dict[str, Any] = {
|
|
162
187
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
163
188
|
"source": "codex_rollout",
|
|
164
|
-
**
|
|
189
|
+
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
165
190
|
"threadCount": len(accepted_thread_ids),
|
|
191
|
+
"byRole": by_role,
|
|
192
|
+
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
193
|
+
"observedAt": observed_at.isoformat(),
|
|
194
|
+
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
166
195
|
}
|
|
196
|
+
if tokens_updated_at is not None:
|
|
197
|
+
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
167
198
|
if missing_thread_ids:
|
|
168
199
|
result["missingThreadCount"] = len(missing_thread_ids)
|
|
169
200
|
if warnings:
|
|
@@ -177,6 +208,7 @@ def _scan_root_thread_ids(
|
|
|
177
208
|
supplied_thread_id: str | None,
|
|
178
209
|
*,
|
|
179
210
|
include_owner_threads: bool = True,
|
|
211
|
+
include_workers: bool = True,
|
|
180
212
|
) -> list[str]:
|
|
181
213
|
candidates: list[str | None] = [supplied_thread_id]
|
|
182
214
|
if include_owner_threads:
|
|
@@ -190,7 +222,7 @@ def _scan_root_thread_ids(
|
|
|
190
222
|
).fetchone()
|
|
191
223
|
if workspace is not None:
|
|
192
224
|
candidates.append(workspace["thread_id"])
|
|
193
|
-
if scan["mode"] == "deep":
|
|
225
|
+
if include_workers and scan["mode"] == "deep":
|
|
194
226
|
candidates.extend(
|
|
195
227
|
row["sdk_thread_id"]
|
|
196
228
|
for row in connection.execute(
|
|
@@ -440,9 +472,10 @@ def _read_rollout_usage(
|
|
|
440
472
|
started_at: datetime,
|
|
441
473
|
completed_at: datetime | None,
|
|
442
474
|
completed_turn_id: str | None = None,
|
|
443
|
-
) ->
|
|
444
|
-
|
|
445
|
-
|
|
475
|
+
) -> RolloutUsage:
|
|
476
|
+
result = RolloutUsage(_empty_token_usage(), set(), None, {})
|
|
477
|
+
total = result.tokens
|
|
478
|
+
warnings = result.warnings
|
|
446
479
|
previous = _empty_token_usage()
|
|
447
480
|
boundary_reached = False
|
|
448
481
|
legacy_usage = _empty_token_usage()
|
|
@@ -451,6 +484,10 @@ def _read_rollout_usage(
|
|
|
451
484
|
response_ids: set[str] = set()
|
|
452
485
|
current_turn_id: str | None = None
|
|
453
486
|
completed_turn_measured = False
|
|
487
|
+
started_turns: set[str] = set()
|
|
488
|
+
open_turns: set[str] = set()
|
|
489
|
+
closed_turns: set[str] = set()
|
|
490
|
+
measured_turns: set[str] = set()
|
|
454
491
|
|
|
455
492
|
with session.path.open("rb") as source:
|
|
456
493
|
for line_number, raw_line in enumerate(source, start=1):
|
|
@@ -473,16 +510,16 @@ def _read_rollout_usage(
|
|
|
473
510
|
if line_number == 1:
|
|
474
511
|
if event.get("type") != "session_meta" or not isinstance(payload, dict):
|
|
475
512
|
warnings.add("thread_identity_mismatch")
|
|
476
|
-
return
|
|
513
|
+
return result
|
|
477
514
|
recorded_id = payload.get("id") or payload.get("session_id")
|
|
478
515
|
if recorded_id != session.thread_id:
|
|
479
516
|
warnings.add("thread_identity_mismatch")
|
|
480
|
-
return
|
|
517
|
+
return result
|
|
481
518
|
recorded_parent = _session_parent_thread_id(payload)
|
|
482
519
|
if session.parent_thread_id is not None:
|
|
483
520
|
if recorded_parent != session.parent_thread_id:
|
|
484
521
|
warnings.add("thread_identity_mismatch")
|
|
485
|
-
return
|
|
522
|
+
return result
|
|
486
523
|
boundary_reached = (
|
|
487
524
|
session.parent_thread_id is None
|
|
488
525
|
and not recorded_parent
|
|
@@ -497,21 +534,21 @@ def _read_rollout_usage(
|
|
|
497
534
|
task_started_at = _timestamp(event.get("timestamp"))
|
|
498
535
|
if task_started_at is None:
|
|
499
536
|
warnings.add("thread_ownership_unavailable")
|
|
500
|
-
return
|
|
537
|
+
return result
|
|
501
538
|
if task_started_at < started_at or (
|
|
502
539
|
completed_at is not None
|
|
503
540
|
and task_started_at > completed_at
|
|
504
541
|
and payload.get("turn_id") != completed_turn_id
|
|
505
542
|
):
|
|
506
543
|
warnings.add("thread_outside_scan_window")
|
|
507
|
-
return
|
|
544
|
+
return result
|
|
508
545
|
boundary_reached = True
|
|
509
|
-
current_turn_id = payload.get("turn_id")
|
|
510
546
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
511
547
|
inherited_usage = _token_snapshot(payload)
|
|
512
548
|
if inherited_usage is not None:
|
|
513
549
|
previous = inherited_usage
|
|
514
|
-
|
|
550
|
+
if not boundary_reached:
|
|
551
|
+
continue
|
|
515
552
|
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
516
553
|
if (
|
|
517
554
|
completed_turn_id is not None
|
|
@@ -527,7 +564,41 @@ def _read_rollout_usage(
|
|
|
527
564
|
if payload.get("turn_id") != current_turn_id:
|
|
528
565
|
observed_response_usage = _empty_token_usage()
|
|
529
566
|
current_turn_id = payload.get("turn_id")
|
|
567
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
568
|
+
if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
|
|
569
|
+
warnings.add("turn_record_invalid")
|
|
570
|
+
elif (
|
|
571
|
+
completed_at is None
|
|
572
|
+
or timestamp <= completed_at
|
|
573
|
+
or current_turn_id == completed_turn_id
|
|
574
|
+
):
|
|
575
|
+
started_turns.add(current_turn_id)
|
|
576
|
+
open_turns.add(current_turn_id)
|
|
530
577
|
continue
|
|
578
|
+
if event.get("type") == "event_msg" and payload.get("type") in {
|
|
579
|
+
"task_complete",
|
|
580
|
+
"turn_aborted",
|
|
581
|
+
}:
|
|
582
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
583
|
+
terminal_turn_id = payload.get("turn_id") or current_turn_id
|
|
584
|
+
if (
|
|
585
|
+
not isinstance(terminal_turn_id, str)
|
|
586
|
+
or not terminal_turn_id
|
|
587
|
+
or timestamp is None
|
|
588
|
+
):
|
|
589
|
+
warnings.add("turn_record_invalid")
|
|
590
|
+
elif (
|
|
591
|
+
completed_at is None
|
|
592
|
+
or timestamp <= completed_at
|
|
593
|
+
or terminal_turn_id == completed_turn_id
|
|
594
|
+
):
|
|
595
|
+
open_turns.discard(terminal_turn_id)
|
|
596
|
+
if timestamp >= started_at:
|
|
597
|
+
if terminal_turn_id in started_turns:
|
|
598
|
+
closed_turns.add(terminal_turn_id)
|
|
599
|
+
if payload.get("type") == "turn_aborted":
|
|
600
|
+
# Later successful work does not recover interrupted request usage.
|
|
601
|
+
warnings.add("turn_interrupted")
|
|
531
602
|
if (
|
|
532
603
|
completed_turn_id is not None
|
|
533
604
|
and event.get("type") == "event_msg"
|
|
@@ -576,11 +647,16 @@ def _read_rollout_usage(
|
|
|
576
647
|
response_usage = _empty_token_usage()
|
|
577
648
|
if current_turn_id is None:
|
|
578
649
|
current_turn_id = payload.get("turn_id")
|
|
650
|
+
if isinstance(current_turn_id, str):
|
|
651
|
+
open_turns.add(current_turn_id)
|
|
579
652
|
if response_id in response_ids:
|
|
580
653
|
continue
|
|
581
654
|
response_ids.add(response_id)
|
|
582
655
|
_add_token_usage(observed_response_usage, usage)
|
|
583
656
|
turn_usage = _usage_counts(payload.get("turn_token_usage"))
|
|
657
|
+
receipt_turn_id = payload.get("turn_id")
|
|
658
|
+
if isinstance(receipt_turn_id, str):
|
|
659
|
+
measured_turns.add(receipt_turn_id)
|
|
584
660
|
if timestamp < started_at:
|
|
585
661
|
if turn_usage is not None:
|
|
586
662
|
observed_response_usage = turn_usage
|
|
@@ -591,10 +667,30 @@ def _read_rollout_usage(
|
|
|
591
667
|
turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
|
|
592
668
|
):
|
|
593
669
|
warnings.add("token_record_incomplete")
|
|
670
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
594
671
|
_add_token_usage(response_usage, usage)
|
|
595
672
|
continue
|
|
596
673
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
597
674
|
continue
|
|
675
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
676
|
+
in_window = (
|
|
677
|
+
timestamp is not None
|
|
678
|
+
and timestamp >= started_at
|
|
679
|
+
and (
|
|
680
|
+
completed_at is None
|
|
681
|
+
or timestamp <= completed_at
|
|
682
|
+
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
683
|
+
)
|
|
684
|
+
)
|
|
685
|
+
if in_window:
|
|
686
|
+
rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
|
|
687
|
+
if rate_limits is not None:
|
|
688
|
+
key = rate_limits["limitId"]
|
|
689
|
+
if (
|
|
690
|
+
key not in result.rate_limits
|
|
691
|
+
or rate_limits["observedAt"] >= result.rate_limits[key]["observedAt"]
|
|
692
|
+
):
|
|
693
|
+
result.rate_limits[key] = rate_limits
|
|
598
694
|
if "info" in payload and payload["info"] is None:
|
|
599
695
|
continue
|
|
600
696
|
timestamp = _timestamp(event.get("timestamp"))
|
|
@@ -607,15 +703,18 @@ def _read_rollout_usage(
|
|
|
607
703
|
for key, value in snapshot.items()
|
|
608
704
|
}
|
|
609
705
|
previous = snapshot
|
|
610
|
-
if timestamp < started_at:
|
|
611
|
-
continue
|
|
612
706
|
is_completion_turn = (
|
|
613
707
|
completed_turn_id is not None and current_turn_id == completed_turn_id
|
|
614
708
|
)
|
|
615
709
|
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
616
710
|
continue
|
|
711
|
+
if isinstance(current_turn_id, str):
|
|
712
|
+
measured_turns.add(current_turn_id)
|
|
713
|
+
if timestamp < started_at:
|
|
714
|
+
continue
|
|
617
715
|
if is_completion_turn:
|
|
618
716
|
completed_turn_measured = True
|
|
717
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
619
718
|
if delta["totalTokens"] <= 0:
|
|
620
719
|
continue
|
|
621
720
|
_add_token_usage(legacy_usage, delta)
|
|
@@ -623,9 +722,70 @@ def _read_rollout_usage(
|
|
|
623
722
|
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
624
723
|
if not boundary_reached:
|
|
625
724
|
warnings.add("thread_ownership_unavailable")
|
|
725
|
+
if completed_turn_id is not None:
|
|
726
|
+
open_turns.discard(completed_turn_id)
|
|
727
|
+
if open_turns:
|
|
728
|
+
warnings.add("turn_in_progress")
|
|
729
|
+
if boundary_reached and result.tokens_updated_at is None:
|
|
730
|
+
warnings.add("token_record_missing")
|
|
731
|
+
if closed_turns - measured_turns:
|
|
732
|
+
warnings.add("token_record_missing")
|
|
626
733
|
if completed_turn_id is not None and not completed_turn_measured:
|
|
627
734
|
warnings.add("completion_turn_unavailable")
|
|
628
|
-
return
|
|
735
|
+
return result
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None:
|
|
739
|
+
if not isinstance(value, dict):
|
|
740
|
+
return None
|
|
741
|
+
result: dict[str, Any] = {
|
|
742
|
+
"limitId": value.get("limit_id") or "codex",
|
|
743
|
+
"observedAt": timestamp.isoformat(),
|
|
744
|
+
}
|
|
745
|
+
for name in ("primary", "secondary"):
|
|
746
|
+
window = value.get(name)
|
|
747
|
+
if window is None:
|
|
748
|
+
result[name] = None
|
|
749
|
+
continue
|
|
750
|
+
if not isinstance(window, dict):
|
|
751
|
+
return None
|
|
752
|
+
result[name] = {
|
|
753
|
+
"usedPercent": window.get("used_percent"),
|
|
754
|
+
"windowMinutes": window.get("window_minutes"),
|
|
755
|
+
"resetsAt": window.get("resets_at"),
|
|
756
|
+
}
|
|
757
|
+
return result if valid_account_rate_limit(result) else None
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def valid_account_rate_limit(value: object) -> bool:
|
|
761
|
+
if not isinstance(value, dict) or set(value) != {
|
|
762
|
+
"limitId",
|
|
763
|
+
"observedAt",
|
|
764
|
+
"primary",
|
|
765
|
+
"secondary",
|
|
766
|
+
}:
|
|
767
|
+
return False
|
|
768
|
+
if not isinstance(value["limitId"], str) or _timestamp(value["observedAt"]) is None:
|
|
769
|
+
return False
|
|
770
|
+
for name in ("primary", "secondary"):
|
|
771
|
+
window = value[name]
|
|
772
|
+
if window is None:
|
|
773
|
+
continue
|
|
774
|
+
if not isinstance(window, dict) or set(window) != {
|
|
775
|
+
"usedPercent",
|
|
776
|
+
"windowMinutes",
|
|
777
|
+
"resetsAt",
|
|
778
|
+
}:
|
|
779
|
+
return False
|
|
780
|
+
percent = window["usedPercent"]
|
|
781
|
+
if type(percent) not in (int, float) or not math.isfinite(percent):
|
|
782
|
+
return False
|
|
783
|
+
if any(
|
|
784
|
+
window[key] is not None and type(window[key]) is not int
|
|
785
|
+
for key in ("windowMinutes", "resetsAt")
|
|
786
|
+
):
|
|
787
|
+
return False
|
|
788
|
+
return True
|
|
629
789
|
|
|
630
790
|
|
|
631
791
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
@@ -75,7 +75,7 @@ def apply_migrations(
|
|
|
75
75
|
elif version == 16:
|
|
76
76
|
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
77
77
|
elif version in applied:
|
|
78
|
-
if version in (2, 12, 13, 26, 28, 31, 32, 47):
|
|
78
|
+
if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50):
|
|
79
79
|
repair_additive_migration(connection, version)
|
|
80
80
|
elif version == 11:
|
|
81
81
|
repair_deep_scan_migration(connection)
|
|
@@ -8,9 +8,16 @@ import sys
|
|
|
8
8
|
from pathlib import Path, PurePosixPath
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
|
-
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
12
11
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
13
|
-
from
|
|
12
|
+
from workbench_source_scopes import (
|
|
13
|
+
load_source_scopes,
|
|
14
|
+
normalized_path_component,
|
|
15
|
+
offline_git_bytes,
|
|
16
|
+
relative_path,
|
|
17
|
+
safe_source_path,
|
|
18
|
+
source_object_for_path,
|
|
19
|
+
)
|
|
20
|
+
from workbench_target import clean_worktree_content_digest
|
|
14
21
|
|
|
15
22
|
CONTEXT_LINES = 3
|
|
16
23
|
MAX_BYTES = 16_000
|
|
@@ -21,23 +28,70 @@ def finding_source_excerpt(
|
|
|
21
28
|
scan: sqlite3.Row,
|
|
22
29
|
target: Path | None,
|
|
23
30
|
locations: list[dict[str, Any]],
|
|
31
|
+
scopes: list[str],
|
|
24
32
|
) -> str | None:
|
|
25
|
-
if target is None or not locations:
|
|
33
|
+
if target is None or not locations or scan["target_revision"] == "unversioned":
|
|
26
34
|
return None
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
35
|
+
snapshot = scan["target_snapshot_digest"]
|
|
36
|
+
if snapshot is not None and snapshot != clean_worktree_content_digest():
|
|
37
|
+
return None
|
|
38
|
+
locations = [
|
|
39
|
+
location
|
|
40
|
+
for location in locations
|
|
41
|
+
if isinstance(path := location.get("path"), str)
|
|
42
|
+
and isinstance(location.get("startLine"), int)
|
|
43
|
+
and safe_source_path(target, path) is not None
|
|
44
|
+
]
|
|
45
|
+
if not locations:
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
context = load_source_scopes(scan, target, scopes)
|
|
49
|
+
except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
|
|
50
|
+
return None
|
|
51
|
+
if context is None:
|
|
52
|
+
return None
|
|
53
|
+
repository, tree, records = context
|
|
54
|
+
indexed: dict[tuple[str, ...], list[dict[str, str]]] = {}
|
|
55
|
+
for record in records:
|
|
56
|
+
for name in ("path", "canonicalPath"):
|
|
57
|
+
parts = tuple(
|
|
58
|
+
normalized_path_component(part) for part in PurePosixPath(record[name]).parts
|
|
59
|
+
)
|
|
60
|
+
indexed.setdefault(parts, []).append(record)
|
|
61
|
+
lengths = sorted({len(parts) for parts in indexed})
|
|
62
|
+
|
|
63
|
+
def source_object(path: str) -> str | None:
|
|
64
|
+
parsed = relative_path(path)
|
|
65
|
+
if parsed is None:
|
|
66
|
+
return None
|
|
67
|
+
parts = tuple(normalized_path_component(part) for part in parsed.parts)
|
|
68
|
+
try:
|
|
69
|
+
for length in lengths:
|
|
70
|
+
if length > len(parts):
|
|
71
|
+
break
|
|
72
|
+
for record in indexed.get(parts[:length], []):
|
|
73
|
+
object_id = source_object_for_path(repository, tree, target, path, record)
|
|
74
|
+
if object_id is not None:
|
|
75
|
+
return object_id
|
|
76
|
+
except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
|
|
77
|
+
# An unavailable source object leaves the optional excerpt absent.
|
|
78
|
+
pass
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
locations.sort(
|
|
82
|
+
key=lambda location: (
|
|
83
|
+
location.get("role") != "root_control",
|
|
84
|
+
"root_control" not in str(location.get("role") or "").lower(),
|
|
85
|
+
)
|
|
34
86
|
)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
87
|
+
for location in locations:
|
|
88
|
+
object_id = source_object(location["path"])
|
|
89
|
+
if object_id is not None:
|
|
90
|
+
break
|
|
91
|
+
else:
|
|
39
92
|
return None
|
|
40
|
-
|
|
93
|
+
start_line, end_line = location["startLine"], location.get("endLine")
|
|
94
|
+
source = scanned_source_text(repository, object_id)
|
|
41
95
|
if not source or "\0" in source:
|
|
42
96
|
return None
|
|
43
97
|
# Match location validation and SARIF hashing: CR, LF, and CRLF are line breaks.
|
|
@@ -58,46 +112,15 @@ def finding_source_excerpt(
|
|
|
58
112
|
f"{line_number:>{width}} {lines[line_number - 1]}"
|
|
59
113
|
for line_number in range(excerpt_start, excerpt_end + 1)
|
|
60
114
|
)
|
|
61
|
-
|
|
62
|
-
return encoded.decode("utf-8", errors="ignore")
|
|
115
|
+
return excerpt.encode("utf-8")[:MAX_BYTES].decode("utf-8", errors="ignore")
|
|
63
116
|
|
|
64
117
|
|
|
65
|
-
def scanned_source_text(
|
|
66
|
-
if safe_source_path(target, path) is None:
|
|
67
|
-
return None
|
|
68
|
-
revision = scan["target_revision"]
|
|
69
|
-
if revision == "unversioned":
|
|
70
|
-
return None
|
|
71
|
-
snapshot_digest = scan["target_snapshot_digest"]
|
|
72
|
-
if (
|
|
73
|
-
scan["diff_target_kind"] == "working_tree"
|
|
74
|
-
and scan["diff_content_digest"] != clean_worktree_content_digest()
|
|
75
|
-
):
|
|
76
|
-
return None
|
|
77
|
-
if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
|
|
78
|
-
return None
|
|
79
|
-
if (
|
|
80
|
-
scan["diff_target_kind"] == "working_tree"
|
|
81
|
-
and scan["diff_content_digest"] != clean_worktree_content_digest()
|
|
82
|
-
):
|
|
83
|
-
return None
|
|
84
|
-
object_name = f"{revision}:./{path}"
|
|
85
|
-
content = git_bytes(target, "cat-file", "blob", object_name)
|
|
86
|
-
return content.decode("utf-8", errors="replace") if content is not None else None
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def safe_source_path(target: Path, relative_path: str) -> Path | None:
|
|
90
|
-
if "\\" in relative_path:
|
|
91
|
-
return None
|
|
92
|
-
parsed = PurePosixPath(relative_path)
|
|
93
|
-
if parsed.is_absolute() or ".." in parsed.parts:
|
|
94
|
-
return None
|
|
118
|
+
def scanned_source_text(repository: Path, object_id: str) -> str | None:
|
|
95
119
|
try:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
except (OSError, RuntimeError, ValueError):
|
|
120
|
+
content = offline_git_bytes(repository, "cat-file", "blob", object_id)
|
|
121
|
+
except (OSError, RuntimeError, SystemExit):
|
|
99
122
|
return None
|
|
100
|
-
return
|
|
123
|
+
return content.decode("utf-8", errors="replace") if content is not None else None
|
|
101
124
|
|
|
102
125
|
|
|
103
126
|
if __name__ == "__main__":
|