@openai/codex-security 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -9
- package/_bundled_plugin/.codex-plugin/plugin.json +5 -5
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +13 -20
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/core-scan.md +2 -0
- package/_bundled_plugin/references/final-report.md +5 -1
- package/_bundled_plugin/references/investigation-time-budget.md +11 -0
- package/_bundled_plugin/references/sarif-adapter.md +1 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -6
- package/_bundled_plugin/references/scan-runtime.md +15 -0
- package/_bundled_plugin/references/threat-model.md +2 -0
- package/_bundled_plugin/schemas/coverage.schema.json +45 -126
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +5 -1
- package/_bundled_plugin/schemas/findings.schema.json +140 -427
- package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +106 -119
- package/_bundled_plugin/schemas/scan-manifest.schema.json +73 -188
- package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +87 -49
- package/_bundled_plugin/scripts/finding_preview.py +5 -3
- package/_bundled_plugin/scripts/generate_rank_input.py +26 -106
- package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
- package/_bundled_plugin/scripts/report_projection.py +35 -13
- package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
- package/_bundled_plugin/scripts/workbench_cli.py +9 -9
- package/_bundled_plugin/scripts/workbench_constants.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +103 -45
- package/_bundled_plugin/scripts/workbench_progress.py +0 -50
- package/_bundled_plugin/scripts/workbench_publication.py +216 -247
- package/_bundled_plugin/scripts/workbench_scan_history.py +3 -2
- package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
- package/_bundled_plugin/scripts/workbench_scan_usage.py +360 -37
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
- package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
- package/_bundled_plugin/scripts/workbench_target.py +7 -5
- package/_bundled_plugin/scripts/workbench_validation.py +52 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
- package/_bundled_plugin/shared/workbench-migrations.json +22 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -3
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +2 -0
- package/_bundled_plugin/skills/track-findings/SKILL.md +8 -4
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +1 -1
- package/dist/api.d.ts +6 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +266 -39
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +2 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +38 -23
- package/dist/artifact-export.js.map +1 -1
- package/dist/classify-severity-records.d.ts +76 -0
- package/dist/classify-severity-records.d.ts.map +1 -0
- package/dist/classify-severity-records.js +141 -0
- package/dist/classify-severity-records.js.map +1 -0
- package/dist/classify-severity.d.ts +1 -23
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -24
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +2 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +4 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +148 -149
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +1 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-sdk-environment.js +1 -1
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.js +1 -1
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +5 -4
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -4
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +6 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +69 -20
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +7 -2
- package/dist/cost-model.js.map +1 -1
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +1 -5
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +2 -2
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +1 -1
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts +3 -8
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +11 -171
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.js +1 -1
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +10 -9
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +4 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +21 -1
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +15 -1
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +15 -16
- package/dist/deep-progress.js.map +1 -1
- package/dist/executable-path.d.ts +2 -0
- package/dist/executable-path.d.ts.map +1 -0
- package/dist/executable-path.js +12 -0
- package/dist/executable-path.js.map +1 -0
- package/dist/feedback.d.ts +1 -4
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +1 -1
- package/dist/feedback.js.map +1 -1
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +3 -3
- package/dist/findings-import.js.map +1 -1
- package/dist/git-remote.d.ts +10 -0
- package/dist/git-remote.d.ts.map +1 -0
- package/dist/git-remote.js +24 -0
- package/dist/git-remote.js.map +1 -0
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +5 -7
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +19 -0
- package/dist/host-protocol.d.ts.map +1 -0
- package/dist/host-protocol.js +270 -0
- package/dist/host-protocol.js.map +1 -0
- package/dist/host-records-protocol.d.ts +15 -0
- package/dist/host-records-protocol.d.ts.map +1 -0
- package/dist/host-records-protocol.js +15 -0
- package/dist/host-records-protocol.js.map +1 -0
- package/dist/hosted-scan-protocol.d.ts +4 -0
- package/dist/hosted-scan-protocol.d.ts.map +1 -0
- package/dist/hosted-scan-protocol.js +25 -0
- package/dist/hosted-scan-protocol.js.map +1 -0
- package/dist/hosted-scan.d.ts +112 -0
- package/dist/hosted-scan.d.ts.map +1 -0
- package/dist/hosted-scan.js +260 -0
- package/dist/hosted-scan.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +87 -7
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +108 -36
- package/dist/multiscan.js.map +1 -1
- package/dist/patch-publication.d.ts +26 -0
- package/dist/patch-publication.d.ts.map +1 -0
- package/dist/patch-publication.js +105 -0
- package/dist/patch-publication.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +4 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +16 -6
- package/dist/project-config.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +73 -63
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts +0 -2
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +135 -24
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +39 -25
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +5 -2
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +86 -58
- package/dist/runtime.js.map +1 -1
- package/dist/scan-executor.d.ts +104 -0
- package/dist/scan-executor.d.ts.map +1 -0
- package/dist/scan-executor.js +62 -0
- package/dist/scan-executor.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +7 -1
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-time-budget.d.ts +9 -0
- package/dist/scan-time-budget.d.ts.map +1 -0
- package/dist/scan-time-budget.js +16 -0
- package/dist/scan-time-budget.js.map +1 -0
- package/dist/severity-review.d.ts +29 -0
- package/dist/severity-review.d.ts.map +1 -0
- package/dist/severity-review.js +47 -0
- package/dist/severity-review.js.map +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +17 -25
- package/dist/targets.js.map +1 -1
- package/dist/validation-records.d.ts +38 -0
- package/dist/validation-records.d.ts.map +1 -0
- package/dist/validation-records.js +76 -0
- package/dist/validation-records.js.map +1 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +4 -2
- package/dist/version.js.map +1 -1
- package/dist/worker-progress.d.ts +43 -15
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +22 -80
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +89 -0
- package/docs/cli.md +281 -35
- package/docs/hosted-scans.md +242 -0
- package/docs/validate-records.md +69 -0
- package/package.json +18 -14
- package/schemas/hosted-scan-v2.fixture.json +145 -0
- package/schemas/project-config.schema.json +7 -1
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
- package/_bundled_plugin/references/config-preflight.md +0 -93
- package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
- package/_bundled_plugin/scripts/config_preflight.py +0 -911
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
|
+
import math
|
|
7
8
|
import os
|
|
8
9
|
import re
|
|
9
10
|
import sqlite3
|
|
@@ -34,6 +35,14 @@ class RolloutSession:
|
|
|
34
35
|
path: Path
|
|
35
36
|
|
|
36
37
|
|
|
38
|
+
@dataclass
|
|
39
|
+
class RolloutUsage:
|
|
40
|
+
tokens: dict[str, int]
|
|
41
|
+
warnings: set[str]
|
|
42
|
+
tokens_updated_at: datetime | None
|
|
43
|
+
rate_limits: dict[str, dict[str, Any]]
|
|
44
|
+
|
|
45
|
+
|
|
37
46
|
def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
38
47
|
"""Project measured usage without changing the existing legacy cost contract."""
|
|
39
48
|
|
|
@@ -50,12 +59,6 @@ def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
|
50
59
|
}
|
|
51
60
|
|
|
52
61
|
|
|
53
|
-
def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
|
|
54
|
-
"""Keep usage in the already-migrated scans.cost_json column."""
|
|
55
|
-
|
|
56
|
-
return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
|
|
57
|
-
|
|
58
|
-
|
|
59
62
|
def reconcile_completed_scan_cost(
|
|
60
63
|
connection: sqlite3.Connection,
|
|
61
64
|
scan: sqlite3.Row,
|
|
@@ -84,11 +87,14 @@ def collect_scan_usage(
|
|
|
84
87
|
*,
|
|
85
88
|
thread_id: str | None = None,
|
|
86
89
|
completed_at: str | None = None,
|
|
90
|
+
completed_turn_id: str | None = None,
|
|
87
91
|
) -> dict[str, Any]:
|
|
88
|
-
"""Count
|
|
92
|
+
"""Count attributable receipts and flag observable gaps inside the scan window."""
|
|
89
93
|
|
|
90
94
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
91
|
-
|
|
95
|
+
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
96
|
+
worker_directories = _scan_worker_directories(connection, scan)
|
|
97
|
+
if not roots and not worker_directories:
|
|
92
98
|
return _unavailable_usage("scan_thread_unavailable")
|
|
93
99
|
|
|
94
100
|
state_database = _codex_state_database()
|
|
@@ -96,7 +102,8 @@ def collect_scan_usage(
|
|
|
96
102
|
return _unavailable_usage("codex_state_unavailable")
|
|
97
103
|
|
|
98
104
|
started_at = _timestamp(scan["started_at"])
|
|
99
|
-
|
|
105
|
+
observed_at = datetime.now(timezone.utc)
|
|
106
|
+
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
100
107
|
if started_at is None:
|
|
101
108
|
return _unavailable_usage("scan_window_unavailable")
|
|
102
109
|
|
|
@@ -106,6 +113,9 @@ def collect_scan_usage(
|
|
|
106
113
|
state_database,
|
|
107
114
|
roots,
|
|
108
115
|
warnings,
|
|
116
|
+
worker_directories=worker_directories,
|
|
117
|
+
started_at=started_at,
|
|
118
|
+
completed_at=stopped_at,
|
|
109
119
|
)
|
|
110
120
|
except (OSError, sqlite3.Error, ValueError):
|
|
111
121
|
return _unavailable_usage("codex_state_unavailable")
|
|
@@ -113,7 +123,9 @@ def collect_scan_usage(
|
|
|
113
123
|
if not sessions:
|
|
114
124
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
115
125
|
|
|
116
|
-
|
|
126
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
|
+
tokens_updated_at = None
|
|
117
129
|
accepted_thread_ids: set[str] = set()
|
|
118
130
|
excluded_thread_ids: set[str] = set()
|
|
119
131
|
for session in sessions:
|
|
@@ -128,36 +140,61 @@ def collect_scan_usage(
|
|
|
128
140
|
warnings.add("thread_lineage_incomplete")
|
|
129
141
|
continue
|
|
130
142
|
try:
|
|
131
|
-
|
|
143
|
+
usage = _read_rollout_usage(
|
|
132
144
|
session,
|
|
133
145
|
started_at=started_at,
|
|
134
146
|
completed_at=stopped_at,
|
|
147
|
+
completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
|
|
135
148
|
)
|
|
136
149
|
except (OSError, UnicodeError, ValueError):
|
|
137
150
|
missing_thread_ids.add(session.thread_id)
|
|
138
151
|
warnings.add("rollout_unavailable")
|
|
139
152
|
continue
|
|
140
|
-
if "thread_outside_scan_window" in
|
|
153
|
+
if "thread_outside_scan_window" in usage.warnings:
|
|
141
154
|
excluded_thread_ids.add(session.thread_id)
|
|
142
155
|
continue
|
|
143
|
-
warnings.update(
|
|
144
|
-
if "thread_identity_mismatch" in
|
|
145
|
-
"thread_ownership_unavailable" in
|
|
156
|
+
warnings.update(usage.warnings)
|
|
157
|
+
if "thread_identity_mismatch" in usage.warnings or (
|
|
158
|
+
"thread_ownership_unavailable" in usage.warnings
|
|
146
159
|
):
|
|
147
160
|
missing_thread_ids.add(session.thread_id)
|
|
148
161
|
continue
|
|
149
162
|
accepted_thread_ids.add(session.thread_id)
|
|
150
|
-
|
|
163
|
+
role = by_role["parent" if session.thread_id in parent_ids else "workers"]
|
|
164
|
+
_add_token_usage(role, usage.tokens)
|
|
165
|
+
role["threadCount"] += 1
|
|
166
|
+
if usage.tokens_updated_at is not None:
|
|
167
|
+
tokens_updated_at = max(
|
|
168
|
+
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
169
|
+
)
|
|
170
|
+
for key, snapshot in usage.rate_limits.items():
|
|
171
|
+
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
172
|
+
rate_limits[key] = snapshot
|
|
151
173
|
|
|
152
174
|
if not accepted_thread_ids:
|
|
153
175
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
154
176
|
|
|
177
|
+
if any(
|
|
178
|
+
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
179
|
+
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
180
|
+
for usage in by_role.values()
|
|
181
|
+
):
|
|
182
|
+
return _unavailable_usage("token_breakdown_unavailable", warnings=warnings)
|
|
183
|
+
|
|
184
|
+
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
185
|
+
stopped_at = max(stopped_at, tokens_updated_at)
|
|
155
186
|
result: dict[str, Any] = {
|
|
156
187
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
157
188
|
"source": "codex_rollout",
|
|
158
|
-
**
|
|
189
|
+
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
159
190
|
"threadCount": len(accepted_thread_ids),
|
|
191
|
+
"byRole": by_role,
|
|
192
|
+
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
193
|
+
"observedAt": observed_at.isoformat(),
|
|
194
|
+
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
160
195
|
}
|
|
196
|
+
if tokens_updated_at is not None:
|
|
197
|
+
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
161
198
|
if missing_thread_ids:
|
|
162
199
|
result["missingThreadCount"] = len(missing_thread_ids)
|
|
163
200
|
if warnings:
|
|
@@ -171,6 +208,7 @@ def _scan_root_thread_ids(
|
|
|
171
208
|
supplied_thread_id: str | None,
|
|
172
209
|
*,
|
|
173
210
|
include_owner_threads: bool = True,
|
|
211
|
+
include_workers: bool = True,
|
|
174
212
|
) -> list[str]:
|
|
175
213
|
candidates: list[str | None] = [supplied_thread_id]
|
|
176
214
|
if include_owner_threads:
|
|
@@ -184,7 +222,7 @@ def _scan_root_thread_ids(
|
|
|
184
222
|
).fetchone()
|
|
185
223
|
if workspace is not None:
|
|
186
224
|
candidates.append(workspace["thread_id"])
|
|
187
|
-
if scan["mode"] == "deep":
|
|
225
|
+
if include_workers and scan["mode"] == "deep":
|
|
188
226
|
candidates.extend(
|
|
189
227
|
row["sdk_thread_id"]
|
|
190
228
|
for row in connection.execute(
|
|
@@ -214,6 +252,21 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
|
|
|
214
252
|
)
|
|
215
253
|
|
|
216
254
|
|
|
255
|
+
def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]:
|
|
256
|
+
if scan["mode"] != "deep":
|
|
257
|
+
return set()
|
|
258
|
+
return {
|
|
259
|
+
row["artifact_dir"]
|
|
260
|
+
if row["kind"] == "discovery"
|
|
261
|
+
else str(Path(scan["scan_dir"]) / "artifacts")
|
|
262
|
+
for row in connection.execute(
|
|
263
|
+
"SELECT kind, artifact_dir FROM deep_scan_workers "
|
|
264
|
+
"WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
|
|
265
|
+
(scan["id"],),
|
|
266
|
+
)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
|
|
217
270
|
def _codex_state_database() -> Path | None:
|
|
218
271
|
configured_database = os.environ.get("CODEX_STATE_DB", "").strip()
|
|
219
272
|
if configured_database:
|
|
@@ -255,6 +308,10 @@ def _discover_rollout_sessions(
|
|
|
255
308
|
state_database: Path,
|
|
256
309
|
roots: list[str],
|
|
257
310
|
warnings: set[str],
|
|
311
|
+
*,
|
|
312
|
+
worker_directories: set[str],
|
|
313
|
+
started_at: datetime,
|
|
314
|
+
completed_at: datetime | None,
|
|
258
315
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
259
316
|
database = sqlite3.connect(
|
|
260
317
|
f"file:{quote(str(state_database), safe='')}?mode=ro",
|
|
@@ -273,7 +330,27 @@ def _discover_rollout_sessions(
|
|
|
273
330
|
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
274
331
|
sessions: dict[str, RolloutSession] = {}
|
|
275
332
|
missing_thread_ids: set[str] = set()
|
|
276
|
-
|
|
333
|
+
root_ids = dict.fromkeys(roots)
|
|
334
|
+
started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
|
|
335
|
+
completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
|
|
336
|
+
directory_expression = "cwd"
|
|
337
|
+
if worker_directories and os.name == "nt":
|
|
338
|
+
from windows_scan_local_files import _normalized_windows_path
|
|
339
|
+
|
|
340
|
+
database.create_function(
|
|
341
|
+
"codex_security_path_key", 1, _normalized_windows_path, deterministic=True
|
|
342
|
+
)
|
|
343
|
+
directory_expression = "codex_security_path_key(cwd)"
|
|
344
|
+
worker_directories = {_normalized_windows_path(value) for value in worker_directories}
|
|
345
|
+
for directory in sorted(worker_directories):
|
|
346
|
+
for row in database.execute(
|
|
347
|
+
"SELECT id FROM threads WHERE cwd IS NOT NULL "
|
|
348
|
+
f"AND {directory_expression} = ? AND created_at >= ? "
|
|
349
|
+
"AND (? IS NULL OR created_at <= ?) ORDER BY id",
|
|
350
|
+
(directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
|
|
351
|
+
):
|
|
352
|
+
root_ids[row["id"]] = None
|
|
353
|
+
for root in root_ids:
|
|
277
354
|
row = database.execute(
|
|
278
355
|
"SELECT id, rollout_path FROM threads WHERE id = ?",
|
|
279
356
|
(root,),
|
|
@@ -282,13 +359,29 @@ def _discover_rollout_sessions(
|
|
|
282
359
|
missing_thread_ids.add(root)
|
|
283
360
|
warnings.add("scan_root_unavailable")
|
|
284
361
|
continue
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
362
|
+
path = _rollout_path(row["rollout_path"])
|
|
363
|
+
if path is None:
|
|
364
|
+
missing_thread_ids.add(root)
|
|
365
|
+
warnings.add("rollout_unavailable")
|
|
366
|
+
continue
|
|
367
|
+
if root not in roots:
|
|
368
|
+
try:
|
|
369
|
+
with path.open("rb") as source:
|
|
370
|
+
metadata = json.loads(source.readline())
|
|
371
|
+
except (OSError, ValueError):
|
|
288
372
|
missing_thread_ids.add(root)
|
|
289
373
|
warnings.add("rollout_unavailable")
|
|
290
374
|
continue
|
|
291
|
-
|
|
375
|
+
if (
|
|
376
|
+
isinstance(metadata, dict)
|
|
377
|
+
and metadata.get("type") == "session_meta"
|
|
378
|
+
and isinstance(payload := metadata.get("payload"), dict)
|
|
379
|
+
and (payload.get("id") or payload.get("session_id")) == root
|
|
380
|
+
and (created_at := _timestamp(payload.get("timestamp"))) is not None
|
|
381
|
+
and created_at < started_millisecond
|
|
382
|
+
):
|
|
383
|
+
continue
|
|
384
|
+
sessions[root] = RolloutSession(root, None, path)
|
|
292
385
|
descendants = database.execute(
|
|
293
386
|
"""
|
|
294
387
|
WITH RECURSIVE descendants(
|
|
@@ -378,11 +471,23 @@ def _read_rollout_usage(
|
|
|
378
471
|
*,
|
|
379
472
|
started_at: datetime,
|
|
380
473
|
completed_at: datetime | None,
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
474
|
+
completed_turn_id: str | None = None,
|
|
475
|
+
) -> RolloutUsage:
|
|
476
|
+
result = RolloutUsage(_empty_token_usage(), set(), None, {})
|
|
477
|
+
total = result.tokens
|
|
478
|
+
warnings = result.warnings
|
|
384
479
|
previous = _empty_token_usage()
|
|
385
480
|
boundary_reached = False
|
|
481
|
+
legacy_usage = _empty_token_usage()
|
|
482
|
+
response_usage: dict[str, int] | None = None
|
|
483
|
+
observed_response_usage = _empty_token_usage()
|
|
484
|
+
response_ids: set[str] = set()
|
|
485
|
+
current_turn_id: str | None = None
|
|
486
|
+
completed_turn_measured = False
|
|
487
|
+
started_turns: set[str] = set()
|
|
488
|
+
open_turns: set[str] = set()
|
|
489
|
+
closed_turns: set[str] = set()
|
|
490
|
+
measured_turns: set[str] = set()
|
|
386
491
|
|
|
387
492
|
with session.path.open("rb") as source:
|
|
388
493
|
for line_number, raw_line in enumerate(source, start=1):
|
|
@@ -405,16 +510,16 @@ def _read_rollout_usage(
|
|
|
405
510
|
if line_number == 1:
|
|
406
511
|
if event.get("type") != "session_meta" or not isinstance(payload, dict):
|
|
407
512
|
warnings.add("thread_identity_mismatch")
|
|
408
|
-
return
|
|
513
|
+
return result
|
|
409
514
|
recorded_id = payload.get("id") or payload.get("session_id")
|
|
410
515
|
if recorded_id != session.thread_id:
|
|
411
516
|
warnings.add("thread_identity_mismatch")
|
|
412
|
-
return
|
|
517
|
+
return result
|
|
413
518
|
recorded_parent = _session_parent_thread_id(payload)
|
|
414
519
|
if session.parent_thread_id is not None:
|
|
415
520
|
if recorded_parent != session.parent_thread_id:
|
|
416
521
|
warnings.add("thread_identity_mismatch")
|
|
417
|
-
return
|
|
522
|
+
return result
|
|
418
523
|
boundary_reached = (
|
|
419
524
|
session.parent_thread_id is None
|
|
420
525
|
and not recorded_parent
|
|
@@ -429,20 +534,163 @@ def _read_rollout_usage(
|
|
|
429
534
|
task_started_at = _timestamp(event.get("timestamp"))
|
|
430
535
|
if task_started_at is None:
|
|
431
536
|
warnings.add("thread_ownership_unavailable")
|
|
432
|
-
return
|
|
537
|
+
return result
|
|
433
538
|
if task_started_at < started_at or (
|
|
434
|
-
completed_at is not None
|
|
539
|
+
completed_at is not None
|
|
540
|
+
and task_started_at > completed_at
|
|
541
|
+
and payload.get("turn_id") != completed_turn_id
|
|
435
542
|
):
|
|
436
543
|
warnings.add("thread_outside_scan_window")
|
|
437
|
-
return
|
|
544
|
+
return result
|
|
438
545
|
boundary_reached = True
|
|
439
546
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
440
547
|
inherited_usage = _token_snapshot(payload)
|
|
441
548
|
if inherited_usage is not None:
|
|
442
549
|
previous = inherited_usage
|
|
550
|
+
if not boundary_reached:
|
|
551
|
+
continue
|
|
552
|
+
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
553
|
+
if (
|
|
554
|
+
completed_turn_id is not None
|
|
555
|
+
and current_turn_id == completed_turn_id
|
|
556
|
+
and payload.get("turn_id") != completed_turn_id
|
|
557
|
+
):
|
|
558
|
+
break
|
|
559
|
+
_add_token_usage(
|
|
560
|
+
total, response_usage if response_usage is not None else legacy_usage
|
|
561
|
+
)
|
|
562
|
+
legacy_usage = _empty_token_usage()
|
|
563
|
+
response_usage = None
|
|
564
|
+
if payload.get("turn_id") != current_turn_id:
|
|
565
|
+
observed_response_usage = _empty_token_usage()
|
|
566
|
+
current_turn_id = payload.get("turn_id")
|
|
567
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
568
|
+
if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
|
|
569
|
+
warnings.add("turn_record_invalid")
|
|
570
|
+
elif (
|
|
571
|
+
completed_at is None
|
|
572
|
+
or timestamp <= completed_at
|
|
573
|
+
or current_turn_id == completed_turn_id
|
|
574
|
+
):
|
|
575
|
+
started_turns.add(current_turn_id)
|
|
576
|
+
open_turns.add(current_turn_id)
|
|
577
|
+
continue
|
|
578
|
+
if event.get("type") == "event_msg" and payload.get("type") in {
|
|
579
|
+
"task_complete",
|
|
580
|
+
"turn_aborted",
|
|
581
|
+
}:
|
|
582
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
583
|
+
terminal_turn_id = payload.get("turn_id") or current_turn_id
|
|
584
|
+
if (
|
|
585
|
+
not isinstance(terminal_turn_id, str)
|
|
586
|
+
or not terminal_turn_id
|
|
587
|
+
or timestamp is None
|
|
588
|
+
):
|
|
589
|
+
warnings.add("turn_record_invalid")
|
|
590
|
+
elif (
|
|
591
|
+
completed_at is None
|
|
592
|
+
or timestamp <= completed_at
|
|
593
|
+
or terminal_turn_id == completed_turn_id
|
|
594
|
+
):
|
|
595
|
+
open_turns.discard(terminal_turn_id)
|
|
596
|
+
if timestamp >= started_at:
|
|
597
|
+
if terminal_turn_id in started_turns:
|
|
598
|
+
closed_turns.add(terminal_turn_id)
|
|
599
|
+
if payload.get("type") == "turn_aborted":
|
|
600
|
+
# Later successful work does not recover interrupted request usage.
|
|
601
|
+
warnings.add("turn_interrupted")
|
|
602
|
+
if (
|
|
603
|
+
completed_turn_id is not None
|
|
604
|
+
and event.get("type") == "event_msg"
|
|
605
|
+
and payload.get("type") == "task_complete"
|
|
606
|
+
and payload.get("turn_id") == completed_turn_id
|
|
607
|
+
):
|
|
608
|
+
break
|
|
609
|
+
if event.get("type") == "compacted":
|
|
610
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
611
|
+
if (
|
|
612
|
+
timestamp is not None
|
|
613
|
+
and timestamp >= started_at
|
|
614
|
+
and (
|
|
615
|
+
completed_at is None
|
|
616
|
+
or timestamp <= completed_at
|
|
617
|
+
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
618
|
+
)
|
|
619
|
+
and "compaction_response_id" not in payload
|
|
620
|
+
and response_usage is None
|
|
621
|
+
):
|
|
622
|
+
warnings.add("compaction_usage_unavailable")
|
|
623
|
+
continue
|
|
624
|
+
if event.get("type") == "token_usage_record":
|
|
625
|
+
if not isinstance(payload.get("thread_id"), str):
|
|
626
|
+
warnings.add("token_record_invalid")
|
|
627
|
+
continue
|
|
628
|
+
if payload.get("thread_id") != session.thread_id:
|
|
629
|
+
continue
|
|
630
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
631
|
+
usage = _usage_counts(payload.get("usage"))
|
|
632
|
+
response_id = payload.get("response_id")
|
|
633
|
+
if (
|
|
634
|
+
timestamp is None
|
|
635
|
+
or usage is None
|
|
636
|
+
or not isinstance(response_id, str)
|
|
637
|
+
or not response_id
|
|
638
|
+
):
|
|
639
|
+
warnings.add("token_record_invalid")
|
|
640
|
+
continue
|
|
641
|
+
is_completion_turn = (
|
|
642
|
+
completed_turn_id is not None and payload.get("turn_id") == completed_turn_id
|
|
643
|
+
)
|
|
644
|
+
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
645
|
+
continue
|
|
646
|
+
if response_usage is None:
|
|
647
|
+
response_usage = _empty_token_usage()
|
|
648
|
+
if current_turn_id is None:
|
|
649
|
+
current_turn_id = payload.get("turn_id")
|
|
650
|
+
if isinstance(current_turn_id, str):
|
|
651
|
+
open_turns.add(current_turn_id)
|
|
652
|
+
if response_id in response_ids:
|
|
653
|
+
continue
|
|
654
|
+
response_ids.add(response_id)
|
|
655
|
+
_add_token_usage(observed_response_usage, usage)
|
|
656
|
+
turn_usage = _usage_counts(payload.get("turn_token_usage"))
|
|
657
|
+
receipt_turn_id = payload.get("turn_id")
|
|
658
|
+
if isinstance(receipt_turn_id, str):
|
|
659
|
+
measured_turns.add(receipt_turn_id)
|
|
660
|
+
if timestamp < started_at:
|
|
661
|
+
if turn_usage is not None:
|
|
662
|
+
observed_response_usage = turn_usage
|
|
663
|
+
continue
|
|
664
|
+
if is_completion_turn:
|
|
665
|
+
completed_turn_measured = True
|
|
666
|
+
if turn_usage is not None and any(
|
|
667
|
+
turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
|
|
668
|
+
):
|
|
669
|
+
warnings.add("token_record_incomplete")
|
|
670
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
671
|
+
_add_token_usage(response_usage, usage)
|
|
443
672
|
continue
|
|
444
673
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
445
674
|
continue
|
|
675
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
676
|
+
in_window = (
|
|
677
|
+
timestamp is not None
|
|
678
|
+
and timestamp >= started_at
|
|
679
|
+
and (
|
|
680
|
+
completed_at is None
|
|
681
|
+
or timestamp <= completed_at
|
|
682
|
+
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
683
|
+
)
|
|
684
|
+
)
|
|
685
|
+
if in_window:
|
|
686
|
+
rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
|
|
687
|
+
if rate_limits is not None:
|
|
688
|
+
key = rate_limits["limitId"]
|
|
689
|
+
if (
|
|
690
|
+
key not in result.rate_limits
|
|
691
|
+
or rate_limits["observedAt"] >= result.rate_limits[key]["observedAt"]
|
|
692
|
+
):
|
|
693
|
+
result.rate_limits[key] = rate_limits
|
|
446
694
|
if "info" in payload and payload["info"] is None:
|
|
447
695
|
continue
|
|
448
696
|
timestamp = _timestamp(event.get("timestamp"))
|
|
@@ -455,17 +703,89 @@ def _read_rollout_usage(
|
|
|
455
703
|
for key, value in snapshot.items()
|
|
456
704
|
}
|
|
457
705
|
previous = snapshot
|
|
458
|
-
|
|
706
|
+
is_completion_turn = (
|
|
707
|
+
completed_turn_id is not None and current_turn_id == completed_turn_id
|
|
708
|
+
)
|
|
709
|
+
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
459
710
|
continue
|
|
460
|
-
if
|
|
711
|
+
if isinstance(current_turn_id, str):
|
|
712
|
+
measured_turns.add(current_turn_id)
|
|
713
|
+
if timestamp < started_at:
|
|
461
714
|
continue
|
|
715
|
+
if is_completion_turn:
|
|
716
|
+
completed_turn_measured = True
|
|
717
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
462
718
|
if delta["totalTokens"] <= 0:
|
|
463
719
|
continue
|
|
464
|
-
_add_token_usage(
|
|
720
|
+
_add_token_usage(legacy_usage, delta)
|
|
465
721
|
|
|
722
|
+
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
466
723
|
if not boundary_reached:
|
|
467
724
|
warnings.add("thread_ownership_unavailable")
|
|
468
|
-
|
|
725
|
+
if completed_turn_id is not None:
|
|
726
|
+
open_turns.discard(completed_turn_id)
|
|
727
|
+
if open_turns:
|
|
728
|
+
warnings.add("turn_in_progress")
|
|
729
|
+
if boundary_reached and result.tokens_updated_at is None:
|
|
730
|
+
warnings.add("token_record_missing")
|
|
731
|
+
if closed_turns - measured_turns:
|
|
732
|
+
warnings.add("token_record_missing")
|
|
733
|
+
if completed_turn_id is not None and not completed_turn_measured:
|
|
734
|
+
warnings.add("completion_turn_unavailable")
|
|
735
|
+
return result
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None:
|
|
739
|
+
if not isinstance(value, dict):
|
|
740
|
+
return None
|
|
741
|
+
result: dict[str, Any] = {
|
|
742
|
+
"limitId": value.get("limit_id") or "codex",
|
|
743
|
+
"observedAt": timestamp.isoformat(),
|
|
744
|
+
}
|
|
745
|
+
for name in ("primary", "secondary"):
|
|
746
|
+
window = value.get(name)
|
|
747
|
+
if window is None:
|
|
748
|
+
result[name] = None
|
|
749
|
+
continue
|
|
750
|
+
if not isinstance(window, dict):
|
|
751
|
+
return None
|
|
752
|
+
result[name] = {
|
|
753
|
+
"usedPercent": window.get("used_percent"),
|
|
754
|
+
"windowMinutes": window.get("window_minutes"),
|
|
755
|
+
"resetsAt": window.get("resets_at"),
|
|
756
|
+
}
|
|
757
|
+
return result if valid_account_rate_limit(result) else None
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def valid_account_rate_limit(value: object) -> bool:
|
|
761
|
+
if not isinstance(value, dict) or set(value) != {
|
|
762
|
+
"limitId",
|
|
763
|
+
"observedAt",
|
|
764
|
+
"primary",
|
|
765
|
+
"secondary",
|
|
766
|
+
}:
|
|
767
|
+
return False
|
|
768
|
+
if not isinstance(value["limitId"], str) or _timestamp(value["observedAt"]) is None:
|
|
769
|
+
return False
|
|
770
|
+
for name in ("primary", "secondary"):
|
|
771
|
+
window = value[name]
|
|
772
|
+
if window is None:
|
|
773
|
+
continue
|
|
774
|
+
if not isinstance(window, dict) or set(window) != {
|
|
775
|
+
"usedPercent",
|
|
776
|
+
"windowMinutes",
|
|
777
|
+
"resetsAt",
|
|
778
|
+
}:
|
|
779
|
+
return False
|
|
780
|
+
percent = window["usedPercent"]
|
|
781
|
+
if type(percent) not in (int, float) or not math.isfinite(percent):
|
|
782
|
+
return False
|
|
783
|
+
if any(
|
|
784
|
+
window[key] is not None and type(window[key]) is not int
|
|
785
|
+
for key in ("windowMinutes", "resetsAt")
|
|
786
|
+
):
|
|
787
|
+
return False
|
|
788
|
+
return True
|
|
469
789
|
|
|
470
790
|
|
|
471
791
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
@@ -515,7 +835,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
515
835
|
info = payload.get("info")
|
|
516
836
|
if not isinstance(info, dict):
|
|
517
837
|
return None
|
|
518
|
-
|
|
838
|
+
return _usage_counts(info.get("total_token_usage"))
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
519
842
|
if not isinstance(usage, dict):
|
|
520
843
|
return None
|
|
521
844
|
cache_write = usage.get("cache_write_input_tokens", usage.get("cache_write_tokens", 0))
|
|
@@ -75,7 +75,7 @@ def apply_migrations(
|
|
|
75
75
|
elif version == 16:
|
|
76
76
|
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
77
77
|
elif version in applied:
|
|
78
|
-
if version in (2, 12, 13, 26, 28, 31, 32, 47):
|
|
78
|
+
if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50):
|
|
79
79
|
repair_additive_migration(connection, version)
|
|
80
80
|
elif version == 11:
|
|
81
81
|
repair_deep_scan_migration(connection)
|