@openai/codex-security 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -9
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/core-scan.md +2 -0
- package/_bundled_plugin/references/final-report.md +5 -1
- package/_bundled_plugin/references/investigation-time-budget.md +11 -0
- package/_bundled_plugin/references/sarif-adapter.md +1 -0
- package/_bundled_plugin/references/scan-prologue.md +2 -6
- package/_bundled_plugin/references/scan-runtime.md +15 -0
- package/_bundled_plugin/references/threat-model.md +2 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +6 -2
- package/_bundled_plugin/scripts/deep_scan_workbench.py +7 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +92 -44
- package/_bundled_plugin/scripts/finding_preview.py +5 -3
- package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
- package/_bundled_plugin/scripts/generate_rank_input.py +13 -6
- package/_bundled_plugin/scripts/launch_codex_security_mcp +39 -4
- package/_bundled_plugin/scripts/report_projection.py +2 -1
- package/_bundled_plugin/scripts/snapshot_sqlite.py +7 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
- package/_bundled_plugin/scripts/workbench_cli.py +5 -10
- package/_bundled_plugin/scripts/workbench_constants.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +68 -39
- package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
- package/_bundled_plugin/scripts/workbench_progress.py +0 -50
- package/_bundled_plugin/scripts/workbench_publication.py +4 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
- package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
- package/_bundled_plugin/scripts/workbench_scan_history.py +82 -80
- package/_bundled_plugin/scripts/workbench_scan_start.py +12 -4
- package/_bundled_plugin/scripts/workbench_scan_usage.py +246 -38
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +74 -51
- package/_bundled_plugin/scripts/workbench_source_scopes.py +388 -0
- package/_bundled_plugin/scripts/workbench_target.py +28 -12
- package/_bundled_plugin/scripts/workbench_validation.py +52 -13
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -0
- package/_bundled_plugin/shared/workbench-migrations.json +15 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +4 -2
- package/_bundled_plugin/skills/security-scan/SKILL.md +3 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/dist/api.d.ts +4 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +215 -93
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +2 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +38 -23
- package/dist/artifact-export.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +16 -18
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity-records.d.ts +76 -0
- package/dist/classify-severity-records.d.ts.map +1 -0
- package/dist/classify-severity-records.js +141 -0
- package/dist/classify-severity-records.js.map +1 -0
- package/dist/classify-severity.d.ts +1 -23
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -24
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-daybreak.d.ts +5 -0
- package/dist/cli-daybreak.d.ts.map +1 -0
- package/dist/cli-daybreak.js +89 -0
- package/dist/cli-daybreak.js.map +1 -0
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +2 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +957 -738
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +1 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.js +2 -2
- package/dist/codeowners.js.map +1 -1
- package/dist/codex-sdk-environment.js +1 -1
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +3 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.js +2 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -5
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +5 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +65 -17
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +7 -2
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +2 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +17 -8
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +2 -5
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +8 -6
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.js +1 -1
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/retry.d.ts.map +1 -1
- package/dist/deduplication/retry.js +5 -0
- package/dist/deduplication/retry.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +15 -10
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +4 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +21 -1
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan.d.ts +6 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +8 -6
- package/dist/deep-scan.js.map +1 -1
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +26 -6
- package/dist/findings-import.js.map +1 -1
- package/dist/git-remote.d.ts +10 -0
- package/dist/git-remote.d.ts.map +1 -0
- package/dist/git-remote.js +24 -0
- package/dist/git-remote.js.map +1 -0
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +5 -7
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -0
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +2 -2
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +15 -0
- package/dist/host-records-protocol.d.ts.map +1 -0
- package/dist/host-records-protocol.js +15 -0
- package/dist/host-records-protocol.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +5 -0
- package/dist/hosted-remediation-protocol.d.ts.map +1 -0
- package/dist/hosted-remediation-protocol.js +26 -0
- package/dist/hosted-remediation-protocol.js.map +1 -0
- package/dist/hosted-remediation.d.ts +89 -0
- package/dist/hosted-remediation.d.ts.map +1 -0
- package/dist/hosted-remediation.js +122 -0
- package/dist/hosted-remediation.js.map +1 -0
- package/dist/hosted-scan.d.ts +1 -6
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +2 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.d.ts +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +5 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +87 -7
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +65 -29
- package/dist/multiscan.js.map +1 -1
- package/dist/patch-publication.d.ts +26 -0
- package/dist/patch-publication.d.ts.map +1 -0
- package/dist/patch-publication.js +105 -0
- package/dist/patch-publication.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +4 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +29 -15
- package/dist/project-config.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +135 -19
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +48 -25
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +63 -0
- package/dist/remediation.d.ts.map +1 -0
- package/dist/remediation.js +110 -0
- package/dist/remediation.js.map +1 -0
- package/dist/request-metadata.d.ts +17 -7
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +43 -8
- package/dist/request-metadata.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +104 -36
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +16 -4
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.js +8 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +15 -9
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +12 -6
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +5 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +9 -8
- package/dist/scan-executor.js.map +1 -1
- package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
- package/dist/scan-progress.d.ts.map +1 -0
- package/dist/scan-progress.js +138 -0
- package/dist/scan-progress.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +7 -1
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-time-budget.d.ts +9 -0
- package/dist/scan-time-budget.d.ts.map +1 -0
- package/dist/scan-time-budget.js +16 -0
- package/dist/scan-time-budget.js.map +1 -0
- package/dist/security-policy.js +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +4 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +14 -2
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-review.d.ts +29 -0
- package/dist/severity-review.d.ts.map +1 -0
- package/dist/severity-review.js +47 -0
- package/dist/severity-review.js.map +1 -0
- package/dist/severity-store.js +4 -1
- package/dist/severity-store.js.map +1 -1
- package/dist/timestamps.d.ts +3 -0
- package/dist/timestamps.d.ts.map +1 -0
- package/dist/timestamps.js +14 -0
- package/dist/timestamps.js.map +1 -0
- package/dist/validation-records.d.ts +38 -0
- package/dist/validation-records.d.ts.map +1 -0
- package/dist/validation-records.js +76 -0
- package/dist/validation-records.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +17 -4
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts.map +1 -1
- package/dist/workbench-arguments.js +9 -3
- package/dist/workbench-arguments.js.map +1 -1
- package/dist/worker-progress.d.ts +13 -7
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +12 -44
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +89 -0
- package/docs/cli.md +352 -45
- package/docs/findings-service.md +7 -2
- package/docs/hosted-remediation.md +117 -0
- package/docs/hosted-scans.md +8 -3
- package/docs/validate-records.md +69 -0
- package/package.json +13 -10
- package/schemas/project-config.schema.json +7 -1
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -76
- package/_bundled_plugin/references/config-preflight.md +0 -93
- package/_bundled_plugin/references/desktop-config-preflight.md +0 -11
- package/_bundled_plugin/scripts/config_preflight.py +0 -911
- package/dist/deep-progress.d.ts.map +0 -1
- package/dist/deep-progress.js +0 -92
- package/dist/deep-progress.js.map +0 -1
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
|
+
import math
|
|
7
8
|
import os
|
|
8
9
|
import re
|
|
9
10
|
import sqlite3
|
|
@@ -34,6 +35,14 @@ class RolloutSession:
|
|
|
34
35
|
path: Path
|
|
35
36
|
|
|
36
37
|
|
|
38
|
+
@dataclass
|
|
39
|
+
class RolloutUsage:
|
|
40
|
+
tokens: dict[str, int]
|
|
41
|
+
warnings: set[str]
|
|
42
|
+
tokens_updated_at: datetime | None
|
|
43
|
+
rate_limits: dict[str, dict[str, Any]]
|
|
44
|
+
|
|
45
|
+
|
|
37
46
|
def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
38
47
|
"""Project measured usage without changing the existing legacy cost contract."""
|
|
39
48
|
|
|
@@ -50,12 +59,6 @@ def stored_scan_cost_fields(value: str | None) -> dict[str, Any]:
|
|
|
50
59
|
}
|
|
51
60
|
|
|
52
61
|
|
|
53
|
-
def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
|
|
54
|
-
"""Keep usage in the already-migrated scans.cost_json column."""
|
|
55
|
-
|
|
56
|
-
return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
|
|
57
|
-
|
|
58
|
-
|
|
59
62
|
def reconcile_completed_scan_cost(
|
|
60
63
|
connection: sqlite3.Connection,
|
|
61
64
|
scan: sqlite3.Row,
|
|
@@ -86,9 +89,10 @@ def collect_scan_usage(
|
|
|
86
89
|
completed_at: str | None = None,
|
|
87
90
|
completed_turn_id: str | None = None,
|
|
88
91
|
) -> dict[str, Any]:
|
|
89
|
-
"""Count
|
|
92
|
+
"""Count attributable receipts and flag observable gaps inside the scan window."""
|
|
90
93
|
|
|
91
94
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
95
|
+
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
92
96
|
worker_directories = _scan_worker_directories(connection, scan)
|
|
93
97
|
if not roots and not worker_directories:
|
|
94
98
|
return _unavailable_usage("scan_thread_unavailable")
|
|
@@ -98,7 +102,8 @@ def collect_scan_usage(
|
|
|
98
102
|
return _unavailable_usage("codex_state_unavailable")
|
|
99
103
|
|
|
100
104
|
started_at = _timestamp(scan["started_at"])
|
|
101
|
-
|
|
105
|
+
observed_at = datetime.now(timezone.utc)
|
|
106
|
+
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
102
107
|
if started_at is None:
|
|
103
108
|
return _unavailable_usage("scan_window_unavailable")
|
|
104
109
|
|
|
@@ -118,8 +123,11 @@ def collect_scan_usage(
|
|
|
118
123
|
if not sessions:
|
|
119
124
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
120
125
|
|
|
121
|
-
|
|
126
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
|
+
tokens_updated_at = None
|
|
122
129
|
accepted_thread_ids: set[str] = set()
|
|
130
|
+
measured_thread_ids: set[str] = set()
|
|
123
131
|
excluded_thread_ids: set[str] = set()
|
|
124
132
|
for session in sessions:
|
|
125
133
|
if session.parent_thread_id in excluded_thread_ids:
|
|
@@ -133,37 +141,75 @@ def collect_scan_usage(
|
|
|
133
141
|
warnings.add("thread_lineage_incomplete")
|
|
134
142
|
continue
|
|
135
143
|
try:
|
|
136
|
-
|
|
144
|
+
usage = _read_rollout_usage(
|
|
137
145
|
session,
|
|
138
146
|
started_at=started_at,
|
|
139
147
|
completed_at=stopped_at,
|
|
140
148
|
completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
|
|
149
|
+
is_scan_owner=(
|
|
150
|
+
scan["mode"] == "deep"
|
|
151
|
+
and session.thread_id == scan["deep_scan_owner_thread_id"]
|
|
152
|
+
),
|
|
141
153
|
)
|
|
142
154
|
except (OSError, UnicodeError, ValueError):
|
|
143
155
|
missing_thread_ids.add(session.thread_id)
|
|
144
156
|
warnings.add("rollout_unavailable")
|
|
145
157
|
continue
|
|
146
|
-
if "thread_outside_scan_window" in
|
|
158
|
+
if "thread_outside_scan_window" in usage.warnings:
|
|
147
159
|
excluded_thread_ids.add(session.thread_id)
|
|
148
160
|
continue
|
|
149
|
-
warnings.update(
|
|
150
|
-
if "thread_identity_mismatch" in
|
|
151
|
-
"thread_ownership_unavailable" in
|
|
161
|
+
warnings.update(usage.warnings)
|
|
162
|
+
if "thread_identity_mismatch" in usage.warnings or (
|
|
163
|
+
"thread_ownership_unavailable" in usage.warnings
|
|
152
164
|
):
|
|
153
165
|
missing_thread_ids.add(session.thread_id)
|
|
154
166
|
continue
|
|
155
167
|
accepted_thread_ids.add(session.thread_id)
|
|
156
|
-
|
|
168
|
+
for key, snapshot in usage.rate_limits.items():
|
|
169
|
+
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
170
|
+
rate_limits[key] = snapshot
|
|
171
|
+
if "token_usage_unavailable" in usage.warnings:
|
|
172
|
+
missing_thread_ids.add(session.thread_id)
|
|
173
|
+
continue
|
|
174
|
+
measured_thread_ids.add(session.thread_id)
|
|
175
|
+
role = by_role["parent" if session.thread_id in parent_ids else "workers"]
|
|
176
|
+
_add_token_usage(role, usage.tokens)
|
|
177
|
+
role["threadCount"] += 1
|
|
178
|
+
if usage.tokens_updated_at is not None:
|
|
179
|
+
tokens_updated_at = max(
|
|
180
|
+
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
181
|
+
)
|
|
157
182
|
|
|
158
183
|
if not accepted_thread_ids:
|
|
159
184
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
185
|
+
if not measured_thread_ids:
|
|
186
|
+
return _unavailable_usage(
|
|
187
|
+
"token_usage_unavailable", warnings=warnings, rate_limits=rate_limits
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
if any(
|
|
191
|
+
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
192
|
+
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
193
|
+
for usage in by_role.values()
|
|
194
|
+
):
|
|
195
|
+
return _unavailable_usage(
|
|
196
|
+
"token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
|
|
197
|
+
)
|
|
160
198
|
|
|
199
|
+
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
200
|
+
stopped_at = max(stopped_at, tokens_updated_at)
|
|
161
201
|
result: dict[str, Any] = {
|
|
162
202
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
163
203
|
"source": "codex_rollout",
|
|
164
|
-
**
|
|
165
|
-
"threadCount": len(
|
|
204
|
+
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
205
|
+
"threadCount": len(measured_thread_ids),
|
|
206
|
+
"byRole": by_role,
|
|
207
|
+
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
208
|
+
"observedAt": observed_at.isoformat(),
|
|
209
|
+
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
166
210
|
}
|
|
211
|
+
if tokens_updated_at is not None:
|
|
212
|
+
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
167
213
|
if missing_thread_ids:
|
|
168
214
|
result["missingThreadCount"] = len(missing_thread_ids)
|
|
169
215
|
if warnings:
|
|
@@ -177,6 +223,7 @@ def _scan_root_thread_ids(
|
|
|
177
223
|
supplied_thread_id: str | None,
|
|
178
224
|
*,
|
|
179
225
|
include_owner_threads: bool = True,
|
|
226
|
+
include_workers: bool = True,
|
|
180
227
|
) -> list[str]:
|
|
181
228
|
candidates: list[str | None] = [supplied_thread_id]
|
|
182
229
|
if include_owner_threads:
|
|
@@ -190,14 +237,14 @@ def _scan_root_thread_ids(
|
|
|
190
237
|
).fetchone()
|
|
191
238
|
if workspace is not None:
|
|
192
239
|
candidates.append(workspace["thread_id"])
|
|
193
|
-
if scan["mode"] == "deep":
|
|
240
|
+
if include_workers and scan["mode"] == "deep":
|
|
194
241
|
candidates.extend(
|
|
195
242
|
row["sdk_thread_id"]
|
|
196
243
|
for row in connection.execute(
|
|
197
244
|
"""
|
|
198
245
|
SELECT DISTINCT sdk_thread_id
|
|
199
|
-
FROM
|
|
200
|
-
WHERE scan_id = ?
|
|
246
|
+
FROM deep_scan_worker_threads
|
|
247
|
+
WHERE scan_id = ?
|
|
201
248
|
ORDER BY sdk_thread_id
|
|
202
249
|
""",
|
|
203
250
|
(scan["id"],),
|
|
@@ -440,9 +487,11 @@ def _read_rollout_usage(
|
|
|
440
487
|
started_at: datetime,
|
|
441
488
|
completed_at: datetime | None,
|
|
442
489
|
completed_turn_id: str | None = None,
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
490
|
+
is_scan_owner: bool = False,
|
|
491
|
+
) -> RolloutUsage:
|
|
492
|
+
result = RolloutUsage(_empty_token_usage(), set(), None, {})
|
|
493
|
+
total = result.tokens
|
|
494
|
+
warnings = result.warnings
|
|
446
495
|
previous = _empty_token_usage()
|
|
447
496
|
boundary_reached = False
|
|
448
497
|
legacy_usage = _empty_token_usage()
|
|
@@ -451,6 +500,11 @@ def _read_rollout_usage(
|
|
|
451
500
|
response_ids: set[str] = set()
|
|
452
501
|
current_turn_id: str | None = None
|
|
453
502
|
completed_turn_measured = False
|
|
503
|
+
direct_control_session = False
|
|
504
|
+
started_turns: set[str] = set()
|
|
505
|
+
open_turns: set[str] = set()
|
|
506
|
+
closed_turns: set[str] = set()
|
|
507
|
+
measured_turns: set[str] = set()
|
|
454
508
|
|
|
455
509
|
with session.path.open("rb") as source:
|
|
456
510
|
for line_number, raw_line in enumerate(source, start=1):
|
|
@@ -473,21 +527,28 @@ def _read_rollout_usage(
|
|
|
473
527
|
if line_number == 1:
|
|
474
528
|
if event.get("type") != "session_meta" or not isinstance(payload, dict):
|
|
475
529
|
warnings.add("thread_identity_mismatch")
|
|
476
|
-
return
|
|
530
|
+
return result
|
|
477
531
|
recorded_id = payload.get("id") or payload.get("session_id")
|
|
478
532
|
if recorded_id != session.thread_id:
|
|
479
533
|
warnings.add("thread_identity_mismatch")
|
|
480
|
-
return
|
|
534
|
+
return result
|
|
481
535
|
recorded_parent = _session_parent_thread_id(payload)
|
|
482
536
|
if session.parent_thread_id is not None:
|
|
483
537
|
if recorded_parent != session.parent_thread_id:
|
|
484
538
|
warnings.add("thread_identity_mismatch")
|
|
485
|
-
return
|
|
539
|
+
return result
|
|
486
540
|
boundary_reached = (
|
|
487
541
|
session.parent_thread_id is None
|
|
488
542
|
and not recorded_parent
|
|
489
543
|
and not payload.get("forked_from_id")
|
|
490
544
|
)
|
|
545
|
+
# prepareCliDeepScanSession creates this owner through app-server
|
|
546
|
+
# without a model turn, so its initial token usage is known zero.
|
|
547
|
+
direct_control_session = (
|
|
548
|
+
boundary_reached
|
|
549
|
+
and is_scan_owner
|
|
550
|
+
and payload.get("originator") == "codex_security_deep_scan"
|
|
551
|
+
)
|
|
491
552
|
continue
|
|
492
553
|
|
|
493
554
|
if not isinstance(payload, dict):
|
|
@@ -497,22 +558,35 @@ def _read_rollout_usage(
|
|
|
497
558
|
task_started_at = _timestamp(event.get("timestamp"))
|
|
498
559
|
if task_started_at is None:
|
|
499
560
|
warnings.add("thread_ownership_unavailable")
|
|
500
|
-
return
|
|
561
|
+
return result
|
|
501
562
|
if task_started_at < started_at or (
|
|
502
563
|
completed_at is not None
|
|
503
564
|
and task_started_at > completed_at
|
|
504
565
|
and payload.get("turn_id") != completed_turn_id
|
|
505
566
|
):
|
|
506
567
|
warnings.add("thread_outside_scan_window")
|
|
507
|
-
return
|
|
568
|
+
return result
|
|
508
569
|
boundary_reached = True
|
|
509
|
-
current_turn_id = payload.get("turn_id")
|
|
510
570
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
511
571
|
inherited_usage = _token_snapshot(payload)
|
|
512
572
|
if inherited_usage is not None:
|
|
513
573
|
previous = inherited_usage
|
|
514
|
-
|
|
574
|
+
if not boundary_reached:
|
|
575
|
+
continue
|
|
515
576
|
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
577
|
+
# The same owner can resume as a model-backed coordinator. Only
|
|
578
|
+
# a provably post-scan turn leaves the original control zero intact.
|
|
579
|
+
task_started_at = _timestamp(event.get("timestamp"))
|
|
580
|
+
if (
|
|
581
|
+
completed_at is None
|
|
582
|
+
or task_started_at is None
|
|
583
|
+
or task_started_at <= completed_at
|
|
584
|
+
or (
|
|
585
|
+
completed_turn_id is not None
|
|
586
|
+
and payload.get("turn_id") == completed_turn_id
|
|
587
|
+
)
|
|
588
|
+
):
|
|
589
|
+
direct_control_session = False
|
|
516
590
|
if (
|
|
517
591
|
completed_turn_id is not None
|
|
518
592
|
and current_turn_id == completed_turn_id
|
|
@@ -527,7 +601,41 @@ def _read_rollout_usage(
|
|
|
527
601
|
if payload.get("turn_id") != current_turn_id:
|
|
528
602
|
observed_response_usage = _empty_token_usage()
|
|
529
603
|
current_turn_id = payload.get("turn_id")
|
|
604
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
605
|
+
if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
|
|
606
|
+
warnings.add("turn_record_invalid")
|
|
607
|
+
elif (
|
|
608
|
+
completed_at is None
|
|
609
|
+
or timestamp <= completed_at
|
|
610
|
+
or current_turn_id == completed_turn_id
|
|
611
|
+
):
|
|
612
|
+
started_turns.add(current_turn_id)
|
|
613
|
+
open_turns.add(current_turn_id)
|
|
530
614
|
continue
|
|
615
|
+
if event.get("type") == "event_msg" and payload.get("type") in {
|
|
616
|
+
"task_complete",
|
|
617
|
+
"turn_aborted",
|
|
618
|
+
}:
|
|
619
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
620
|
+
terminal_turn_id = payload.get("turn_id") or current_turn_id
|
|
621
|
+
if (
|
|
622
|
+
not isinstance(terminal_turn_id, str)
|
|
623
|
+
or not terminal_turn_id
|
|
624
|
+
or timestamp is None
|
|
625
|
+
):
|
|
626
|
+
warnings.add("turn_record_invalid")
|
|
627
|
+
elif (
|
|
628
|
+
completed_at is None
|
|
629
|
+
or timestamp <= completed_at
|
|
630
|
+
or terminal_turn_id == completed_turn_id
|
|
631
|
+
):
|
|
632
|
+
open_turns.discard(terminal_turn_id)
|
|
633
|
+
if timestamp >= started_at:
|
|
634
|
+
if terminal_turn_id in started_turns:
|
|
635
|
+
closed_turns.add(terminal_turn_id)
|
|
636
|
+
if payload.get("type") == "turn_aborted":
|
|
637
|
+
# Later successful work does not recover interrupted request usage.
|
|
638
|
+
warnings.add("turn_interrupted")
|
|
531
639
|
if (
|
|
532
640
|
completed_turn_id is not None
|
|
533
641
|
and event.get("type") == "event_msg"
|
|
@@ -576,11 +684,16 @@ def _read_rollout_usage(
|
|
|
576
684
|
response_usage = _empty_token_usage()
|
|
577
685
|
if current_turn_id is None:
|
|
578
686
|
current_turn_id = payload.get("turn_id")
|
|
687
|
+
if isinstance(current_turn_id, str):
|
|
688
|
+
open_turns.add(current_turn_id)
|
|
579
689
|
if response_id in response_ids:
|
|
580
690
|
continue
|
|
581
691
|
response_ids.add(response_id)
|
|
582
692
|
_add_token_usage(observed_response_usage, usage)
|
|
583
693
|
turn_usage = _usage_counts(payload.get("turn_token_usage"))
|
|
694
|
+
receipt_turn_id = payload.get("turn_id")
|
|
695
|
+
if isinstance(receipt_turn_id, str):
|
|
696
|
+
measured_turns.add(receipt_turn_id)
|
|
584
697
|
if timestamp < started_at:
|
|
585
698
|
if turn_usage is not None:
|
|
586
699
|
observed_response_usage = turn_usage
|
|
@@ -591,10 +704,30 @@ def _read_rollout_usage(
|
|
|
591
704
|
turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
|
|
592
705
|
):
|
|
593
706
|
warnings.add("token_record_incomplete")
|
|
707
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
594
708
|
_add_token_usage(response_usage, usage)
|
|
595
709
|
continue
|
|
596
710
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
597
711
|
continue
|
|
712
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
713
|
+
in_window = (
|
|
714
|
+
timestamp is not None
|
|
715
|
+
and timestamp >= started_at
|
|
716
|
+
and (
|
|
717
|
+
completed_at is None
|
|
718
|
+
or timestamp <= completed_at
|
|
719
|
+
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
720
|
+
)
|
|
721
|
+
)
|
|
722
|
+
if in_window:
|
|
723
|
+
rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
|
|
724
|
+
if rate_limits is not None:
|
|
725
|
+
key = rate_limits["limitId"]
|
|
726
|
+
if (
|
|
727
|
+
key not in result.rate_limits
|
|
728
|
+
or rate_limits["observedAt"] >= result.rate_limits[key]["observedAt"]
|
|
729
|
+
):
|
|
730
|
+
result.rate_limits[key] = rate_limits
|
|
598
731
|
if "info" in payload and payload["info"] is None:
|
|
599
732
|
continue
|
|
600
733
|
timestamp = _timestamp(event.get("timestamp"))
|
|
@@ -607,15 +740,18 @@ def _read_rollout_usage(
|
|
|
607
740
|
for key, value in snapshot.items()
|
|
608
741
|
}
|
|
609
742
|
previous = snapshot
|
|
610
|
-
if timestamp < started_at:
|
|
611
|
-
continue
|
|
612
743
|
is_completion_turn = (
|
|
613
744
|
completed_turn_id is not None and current_turn_id == completed_turn_id
|
|
614
745
|
)
|
|
615
746
|
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
616
747
|
continue
|
|
748
|
+
if isinstance(current_turn_id, str):
|
|
749
|
+
measured_turns.add(current_turn_id)
|
|
750
|
+
if timestamp < started_at:
|
|
751
|
+
continue
|
|
617
752
|
if is_completion_turn:
|
|
618
753
|
completed_turn_measured = True
|
|
754
|
+
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
619
755
|
if delta["totalTokens"] <= 0:
|
|
620
756
|
continue
|
|
621
757
|
_add_token_usage(legacy_usage, delta)
|
|
@@ -623,9 +759,72 @@ def _read_rollout_usage(
|
|
|
623
759
|
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
624
760
|
if not boundary_reached:
|
|
625
761
|
warnings.add("thread_ownership_unavailable")
|
|
762
|
+
elif result.tokens_updated_at is None and not direct_control_session:
|
|
763
|
+
warnings.add("token_usage_unavailable")
|
|
764
|
+
if completed_turn_id is not None:
|
|
765
|
+
open_turns.discard(completed_turn_id)
|
|
766
|
+
if open_turns:
|
|
767
|
+
warnings.add("turn_in_progress")
|
|
768
|
+
if boundary_reached and result.tokens_updated_at is None and not direct_control_session:
|
|
769
|
+
warnings.add("token_record_missing")
|
|
770
|
+
if closed_turns - measured_turns:
|
|
771
|
+
warnings.add("token_record_missing")
|
|
626
772
|
if completed_turn_id is not None and not completed_turn_measured:
|
|
627
773
|
warnings.add("completion_turn_unavailable")
|
|
628
|
-
return
|
|
774
|
+
return result
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _rate_limit_snapshot(value: object, timestamp: datetime) -> dict[str, Any] | None:
|
|
778
|
+
if not isinstance(value, dict):
|
|
779
|
+
return None
|
|
780
|
+
result: dict[str, Any] = {
|
|
781
|
+
"limitId": value.get("limit_id") or "codex",
|
|
782
|
+
"observedAt": timestamp.isoformat(),
|
|
783
|
+
}
|
|
784
|
+
for name in ("primary", "secondary"):
|
|
785
|
+
window = value.get(name)
|
|
786
|
+
if window is None:
|
|
787
|
+
result[name] = None
|
|
788
|
+
continue
|
|
789
|
+
if not isinstance(window, dict):
|
|
790
|
+
return None
|
|
791
|
+
result[name] = {
|
|
792
|
+
"usedPercent": window.get("used_percent"),
|
|
793
|
+
"windowMinutes": window.get("window_minutes"),
|
|
794
|
+
"resetsAt": window.get("resets_at"),
|
|
795
|
+
}
|
|
796
|
+
return result if valid_account_rate_limit(result) else None
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def valid_account_rate_limit(value: object) -> bool:
|
|
800
|
+
if not isinstance(value, dict) or set(value) != {
|
|
801
|
+
"limitId",
|
|
802
|
+
"observedAt",
|
|
803
|
+
"primary",
|
|
804
|
+
"secondary",
|
|
805
|
+
}:
|
|
806
|
+
return False
|
|
807
|
+
if not isinstance(value["limitId"], str) or _timestamp(value["observedAt"]) is None:
|
|
808
|
+
return False
|
|
809
|
+
for name in ("primary", "secondary"):
|
|
810
|
+
window = value[name]
|
|
811
|
+
if window is None:
|
|
812
|
+
continue
|
|
813
|
+
if not isinstance(window, dict) or set(window) != {
|
|
814
|
+
"usedPercent",
|
|
815
|
+
"windowMinutes",
|
|
816
|
+
"resetsAt",
|
|
817
|
+
}:
|
|
818
|
+
return False
|
|
819
|
+
percent = window["usedPercent"]
|
|
820
|
+
if type(percent) not in (int, float) or not math.isfinite(percent):
|
|
821
|
+
return False
|
|
822
|
+
if any(
|
|
823
|
+
window[key] is not None and type(window[key]) is not int
|
|
824
|
+
for key in ("windowMinutes", "resetsAt")
|
|
825
|
+
):
|
|
826
|
+
return False
|
|
827
|
+
return True
|
|
629
828
|
|
|
630
829
|
|
|
631
830
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
@@ -686,7 +885,8 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
|
686
885
|
input_tokens = usage.get("input_tokens")
|
|
687
886
|
cached_input_tokens = usage.get("cached_input_tokens", 0)
|
|
688
887
|
if (
|
|
689
|
-
cache_write
|
|
888
|
+
type(cache_write) is int
|
|
889
|
+
and cache_write == 0
|
|
690
890
|
and type(legacy_cache_write) is int
|
|
691
891
|
and legacy_cache_write > 0
|
|
692
892
|
and type(input_tokens) is int
|
|
@@ -730,18 +930,26 @@ def _timestamp(value: object) -> datetime | None:
|
|
|
730
930
|
if value.endswith("Z")
|
|
731
931
|
else datetime.fromisoformat(value)
|
|
732
932
|
)
|
|
733
|
-
|
|
933
|
+
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
934
|
+
except (ValueError, OverflowError):
|
|
734
935
|
return None
|
|
735
|
-
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
736
936
|
|
|
737
937
|
|
|
738
|
-
def _unavailable_usage(
|
|
739
|
-
|
|
938
|
+
def _unavailable_usage(
|
|
939
|
+
reason: str,
|
|
940
|
+
*,
|
|
941
|
+
warnings: set[str] | None = None,
|
|
942
|
+
rate_limits: dict[str, dict[str, Any]] | None = None,
|
|
943
|
+
) -> dict[str, Any]:
|
|
944
|
+
result: dict[str, Any] = {
|
|
740
945
|
"coverage": "unavailable",
|
|
741
946
|
"source": "codex_rollout",
|
|
742
947
|
"threadCount": 0,
|
|
743
948
|
"warnings": sorted({reason, *(warnings or set())}),
|
|
744
949
|
}
|
|
950
|
+
if rate_limits:
|
|
951
|
+
result["accountRateLimits"] = [rate_limits[key] for key in sorted(rate_limits)]
|
|
952
|
+
return result
|
|
745
953
|
|
|
746
954
|
|
|
747
955
|
def _reject_nonstandard_json_number(value: str) -> None:
|
|
@@ -75,7 +75,7 @@ def apply_migrations(
|
|
|
75
75
|
elif version == 16:
|
|
76
76
|
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
77
77
|
elif version in applied:
|
|
78
|
-
if version in (2, 12, 13, 26, 28, 31, 32, 47):
|
|
78
|
+
if version in (2, 12, 13, 26, 28, 31, 32, 42, 47, 50):
|
|
79
79
|
repair_additive_migration(connection, version)
|
|
80
80
|
elif version == 11:
|
|
81
81
|
repair_deep_scan_migration(connection)
|
|
@@ -8,9 +8,16 @@ import sys
|
|
|
8
8
|
from pathlib import Path, PurePosixPath
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
|
-
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
12
11
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
13
|
-
from
|
|
12
|
+
from workbench_source_scopes import (
|
|
13
|
+
load_source_scopes,
|
|
14
|
+
normalized_path_component,
|
|
15
|
+
offline_git_bytes,
|
|
16
|
+
relative_path,
|
|
17
|
+
safe_source_path,
|
|
18
|
+
source_object_for_path,
|
|
19
|
+
)
|
|
20
|
+
from workbench_target import clean_worktree_content_digest
|
|
14
21
|
|
|
15
22
|
CONTEXT_LINES = 3
|
|
16
23
|
MAX_BYTES = 16_000
|
|
@@ -21,23 +28,70 @@ def finding_source_excerpt(
|
|
|
21
28
|
scan: sqlite3.Row,
|
|
22
29
|
target: Path | None,
|
|
23
30
|
locations: list[dict[str, Any]],
|
|
31
|
+
scopes: list[str],
|
|
24
32
|
) -> str | None:
|
|
25
|
-
if target is None or not locations:
|
|
33
|
+
if target is None or not locations or scan["target_revision"] == "unversioned":
|
|
26
34
|
return None
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
35
|
+
snapshot = scan["target_snapshot_digest"]
|
|
36
|
+
if snapshot is not None and snapshot != clean_worktree_content_digest():
|
|
37
|
+
return None
|
|
38
|
+
locations = [
|
|
39
|
+
location
|
|
40
|
+
for location in locations
|
|
41
|
+
if isinstance(path := location.get("path"), str)
|
|
42
|
+
and isinstance(location.get("startLine"), int)
|
|
43
|
+
and safe_source_path(target, path) is not None
|
|
44
|
+
]
|
|
45
|
+
if not locations:
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
context = load_source_scopes(scan, target, scopes)
|
|
49
|
+
except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
|
|
50
|
+
return None
|
|
51
|
+
if context is None:
|
|
52
|
+
return None
|
|
53
|
+
repository, tree, records = context
|
|
54
|
+
indexed: dict[tuple[str, ...], list[dict[str, str]]] = {}
|
|
55
|
+
for record in records:
|
|
56
|
+
for name in ("path", "canonicalPath"):
|
|
57
|
+
parts = tuple(
|
|
58
|
+
normalized_path_component(part) for part in PurePosixPath(record[name]).parts
|
|
59
|
+
)
|
|
60
|
+
indexed.setdefault(parts, []).append(record)
|
|
61
|
+
lengths = sorted({len(parts) for parts in indexed})
|
|
62
|
+
|
|
63
|
+
def source_object(path: str) -> str | None:
|
|
64
|
+
parsed = relative_path(path)
|
|
65
|
+
if parsed is None:
|
|
66
|
+
return None
|
|
67
|
+
parts = tuple(normalized_path_component(part) for part in parsed.parts)
|
|
68
|
+
try:
|
|
69
|
+
for length in lengths:
|
|
70
|
+
if length > len(parts):
|
|
71
|
+
break
|
|
72
|
+
for record in indexed.get(parts[:length], []):
|
|
73
|
+
object_id = source_object_for_path(repository, tree, target, path, record)
|
|
74
|
+
if object_id is not None:
|
|
75
|
+
return object_id
|
|
76
|
+
except (OSError, RuntimeError, SystemExit, UnicodeError, ValueError):
|
|
77
|
+
# An unavailable source object leaves the optional excerpt absent.
|
|
78
|
+
pass
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
locations.sort(
|
|
82
|
+
key=lambda location: (
|
|
83
|
+
location.get("role") != "root_control",
|
|
84
|
+
"root_control" not in str(location.get("role") or "").lower(),
|
|
85
|
+
)
|
|
34
86
|
)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
87
|
+
for location in locations:
|
|
88
|
+
object_id = source_object(location["path"])
|
|
89
|
+
if object_id is not None:
|
|
90
|
+
break
|
|
91
|
+
else:
|
|
39
92
|
return None
|
|
40
|
-
|
|
93
|
+
start_line, end_line = location["startLine"], location.get("endLine")
|
|
94
|
+
source = scanned_source_text(repository, object_id)
|
|
41
95
|
if not source or "\0" in source:
|
|
42
96
|
return None
|
|
43
97
|
# Match location validation and SARIF hashing: CR, LF, and CRLF are line breaks.
|
|
@@ -58,46 +112,15 @@ def finding_source_excerpt(
|
|
|
58
112
|
f"{line_number:>{width}} {lines[line_number - 1]}"
|
|
59
113
|
for line_number in range(excerpt_start, excerpt_end + 1)
|
|
60
114
|
)
|
|
61
|
-
|
|
62
|
-
return encoded.decode("utf-8", errors="ignore")
|
|
115
|
+
return excerpt.encode("utf-8")[:MAX_BYTES].decode("utf-8", errors="ignore")
|
|
63
116
|
|
|
64
117
|
|
|
65
|
-
def scanned_source_text(
|
|
66
|
-
if safe_source_path(target, path) is None:
|
|
67
|
-
return None
|
|
68
|
-
revision = scan["target_revision"]
|
|
69
|
-
if revision == "unversioned":
|
|
70
|
-
return None
|
|
71
|
-
snapshot_digest = scan["target_snapshot_digest"]
|
|
72
|
-
if (
|
|
73
|
-
scan["diff_target_kind"] == "working_tree"
|
|
74
|
-
and scan["diff_content_digest"] != clean_worktree_content_digest()
|
|
75
|
-
):
|
|
76
|
-
return None
|
|
77
|
-
if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
|
|
78
|
-
return None
|
|
79
|
-
if (
|
|
80
|
-
scan["diff_target_kind"] == "working_tree"
|
|
81
|
-
and scan["diff_content_digest"] != clean_worktree_content_digest()
|
|
82
|
-
):
|
|
83
|
-
return None
|
|
84
|
-
object_name = f"{revision}:./{path}"
|
|
85
|
-
content = git_bytes(target, "cat-file", "blob", object_name)
|
|
86
|
-
return content.decode("utf-8", errors="replace") if content is not None else None
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def safe_source_path(target: Path, relative_path: str) -> Path | None:
|
|
90
|
-
if "\\" in relative_path:
|
|
91
|
-
return None
|
|
92
|
-
parsed = PurePosixPath(relative_path)
|
|
93
|
-
if parsed.is_absolute() or ".." in parsed.parts:
|
|
94
|
-
return None
|
|
118
|
+
def scanned_source_text(repository: Path, object_id: str) -> str | None:
|
|
95
119
|
try:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
except (OSError, RuntimeError, ValueError):
|
|
120
|
+
content = offline_git_bytes(repository, "cat-file", "blob", object_id)
|
|
121
|
+
except (OSError, RuntimeError, SystemExit):
|
|
99
122
|
return None
|
|
100
|
-
return
|
|
123
|
+
return content.decode("utf-8", errors="replace") if content is not None else None
|
|
101
124
|
|
|
102
125
|
|
|
103
126
|
if __name__ == "__main__":
|