@openai/codex-security 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -33
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
- package/_bundled_plugin/references/config-preflight.md +11 -13
- package/_bundled_plugin/references/core-scan.md +94 -0
- package/_bundled_plugin/references/final-report.md +6 -6
- package/_bundled_plugin/references/scan-artifacts.md +15 -14
- package/_bundled_plugin/references/scan-prologue.md +27 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
- package/_bundled_plugin/scripts/config_preflight.py +40 -3
- package/_bundled_plugin/scripts/deep_scan_config.py +8 -5
- package/_bundled_plugin/scripts/deep_scan_workbench.py +103 -35
- package/_bundled_plugin/scripts/finalize_scan_contract.py +29 -17
- package/_bundled_plugin/scripts/finding_preview.py +122 -6
- package/_bundled_plugin/scripts/generate_in_scope_files.py +39 -17
- package/_bundled_plugin/scripts/generate_rank_input.py +21 -3
- package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
- package/_bundled_plugin/scripts/report_projection.py +1 -19
- package/_bundled_plugin/scripts/workbench_db.py +22 -29
- package/_bundled_plugin/scripts/workbench_progress.py +0 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +43 -8
- package/_bundled_plugin/scripts/workbench_schema.py +1 -20
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -110
- package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
- package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
- package/_bundled_plugin/skills/validation/SKILL.md +9 -12
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
- package/dist/api.d.ts +1 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +179 -56
- package/dist/api.js.map +1 -1
- package/dist/bulk-scan-discovery.js +14 -2
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +6 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +188 -42
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +10 -1
- package/dist/cost.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +10 -3
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +77 -18
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +15 -44
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +65 -16
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-logs.d.ts +2 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +97 -37
- package/dist/scan-logs.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +20 -11
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.js +1 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +3 -3
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +6 -4
- package/dist/version.js.map +1 -1
- package/package.json +4 -4
- package/_bundled_plugin/references/shared-hard-rules.md +0 -15
- package/_bundled_plugin/scripts/validate_report_format.py +0 -138
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
|
@@ -17,6 +17,7 @@ import re
|
|
|
17
17
|
import secrets
|
|
18
18
|
import stat
|
|
19
19
|
import sys
|
|
20
|
+
import time
|
|
20
21
|
from collections.abc import Iterator
|
|
21
22
|
from datetime import datetime, timezone
|
|
22
23
|
from pathlib import Path, PurePosixPath
|
|
@@ -71,6 +72,10 @@ class ContractError(ValueError):
|
|
|
71
72
|
"""Raised when a completed scan does not satisfy the additive contract."""
|
|
72
73
|
|
|
73
74
|
|
|
75
|
+
class RecoverableContractError(ContractError):
|
|
76
|
+
"""Raised when report projection can safely be retried before publication."""
|
|
77
|
+
|
|
78
|
+
|
|
74
79
|
def _reject_non_finite_json(value: str) -> None:
|
|
75
80
|
raise ValueError(f"non-finite JSON number {value!r} is not supported")
|
|
76
81
|
|
|
@@ -94,11 +99,6 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|
|
94
99
|
return payload
|
|
95
100
|
|
|
96
101
|
|
|
97
|
-
def _write_json(path: Path, payload: Any) -> None:
|
|
98
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
99
|
-
path.write_bytes(_json_bytes(payload))
|
|
100
|
-
|
|
101
|
-
|
|
102
102
|
def _generate_report_projection(
|
|
103
103
|
manifest: dict[str, Any],
|
|
104
104
|
findings: dict[str, Any],
|
|
@@ -110,10 +110,21 @@ def _generate_report_projection(
|
|
|
110
110
|
raise ContractError(f"could not load report projection helper: {script}")
|
|
111
111
|
module = importlib.util.module_from_spec(spec)
|
|
112
112
|
spec.loader.exec_module(module)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
113
|
+
attempts = (
|
|
114
|
+
getattr(sys.modules.get("workbench_constants"), "SQLITE_RETRY_ATTEMPTS", 1)
|
|
115
|
+
if coverage.get("mode") == "deep_repository"
|
|
116
|
+
else 1
|
|
117
|
+
)
|
|
118
|
+
for attempt in range(attempts):
|
|
119
|
+
try:
|
|
120
|
+
return module.generate_report_markdown(manifest, findings, coverage)
|
|
121
|
+
except OSError as exc:
|
|
122
|
+
if attempt == attempts - 1:
|
|
123
|
+
raise RecoverableContractError(f"report projection failed: {exc}") from exc
|
|
124
|
+
time.sleep(0.05 * (2**attempt))
|
|
125
|
+
except ValueError as exc:
|
|
126
|
+
raise ContractError(f"report projection failed: {exc}") from exc
|
|
127
|
+
raise AssertionError("Report projection retry loop exhausted unexpectedly.")
|
|
117
128
|
|
|
118
129
|
|
|
119
130
|
def _validate_report_output_paths(scan_dir: Path) -> None:
|
|
@@ -178,14 +189,6 @@ def _sha256_bytes(value: bytes) -> str:
|
|
|
178
189
|
return hashlib.sha256(value).hexdigest()
|
|
179
190
|
|
|
180
191
|
|
|
181
|
-
def _sha256_file(path: Path) -> str:
|
|
182
|
-
digest = hashlib.sha256()
|
|
183
|
-
with path.open("rb") as handle:
|
|
184
|
-
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
185
|
-
digest.update(chunk)
|
|
186
|
-
return digest.hexdigest()
|
|
187
|
-
|
|
188
|
-
|
|
189
192
|
def _require_dict(payload: dict[str, Any], key: str, context: str) -> dict[str, Any]:
|
|
190
193
|
value = payload.get(key)
|
|
191
194
|
if not isinstance(value, dict):
|
|
@@ -858,6 +861,15 @@ def _recover_unsealed_coverage(
|
|
|
858
861
|
partial = completeness not in ("complete", "partial", "unknown")
|
|
859
862
|
if partial:
|
|
860
863
|
warnings.append("Recovered malformed coverage completeness; marked coverage as partial.")
|
|
864
|
+
if (
|
|
865
|
+
coverage.get("mode") == "deep_repository"
|
|
866
|
+
and coverage.get("inventoryStrategy") != "repository"
|
|
867
|
+
):
|
|
868
|
+
coverage["inventoryStrategy"] = "repository"
|
|
869
|
+
warnings.append(
|
|
870
|
+
"Recovered malformed Deep Scan inventory strategy; marked coverage as partial."
|
|
871
|
+
)
|
|
872
|
+
partial = True
|
|
861
873
|
|
|
862
874
|
surface_ids: set[str] = set()
|
|
863
875
|
for field, label in (
|
|
@@ -111,8 +111,14 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
111
111
|
):
|
|
112
112
|
key = next((alias for alias in aliases if alias in value), None)
|
|
113
113
|
if key is not None:
|
|
114
|
+
section = value[key]
|
|
115
|
+
if key == "attackPath" and isinstance(section, dict):
|
|
116
|
+
section = dict(section)
|
|
117
|
+
for assessment in ("impact", "likelihood"):
|
|
118
|
+
if isinstance(section.get(assessment), str):
|
|
119
|
+
section[assessment] = {"level": section[assessment]}
|
|
114
120
|
prepared[key] = bounded_finding_section(
|
|
115
|
-
|
|
121
|
+
section,
|
|
116
122
|
maximum_bytes,
|
|
117
123
|
priority_keys,
|
|
118
124
|
reserved_fields,
|
|
@@ -140,6 +146,8 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
140
146
|
"severity",
|
|
141
147
|
"status",
|
|
142
148
|
"taxonomy",
|
|
149
|
+
"preventiveControls",
|
|
150
|
+
"remediationTests",
|
|
143
151
|
):
|
|
144
152
|
if key in value:
|
|
145
153
|
prepared[key] = (
|
|
@@ -148,8 +156,64 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
148
156
|
else value[key]
|
|
149
157
|
)
|
|
150
158
|
|
|
151
|
-
|
|
152
|
-
|
|
159
|
+
guidance = {
|
|
160
|
+
key: prepared[key]
|
|
161
|
+
for key in ("remediationTests", "preventiveControls")
|
|
162
|
+
if key in prepared and isinstance(prepared[key], list)
|
|
163
|
+
}
|
|
164
|
+
diagnostics = (
|
|
165
|
+
"rootCause",
|
|
166
|
+
"root_cause",
|
|
167
|
+
"validation",
|
|
168
|
+
"attackPath",
|
|
169
|
+
"codeEvidence",
|
|
170
|
+
"code_evidence",
|
|
171
|
+
)
|
|
172
|
+
core_keys = (
|
|
173
|
+
"writeup",
|
|
174
|
+
*diagnostics,
|
|
175
|
+
"confidence",
|
|
176
|
+
"detectedAt",
|
|
177
|
+
"identity",
|
|
178
|
+
"provenance",
|
|
179
|
+
"ruleId",
|
|
180
|
+
"severity",
|
|
181
|
+
"status",
|
|
182
|
+
"taxonomy",
|
|
183
|
+
"evidence",
|
|
184
|
+
"evidenceExcerpt",
|
|
185
|
+
)
|
|
186
|
+
core = {key: prepared[key] for key in core_keys if key in prepared}
|
|
187
|
+
extras = {
|
|
188
|
+
key: item
|
|
189
|
+
for key, item in prepared.items()
|
|
190
|
+
if key not in core and key not in guidance
|
|
191
|
+
}
|
|
192
|
+
complete_guidance = {key: items[:1] for key, items in guidance.items()}
|
|
193
|
+
minimum_guidance = {
|
|
194
|
+
key: [items[0][:1]] if items and isinstance(items[0], str) else []
|
|
195
|
+
for key, items in guidance.items()
|
|
196
|
+
}
|
|
197
|
+
projected_core = {}
|
|
198
|
+
for selected_guidance in (complete_guidance, minimum_guidance):
|
|
199
|
+
reserved = (
|
|
200
|
+
len(json.dumps(selected_guidance, separators=(",", ":")).encode("utf-8")) - 1
|
|
201
|
+
if selected_guidance
|
|
202
|
+
else 0
|
|
203
|
+
)
|
|
204
|
+
if reserved >= FINDING_DETAILS_PREVIEW_BYTES:
|
|
205
|
+
continue
|
|
206
|
+
projected_core = bounded_json_value(
|
|
207
|
+
core,
|
|
208
|
+
[FINDING_DETAILS_PREVIEW_BYTES - reserved],
|
|
209
|
+
)
|
|
210
|
+
if all(key in projected_core for key in core):
|
|
211
|
+
break
|
|
212
|
+
ordered_guidance = dict(sorted(guidance.items(), key=lambda entry: bool(entry[1])))
|
|
213
|
+
bounded = bounded_json_value(
|
|
214
|
+
{**projected_core, **ordered_guidance, **extras},
|
|
215
|
+
[FINDING_DETAILS_PREVIEW_BYTES],
|
|
216
|
+
)
|
|
153
217
|
return bounded if isinstance(bounded, dict) else {}
|
|
154
218
|
|
|
155
219
|
|
|
@@ -215,11 +279,20 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
215
279
|
if not consume_json_budget(budget, 2):
|
|
216
280
|
return []
|
|
217
281
|
result = []
|
|
218
|
-
for item in value
|
|
282
|
+
for item in value:
|
|
283
|
+
remaining = budget[0]
|
|
219
284
|
separator = 0 if not result else 1
|
|
220
285
|
if not consume_json_budget(budget, separator):
|
|
221
286
|
break
|
|
222
|
-
|
|
287
|
+
bounded_item = bounded_json_value(item, budget, depth=depth + 1)
|
|
288
|
+
size = len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
289
|
+
if separator + size > remaining or (
|
|
290
|
+
isinstance(item, str) and item and bounded_item == ""
|
|
291
|
+
):
|
|
292
|
+
budget[0] = remaining
|
|
293
|
+
break
|
|
294
|
+
budget[0] = remaining - separator - size
|
|
295
|
+
result.append(bounded_item)
|
|
223
296
|
return result
|
|
224
297
|
if isinstance(value, dict):
|
|
225
298
|
if not consume_json_budget(budget, 2):
|
|
@@ -228,13 +301,56 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
228
301
|
for key, item in list(value.items())[:20]:
|
|
229
302
|
if budget[0] <= 0 or not isinstance(key, str):
|
|
230
303
|
break
|
|
304
|
+
remaining = budget[0]
|
|
231
305
|
separator = 0 if not result else 1
|
|
232
306
|
if not consume_json_budget(budget, separator):
|
|
307
|
+
budget[0] = remaining
|
|
233
308
|
break
|
|
234
309
|
bounded_key, key_size = bounded_json_text(key, min(budget[0], 512))
|
|
235
310
|
if not consume_json_budget(budget, key_size + 1):
|
|
311
|
+
budget[0] = remaining
|
|
312
|
+
break
|
|
313
|
+
item_budget = budget
|
|
314
|
+
if depth == 0 and key == "remediationTests":
|
|
315
|
+
controls = value.get("preventiveControls")
|
|
316
|
+
if (
|
|
317
|
+
isinstance(item, list)
|
|
318
|
+
and item
|
|
319
|
+
and isinstance(item[0], str)
|
|
320
|
+
and item[0]
|
|
321
|
+
and isinstance(controls, list)
|
|
322
|
+
and controls
|
|
323
|
+
and isinstance(controls[0], str)
|
|
324
|
+
and controls[0]
|
|
325
|
+
):
|
|
326
|
+
minimum_tests = len(
|
|
327
|
+
json.dumps([item[0][0]], separators=(",", ":")).encode("utf-8")
|
|
328
|
+
)
|
|
329
|
+
for control in (controls[0], controls[0][0]):
|
|
330
|
+
reserved = (
|
|
331
|
+
len(
|
|
332
|
+
json.dumps(
|
|
333
|
+
{"preventiveControls": [control]},
|
|
334
|
+
separators=(",", ":"),
|
|
335
|
+
).encode("utf-8")
|
|
336
|
+
)
|
|
337
|
+
- 1
|
|
338
|
+
)
|
|
339
|
+
if budget[0] >= minimum_tests + reserved:
|
|
340
|
+
item_budget = [budget[0] - reserved]
|
|
341
|
+
break
|
|
342
|
+
bounded_item = bounded_json_value(item, item_budget, depth=depth + 1)
|
|
343
|
+
size = (
|
|
344
|
+
separator
|
|
345
|
+
+ key_size
|
|
346
|
+
+ 1
|
|
347
|
+
+ len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
348
|
+
)
|
|
349
|
+
if size > remaining or (isinstance(item, str) and item and bounded_item == ""):
|
|
350
|
+
budget[0] = remaining
|
|
236
351
|
break
|
|
237
|
-
|
|
352
|
+
budget[0] = remaining - size
|
|
353
|
+
result[bounded_key] = bounded_item
|
|
238
354
|
return result
|
|
239
355
|
consume_json_budget(budget, 4)
|
|
240
356
|
return None
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import argparse
|
|
7
|
+
import os
|
|
7
8
|
import subprocess
|
|
8
9
|
import sys
|
|
9
10
|
import tempfile
|
|
@@ -105,6 +106,38 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
105
106
|
return write_inventory(output, rows)
|
|
106
107
|
|
|
107
108
|
|
|
109
|
+
def committed_changed_paths(repository: Path, base: str, head: str) -> list[tuple[Path, str]]:
|
|
110
|
+
result = subprocess.run(
|
|
111
|
+
[
|
|
112
|
+
"git",
|
|
113
|
+
"-C",
|
|
114
|
+
str(repository),
|
|
115
|
+
"diff",
|
|
116
|
+
"--raw",
|
|
117
|
+
"-z",
|
|
118
|
+
"--diff-filter=ACMRD",
|
|
119
|
+
f"{base}..{head}",
|
|
120
|
+
],
|
|
121
|
+
capture_output=True,
|
|
122
|
+
check=True,
|
|
123
|
+
)
|
|
124
|
+
fields = result.stdout.split(b"\0")
|
|
125
|
+
changed: list[tuple[Path, str]] = []
|
|
126
|
+
index = 0
|
|
127
|
+
while index < len(fields) - 1:
|
|
128
|
+
metadata = fields[index].split()
|
|
129
|
+
status = chr(metadata[-1][0])
|
|
130
|
+
index += 1
|
|
131
|
+
if status in {"C", "R"}:
|
|
132
|
+
index += 1
|
|
133
|
+
path = os.fsdecode(fields[index])
|
|
134
|
+
index += 1
|
|
135
|
+
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
|
|
136
|
+
if selected_mode != b"120000":
|
|
137
|
+
changed.append((repository / path, status))
|
|
138
|
+
return changed
|
|
139
|
+
|
|
140
|
+
|
|
108
141
|
def generate_diff_in_scope_files(
|
|
109
142
|
repository: Path,
|
|
110
143
|
base: str,
|
|
@@ -114,7 +147,7 @@ def generate_diff_in_scope_files(
|
|
|
114
147
|
) -> int:
|
|
115
148
|
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
|
|
116
149
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
117
|
-
from generate_rank_input import git_changed_paths, path_is_excluded
|
|
150
|
+
from generate_rank_input import git_changed_paths, path_is_excluded
|
|
118
151
|
from rank_preview import (
|
|
119
152
|
DEFAULT_PREVIEW_BYTES,
|
|
120
153
|
TEXT_CODE_EXTENSIONS,
|
|
@@ -125,22 +158,11 @@ def generate_diff_in_scope_files(
|
|
|
125
158
|
|
|
126
159
|
rows: list[bytes] = []
|
|
127
160
|
try:
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
text=True,
|
|
134
|
-
check=True,
|
|
135
|
-
)
|
|
136
|
-
changed.extend(
|
|
137
|
-
(repository / relative, "A")
|
|
138
|
-
for relative in untracked.stdout.split("\0")
|
|
139
|
-
if relative
|
|
140
|
-
)
|
|
141
|
-
else:
|
|
142
|
-
changed = git_changed_paths(repository, base, head, mode)
|
|
143
|
-
|
|
161
|
+
changed = (
|
|
162
|
+
committed_changed_paths(repository, base, head)
|
|
163
|
+
if mode == "revisions"
|
|
164
|
+
else git_changed_paths(repository, base, head, mode)
|
|
165
|
+
)
|
|
144
166
|
eligible = [
|
|
145
167
|
(path, status)
|
|
146
168
|
for path, status in changed
|
|
@@ -646,8 +646,19 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
|
|
|
646
646
|
if mode == "local-patch":
|
|
647
647
|
unstaged = run_git_changed_paths(repo, [base])
|
|
648
648
|
staged = run_git_changed_paths(repo, ["--cached", base])
|
|
649
|
+
untracked = subprocess.run(
|
|
650
|
+
["git", "-C", str(repo), "ls-files", "--others", "--exclude-standard", "-z"],
|
|
651
|
+
capture_output=True,
|
|
652
|
+
text=True,
|
|
653
|
+
check=True,
|
|
654
|
+
)
|
|
649
655
|
combined = dict(staged)
|
|
650
656
|
combined.update(unstaged)
|
|
657
|
+
combined.update(
|
|
658
|
+
(repo / relative, "A")
|
|
659
|
+
for relative in untracked.stdout.split("\0")
|
|
660
|
+
if relative
|
|
661
|
+
)
|
|
651
662
|
return sorted(combined.items())
|
|
652
663
|
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
653
664
|
|
|
@@ -693,10 +704,17 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
693
704
|
preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
|
|
694
705
|
if is_binary:
|
|
695
706
|
continue
|
|
707
|
+
elif path.is_symlink():
|
|
708
|
+
preview = ""
|
|
696
709
|
elif path.is_file():
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
710
|
+
try:
|
|
711
|
+
path.resolve(strict=True).relative_to(repo)
|
|
712
|
+
except (OSError, ValueError):
|
|
713
|
+
preview = ""
|
|
714
|
+
else:
|
|
715
|
+
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
716
|
+
if is_binary:
|
|
717
|
+
continue
|
|
700
718
|
else:
|
|
701
719
|
preview = ""
|
|
702
720
|
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
@@ -308,10 +308,6 @@ def main() -> None:
|
|
|
308
308
|
raise ValueError(f"{source} row {number}: {error}") from error
|
|
309
309
|
combined = combine(rows)
|
|
310
310
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
311
|
-
payload = "".join(
|
|
312
|
-
json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n"
|
|
313
|
-
for row in combined
|
|
314
|
-
)
|
|
315
311
|
temporary: Path | None = None
|
|
316
312
|
try:
|
|
317
313
|
with tempfile.NamedTemporaryFile(
|
|
@@ -323,7 +319,11 @@ def main() -> None:
|
|
|
323
319
|
delete=False,
|
|
324
320
|
) as handle:
|
|
325
321
|
temporary = Path(handle.name)
|
|
326
|
-
|
|
322
|
+
for row in combined:
|
|
323
|
+
handle.write(
|
|
324
|
+
json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
325
|
+
+ "\n"
|
|
326
|
+
)
|
|
327
327
|
temporary.replace(output)
|
|
328
328
|
finally:
|
|
329
329
|
if temporary is not None:
|
|
@@ -4,11 +4,8 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import argparse
|
|
7
|
-
import importlib.util
|
|
8
7
|
import re
|
|
9
8
|
from collections import Counter
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
from types import ModuleType
|
|
12
9
|
from typing import Any
|
|
13
10
|
|
|
14
11
|
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "informational": 4}
|
|
@@ -28,16 +25,6 @@ class ReportProjectionError(ValueError):
|
|
|
28
25
|
"""Raised when a canonical scan cannot be projected into a valid report."""
|
|
29
26
|
|
|
30
27
|
|
|
31
|
-
def _load_script(name: str) -> ModuleType:
|
|
32
|
-
path = Path(__file__).resolve().parent / f"{name}.py"
|
|
33
|
-
spec = importlib.util.spec_from_file_location(f"codex_security_{name}", path)
|
|
34
|
-
if spec is None or spec.loader is None:
|
|
35
|
-
raise ReportProjectionError(f"could not load report helper: {path}")
|
|
36
|
-
module = importlib.util.module_from_spec(spec)
|
|
37
|
-
spec.loader.exec_module(module)
|
|
38
|
-
return module
|
|
39
|
-
|
|
40
|
-
|
|
41
28
|
def _text(value: Any, fallback: str) -> str:
|
|
42
29
|
candidate = value if isinstance(value, str) and value.strip() else fallback
|
|
43
30
|
normalized = " ".join(candidate.split())
|
|
@@ -864,12 +851,7 @@ def generate_report_markdown(
|
|
|
864
851
|
findings: dict[str, Any],
|
|
865
852
|
coverage: dict[str, Any],
|
|
866
853
|
) -> bytes:
|
|
867
|
-
|
|
868
|
-
validator = _load_script("validate_report_format")
|
|
869
|
-
errors = validator.validate_report(markdown)
|
|
870
|
-
if errors:
|
|
871
|
-
raise ReportProjectionError("generated report failed validation: " + "; ".join(errors))
|
|
872
|
-
return markdown.encode("utf-8")
|
|
854
|
+
return build_report_markdown(manifest, findings, coverage).encode("utf-8")
|
|
873
855
|
|
|
874
856
|
|
|
875
857
|
def main() -> int:
|
|
@@ -97,7 +97,7 @@ from workbench_schema import (
|
|
|
97
97
|
from workbench_schema import (
|
|
98
98
|
sql_statements as sql_statements,
|
|
99
99
|
)
|
|
100
|
-
from workbench_source_excerpt import finding_source_excerpt
|
|
100
|
+
from workbench_source_excerpt import finding_source_excerpt, safe_source_path
|
|
101
101
|
from workbench_target import (
|
|
102
102
|
clean_worktree_content_digest,
|
|
103
103
|
copy_directory_excluding,
|
|
@@ -1175,7 +1175,11 @@ def complete_budget_exhausted_scan(
|
|
|
1175
1175
|
"discovery."
|
|
1176
1176
|
)
|
|
1177
1177
|
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1178
|
-
candidates =
|
|
1178
|
+
candidates = (
|
|
1179
|
+
[]
|
|
1180
|
+
if run["manifest_path"] == str(scan_dir / "scan-manifest.json")
|
|
1181
|
+
else budget_exhausted_candidates(scan, scan_dir)
|
|
1182
|
+
)
|
|
1179
1183
|
warning = optional_text(args.message, maximum=2400)
|
|
1180
1184
|
if warning is None:
|
|
1181
1185
|
warning = (
|
|
@@ -1450,11 +1454,14 @@ def complete_scan_locked(
|
|
|
1450
1454
|
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1451
1455
|
warnings = json.loads(scan["completion_warnings_json"])
|
|
1452
1456
|
target_warnings: list[str] = []
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1457
|
+
|
|
1458
|
+
def add_warning() -> None:
|
|
1459
|
+
if (warning := scan_target_warning(scan)) is not None:
|
|
1460
|
+
for items in (target_warnings, warnings):
|
|
1461
|
+
if warning not in items:
|
|
1462
|
+
items.append(warning)
|
|
1463
|
+
|
|
1464
|
+
add_warning()
|
|
1458
1465
|
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1459
1466
|
completion_timestamp = now()
|
|
1460
1467
|
completion_binding = workbench_completion_binding(scan, completion_timestamp)
|
|
@@ -1492,12 +1499,7 @@ def complete_scan_locked(
|
|
|
1492
1499
|
completion_binding=completion_binding,
|
|
1493
1500
|
completion_warnings=warnings,
|
|
1494
1501
|
)
|
|
1495
|
-
|
|
1496
|
-
if warning is not None:
|
|
1497
|
-
if warning not in target_warnings:
|
|
1498
|
-
target_warnings.append(warning)
|
|
1499
|
-
if warning not in warnings:
|
|
1500
|
-
warnings.append(warning)
|
|
1502
|
+
add_warning()
|
|
1501
1503
|
manifest, findings, _ = _write_prepared_scan_finalization(prepared)
|
|
1502
1504
|
except ContractError as exc:
|
|
1503
1505
|
raise SystemExit(str(exc)) from exc
|
|
@@ -2905,10 +2907,15 @@ def scan_context(
|
|
|
2905
2907
|
occurrence_id: str | None = None,
|
|
2906
2908
|
) -> dict[str, Any]:
|
|
2907
2909
|
scan = require_scan(connection, scan_id)
|
|
2910
|
+
workspace = workspace_state(connection, scan["workspace_id"], result_scan_id=scan["id"])
|
|
2908
2911
|
context = {
|
|
2909
2912
|
"otherRunningDeepScans": deep_scan.other_running_deep_scans(connection, scan["id"]),
|
|
2910
|
-
"scan":
|
|
2911
|
-
|
|
2913
|
+
"scan": (
|
|
2914
|
+
workspace["results"]
|
|
2915
|
+
if occurrence_id is None
|
|
2916
|
+
else scan_result(connection, scan, occurrence_id=occurrence_id)
|
|
2917
|
+
),
|
|
2918
|
+
"workspace": workspace,
|
|
2912
2919
|
}
|
|
2913
2920
|
if scan["recipe_json"] is not None:
|
|
2914
2921
|
context["parentScanId"] = scan["parent_scan_id"]
|
|
@@ -3462,20 +3469,6 @@ def patch_artifact_preview(
|
|
|
3462
3469
|
}
|
|
3463
3470
|
|
|
3464
3471
|
|
|
3465
|
-
def safe_source_path(target: Path, relative_path: str) -> Path | None:
|
|
3466
|
-
if "\\" in relative_path:
|
|
3467
|
-
return None
|
|
3468
|
-
parsed = PurePosixPath(relative_path)
|
|
3469
|
-
if parsed.is_absolute() or ".." in parsed.parts:
|
|
3470
|
-
return None
|
|
3471
|
-
try:
|
|
3472
|
-
path = (target / parsed.as_posix()).resolve()
|
|
3473
|
-
path.relative_to(target)
|
|
3474
|
-
except (OSError, RuntimeError, ValueError):
|
|
3475
|
-
return None
|
|
3476
|
-
return path
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
3472
|
def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None:
|
|
3480
3473
|
try:
|
|
3481
3474
|
resolved_scan_dir = require_canonical_scan_directory(scan_dir)
|
|
@@ -13,7 +13,6 @@ from workbench.handoff import require_current_continuation
|
|
|
13
13
|
from workbench_constants import PHASES
|
|
14
14
|
from workbench_validation import optional_text, require_uuid, user_text
|
|
15
15
|
|
|
16
|
-
MAX_PREFLIGHT_ISSUES_JSON_BYTES = 64 * 1024
|
|
17
16
|
MAX_PREFLIGHT_ISSUES = 32
|
|
18
17
|
|
|
19
18
|
|
|
@@ -33,10 +32,6 @@ def _preflight_issue_text(value: Any, maximum: int, label: str) -> str:
|
|
|
33
32
|
def preflight_issues_json(value: str | None) -> str | None:
|
|
34
33
|
if value is None:
|
|
35
34
|
return None
|
|
36
|
-
if len(value.encode("utf-8")) > MAX_PREFLIGHT_ISSUES_JSON_BYTES:
|
|
37
|
-
raise SystemExit(
|
|
38
|
-
f"Preflight issues must be no larger than {MAX_PREFLIGHT_ISSUES_JSON_BYTES} bytes."
|
|
39
|
-
)
|
|
40
35
|
try:
|
|
41
36
|
payload = json.loads(value)
|
|
42
37
|
except json.JSONDecodeError as exc:
|
|
@@ -189,10 +189,18 @@ def _codex_state_database() -> Path | None:
|
|
|
189
189
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
190
190
|
|
|
191
191
|
configured_home = os.environ.get("CODEX_HOME", "").strip()
|
|
192
|
-
codex_home =
|
|
192
|
+
codex_home = (
|
|
193
|
+
Path(configured_home).expanduser()
|
|
194
|
+
if configured_home
|
|
195
|
+
else Path.home() / ".codex"
|
|
196
|
+
)
|
|
193
197
|
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip()
|
|
194
198
|
search_roots = [
|
|
195
|
-
*(
|
|
199
|
+
*(
|
|
200
|
+
[Path(configured_sqlite_home).expanduser()]
|
|
201
|
+
if configured_sqlite_home
|
|
202
|
+
else []
|
|
203
|
+
),
|
|
196
204
|
codex_home,
|
|
197
205
|
codex_home / "sqlite",
|
|
198
206
|
]
|
|
@@ -329,7 +337,9 @@ def _require_state_columns(
|
|
|
329
337
|
}
|
|
330
338
|
columns = {str(row["name"]) for row in connection.execute(statements[table])}
|
|
331
339
|
if not required.issubset(columns):
|
|
332
|
-
raise ValueError(
|
|
340
|
+
raise ValueError(
|
|
341
|
+
"Codex state graph does not expose the required thread columns."
|
|
342
|
+
)
|
|
333
343
|
|
|
334
344
|
|
|
335
345
|
def _rollout_path(value: object) -> Path | None:
|
|
@@ -375,7 +385,9 @@ def _read_rollout_usage(
|
|
|
375
385
|
event = json.loads(raw_line)
|
|
376
386
|
except (UnicodeError, ValueError):
|
|
377
387
|
if line_number == 1:
|
|
378
|
-
raise ValueError(
|
|
388
|
+
raise ValueError(
|
|
389
|
+
"The rollout session metadata is unreadable."
|
|
390
|
+
) from None
|
|
379
391
|
if boundary_reached:
|
|
380
392
|
warnings.add("rollout_record_invalid")
|
|
381
393
|
continue
|
|
@@ -418,7 +430,10 @@ def _read_rollout_usage(
|
|
|
418
430
|
warnings.add("thread_outside_scan_window")
|
|
419
431
|
return total, warnings
|
|
420
432
|
boundary_reached = True
|
|
421
|
-
elif
|
|
433
|
+
elif (
|
|
434
|
+
event.get("type") == "event_msg"
|
|
435
|
+
and payload.get("type") == "token_count"
|
|
436
|
+
):
|
|
422
437
|
inherited_usage = _token_snapshot(payload)
|
|
423
438
|
if inherited_usage is not None:
|
|
424
439
|
previous = inherited_usage
|
|
@@ -497,10 +512,25 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
497
512
|
usage = info.get("total_token_usage")
|
|
498
513
|
if not isinstance(usage, dict):
|
|
499
514
|
return None
|
|
515
|
+
cache_write = usage.get(
|
|
516
|
+
"cache_write_input_tokens", usage.get("cache_write_tokens", 0)
|
|
517
|
+
)
|
|
518
|
+
legacy_cache_write = usage.get("cache_write_tokens")
|
|
519
|
+
input_tokens = usage.get("input_tokens")
|
|
520
|
+
cached_input_tokens = usage.get("cached_input_tokens", 0)
|
|
521
|
+
if (
|
|
522
|
+
cache_write == 0
|
|
523
|
+
and type(legacy_cache_write) is int
|
|
524
|
+
and legacy_cache_write > 0
|
|
525
|
+
and type(input_tokens) is int
|
|
526
|
+
and type(cached_input_tokens) is int
|
|
527
|
+
and cached_input_tokens + legacy_cache_write <= input_tokens
|
|
528
|
+
):
|
|
529
|
+
cache_write = legacy_cache_write
|
|
500
530
|
result: dict[str, int] = {}
|
|
501
531
|
for source_key, result_key in TOKEN_FIELDS.items():
|
|
502
532
|
value = (
|
|
503
|
-
|
|
533
|
+
cache_write
|
|
504
534
|
if source_key == "cache_write_input_tokens"
|
|
505
535
|
else usage.get(source_key, 0)
|
|
506
536
|
)
|
|
@@ -511,7 +541,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
511
541
|
):
|
|
512
542
|
return None
|
|
513
543
|
result[result_key] = value
|
|
514
|
-
if
|
|
544
|
+
if (
|
|
545
|
+
result["cachedInputTokens"] + result["cacheWriteInputTokens"]
|
|
546
|
+
> result["inputTokens"]
|
|
547
|
+
):
|
|
515
548
|
return None
|
|
516
549
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
517
550
|
return result
|
|
@@ -540,7 +573,9 @@ def _timestamp(value: object) -> datetime | None:
|
|
|
540
573
|
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
541
574
|
|
|
542
575
|
|
|
543
|
-
def _unavailable_usage(
|
|
576
|
+
def _unavailable_usage(
|
|
577
|
+
reason: str, *, warnings: set[str] | None = None
|
|
578
|
+
) -> dict[str, Any]:
|
|
544
579
|
return {
|
|
545
580
|
"coverage": "unavailable",
|
|
546
581
|
"source": "codex_rollout",
|