@openai/codex-security 0.1.23 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -5
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/scan-contract.md +10 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +27 -78
- package/_bundled_plugin/scripts/generate_in_scope_files.py +2 -8
- package/_bundled_plugin/scripts/generate_rank_input.py +1 -5
- package/_bundled_plugin/scripts/rank_preview.py +1 -0
- package/_bundled_plugin/scripts/report_projection.py +8 -14
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +5 -15
- package/_bundled_plugin/scripts/workbench_cli.py +6 -0
- package/_bundled_plugin/scripts/workbench_dashboard.py +41 -18
- package/_bundled_plugin/scripts/workbench_db.py +83 -539
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +31 -13
- package/_bundled_plugin/scripts/workbench_findings.py +8 -3
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -2
- package/_bundled_plugin/scripts/workbench_publication.py +520 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +37 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +298 -90
- package/_bundled_plugin/scripts/workbench_scan_history.py +1 -2
- package/_bundled_plugin/scripts/workbench_scan_usage.py +9 -33
- package/_bundled_plugin/scripts/workbench_schema.py +34 -21
- package/_bundled_plugin/scripts/workbench_target.py +10 -5
- package/_bundled_plugin/scripts/workbench_validation.py +1 -5
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +1 -5
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +6 -1
- package/dist/api.d.ts +3 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +20 -0
- package/dist/api.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +34 -4
- package/dist/cli.js.map +1 -1
- package/dist/deep-progress.d.ts +21 -0
- package/dist/deep-progress.d.ts.map +1 -0
- package/dist/deep-progress.js +108 -0
- package/dist/deep-progress.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +6 -1
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +12 -2
- package/dist/scan-activity.js.map +1 -1
- package/dist/server/index.js +2 -28
- package/dist/server/index.js.map +1 -1
- package/dist/server/serve.d.ts +2 -0
- package/dist/server/serve.d.ts.map +1 -0
- package/dist/server/serve.js +28 -0
- package/dist/server/serve.js.map +1 -0
- package/package.json +3 -3
|
@@ -98,6 +98,238 @@ def _children(scan_dir: Path, relative: str) -> list[str]:
|
|
|
98
98
|
return sorted(child.name for child in cursor.iterdir())
|
|
99
99
|
|
|
100
100
|
|
|
101
|
+
def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
102
|
+
return max(
|
|
103
|
+
(
|
|
104
|
+
worker
|
|
105
|
+
for worker in workers
|
|
106
|
+
if worker["kind"] == "dedup"
|
|
107
|
+
and worker["status"] == "succeeded"
|
|
108
|
+
and worker["result_manifest_path"]
|
|
109
|
+
),
|
|
110
|
+
key=lambda worker: (worker["completed_at"] or "", worker["id"]),
|
|
111
|
+
default=None,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[str]:
|
|
116
|
+
latest_reducer = _latest_successful_reducer(workers)
|
|
117
|
+
|
|
118
|
+
def checkpoints(directory: str) -> Iterator[str]:
|
|
119
|
+
for name in _children(scan_dir, directory):
|
|
120
|
+
if re.fullmatch(r"[0-9a-f]{64}\.json", name):
|
|
121
|
+
yield f"{directory}/{name}"
|
|
122
|
+
|
|
123
|
+
yield from checkpoints("checkpoints")
|
|
124
|
+
for worker in workers:
|
|
125
|
+
if worker["kind"] not in {"dedup", "discovery"}:
|
|
126
|
+
continue
|
|
127
|
+
try:
|
|
128
|
+
output = Path(worker["artifact_dir"]).relative_to(scan_dir).as_posix()
|
|
129
|
+
except (TypeError, ValueError):
|
|
130
|
+
continue
|
|
131
|
+
attempts = (Path(output).parent if Path(output).name == "output" else Path(output)) / (
|
|
132
|
+
"attempts"
|
|
133
|
+
)
|
|
134
|
+
directories = [output] + [
|
|
135
|
+
(attempts / name).as_posix()
|
|
136
|
+
for name in _children(scan_dir, attempts.as_posix())
|
|
137
|
+
if re.fullmatch(r"attempt-\d+", name)
|
|
138
|
+
]
|
|
139
|
+
for directory in directories:
|
|
140
|
+
checkpoint_paths = list(checkpoints(f"{directory}/checkpoints"))
|
|
141
|
+
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
142
|
+
yield f"{directory}/result.json"
|
|
143
|
+
yield from checkpoint_paths
|
|
144
|
+
if worker["result_manifest_path"] and (
|
|
145
|
+
worker["kind"] == "discovery"
|
|
146
|
+
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
147
|
+
):
|
|
148
|
+
try:
|
|
149
|
+
yield Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
150
|
+
except ValueError:
|
|
151
|
+
continue
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _read_saved_result(scan_dir: Path, relative: str, scan_id: str) -> tuple[dict[str, Any], str]:
|
|
155
|
+
draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint")
|
|
156
|
+
if draft.get("scanId") != scan_id:
|
|
157
|
+
raise ContractError("checkpoint belongs to a different scan")
|
|
158
|
+
if not isinstance(draft.get("findings"), list) or not isinstance(draft.get("coverage"), dict):
|
|
159
|
+
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
160
|
+
return draft, _digest(draft)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _read_saved_parent_result(
|
|
164
|
+
scan_dir: Path, scan_id: str
|
|
165
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
166
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "Saved parent manifest")
|
|
167
|
+
findings = _read_scan_local_json(scan_dir, "findings.json", "Saved parent findings")
|
|
168
|
+
coverage = _read_scan_local_json(scan_dir, "coverage.json", "Saved parent coverage")
|
|
169
|
+
parent_scan = manifest.get("scan")
|
|
170
|
+
if not isinstance(parent_scan, dict):
|
|
171
|
+
raise ContractError("Saved parent manifest has no scan object")
|
|
172
|
+
if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
|
|
173
|
+
parent_scan.get("id", scan_id) != scan_id
|
|
174
|
+
or findings.get("scanId", scan_id) != scan_id
|
|
175
|
+
or coverage.get("scanId", scan_id) != scan_id
|
|
176
|
+
):
|
|
177
|
+
raise ContractError("Saved parent documents belong to a different scan")
|
|
178
|
+
parent = {
|
|
179
|
+
"scanId": scan_id,
|
|
180
|
+
"findings": findings.get("findings"),
|
|
181
|
+
"coverage": coverage,
|
|
182
|
+
**{
|
|
183
|
+
key: parent_scan[key]
|
|
184
|
+
for key in ("scope", "threatModel", "complete")
|
|
185
|
+
if key in parent_scan
|
|
186
|
+
},
|
|
187
|
+
}
|
|
188
|
+
if not isinstance(parent["findings"], list):
|
|
189
|
+
raise ContractError("Saved parent draft has no findings array")
|
|
190
|
+
return manifest, parent
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _source_digests(value: Any, label: str) -> dict[str, str]:
|
|
194
|
+
if not isinstance(value, dict) or not all(
|
|
195
|
+
isinstance(relative, str) and isinstance(digest, str) for relative, digest in value.items()
|
|
196
|
+
):
|
|
197
|
+
raise ContractError(f"{label} source digests are malformed.")
|
|
198
|
+
return value
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
202
|
+
try:
|
|
203
|
+
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
204
|
+
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
205
|
+
workers = connection.execute(
|
|
206
|
+
"SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
|
|
207
|
+
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
208
|
+
(scan["id"],),
|
|
209
|
+
).fetchall()
|
|
210
|
+
paths = set(_saved_result_paths(scan_dir, workers))
|
|
211
|
+
frozen_sources = scan["retained_source_digests_json"]
|
|
212
|
+
|
|
213
|
+
def has_saved_source() -> bool:
|
|
214
|
+
for path in paths:
|
|
215
|
+
try:
|
|
216
|
+
_read_saved_result(scan_dir, path, scan["id"])
|
|
217
|
+
return True
|
|
218
|
+
except (ContractError, OSError, ValueError):
|
|
219
|
+
continue
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
if manifest_path is None:
|
|
223
|
+
if frozen_sources is not None:
|
|
224
|
+
return bool(_source_digests(json.loads(frozen_sources), "Frozen stopped-scan"))
|
|
225
|
+
return has_saved_source()
|
|
226
|
+
if scan["seal_manifest_digest"] is None:
|
|
227
|
+
try:
|
|
228
|
+
_read_saved_parent_result(scan_dir, scan["id"])
|
|
229
|
+
return True
|
|
230
|
+
except (ContractError, OSError, ValueError):
|
|
231
|
+
pass
|
|
232
|
+
return has_saved_source()
|
|
233
|
+
manifest = _read_scan_local_json(
|
|
234
|
+
scan_dir,
|
|
235
|
+
manifest_path.relative_to(scan_dir).as_posix(),
|
|
236
|
+
"Saved scan manifest",
|
|
237
|
+
)
|
|
238
|
+
manifest_scan = manifest.get("scan")
|
|
239
|
+
if not isinstance(manifest_scan, dict):
|
|
240
|
+
return True
|
|
241
|
+
published_sources = _source_digests(
|
|
242
|
+
manifest_scan.get("preservedSources", {}), "Published scan"
|
|
243
|
+
)
|
|
244
|
+
current_sources = dict(published_sources)
|
|
245
|
+
for path in paths:
|
|
246
|
+
try:
|
|
247
|
+
_, current_sources[path] = _read_saved_result(scan_dir, path, scan["id"])
|
|
248
|
+
except (ContractError, OSError, ValueError):
|
|
249
|
+
continue
|
|
250
|
+
return current_sources != published_sources
|
|
251
|
+
except (ContractError, OSError, SystemExit, ValueError):
|
|
252
|
+
return False
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[str, str], bool]:
|
|
256
|
+
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
257
|
+
frozen_sources: dict[str, str] | None = None
|
|
258
|
+
include_parent = True
|
|
259
|
+
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
260
|
+
if raw_frozen_sources is not None:
|
|
261
|
+
frozen_sources = _source_digests(json.loads(raw_frozen_sources), "Saved stopped-scan")
|
|
262
|
+
include_parent = False
|
|
263
|
+
|
|
264
|
+
manifest_path = db.artifact_path(scan_dir, db.ARTIFACTS["manifest"], required=False)
|
|
265
|
+
if manifest_path is not None:
|
|
266
|
+
manifest = _read_scan_local_json(
|
|
267
|
+
scan_dir,
|
|
268
|
+
manifest_path.relative_to(scan_dir).as_posix(),
|
|
269
|
+
"Saved scan manifest",
|
|
270
|
+
)
|
|
271
|
+
manifest_scan = manifest.get("scan")
|
|
272
|
+
if not isinstance(manifest_scan, dict):
|
|
273
|
+
raise ContractError("Saved scan manifest has no scan object")
|
|
274
|
+
if scan["seal_manifest_digest"] is not None or (
|
|
275
|
+
manifest_scan.get("sealedAt") is not None or manifest_scan.get("artifacts") is not None
|
|
276
|
+
):
|
|
277
|
+
if "preservedSources" in manifest_scan:
|
|
278
|
+
published_sources = _source_digests(
|
|
279
|
+
manifest_scan["preservedSources"], "Published scan"
|
|
280
|
+
)
|
|
281
|
+
include_parent = not published_sources
|
|
282
|
+
if published_sources:
|
|
283
|
+
if frozen_sources is not None and frozen_sources != published_sources:
|
|
284
|
+
raise ContractError(
|
|
285
|
+
"Stopped scan sources changed after terminal publication."
|
|
286
|
+
)
|
|
287
|
+
frozen_sources = published_sources
|
|
288
|
+
elif frozen_sources is None:
|
|
289
|
+
frozen_sources = {}
|
|
290
|
+
else:
|
|
291
|
+
include_parent = True
|
|
292
|
+
|
|
293
|
+
recovery_sources = dict(frozen_sources or {})
|
|
294
|
+
for relative, expected_digest in recovery_sources.items():
|
|
295
|
+
try:
|
|
296
|
+
_, digest = _read_saved_result(scan_dir, relative, scan["id"])
|
|
297
|
+
except (ContractError, OSError, ValueError) as exc:
|
|
298
|
+
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
299
|
+
if digest != expected_digest:
|
|
300
|
+
raise ContractError("checkpoint changed after the scan stopped")
|
|
301
|
+
|
|
302
|
+
workers = connection.execute(
|
|
303
|
+
"SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
|
|
304
|
+
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
305
|
+
(scan["id"],),
|
|
306
|
+
).fetchall()
|
|
307
|
+
for relative in set(_saved_result_paths(scan_dir, workers)) - recovery_sources.keys():
|
|
308
|
+
try:
|
|
309
|
+
_, recovery_sources[relative] = _read_saved_result(scan_dir, relative, scan["id"])
|
|
310
|
+
except (ContractError, OSError, ValueError):
|
|
311
|
+
continue
|
|
312
|
+
return recovery_sources, include_parent
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def scan_results_recovery_needed(db: Any, connection: Any, scan: Any) -> bool:
|
|
316
|
+
if scan["status"] != "failed" or scan["canceled_at"] is not None:
|
|
317
|
+
return False
|
|
318
|
+
warnings = json.loads(scan["completion_warnings_json"])
|
|
319
|
+
if any(
|
|
320
|
+
isinstance(warning, str) and warning.startswith(_PUBLICATION_FOLLOW_UP_WARNING)
|
|
321
|
+
for warning in warnings
|
|
322
|
+
):
|
|
323
|
+
return True
|
|
324
|
+
publication_error = connection.execute(
|
|
325
|
+
"SELECT publication_error_message FROM deep_scan_runs WHERE scan_id = ?",
|
|
326
|
+
(scan["id"],),
|
|
327
|
+
).fetchone()
|
|
328
|
+
if publication_error is not None and publication_error["publication_error_message"]:
|
|
329
|
+
return True
|
|
330
|
+
return _saved_results_changed(db, connection, scan)
|
|
331
|
+
|
|
332
|
+
|
|
101
333
|
def _finding_key(finding: dict[str, Any]) -> str:
|
|
102
334
|
# Wording and evidence may improve between checkpoints; distinct source locations
|
|
103
335
|
# must not collide merely because two workers chose the same semantic identity.
|
|
@@ -227,43 +459,7 @@ def merge_saved_results(
|
|
|
227
459
|
parent_manifest: dict[str, Any] | None = None
|
|
228
460
|
if frozen_source_digests is None or allow_frozen_legacy_parent:
|
|
229
461
|
try:
|
|
230
|
-
parent_manifest =
|
|
231
|
-
scan_dir, "scan-manifest.json", "Saved parent manifest"
|
|
232
|
-
)
|
|
233
|
-
parent_findings = _read_scan_local_json(
|
|
234
|
-
scan_dir, "findings.json", "Saved parent findings"
|
|
235
|
-
)
|
|
236
|
-
parent_coverage = _read_scan_local_json(
|
|
237
|
-
scan_dir, "coverage.json", "Saved parent coverage"
|
|
238
|
-
)
|
|
239
|
-
parent_scan = parent_manifest.get("scan")
|
|
240
|
-
if not isinstance(parent_scan, dict):
|
|
241
|
-
raise ContractError("Saved parent manifest has no scan object")
|
|
242
|
-
if (parent_scan.get("sealedAt") or parent_scan.get("artifacts")) and (
|
|
243
|
-
parent_scan.get("id", scan_id) != scan_id
|
|
244
|
-
or parent_findings.get("scanId", scan_id) != scan_id
|
|
245
|
-
or parent_coverage.get("scanId", scan_id) != scan_id
|
|
246
|
-
):
|
|
247
|
-
raise ContractError("Saved parent documents belong to a different scan")
|
|
248
|
-
parent = {
|
|
249
|
-
"scanId": scan_id,
|
|
250
|
-
"findings": parent_findings.get("findings"),
|
|
251
|
-
"coverage": parent_coverage,
|
|
252
|
-
**{
|
|
253
|
-
key: parent_scan[key]
|
|
254
|
-
for key in ("scope", "threatModel", "complete")
|
|
255
|
-
if key in parent_scan
|
|
256
|
-
},
|
|
257
|
-
}
|
|
258
|
-
if not isinstance(parent["findings"], list):
|
|
259
|
-
raise ContractError("Saved parent draft has no findings array")
|
|
260
|
-
if not parent_scan.get("sealedAt"):
|
|
261
|
-
payload = _encoded(parent)
|
|
262
|
-
write_scan_local_bytes(
|
|
263
|
-
scan_dir,
|
|
264
|
-
f"checkpoints/{hashlib.sha256(payload).hexdigest()}.json",
|
|
265
|
-
payload,
|
|
266
|
-
)
|
|
462
|
+
parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id)
|
|
267
463
|
except (ContractError, OSError, ValueError) as exc:
|
|
268
464
|
if not stopped:
|
|
269
465
|
raise
|
|
@@ -271,6 +467,18 @@ def merge_saved_results(
|
|
|
271
467
|
warnings.append(f"Could not read the saved parent draft: {exc}")
|
|
272
468
|
parent_manifest = None
|
|
273
469
|
parent = None
|
|
470
|
+
if parent_manifest is not None and parent is not None:
|
|
471
|
+
parent_scan = parent_manifest["scan"]
|
|
472
|
+
if not parent_scan.get("sealedAt") or allow_frozen_legacy_parent:
|
|
473
|
+
payload = _encoded(parent)
|
|
474
|
+
parent_digest = hashlib.sha256(payload).hexdigest()
|
|
475
|
+
parent_checkpoint = f"checkpoints/{parent_digest}.json"
|
|
476
|
+
write_scan_local_bytes(scan_dir, parent_checkpoint, payload)
|
|
477
|
+
if frozen_source_digests is not None:
|
|
478
|
+
frozen_source_digests = {
|
|
479
|
+
**frozen_source_digests,
|
|
480
|
+
parent_checkpoint: parent_digest,
|
|
481
|
+
}
|
|
274
482
|
|
|
275
483
|
sources: list[tuple[str, dict[str, Any], str | None]] = []
|
|
276
484
|
parent_preserved_sources: dict[str, str] = {}
|
|
@@ -283,18 +491,9 @@ def merge_saved_results(
|
|
|
283
491
|
paths: dict[str, str | None] = {}
|
|
284
492
|
current_results: set[str] = set()
|
|
285
493
|
reducer_outputs: list[tuple[Any, str, list[str], int]] = []
|
|
286
|
-
|
|
287
|
-
worker
|
|
288
|
-
for worker in workers
|
|
289
|
-
if worker["kind"] == "dedup"
|
|
290
|
-
and worker["status"] == "succeeded"
|
|
291
|
-
and worker["result_manifest_path"]
|
|
292
|
-
]
|
|
494
|
+
reducer = _latest_successful_reducer(workers)
|
|
293
495
|
latest_reducer: str | None = None
|
|
294
|
-
if
|
|
295
|
-
reducer = max(
|
|
296
|
-
accepted_reducers, key=lambda worker: (worker["completed_at"] or "", worker["id"])
|
|
297
|
-
)
|
|
496
|
+
if reducer is not None:
|
|
298
497
|
try:
|
|
299
498
|
latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
300
499
|
paths[latest_reducer] = None
|
|
@@ -371,14 +570,7 @@ def merge_saved_results(
|
|
|
371
570
|
|
|
372
571
|
for relative, worker_id in paths.items():
|
|
373
572
|
try:
|
|
374
|
-
draft =
|
|
375
|
-
if draft.get("scanId") != scan_id:
|
|
376
|
-
raise ContractError("checkpoint belongs to a different scan")
|
|
377
|
-
if not isinstance(draft.get("findings"), list) or not isinstance(
|
|
378
|
-
draft.get("coverage"), dict
|
|
379
|
-
):
|
|
380
|
-
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
381
|
-
digest = _digest(draft)
|
|
573
|
+
draft, digest = _read_saved_result(scan_dir, relative, scan_id)
|
|
382
574
|
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
383
575
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
384
576
|
source_digests[relative] = digest
|
|
@@ -393,7 +585,7 @@ def merge_saved_results(
|
|
|
393
585
|
drafts_by_path = {relative: draft for relative, draft, _ in sources}
|
|
394
586
|
latest_reducer_key = (
|
|
395
587
|
(reducer["completed_at"] or "", reducer["id"], int(reducer["attempt"] or 0))
|
|
396
|
-
if
|
|
588
|
+
if reducer is not None and latest_reducer in drafts_by_path
|
|
397
589
|
else None
|
|
398
590
|
)
|
|
399
591
|
if latest_reducer_key is None:
|
|
@@ -592,8 +784,10 @@ def merge_saved_results(
|
|
|
592
784
|
and coverage.get("completeness") in {"complete", "unknown"}
|
|
593
785
|
):
|
|
594
786
|
coverage["completeness"] = "partial"
|
|
595
|
-
if
|
|
596
|
-
|
|
787
|
+
if (
|
|
788
|
+
superseded
|
|
789
|
+
and not stopped
|
|
790
|
+
and all(valid_finding(finding) for finding in (parent["findings"] if parent else []))
|
|
597
791
|
):
|
|
598
792
|
continue
|
|
599
793
|
if "threatModel" not in manifest["scan"] and isinstance(draft.get("threatModel"), dict):
|
|
@@ -875,21 +1069,26 @@ def _restore_published_outputs(scan_dir: Path, snapshots: dict[str, bytes | None
|
|
|
875
1069
|
write_scan_local_bytes(scan_dir, relative, contents)
|
|
876
1070
|
|
|
877
1071
|
|
|
878
|
-
def preserve_scan_results_locked(
|
|
1072
|
+
def preserve_scan_results_locked(
|
|
1073
|
+
db: Any,
|
|
1074
|
+
connection: Any,
|
|
1075
|
+
scan_id: str,
|
|
1076
|
+
*,
|
|
1077
|
+
recovery_source_digests: dict[str, str] | None = None,
|
|
1078
|
+
include_parent_with_recovery: bool = False,
|
|
1079
|
+
) -> bool:
|
|
879
1080
|
"""Publish or verify retained terminal results through the workbench host."""
|
|
880
1081
|
scan = db.require_scan(connection, scan_id)
|
|
881
1082
|
if scan["status"] != "failed":
|
|
882
1083
|
return False
|
|
883
1084
|
frozen_source_digests: dict[str, str] | None = None
|
|
884
1085
|
raw_frozen_sources = scan["retained_source_digests_json"]
|
|
885
|
-
if
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
)
|
|
891
|
-
raise ContractError("Saved stopped-scan source digests are malformed.")
|
|
892
|
-
frozen_source_digests = parsed_frozen_sources
|
|
1086
|
+
if recovery_source_digests is not None:
|
|
1087
|
+
frozen_source_digests = recovery_source_digests
|
|
1088
|
+
elif raw_frozen_sources is not None:
|
|
1089
|
+
frozen_source_digests = _source_digests(
|
|
1090
|
+
json.loads(raw_frozen_sources), "Saved stopped-scan"
|
|
1091
|
+
)
|
|
893
1092
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
894
1093
|
deep_run = connection.execute(
|
|
895
1094
|
"SELECT status FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
@@ -942,8 +1141,8 @@ def preserve_scan_results_locked(db: Any, connection: Any, scan_id: str) -> bool
|
|
|
942
1141
|
)
|
|
943
1142
|
db.index_findings(connection, scan_id, findings, scan["completed_at"])
|
|
944
1143
|
connection.execute(
|
|
945
|
-
"UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = "
|
|
946
|
-
"
|
|
1144
|
+
"UPDATE scans SET seal_manifest_digest = ?, retained_source_digests_json = ?, "
|
|
1145
|
+
"completion_warnings_json = ?, "
|
|
947
1146
|
"updated_at = ? WHERE id = ? AND status = 'failed'",
|
|
948
1147
|
(
|
|
949
1148
|
digest,
|
|
@@ -990,7 +1189,8 @@ def preserve_scan_results_locked(db: Any, connection: Any, scan_id: str) -> bool
|
|
|
990
1189
|
return True
|
|
991
1190
|
record_publication(existing, existing_findings)
|
|
992
1191
|
return True
|
|
993
|
-
|
|
1192
|
+
if recovery_source_digests is None:
|
|
1193
|
+
raise ContractError("Stopped scan sources changed after terminal publication.")
|
|
994
1194
|
binding = {**db.workbench_completion_binding(scan, scan["completed_at"]), "status": outcome}
|
|
995
1195
|
documents = merge_saved_results(
|
|
996
1196
|
scan_dir,
|
|
@@ -1008,10 +1208,14 @@ def preserve_scan_results_locked(db: Any, connection: Any, scan_id: str) -> bool
|
|
|
1008
1208
|
).strip(),
|
|
1009
1209
|
frozen_source_digests=frozen_source_digests,
|
|
1010
1210
|
allow_frozen_legacy_parent=(
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1211
|
+
include_parent_with_recovery
|
|
1212
|
+
or (
|
|
1213
|
+
recovery_source_digests is None
|
|
1214
|
+
and frozen_source_digests == {}
|
|
1215
|
+
and isinstance(existing_scan, dict)
|
|
1216
|
+
and existing_scan.get("sealedAt") is not None
|
|
1217
|
+
and "preservedSources" not in existing_scan
|
|
1218
|
+
)
|
|
1015
1219
|
),
|
|
1016
1220
|
)
|
|
1017
1221
|
if documents is None:
|
|
@@ -1056,19 +1260,25 @@ def preserve_scan_results_locked(db: Any, connection: Any, scan_id: str) -> bool
|
|
|
1056
1260
|
return True
|
|
1057
1261
|
|
|
1058
1262
|
|
|
1059
|
-
def
|
|
1060
|
-
|
|
1061
|
-
) -> None:
|
|
1062
|
-
scan = db.require_scan(connection, scan_id)
|
|
1063
|
-
if scan["status"] != "failed":
|
|
1064
|
-
return
|
|
1065
|
-
scan_id = scan["id"]
|
|
1263
|
+
def recover_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
1264
|
+
scan_id = db.require_uuid(args.scan_id, "scan-id")
|
|
1066
1265
|
with db.scan_completion_lock(scan_id):
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1266
|
+
scan = db.require_scan(connection, scan_id)
|
|
1267
|
+
if scan["status"] != "failed":
|
|
1268
|
+
raise SystemExit("Only a stopped scan can recover terminal results.")
|
|
1269
|
+
if scan["canceled_at"] is not None:
|
|
1270
|
+
raise SystemExit("Canceled scans cannot recover terminal results.")
|
|
1271
|
+
recovery_source_digests, include_parent = _recovery_source_digests(db, connection, scan)
|
|
1272
|
+
if not preserve_scan_results_locked(
|
|
1273
|
+
db,
|
|
1274
|
+
connection,
|
|
1275
|
+
scan_id,
|
|
1276
|
+
recovery_source_digests=recovery_source_digests,
|
|
1277
|
+
include_parent_with_recovery=include_parent,
|
|
1278
|
+
):
|
|
1279
|
+
raise SystemExit("No saved stopped-scan results were available to recover.")
|
|
1280
|
+
db.deep_scan.clear_deep_scan_publication_failure(connection, scan_id)
|
|
1281
|
+
return db.scan_context(connection, scan_id)
|
|
1072
1282
|
|
|
1073
1283
|
|
|
1074
1284
|
def preserve_scan_results(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
@@ -1291,9 +1501,7 @@ def cancel_scan_locked(db: Any, connection: Any, args: Any) -> dict[str, Any]:
|
|
|
1291
1501
|
return db.workspace_state(connection, scan["workspace_id"])
|
|
1292
1502
|
|
|
1293
1503
|
|
|
1294
|
-
def preserve_stopped_results_after_transition(
|
|
1295
|
-
db: Any, connection: Any, scan_id: str
|
|
1296
|
-
) -> None:
|
|
1504
|
+
def preserve_stopped_results_after_transition(db: Any, connection: Any, scan_id: str) -> None:
|
|
1297
1505
|
try:
|
|
1298
1506
|
published = preserve_scan_results_locked(db, connection, scan_id)
|
|
1299
1507
|
except (ContractError, OSError, SystemExit, ValueError) as exc:
|
|
@@ -284,8 +284,7 @@ def list_unmatched_scan_pairs(
|
|
|
284
284
|
if scan["id"] not in matching_findings:
|
|
285
285
|
backfill_finding_details(connection, scan)
|
|
286
286
|
matching_findings[scan["id"]] = [
|
|
287
|
-
_matching_input(row)
|
|
288
|
-
for row in _scan_findings(connection, scan["id"]).values()
|
|
287
|
+
_matching_input(row) for row in _scan_findings(connection, scan["id"]).values()
|
|
289
288
|
]
|
|
290
289
|
batches.append(
|
|
291
290
|
{
|
|
@@ -215,18 +215,10 @@ def _codex_state_database() -> Path | None:
|
|
|
215
215
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
216
216
|
|
|
217
217
|
configured_home = os.environ.get("CODEX_HOME", "").strip()
|
|
218
|
-
codex_home = (
|
|
219
|
-
Path(configured_home).expanduser()
|
|
220
|
-
if configured_home
|
|
221
|
-
else Path.home() / ".codex"
|
|
222
|
-
)
|
|
218
|
+
codex_home = Path(configured_home).expanduser() if configured_home else Path.home() / ".codex"
|
|
223
219
|
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip()
|
|
224
220
|
search_roots = [
|
|
225
|
-
*(
|
|
226
|
-
[Path(configured_sqlite_home).expanduser()]
|
|
227
|
-
if configured_sqlite_home
|
|
228
|
-
else []
|
|
229
|
-
),
|
|
221
|
+
*([Path(configured_sqlite_home).expanduser()] if configured_sqlite_home else []),
|
|
230
222
|
codex_home,
|
|
231
223
|
codex_home / "sqlite",
|
|
232
224
|
]
|
|
@@ -363,9 +355,7 @@ def _require_state_columns(
|
|
|
363
355
|
}
|
|
364
356
|
columns = {str(row["name"]) for row in connection.execute(statements[table])}
|
|
365
357
|
if not required.issubset(columns):
|
|
366
|
-
raise ValueError(
|
|
367
|
-
"Codex state graph does not expose the required thread columns."
|
|
368
|
-
)
|
|
358
|
+
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
369
359
|
|
|
370
360
|
|
|
371
361
|
def _rollout_path(value: object) -> Path | None:
|
|
@@ -411,9 +401,7 @@ def _read_rollout_usage(
|
|
|
411
401
|
event = json.loads(raw_line)
|
|
412
402
|
except (UnicodeError, ValueError):
|
|
413
403
|
if line_number == 1:
|
|
414
|
-
raise ValueError(
|
|
415
|
-
"The rollout session metadata is unreadable."
|
|
416
|
-
) from None
|
|
404
|
+
raise ValueError("The rollout session metadata is unreadable.") from None
|
|
417
405
|
if boundary_reached:
|
|
418
406
|
warnings.add("rollout_record_invalid")
|
|
419
407
|
continue
|
|
@@ -456,10 +444,7 @@ def _read_rollout_usage(
|
|
|
456
444
|
warnings.add("thread_outside_scan_window")
|
|
457
445
|
return total, warnings
|
|
458
446
|
boundary_reached = True
|
|
459
|
-
elif (
|
|
460
|
-
event.get("type") == "event_msg"
|
|
461
|
-
and payload.get("type") == "token_count"
|
|
462
|
-
):
|
|
447
|
+
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
463
448
|
inherited_usage = _token_snapshot(payload)
|
|
464
449
|
if inherited_usage is not None:
|
|
465
450
|
previous = inherited_usage
|
|
@@ -538,9 +523,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
538
523
|
usage = info.get("total_token_usage")
|
|
539
524
|
if not isinstance(usage, dict):
|
|
540
525
|
return None
|
|
541
|
-
cache_write = usage.get(
|
|
542
|
-
"cache_write_input_tokens", usage.get("cache_write_tokens", 0)
|
|
543
|
-
)
|
|
526
|
+
cache_write = usage.get("cache_write_input_tokens", usage.get("cache_write_tokens", 0))
|
|
544
527
|
legacy_cache_write = usage.get("cache_write_tokens")
|
|
545
528
|
input_tokens = usage.get("input_tokens")
|
|
546
529
|
cached_input_tokens = usage.get("cached_input_tokens", 0)
|
|
@@ -556,9 +539,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
556
539
|
result: dict[str, int] = {}
|
|
557
540
|
for source_key, result_key in TOKEN_FIELDS.items():
|
|
558
541
|
value = (
|
|
559
|
-
cache_write
|
|
560
|
-
if source_key == "cache_write_input_tokens"
|
|
561
|
-
else usage.get(source_key, 0)
|
|
542
|
+
cache_write if source_key == "cache_write_input_tokens" else usage.get(source_key, 0)
|
|
562
543
|
)
|
|
563
544
|
if type(value) is not int or value < 0:
|
|
564
545
|
return None
|
|
@@ -567,10 +548,7 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
567
548
|
):
|
|
568
549
|
return None
|
|
569
550
|
result[result_key] = value
|
|
570
|
-
if
|
|
571
|
-
result["cachedInputTokens"] + result["cacheWriteInputTokens"]
|
|
572
|
-
> result["inputTokens"]
|
|
573
|
-
):
|
|
551
|
+
if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
|
|
574
552
|
return None
|
|
575
553
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
576
554
|
return result
|
|
@@ -599,9 +577,7 @@ def _timestamp(value: object) -> datetime | None:
|
|
|
599
577
|
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
600
578
|
|
|
601
579
|
|
|
602
|
-
def _unavailable_usage(
|
|
603
|
-
reason: str, *, warnings: set[str] | None = None
|
|
604
|
-
) -> dict[str, Any]:
|
|
580
|
+
def _unavailable_usage(reason: str, *, warnings: set[str] | None = None) -> dict[str, Any]:
|
|
605
581
|
return {
|
|
606
582
|
"coverage": "unavailable",
|
|
607
583
|
"source": "codex_rollout",
|