@openai/codex-security 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -41
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +5 -1
- package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
- package/_bundled_plugin/references/config-preflight.md +11 -13
- package/_bundled_plugin/references/core-scan.md +94 -0
- package/_bundled_plugin/references/final-report.md +6 -6
- package/_bundled_plugin/references/scan-artifacts.md +15 -14
- package/_bundled_plugin/references/scan-prologue.md +27 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
- package/_bundled_plugin/scripts/config_preflight.py +40 -3
- package/_bundled_plugin/scripts/deep_scan_config.py +29 -6
- package/_bundled_plugin/scripts/deep_scan_workbench.py +132 -34
- package/_bundled_plugin/scripts/finalize_scan_contract.py +44 -224
- package/_bundled_plugin/scripts/finding_preview.py +122 -6
- package/_bundled_plugin/scripts/generate_in_scope_files.py +77 -31
- package/_bundled_plugin/scripts/generate_rank_input.py +58 -7
- package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
- package/_bundled_plugin/scripts/rank_preview.py +4 -0
- package/_bundled_plugin/scripts/report_projection.py +40 -20
- package/_bundled_plugin/scripts/workbench_cli.py +5 -0
- package/_bundled_plugin/scripts/workbench_db.py +304 -32
- package/_bundled_plugin/scripts/workbench_progress.py +0 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +43 -8
- package/_bundled_plugin/scripts/workbench_schema.py +16 -20
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +1 -5
- package/_bundled_plugin/scripts/workbench_target.py +82 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -108
- package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
- package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
- package/_bundled_plugin/skills/validation/SKILL.md +9 -12
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
- package/dist/api.d.ts +2 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +393 -145
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts +2 -3
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +59 -270
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.js +14 -2
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +6 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +202 -46
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/contract.js +7 -5
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +33 -2
- package/dist/cost.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +12 -4
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +77 -17
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts +9 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +140 -326
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.js +1 -1
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +65 -16
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-logs.d.ts +2 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +97 -37
- package/dist/scan-logs.js.map +1 -1
- package/dist/targets.d.ts +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +35 -4
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.js +1 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +3 -3
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +6 -4
- package/dist/version.js.map +1 -1
- package/package.json +4 -4
- package/_bundled_plugin/references/shared-hard-rules.md +0 -15
- package/_bundled_plugin/scripts/validate_report_format.py +0 -138
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
|
@@ -97,7 +97,7 @@ from workbench_schema import (
|
|
|
97
97
|
from workbench_schema import (
|
|
98
98
|
sql_statements as sql_statements,
|
|
99
99
|
)
|
|
100
|
-
from workbench_source_excerpt import finding_source_excerpt
|
|
100
|
+
from workbench_source_excerpt import finding_source_excerpt, safe_source_path
|
|
101
101
|
from workbench_target import (
|
|
102
102
|
clean_worktree_content_digest,
|
|
103
103
|
copy_directory_excluding,
|
|
@@ -1136,6 +1136,289 @@ def complete_scan(
|
|
|
1136
1136
|
)
|
|
1137
1137
|
|
|
1138
1138
|
|
|
1139
|
+
def complete_budget_exhausted_scan(
|
|
1140
|
+
connection: sqlite3.Connection, args: argparse.Namespace
|
|
1141
|
+
) -> dict[str, Any]:
|
|
1142
|
+
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1143
|
+
cost_json = parse_scan_cost(args.cost_json)
|
|
1144
|
+
if cost_json is None:
|
|
1145
|
+
raise SystemExit("Budget-exhausted scan completion requires the measured scan cost.")
|
|
1146
|
+
with scan_completion_lock(scan_id):
|
|
1147
|
+
scan = require_scan(connection, scan_id)
|
|
1148
|
+
if scan["status"] != "running" or scan["mode"] != "deep" or scan["recipe_json"] is None:
|
|
1149
|
+
raise SystemExit("Only a running CLI Deep Scan can complete after its cost limit.")
|
|
1150
|
+
recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
|
|
1151
|
+
if not isinstance(recipe, dict) or recipe.get("mode") != "deep":
|
|
1152
|
+
raise SystemExit("Budget-exhausted scan completion requires a Deep Scan launch recipe.")
|
|
1153
|
+
cost = json.loads(cost_json)
|
|
1154
|
+
measured = cost.get("cost", cost)
|
|
1155
|
+
limit = recipe.get("maxCostUsd")
|
|
1156
|
+
if (
|
|
1157
|
+
not isinstance(limit, (int, float))
|
|
1158
|
+
or isinstance(limit, bool)
|
|
1159
|
+
or not isinstance(measured, dict)
|
|
1160
|
+
or measured.get("estimatedUsd", 0) <= limit
|
|
1161
|
+
):
|
|
1162
|
+
raise SystemExit("Deep Scan has not exceeded its configured cost limit.")
|
|
1163
|
+
run = connection.execute(
|
|
1164
|
+
"SELECT status, terminal_reason, manifest_path FROM deep_scan_runs WHERE scan_id = ?",
|
|
1165
|
+
(scan_id,),
|
|
1166
|
+
).fetchone()
|
|
1167
|
+
if (
|
|
1168
|
+
run is None
|
|
1169
|
+
or run["status"] != "succeeded"
|
|
1170
|
+
or run["terminal_reason"] not in {"saturated", "capped"}
|
|
1171
|
+
or not run["manifest_path"]
|
|
1172
|
+
):
|
|
1173
|
+
raise SystemExit(
|
|
1174
|
+
"Budget-exhausted scan completion requires successfully completed Deep Scan "
|
|
1175
|
+
"discovery."
|
|
1176
|
+
)
|
|
1177
|
+
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1178
|
+
candidates = (
|
|
1179
|
+
[]
|
|
1180
|
+
if run["manifest_path"] == str(scan_dir / "scan-manifest.json")
|
|
1181
|
+
else budget_exhausted_candidates(scan, scan_dir)
|
|
1182
|
+
)
|
|
1183
|
+
warning = optional_text(args.message, maximum=2400)
|
|
1184
|
+
if warning is None:
|
|
1185
|
+
warning = (
|
|
1186
|
+
f"Deep Scan reached its cost limit after an estimated "
|
|
1187
|
+
f"${measured['estimatedUsd']:.6g}; completed discovery was preserved."
|
|
1188
|
+
)
|
|
1189
|
+
budget_exhausted_draft(scan, scan_dir, candidates, warning)
|
|
1190
|
+
warnings = json.loads(scan["completion_warnings_json"])
|
|
1191
|
+
if warning not in warnings:
|
|
1192
|
+
connection.execute(
|
|
1193
|
+
"UPDATE scans SET completion_warnings_json = ? WHERE id = ? AND status = 'running'",
|
|
1194
|
+
(json.dumps([*warnings, warning]), scan_id),
|
|
1195
|
+
)
|
|
1196
|
+
connection.commit()
|
|
1197
|
+
return complete_scan_locked(connection, scan_id, None, cost_json)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def budget_exhausted_candidates(scan: sqlite3.Row, scan_dir: Path) -> list[dict[str, Any]]:
|
|
1201
|
+
artifacts = deep_scan.canonical_discovery_artifacts(scan)
|
|
1202
|
+
ledger = Path(artifacts["candidateLedgerPath"])
|
|
1203
|
+
try:
|
|
1204
|
+
inventory = Path(artifacts["inScopeFilesPath"])
|
|
1205
|
+
inventory_descriptor = open_scan_local_file_descriptor(
|
|
1206
|
+
scan_dir,
|
|
1207
|
+
inventory.relative_to(scan_dir).as_posix(),
|
|
1208
|
+
"Canonical Deep Scan in-scope inventory",
|
|
1209
|
+
)
|
|
1210
|
+
with os.fdopen(inventory_descriptor, "rb") as source:
|
|
1211
|
+
lines = re.split(r"\r?\n", source.read().decode("utf-8"))
|
|
1212
|
+
in_scope = {re.sub(r"^(?:\./)+", "", line) for line in lines if line}
|
|
1213
|
+
descriptor = open_scan_local_file_descriptor(
|
|
1214
|
+
scan_dir,
|
|
1215
|
+
ledger.relative_to(scan_dir).as_posix(),
|
|
1216
|
+
"Canonical Deep Scan candidate ledger",
|
|
1217
|
+
)
|
|
1218
|
+
with os.fdopen(descriptor, "r", encoding="utf-8") as source:
|
|
1219
|
+
candidates = [
|
|
1220
|
+
json.loads(line, parse_constant=reject_non_finite_json)
|
|
1221
|
+
for line in source
|
|
1222
|
+
if line.strip()
|
|
1223
|
+
]
|
|
1224
|
+
except (ContractError, OSError, UnicodeError, ValueError) as exc:
|
|
1225
|
+
raise SystemExit(f"Canonical Deep Scan candidate ledger is invalid: {exc}") from exc
|
|
1226
|
+
|
|
1227
|
+
candidate_ids: set[str] = set()
|
|
1228
|
+
for candidate in candidates:
|
|
1229
|
+
if not isinstance(candidate, dict):
|
|
1230
|
+
raise SystemExit("Canonical Deep Scan candidate ledger rows must be objects.")
|
|
1231
|
+
candidate_id = candidate.get("candidate_id")
|
|
1232
|
+
locations = candidate.get("locations")
|
|
1233
|
+
if (
|
|
1234
|
+
not isinstance(candidate_id, str)
|
|
1235
|
+
or not candidate_id.strip()
|
|
1236
|
+
or candidate_id in {".", ".."}
|
|
1237
|
+
or "/" in candidate_id
|
|
1238
|
+
or "\\" in candidate_id
|
|
1239
|
+
or candidate_id in candidate_ids
|
|
1240
|
+
or not isinstance(candidate.get("summary"), str)
|
|
1241
|
+
or not candidate["summary"].strip()
|
|
1242
|
+
or not isinstance(candidate.get("evidence"), str)
|
|
1243
|
+
or not candidate["evidence"].strip()
|
|
1244
|
+
or not isinstance(locations, list)
|
|
1245
|
+
or not locations
|
|
1246
|
+
):
|
|
1247
|
+
raise SystemExit("Canonical Deep Scan candidate ledger contains an invalid candidate.")
|
|
1248
|
+
candidate_ids.add(candidate_id)
|
|
1249
|
+
for location in locations:
|
|
1250
|
+
if not isinstance(location, dict):
|
|
1251
|
+
raise SystemExit("Canonical Deep Scan candidate location must be an object.")
|
|
1252
|
+
path = location.get("path")
|
|
1253
|
+
if (
|
|
1254
|
+
not isinstance(path, str)
|
|
1255
|
+
or not path
|
|
1256
|
+
or "\\" in path
|
|
1257
|
+
or "\x00" in path
|
|
1258
|
+
or re.match(r"^[A-Za-z]:", path)
|
|
1259
|
+
or PurePosixPath(path).is_absolute()
|
|
1260
|
+
or any(part in {".", "..", ""} for part in path.split("/"))
|
|
1261
|
+
):
|
|
1262
|
+
raise SystemExit(
|
|
1263
|
+
"Canonical Deep Scan candidate location must be repository-relative."
|
|
1264
|
+
)
|
|
1265
|
+
if not any(location["path"] in in_scope for location in locations):
|
|
1266
|
+
raise SystemExit(
|
|
1267
|
+
"Canonical Deep Scan candidate must include a location in its in-scope inventory."
|
|
1268
|
+
)
|
|
1269
|
+
return candidates
|
|
1270
|
+
|
|
1271
|
+
|
|
1272
|
+
def budget_exhausted_draft(
|
|
1273
|
+
scan: sqlite3.Row,
|
|
1274
|
+
scan_dir: Path,
|
|
1275
|
+
candidates: list[dict[str, Any]],
|
|
1276
|
+
warning: str,
|
|
1277
|
+
) -> None:
|
|
1278
|
+
documents: dict[str, dict[str, Any]] = {}
|
|
1279
|
+
for name in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
1280
|
+
path = artifact_path(scan_dir, name, required=False)
|
|
1281
|
+
if path is not None:
|
|
1282
|
+
documents[name] = read_json_object(path)
|
|
1283
|
+
if documents and len(documents) != 3:
|
|
1284
|
+
raise SystemExit("Budget-exhausted scan contains an incomplete canonical scan draft.")
|
|
1285
|
+
|
|
1286
|
+
if documents:
|
|
1287
|
+
manifest = documents["scan-manifest.json"]
|
|
1288
|
+
findings = documents["findings.json"]
|
|
1289
|
+
coverage = documents["coverage.json"]
|
|
1290
|
+
if not isinstance(manifest.get("scan"), dict) or not isinstance(
|
|
1291
|
+
findings.get("findings"), list
|
|
1292
|
+
):
|
|
1293
|
+
raise SystemExit("Budget-exhausted scan contains an invalid canonical scan draft.")
|
|
1294
|
+
for key in ("surfaces", "explicitExclusions", "deferred"):
|
|
1295
|
+
if not isinstance(coverage.get(key), list):
|
|
1296
|
+
raise SystemExit("Budget-exhausted scan contains invalid canonical coverage.")
|
|
1297
|
+
if manifest["scan"].get("sealedAt") is not None or manifest["scan"].get("artifacts"):
|
|
1298
|
+
raise SystemExit("Budget-exhausted scan cannot replace an already sealed scan draft.")
|
|
1299
|
+
else:
|
|
1300
|
+
contract = scan_contract(scan)
|
|
1301
|
+
target_contract = contract["target"]
|
|
1302
|
+
target: dict[str, Any] = {
|
|
1303
|
+
"kind": target_contract["allowedKinds"][0],
|
|
1304
|
+
"targetId": target_contract["targetId"],
|
|
1305
|
+
"displayName": target_contract["displayName"],
|
|
1306
|
+
}
|
|
1307
|
+
if scan["target_revision"] != "unversioned":
|
|
1308
|
+
target["revision"] = scan["target_revision"]
|
|
1309
|
+
if "requiredSnapshotDigest" in target_contract:
|
|
1310
|
+
target["snapshotDigest"] = target_contract["requiredSnapshotDigest"]
|
|
1311
|
+
manifest = {
|
|
1312
|
+
"scan": {
|
|
1313
|
+
"target": target,
|
|
1314
|
+
"scope": {"limitations": [warning], "validationMode": "incomplete"},
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
findings = {"findings": []}
|
|
1318
|
+
coverage = {
|
|
1319
|
+
"completeness": "partial",
|
|
1320
|
+
"inventoryStrategy": (
|
|
1321
|
+
"scoped_path" if expected_coverage_mode(scan) == "scoped_path" else "repository"
|
|
1322
|
+
),
|
|
1323
|
+
"surfaces": [],
|
|
1324
|
+
"explicitExclusions": [],
|
|
1325
|
+
"deferred": [],
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
findings_by_candidate = {
|
|
1329
|
+
candidate_id
|
|
1330
|
+
for finding in findings["findings"]
|
|
1331
|
+
if isinstance(finding, dict)
|
|
1332
|
+
and isinstance(candidate_id := finding_candidate_id(finding), str)
|
|
1333
|
+
}
|
|
1334
|
+
existing_deferred = {
|
|
1335
|
+
item.get("candidateId", item.get("id"))
|
|
1336
|
+
for item in coverage["deferred"]
|
|
1337
|
+
if isinstance(item, dict)
|
|
1338
|
+
and isinstance(item.get("candidateId", item.get("id")), str)
|
|
1339
|
+
}
|
|
1340
|
+
existing_surfaces = {
|
|
1341
|
+
item.get("id")
|
|
1342
|
+
for item in coverage["surfaces"]
|
|
1343
|
+
if isinstance(item, dict) and isinstance(item.get("id"), str)
|
|
1344
|
+
}
|
|
1345
|
+
for candidate in candidates:
|
|
1346
|
+
candidate_id = candidate["candidate_id"]
|
|
1347
|
+
if candidate_id in findings_by_candidate or candidate_id in existing_deferred:
|
|
1348
|
+
continue
|
|
1349
|
+
paths = list(dict.fromkeys(location["path"] for location in candidate["locations"]))
|
|
1350
|
+
surface_id = f"candidate-{candidate_id}"
|
|
1351
|
+
validation = candidate.get("validation")
|
|
1352
|
+
validation = validation.get("disposition") if isinstance(validation, dict) else None
|
|
1353
|
+
attack = candidate.get("attack_path")
|
|
1354
|
+
attack = attack.get("decision") if isinstance(attack, dict) else None
|
|
1355
|
+
disposition = (
|
|
1356
|
+
"needs_follow_up"
|
|
1357
|
+
if validation == "deferred" or attack == "deferred"
|
|
1358
|
+
else "not_applicable"
|
|
1359
|
+
if validation == "not_applicable"
|
|
1360
|
+
else "rejected"
|
|
1361
|
+
if validation == "suppressed" or attack == "ignore"
|
|
1362
|
+
else "needs_follow_up"
|
|
1363
|
+
)
|
|
1364
|
+
if surface_id not in existing_surfaces:
|
|
1365
|
+
coverage["surfaces"].append(
|
|
1366
|
+
{
|
|
1367
|
+
"id": surface_id,
|
|
1368
|
+
"label": candidate["summary"],
|
|
1369
|
+
"disposition": disposition,
|
|
1370
|
+
"notes": candidate["evidence"],
|
|
1371
|
+
"receiptRefs": [],
|
|
1372
|
+
}
|
|
1373
|
+
)
|
|
1374
|
+
existing_surfaces.add(surface_id)
|
|
1375
|
+
if disposition != "needs_follow_up":
|
|
1376
|
+
continue
|
|
1377
|
+
coverage["deferred"].append(
|
|
1378
|
+
{
|
|
1379
|
+
"id": candidate_id,
|
|
1380
|
+
"candidateId": candidate_id,
|
|
1381
|
+
"reason": (
|
|
1382
|
+
"Validation was deferred because the scan reached its cost limit: "
|
|
1383
|
+
f"{candidate['summary']}. Evidence: {candidate['evidence']}"
|
|
1384
|
+
),
|
|
1385
|
+
"paths": paths,
|
|
1386
|
+
"surfaceIds": [surface_id],
|
|
1387
|
+
}
|
|
1388
|
+
)
|
|
1389
|
+
if not any(
|
|
1390
|
+
isinstance(item, dict)
|
|
1391
|
+
and isinstance(reason := item.get("reason"), str)
|
|
1392
|
+
and (
|
|
1393
|
+
reason == "Validation was deferred because the scan reached its cost limit."
|
|
1394
|
+
or reason.startswith(
|
|
1395
|
+
"Validation was deferred because the scan reached its cost limit: "
|
|
1396
|
+
)
|
|
1397
|
+
)
|
|
1398
|
+
for item in coverage["deferred"]
|
|
1399
|
+
):
|
|
1400
|
+
coverage["deferred"].append(
|
|
1401
|
+
{
|
|
1402
|
+
"id": "scan-cost-limit",
|
|
1403
|
+
"reason": "Validation was deferred because the scan reached its cost limit.",
|
|
1404
|
+
}
|
|
1405
|
+
)
|
|
1406
|
+
coverage["completeness"] = "partial"
|
|
1407
|
+
for name, payload in (
|
|
1408
|
+
("findings.json", findings),
|
|
1409
|
+
("coverage.json", coverage),
|
|
1410
|
+
("scan-manifest.json", manifest),
|
|
1411
|
+
):
|
|
1412
|
+
try:
|
|
1413
|
+
write_scan_local_bytes(
|
|
1414
|
+
scan_dir,
|
|
1415
|
+
name,
|
|
1416
|
+
(json.dumps(payload, allow_nan=False, indent=2, sort_keys=True) + "\n").encode(),
|
|
1417
|
+
)
|
|
1418
|
+
except (ContractError, OSError, TypeError, ValueError) as exc:
|
|
1419
|
+
raise SystemExit(f"Budget-exhausted scan draft could not be saved: {exc}") from exc
|
|
1420
|
+
|
|
1421
|
+
|
|
1139
1422
|
def complete_scan_locked(
|
|
1140
1423
|
connection: sqlite3.Connection,
|
|
1141
1424
|
scan_id: str,
|
|
@@ -1168,15 +1451,17 @@ def complete_scan_locked(
|
|
|
1168
1451
|
claim_token,
|
|
1169
1452
|
error_message="Scan completion is owned by another continuation.",
|
|
1170
1453
|
)
|
|
1171
|
-
|
|
1172
|
-
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1454
|
+
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1173
1455
|
warnings = json.loads(scan["completion_warnings_json"])
|
|
1174
1456
|
target_warnings: list[str] = []
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1457
|
+
|
|
1458
|
+
def add_warning() -> None:
|
|
1459
|
+
if (warning := scan_target_warning(scan)) is not None:
|
|
1460
|
+
for items in (target_warnings, warnings):
|
|
1461
|
+
if warning not in items:
|
|
1462
|
+
items.append(warning)
|
|
1463
|
+
|
|
1464
|
+
add_warning()
|
|
1180
1465
|
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1181
1466
|
completion_timestamp = now()
|
|
1182
1467
|
completion_binding = workbench_completion_binding(scan, completion_timestamp)
|
|
@@ -1214,12 +1499,7 @@ def complete_scan_locked(
|
|
|
1214
1499
|
completion_binding=completion_binding,
|
|
1215
1500
|
completion_warnings=warnings,
|
|
1216
1501
|
)
|
|
1217
|
-
|
|
1218
|
-
if warning is not None:
|
|
1219
|
-
if warning not in target_warnings:
|
|
1220
|
-
target_warnings.append(warning)
|
|
1221
|
-
if warning not in warnings:
|
|
1222
|
-
warnings.append(warning)
|
|
1502
|
+
add_warning()
|
|
1223
1503
|
manifest, findings, _ = _write_prepared_scan_finalization(prepared)
|
|
1224
1504
|
except ContractError as exc:
|
|
1225
1505
|
raise SystemExit(str(exc)) from exc
|
|
@@ -1262,8 +1542,7 @@ def complete_scan_locked(
|
|
|
1262
1542
|
return scan_context(connection, scan["id"])
|
|
1263
1543
|
if scan["status"] != "running":
|
|
1264
1544
|
raise SystemExit("Only a running scan can be completed.")
|
|
1265
|
-
|
|
1266
|
-
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1545
|
+
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1267
1546
|
handoff.require_current_continuation(
|
|
1268
1547
|
scan,
|
|
1269
1548
|
claim_token,
|
|
@@ -2628,10 +2907,15 @@ def scan_context(
|
|
|
2628
2907
|
occurrence_id: str | None = None,
|
|
2629
2908
|
) -> dict[str, Any]:
|
|
2630
2909
|
scan = require_scan(connection, scan_id)
|
|
2910
|
+
workspace = workspace_state(connection, scan["workspace_id"], result_scan_id=scan["id"])
|
|
2631
2911
|
context = {
|
|
2632
2912
|
"otherRunningDeepScans": deep_scan.other_running_deep_scans(connection, scan["id"]),
|
|
2633
|
-
"scan":
|
|
2634
|
-
|
|
2913
|
+
"scan": (
|
|
2914
|
+
workspace["results"]
|
|
2915
|
+
if occurrence_id is None
|
|
2916
|
+
else scan_result(connection, scan, occurrence_id=occurrence_id)
|
|
2917
|
+
),
|
|
2918
|
+
"workspace": workspace,
|
|
2635
2919
|
}
|
|
2636
2920
|
if scan["recipe_json"] is not None:
|
|
2637
2921
|
context["parentScanId"] = scan["parent_scan_id"]
|
|
@@ -3185,20 +3469,6 @@ def patch_artifact_preview(
|
|
|
3185
3469
|
}
|
|
3186
3470
|
|
|
3187
3471
|
|
|
3188
|
-
def safe_source_path(target: Path, relative_path: str) -> Path | None:
|
|
3189
|
-
if "\\" in relative_path:
|
|
3190
|
-
return None
|
|
3191
|
-
parsed = PurePosixPath(relative_path)
|
|
3192
|
-
if parsed.is_absolute() or ".." in parsed.parts:
|
|
3193
|
-
return None
|
|
3194
|
-
try:
|
|
3195
|
-
path = (target / parsed.as_posix()).resolve()
|
|
3196
|
-
path.relative_to(target)
|
|
3197
|
-
except (OSError, RuntimeError, ValueError):
|
|
3198
|
-
return None
|
|
3199
|
-
return path
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
3472
|
def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None:
|
|
3203
3473
|
try:
|
|
3204
3474
|
resolved_scan_dir = require_canonical_scan_directory(scan_dir)
|
|
@@ -3400,6 +3670,8 @@ def main() -> None:
|
|
|
3400
3670
|
result = complete_scan(
|
|
3401
3671
|
connection, args, prepare_only=args.command == "prepare-scan-completion"
|
|
3402
3672
|
)
|
|
3673
|
+
elif args.command == "complete-budget-exhausted-scan":
|
|
3674
|
+
result = complete_budget_exhausted_scan(connection, args)
|
|
3403
3675
|
elif args.command == "cancel-scan":
|
|
3404
3676
|
result = cancel_scan(connection, args)
|
|
3405
3677
|
elif args.command == "fail-scan":
|
|
@@ -13,7 +13,6 @@ from workbench.handoff import require_current_continuation
|
|
|
13
13
|
from workbench_constants import PHASES
|
|
14
14
|
from workbench_validation import optional_text, require_uuid, user_text
|
|
15
15
|
|
|
16
|
-
MAX_PREFLIGHT_ISSUES_JSON_BYTES = 64 * 1024
|
|
17
16
|
MAX_PREFLIGHT_ISSUES = 32
|
|
18
17
|
|
|
19
18
|
|
|
@@ -33,10 +32,6 @@ def _preflight_issue_text(value: Any, maximum: int, label: str) -> str:
|
|
|
33
32
|
def preflight_issues_json(value: str | None) -> str | None:
|
|
34
33
|
if value is None:
|
|
35
34
|
return None
|
|
36
|
-
if len(value.encode("utf-8")) > MAX_PREFLIGHT_ISSUES_JSON_BYTES:
|
|
37
|
-
raise SystemExit(
|
|
38
|
-
f"Preflight issues must be no larger than {MAX_PREFLIGHT_ISSUES_JSON_BYTES} bytes."
|
|
39
|
-
)
|
|
40
35
|
try:
|
|
41
36
|
payload = json.loads(value)
|
|
42
37
|
except json.JSONDecodeError as exc:
|
|
@@ -189,10 +189,18 @@ def _codex_state_database() -> Path | None:
|
|
|
189
189
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
190
190
|
|
|
191
191
|
configured_home = os.environ.get("CODEX_HOME", "").strip()
|
|
192
|
-
codex_home =
|
|
192
|
+
codex_home = (
|
|
193
|
+
Path(configured_home).expanduser()
|
|
194
|
+
if configured_home
|
|
195
|
+
else Path.home() / ".codex"
|
|
196
|
+
)
|
|
193
197
|
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip()
|
|
194
198
|
search_roots = [
|
|
195
|
-
*(
|
|
199
|
+
*(
|
|
200
|
+
[Path(configured_sqlite_home).expanduser()]
|
|
201
|
+
if configured_sqlite_home
|
|
202
|
+
else []
|
|
203
|
+
),
|
|
196
204
|
codex_home,
|
|
197
205
|
codex_home / "sqlite",
|
|
198
206
|
]
|
|
@@ -329,7 +337,9 @@ def _require_state_columns(
|
|
|
329
337
|
}
|
|
330
338
|
columns = {str(row["name"]) for row in connection.execute(statements[table])}
|
|
331
339
|
if not required.issubset(columns):
|
|
332
|
-
raise ValueError(
|
|
340
|
+
raise ValueError(
|
|
341
|
+
"Codex state graph does not expose the required thread columns."
|
|
342
|
+
)
|
|
333
343
|
|
|
334
344
|
|
|
335
345
|
def _rollout_path(value: object) -> Path | None:
|
|
@@ -375,7 +385,9 @@ def _read_rollout_usage(
|
|
|
375
385
|
event = json.loads(raw_line)
|
|
376
386
|
except (UnicodeError, ValueError):
|
|
377
387
|
if line_number == 1:
|
|
378
|
-
raise ValueError(
|
|
388
|
+
raise ValueError(
|
|
389
|
+
"The rollout session metadata is unreadable."
|
|
390
|
+
) from None
|
|
379
391
|
if boundary_reached:
|
|
380
392
|
warnings.add("rollout_record_invalid")
|
|
381
393
|
continue
|
|
@@ -418,7 +430,10 @@ def _read_rollout_usage(
|
|
|
418
430
|
warnings.add("thread_outside_scan_window")
|
|
419
431
|
return total, warnings
|
|
420
432
|
boundary_reached = True
|
|
421
|
-
elif
|
|
433
|
+
elif (
|
|
434
|
+
event.get("type") == "event_msg"
|
|
435
|
+
and payload.get("type") == "token_count"
|
|
436
|
+
):
|
|
422
437
|
inherited_usage = _token_snapshot(payload)
|
|
423
438
|
if inherited_usage is not None:
|
|
424
439
|
previous = inherited_usage
|
|
@@ -497,10 +512,25 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
497
512
|
usage = info.get("total_token_usage")
|
|
498
513
|
if not isinstance(usage, dict):
|
|
499
514
|
return None
|
|
515
|
+
cache_write = usage.get(
|
|
516
|
+
"cache_write_input_tokens", usage.get("cache_write_tokens", 0)
|
|
517
|
+
)
|
|
518
|
+
legacy_cache_write = usage.get("cache_write_tokens")
|
|
519
|
+
input_tokens = usage.get("input_tokens")
|
|
520
|
+
cached_input_tokens = usage.get("cached_input_tokens", 0)
|
|
521
|
+
if (
|
|
522
|
+
cache_write == 0
|
|
523
|
+
and type(legacy_cache_write) is int
|
|
524
|
+
and legacy_cache_write > 0
|
|
525
|
+
and type(input_tokens) is int
|
|
526
|
+
and type(cached_input_tokens) is int
|
|
527
|
+
and cached_input_tokens + legacy_cache_write <= input_tokens
|
|
528
|
+
):
|
|
529
|
+
cache_write = legacy_cache_write
|
|
500
530
|
result: dict[str, int] = {}
|
|
501
531
|
for source_key, result_key in TOKEN_FIELDS.items():
|
|
502
532
|
value = (
|
|
503
|
-
|
|
533
|
+
cache_write
|
|
504
534
|
if source_key == "cache_write_input_tokens"
|
|
505
535
|
else usage.get(source_key, 0)
|
|
506
536
|
)
|
|
@@ -511,7 +541,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
511
541
|
):
|
|
512
542
|
return None
|
|
513
543
|
result[result_key] = value
|
|
514
|
-
if
|
|
544
|
+
if (
|
|
545
|
+
result["cachedInputTokens"] + result["cacheWriteInputTokens"]
|
|
546
|
+
> result["inputTokens"]
|
|
547
|
+
):
|
|
515
548
|
return None
|
|
516
549
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
517
550
|
return result
|
|
@@ -540,7 +573,9 @@ def _timestamp(value: object) -> datetime | None:
|
|
|
540
573
|
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
541
574
|
|
|
542
575
|
|
|
543
|
-
def _unavailable_usage(
|
|
576
|
+
def _unavailable_usage(
|
|
577
|
+
reason: str, *, warnings: set[str] | None = None
|
|
578
|
+
) -> dict[str, Any]:
|
|
544
579
|
return {
|
|
545
580
|
"coverage": "unavailable",
|
|
546
581
|
"source": "codex_rollout",
|
|
@@ -635,6 +635,14 @@ MIGRATIONS = (
|
|
|
635
635
|
SET stop_after_consecutive_errors = stop_after_no_new;
|
|
636
636
|
""",
|
|
637
637
|
),
|
|
638
|
+
(
|
|
639
|
+
28,
|
|
640
|
+
"persist deep scan discovery time limit",
|
|
641
|
+
"""
|
|
642
|
+
ALTER TABLE deep_scan_runs
|
|
643
|
+
ADD COLUMN max_time_hours REAL NOT NULL DEFAULT 96;
|
|
644
|
+
""",
|
|
645
|
+
),
|
|
638
646
|
)
|
|
639
647
|
|
|
640
648
|
|
|
@@ -689,6 +697,13 @@ def apply_migrations(
|
|
|
689
697
|
"completion_warnings_json",
|
|
690
698
|
"TEXT NOT NULL DEFAULT '[]'",
|
|
691
699
|
)
|
|
700
|
+
elif version == 28:
|
|
701
|
+
add_column_if_missing(
|
|
702
|
+
connection,
|
|
703
|
+
"deep_scan_runs",
|
|
704
|
+
"max_time_hours",
|
|
705
|
+
"REAL NOT NULL DEFAULT 96",
|
|
706
|
+
)
|
|
692
707
|
continue
|
|
693
708
|
if version == 6:
|
|
694
709
|
repair_thread_scoped_workspaces_migration(connection)
|
|
@@ -729,7 +744,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
729
744
|
12: "scan continuation threads",
|
|
730
745
|
}
|
|
731
746
|
model_migration_name = "persist scan model settings"
|
|
732
|
-
warnings_migration_name = "persist scan completion warnings"
|
|
733
747
|
if execution_migrations.get(25) == "dynamic scan execution profiles":
|
|
734
748
|
connection.execute(
|
|
735
749
|
"UPDATE schema_migrations SET name = ? WHERE version = 25 AND name = ?",
|
|
@@ -737,9 +751,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
737
751
|
)
|
|
738
752
|
execution_migrations[25] = model_migration_name
|
|
739
753
|
has_legacy_profile_history = execution_migrations.get(11) == legacy_names[11]
|
|
740
|
-
has_public_warnings_history = (
|
|
741
|
-
execution_migrations.get(25) == warnings_migration_name
|
|
742
|
-
)
|
|
743
754
|
|
|
744
755
|
scan_columns = {
|
|
745
756
|
row["name"] for row in connection.execute("PRAGMA table_info(scans)")
|
|
@@ -750,11 +761,7 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
750
761
|
has_legacy_profile_columns = (
|
|
751
762
|
"execution_model" in scan_columns or "execution_model" in workspace_columns
|
|
752
763
|
)
|
|
753
|
-
if not (
|
|
754
|
-
has_legacy_profile_history
|
|
755
|
-
or has_public_warnings_history
|
|
756
|
-
or has_legacy_profile_columns
|
|
757
|
-
):
|
|
764
|
+
if not (has_legacy_profile_history or has_legacy_profile_columns):
|
|
758
765
|
return
|
|
759
766
|
|
|
760
767
|
if (
|
|
@@ -796,17 +803,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
796
803
|
"The Codex Security database has an unsupported execution-profile migration history."
|
|
797
804
|
)
|
|
798
805
|
|
|
799
|
-
if has_public_warnings_history:
|
|
800
|
-
if execution_migrations.get(26) is not None:
|
|
801
|
-
raise SystemExit(
|
|
802
|
-
"The Codex Security database has an unsupported pre-release migration history."
|
|
803
|
-
)
|
|
804
|
-
connection.execute(
|
|
805
|
-
"UPDATE schema_migrations SET version = 26 WHERE version = 25 AND name = ?",
|
|
806
|
-
(warnings_migration_name,),
|
|
807
|
-
)
|
|
808
|
-
execution_migrations.pop(25)
|
|
809
|
-
|
|
810
806
|
if execution_migrations.get(25) not in (None, model_migration_name):
|
|
811
807
|
raise SystemExit(
|
|
812
808
|
"The Codex Security database has an unsupported execution-profile migration history."
|
|
@@ -10,11 +10,10 @@ from typing import Any
|
|
|
10
10
|
|
|
11
11
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
12
12
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
13
|
-
from workbench_target import clean_worktree_content_digest, git_bytes
|
|
13
|
+
from workbench_target import clean_worktree_content_digest, git_bytes
|
|
14
14
|
|
|
15
15
|
CONTEXT_LINES = 3
|
|
16
16
|
MAX_BYTES = 16_000
|
|
17
|
-
MAX_FILE_BYTES = 1_048_576
|
|
18
17
|
MAX_LINES = 60
|
|
19
18
|
|
|
20
19
|
|
|
@@ -70,9 +69,6 @@ def scanned_source_text(scan: sqlite3.Row, target: Path, path: str) -> str | Non
|
|
|
70
69
|
if snapshot_digest is not None and snapshot_digest != clean_worktree_content_digest():
|
|
71
70
|
return None
|
|
72
71
|
object_name = f"{revision}:{path}"
|
|
73
|
-
size = git_output(target, "cat-file", "-s", object_name)
|
|
74
|
-
if size is None or not size.isdigit() or int(size) > MAX_FILE_BYTES:
|
|
75
|
-
return None
|
|
76
72
|
content = git_bytes(target, "cat-file", "blob", object_name)
|
|
77
73
|
return content.decode("utf-8", errors="replace") if content is not None else None
|
|
78
74
|
|