@openai/codex-security 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -49
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
- package/_bundled_plugin/references/config-preflight.md +11 -13
- package/_bundled_plugin/references/core-scan.md +94 -0
- package/_bundled_plugin/references/final-report.md +6 -6
- package/_bundled_plugin/references/scan-artifacts.md +15 -14
- package/_bundled_plugin/references/scan-prologue.md +27 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
- package/_bundled_plugin/scripts/config_preflight.py +40 -3
- package/_bundled_plugin/scripts/deep_scan_config.py +8 -5
- package/_bundled_plugin/scripts/deep_scan_workbench.py +103 -35
- package/_bundled_plugin/scripts/finalize_scan_contract.py +29 -17
- package/_bundled_plugin/scripts/finding_preview.py +122 -6
- package/_bundled_plugin/scripts/generate_in_scope_files.py +39 -17
- package/_bundled_plugin/scripts/generate_rank_input.py +21 -3
- package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
- package/_bundled_plugin/scripts/report_projection.py +1 -19
- package/_bundled_plugin/scripts/workbench_cli.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +280 -29
- package/_bundled_plugin/scripts/workbench_progress.py +0 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +69 -8
- package/_bundled_plugin/scripts/workbench_schema.py +41 -20
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -110
- package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
- package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
- package/_bundled_plugin/skills/validation/SKILL.md +9 -12
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
- package/dist/api.d.ts +1 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +195 -59
- package/dist/api.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts +5 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +38 -9
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +9 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +764 -60
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +12 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +30 -2
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +10 -1
- package/dist/cost.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +10 -3
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +77 -18
- package/dist/multiscan.js.map +1 -1
- package/dist/publication-events.d.ts +16 -0
- package/dist/publication-events.d.ts.map +1 -0
- package/dist/publication-events.js +141 -0
- package/dist/publication-events.js.map +1 -0
- package/dist/publication-store.d.ts +5 -0
- package/dist/publication-store.d.ts.map +1 -0
- package/dist/publication-store.js +102 -0
- package/dist/publication-store.js.map +1 -0
- package/dist/publication.d.ts +27 -0
- package/dist/publication.d.ts.map +1 -0
- package/dist/publication.js +207 -0
- package/dist/publication.js.map +1 -0
- package/dist/publish.d.ts +77 -0
- package/dist/publish.d.ts.map +1 -0
- package/dist/publish.js +748 -0
- package/dist/publish.js.map +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +15 -44
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +7 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +86 -20
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-logs.d.ts +2 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +97 -37
- package/dist/scan-logs.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +20 -11
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.js +1 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +3 -3
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +6 -4
- package/dist/version.js.map +1 -1
- package/package.json +5 -4
- package/_bundled_plugin/references/shared-hard-rules.md +0 -15
- package/_bundled_plugin/scripts/validate_report_format.py +0 -138
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
|
@@ -97,7 +97,7 @@ from workbench_schema import (
|
|
|
97
97
|
from workbench_schema import (
|
|
98
98
|
sql_statements as sql_statements,
|
|
99
99
|
)
|
|
100
|
-
from workbench_source_excerpt import finding_source_excerpt
|
|
100
|
+
from workbench_source_excerpt import finding_source_excerpt, safe_source_path
|
|
101
101
|
from workbench_target import (
|
|
102
102
|
clean_worktree_content_digest,
|
|
103
103
|
copy_directory_excluding,
|
|
@@ -1175,7 +1175,11 @@ def complete_budget_exhausted_scan(
|
|
|
1175
1175
|
"discovery."
|
|
1176
1176
|
)
|
|
1177
1177
|
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1178
|
-
candidates =
|
|
1178
|
+
candidates = (
|
|
1179
|
+
[]
|
|
1180
|
+
if run["manifest_path"] == str(scan_dir / "scan-manifest.json")
|
|
1181
|
+
else budget_exhausted_candidates(scan, scan_dir)
|
|
1182
|
+
)
|
|
1179
1183
|
warning = optional_text(args.message, maximum=2400)
|
|
1180
1184
|
if warning is None:
|
|
1181
1185
|
warning = (
|
|
@@ -1439,6 +1443,8 @@ def complete_scan_locked(
|
|
|
1439
1443
|
verify_manifest_binding(scan, manifest)
|
|
1440
1444
|
manifest_digest = published_manifest_digest(scan_dir, manifest)
|
|
1441
1445
|
pin_legacy_manifest_digest(connection, scan["id"], manifest_digest)
|
|
1446
|
+
if cost_json is not None and scan["recipe_json"] is not None:
|
|
1447
|
+
scan_usage.reconcile_completed_scan_cost(connection, scan, cost_json)
|
|
1442
1448
|
return scan_context(connection, scan["id"])
|
|
1443
1449
|
if scan["status"] != "running":
|
|
1444
1450
|
raise SystemExit("Only a running scan can be completed.")
|
|
@@ -1450,11 +1456,14 @@ def complete_scan_locked(
|
|
|
1450
1456
|
deep_scan.require_deep_scan_ready_for_parent_completion(connection, scan)
|
|
1451
1457
|
warnings = json.loads(scan["completion_warnings_json"])
|
|
1452
1458
|
target_warnings: list[str] = []
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1459
|
+
|
|
1460
|
+
def add_warning() -> None:
|
|
1461
|
+
if (warning := scan_target_warning(scan)) is not None:
|
|
1462
|
+
for items in (target_warnings, warnings):
|
|
1463
|
+
if warning not in items:
|
|
1464
|
+
items.append(warning)
|
|
1465
|
+
|
|
1466
|
+
add_warning()
|
|
1458
1467
|
scan_dir = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
1459
1468
|
completion_timestamp = now()
|
|
1460
1469
|
completion_binding = workbench_completion_binding(scan, completion_timestamp)
|
|
@@ -1492,12 +1501,7 @@ def complete_scan_locked(
|
|
|
1492
1501
|
completion_binding=completion_binding,
|
|
1493
1502
|
completion_warnings=warnings,
|
|
1494
1503
|
)
|
|
1495
|
-
|
|
1496
|
-
if warning is not None:
|
|
1497
|
-
if warning not in target_warnings:
|
|
1498
|
-
target_warnings.append(warning)
|
|
1499
|
-
if warning not in warnings:
|
|
1500
|
-
warnings.append(warning)
|
|
1504
|
+
add_warning()
|
|
1501
1505
|
manifest, findings, _ = _write_prepared_scan_finalization(prepared)
|
|
1502
1506
|
except ContractError as exc:
|
|
1503
1507
|
raise SystemExit(str(exc)) from exc
|
|
@@ -2401,6 +2405,258 @@ def set_finding_remediation(
|
|
|
2401
2405
|
return scan_context(connection, occurrence["scan_id"])
|
|
2402
2406
|
|
|
2403
2407
|
|
|
2408
|
+
def linear_publication_input(
|
|
2409
|
+
args: argparse.Namespace, *, recording: bool
|
|
2410
|
+
) -> tuple[dict[str, Any], dict[str, str], list[dict[str, str]]]:
|
|
2411
|
+
payload = read_json_object(Path(args.input_file))
|
|
2412
|
+
required = {"scanId", "scanDirectory", "destination", "findings"}
|
|
2413
|
+
if recording:
|
|
2414
|
+
required.add("publications")
|
|
2415
|
+
if set(payload) != required:
|
|
2416
|
+
raise SystemExit("Linear publication input contains unexpected or missing fields.")
|
|
2417
|
+
|
|
2418
|
+
scan_id = payload["scanId"]
|
|
2419
|
+
scan_directory = payload["scanDirectory"]
|
|
2420
|
+
destination = payload["destination"]
|
|
2421
|
+
findings = payload["findings"]
|
|
2422
|
+
if not isinstance(scan_id, str) or not isinstance(scan_directory, str):
|
|
2423
|
+
raise SystemExit("Linear publication input must identify the exact completed scan.")
|
|
2424
|
+
if (
|
|
2425
|
+
not isinstance(destination, dict)
|
|
2426
|
+
or not {"type", "teamId"}.issubset(destination)
|
|
2427
|
+
or not set(destination).issubset({"type", "teamId", "projectId"})
|
|
2428
|
+
or destination.get("type") != "linear"
|
|
2429
|
+
or not isinstance(destination.get("teamId"), str)
|
|
2430
|
+
or not destination["teamId"].strip()
|
|
2431
|
+
or (
|
|
2432
|
+
"projectId" in destination
|
|
2433
|
+
and (
|
|
2434
|
+
not isinstance(destination["projectId"], str)
|
|
2435
|
+
or not destination["projectId"].strip()
|
|
2436
|
+
)
|
|
2437
|
+
)
|
|
2438
|
+
):
|
|
2439
|
+
raise SystemExit("Linear publication input must identify the exact team and optional project.")
|
|
2440
|
+
if not isinstance(findings, list):
|
|
2441
|
+
raise SystemExit("Linear publication input must include the planned scan findings.")
|
|
2442
|
+
|
|
2443
|
+
seen_finding_ids: set[str] = set()
|
|
2444
|
+
seen_occurrence_ids: set[str] = set()
|
|
2445
|
+
for finding in findings:
|
|
2446
|
+
if (
|
|
2447
|
+
not isinstance(finding, dict)
|
|
2448
|
+
or set(finding) != {"findingId", "occurrenceId"}
|
|
2449
|
+
or not isinstance(finding.get("findingId"), str)
|
|
2450
|
+
or not finding["findingId"].strip()
|
|
2451
|
+
or not isinstance(finding.get("occurrenceId"), str)
|
|
2452
|
+
or not finding["occurrenceId"].strip()
|
|
2453
|
+
):
|
|
2454
|
+
raise SystemExit("Linear publication input contains an invalid finding identity.")
|
|
2455
|
+
if (
|
|
2456
|
+
finding["findingId"] in seen_finding_ids
|
|
2457
|
+
or finding["occurrenceId"] in seen_occurrence_ids
|
|
2458
|
+
):
|
|
2459
|
+
raise SystemExit("Linear publication input repeats a finding or occurrence.")
|
|
2460
|
+
seen_finding_ids.add(finding["findingId"])
|
|
2461
|
+
seen_occurrence_ids.add(finding["occurrenceId"])
|
|
2462
|
+
|
|
2463
|
+
return payload, destination, findings
|
|
2464
|
+
|
|
2465
|
+
|
|
2466
|
+
def verify_linear_publication_scan(
|
|
2467
|
+
connection: sqlite3.Connection,
|
|
2468
|
+
payload: dict[str, Any],
|
|
2469
|
+
findings: list[dict[str, str]],
|
|
2470
|
+
) -> sqlite3.Row:
|
|
2471
|
+
try:
|
|
2472
|
+
scan = require_scan(connection, payload["scanId"])
|
|
2473
|
+
except SystemExit as exc:
|
|
2474
|
+
raise SystemExit(
|
|
2475
|
+
"The completed scan is not present in the local Codex Security scan-history database. "
|
|
2476
|
+
"Use the state directory where the scan was completed."
|
|
2477
|
+
) from exc
|
|
2478
|
+
if scan["id"] != payload["scanId"]:
|
|
2479
|
+
raise SystemExit("Linear publication must use the exact completed scan identifier.")
|
|
2480
|
+
if scan["status"] != "complete":
|
|
2481
|
+
raise SystemExit("Only completed scans can publish findings to Linear.")
|
|
2482
|
+
|
|
2483
|
+
requested_directory = require_canonical_scan_directory(Path(payload["scanDirectory"]))
|
|
2484
|
+
recorded_directory = require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
2485
|
+
if os.path.normcase(requested_directory) != os.path.normcase(recorded_directory):
|
|
2486
|
+
raise SystemExit(
|
|
2487
|
+
"The selected scan directory does not match its local Codex Security scan history."
|
|
2488
|
+
)
|
|
2489
|
+
|
|
2490
|
+
stored_findings = {
|
|
2491
|
+
row["id"]: row["finding_id"]
|
|
2492
|
+
for row in connection.execute(
|
|
2493
|
+
"SELECT id, finding_id FROM finding_occurrences WHERE scan_id = ?",
|
|
2494
|
+
(scan["id"],),
|
|
2495
|
+
)
|
|
2496
|
+
}
|
|
2497
|
+
for finding in findings:
|
|
2498
|
+
if stored_findings.get(finding["occurrenceId"]) != finding["findingId"]:
|
|
2499
|
+
raise SystemExit(
|
|
2500
|
+
"A selected finding or occurrence does not belong to the completed scan "
|
|
2501
|
+
"in local Codex Security scan history."
|
|
2502
|
+
)
|
|
2503
|
+
if len(stored_findings) != len(findings):
|
|
2504
|
+
raise SystemExit(
|
|
2505
|
+
"The completed scan findings do not exactly match local Codex Security scan history."
|
|
2506
|
+
)
|
|
2507
|
+
return scan
|
|
2508
|
+
|
|
2509
|
+
|
|
2510
|
+
def prepare_linear_publication(
|
|
2511
|
+
connection: sqlite3.Connection, args: argparse.Namespace
|
|
2512
|
+
) -> dict[str, Any]:
|
|
2513
|
+
payload, destination, findings = linear_publication_input(args, recording=False)
|
|
2514
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
2515
|
+
try:
|
|
2516
|
+
scan = verify_linear_publication_scan(connection, payload, findings)
|
|
2517
|
+
result = {
|
|
2518
|
+
"scanId": scan["id"],
|
|
2519
|
+
"destination": destination,
|
|
2520
|
+
"findingCount": len(findings),
|
|
2521
|
+
}
|
|
2522
|
+
connection.commit()
|
|
2523
|
+
except BaseException:
|
|
2524
|
+
connection.rollback()
|
|
2525
|
+
raise
|
|
2526
|
+
return result
|
|
2527
|
+
|
|
2528
|
+
|
|
2529
|
+
def record_linear_publications(
|
|
2530
|
+
connection: sqlite3.Connection, args: argparse.Namespace
|
|
2531
|
+
) -> dict[str, Any]:
|
|
2532
|
+
payload, destination, findings = linear_publication_input(args, recording=True)
|
|
2533
|
+
publications = payload["publications"]
|
|
2534
|
+
if not isinstance(publications, list):
|
|
2535
|
+
raise SystemExit("Linear publication results must be an array.")
|
|
2536
|
+
planned = {finding["findingId"]: finding["occurrenceId"] for finding in findings}
|
|
2537
|
+
current: dict[str, dict[str, str]] = {}
|
|
2538
|
+
external_ids: set[str] = set()
|
|
2539
|
+
for publication in publications:
|
|
2540
|
+
if (
|
|
2541
|
+
not isinstance(publication, dict)
|
|
2542
|
+
or not {"findingId", "occurrenceId", "issueIdentifier"}.issubset(publication)
|
|
2543
|
+
or not set(publication).issubset(
|
|
2544
|
+
{"findingId", "occurrenceId", "issueIdentifier", "url"}
|
|
2545
|
+
)
|
|
2546
|
+
or not isinstance(publication.get("findingId"), str)
|
|
2547
|
+
or not isinstance(publication.get("occurrenceId"), str)
|
|
2548
|
+
or not isinstance(publication.get("issueIdentifier"), str)
|
|
2549
|
+
or not publication["issueIdentifier"].strip()
|
|
2550
|
+
or (
|
|
2551
|
+
"url" in publication
|
|
2552
|
+
and (
|
|
2553
|
+
not isinstance(publication["url"], str)
|
|
2554
|
+
or not publication["url"].strip()
|
|
2555
|
+
)
|
|
2556
|
+
)
|
|
2557
|
+
):
|
|
2558
|
+
raise SystemExit("Linear publication results contain an invalid issue association.")
|
|
2559
|
+
finding_id = publication["findingId"]
|
|
2560
|
+
issue_identifier = publication["issueIdentifier"]
|
|
2561
|
+
if planned.get(finding_id) != publication["occurrenceId"]:
|
|
2562
|
+
raise SystemExit(
|
|
2563
|
+
"A created Linear issue does not match its planned finding and occurrence."
|
|
2564
|
+
)
|
|
2565
|
+
if finding_id in current or issue_identifier in external_ids:
|
|
2566
|
+
raise SystemExit("Linear publication results repeat a finding or issue identifier.")
|
|
2567
|
+
current[finding_id] = publication
|
|
2568
|
+
external_ids.add(issue_identifier)
|
|
2569
|
+
|
|
2570
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
2571
|
+
try:
|
|
2572
|
+
scan = verify_linear_publication_scan(connection, payload, findings)
|
|
2573
|
+
timestamp = now()
|
|
2574
|
+
for publication in publications:
|
|
2575
|
+
conflicting = connection.execute(
|
|
2576
|
+
"""
|
|
2577
|
+
SELECT occurrence_id, external_url
|
|
2578
|
+
FROM finding_publications
|
|
2579
|
+
WHERE destination_type = ? AND team_id = ? AND project_id IS ?
|
|
2580
|
+
AND external_id = ?
|
|
2581
|
+
""",
|
|
2582
|
+
(
|
|
2583
|
+
destination["type"],
|
|
2584
|
+
destination["teamId"],
|
|
2585
|
+
destination.get("projectId"),
|
|
2586
|
+
publication["issueIdentifier"],
|
|
2587
|
+
),
|
|
2588
|
+
).fetchone()
|
|
2589
|
+
if conflicting is not None and conflicting["occurrence_id"] != publication[
|
|
2590
|
+
"occurrenceId"
|
|
2591
|
+
]:
|
|
2592
|
+
raise SystemExit("This Linear issue is already associated with a different finding.")
|
|
2593
|
+
if (
|
|
2594
|
+
conflicting is not None
|
|
2595
|
+
and "url" in publication
|
|
2596
|
+
and conflicting["external_url"] != publication["url"]
|
|
2597
|
+
):
|
|
2598
|
+
raise SystemExit("This Linear issue is already associated with a different URL.")
|
|
2599
|
+
|
|
2600
|
+
connection.execute(
|
|
2601
|
+
"""
|
|
2602
|
+
INSERT INTO finding_publications (
|
|
2603
|
+
scan_id, finding_id, occurrence_id, destination_type,
|
|
2604
|
+
team_id, project_id, external_id, external_url, created_at
|
|
2605
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2606
|
+
ON CONFLICT DO NOTHING
|
|
2607
|
+
""",
|
|
2608
|
+
(
|
|
2609
|
+
scan["id"],
|
|
2610
|
+
publication["findingId"],
|
|
2611
|
+
publication["occurrenceId"],
|
|
2612
|
+
destination["type"],
|
|
2613
|
+
destination["teamId"],
|
|
2614
|
+
destination.get("projectId"),
|
|
2615
|
+
publication["issueIdentifier"],
|
|
2616
|
+
publication.get("url"),
|
|
2617
|
+
timestamp,
|
|
2618
|
+
),
|
|
2619
|
+
)
|
|
2620
|
+
|
|
2621
|
+
created = []
|
|
2622
|
+
for finding in findings:
|
|
2623
|
+
publication = current.get(finding["findingId"])
|
|
2624
|
+
if publication is None:
|
|
2625
|
+
continue
|
|
2626
|
+
row = connection.execute(
|
|
2627
|
+
"""
|
|
2628
|
+
SELECT finding_id, occurrence_id, external_id, external_url
|
|
2629
|
+
FROM finding_publications
|
|
2630
|
+
WHERE scan_id = ? AND occurrence_id = ? AND destination_type = ?
|
|
2631
|
+
AND team_id = ? AND project_id IS ? AND external_id = ?
|
|
2632
|
+
""",
|
|
2633
|
+
(
|
|
2634
|
+
scan["id"],
|
|
2635
|
+
publication["occurrenceId"],
|
|
2636
|
+
destination["type"],
|
|
2637
|
+
destination["teamId"],
|
|
2638
|
+
destination.get("projectId"),
|
|
2639
|
+
publication["issueIdentifier"],
|
|
2640
|
+
),
|
|
2641
|
+
).fetchone()
|
|
2642
|
+
if row is None:
|
|
2643
|
+
raise SystemExit("A created Linear issue could not be read from scan history.")
|
|
2644
|
+
created.append(
|
|
2645
|
+
{
|
|
2646
|
+
"findingId": row["finding_id"],
|
|
2647
|
+
"occurrenceId": row["occurrence_id"],
|
|
2648
|
+
"issueIdentifier": row["external_id"],
|
|
2649
|
+
**({"url": row["external_url"]} if row["external_url"] is not None else {}),
|
|
2650
|
+
}
|
|
2651
|
+
)
|
|
2652
|
+
result = {"scanId": scan["id"], "destination": destination, "created": created}
|
|
2653
|
+
connection.commit()
|
|
2654
|
+
except BaseException:
|
|
2655
|
+
connection.rollback()
|
|
2656
|
+
raise
|
|
2657
|
+
return result
|
|
2658
|
+
|
|
2659
|
+
|
|
2404
2660
|
def export_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
2405
2661
|
scan = require_scan(connection, args.scan_id)
|
|
2406
2662
|
if scan["status"] != "complete":
|
|
@@ -2905,10 +3161,15 @@ def scan_context(
|
|
|
2905
3161
|
occurrence_id: str | None = None,
|
|
2906
3162
|
) -> dict[str, Any]:
|
|
2907
3163
|
scan = require_scan(connection, scan_id)
|
|
3164
|
+
workspace = workspace_state(connection, scan["workspace_id"], result_scan_id=scan["id"])
|
|
2908
3165
|
context = {
|
|
2909
3166
|
"otherRunningDeepScans": deep_scan.other_running_deep_scans(connection, scan["id"]),
|
|
2910
|
-
"scan":
|
|
2911
|
-
|
|
3167
|
+
"scan": (
|
|
3168
|
+
workspace["results"]
|
|
3169
|
+
if occurrence_id is None
|
|
3170
|
+
else scan_result(connection, scan, occurrence_id=occurrence_id)
|
|
3171
|
+
),
|
|
3172
|
+
"workspace": workspace,
|
|
2912
3173
|
}
|
|
2913
3174
|
if scan["recipe_json"] is not None:
|
|
2914
3175
|
context["parentScanId"] = scan["parent_scan_id"]
|
|
@@ -3462,20 +3723,6 @@ def patch_artifact_preview(
|
|
|
3462
3723
|
}
|
|
3463
3724
|
|
|
3464
3725
|
|
|
3465
|
-
def safe_source_path(target: Path, relative_path: str) -> Path | None:
|
|
3466
|
-
if "\\" in relative_path:
|
|
3467
|
-
return None
|
|
3468
|
-
parsed = PurePosixPath(relative_path)
|
|
3469
|
-
if parsed.is_absolute() or ".." in parsed.parts:
|
|
3470
|
-
return None
|
|
3471
|
-
try:
|
|
3472
|
-
path = (target / parsed.as_posix()).resolve()
|
|
3473
|
-
path.relative_to(target)
|
|
3474
|
-
except (OSError, RuntimeError, ValueError):
|
|
3475
|
-
return None
|
|
3476
|
-
return path
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
3726
|
def available_artifact_path(scan_dir: Path, candidate: Path) -> Path | None:
|
|
3480
3727
|
try:
|
|
3481
3728
|
resolved_scan_dir = require_canonical_scan_directory(scan_dir)
|
|
@@ -3735,6 +3982,10 @@ def main() -> None:
|
|
|
3735
3982
|
)
|
|
3736
3983
|
elif args.command == "set-finding-remediation":
|
|
3737
3984
|
result = set_finding_remediation(connection, args)
|
|
3985
|
+
elif args.command == "prepare-linear-publication":
|
|
3986
|
+
result = prepare_linear_publication(connection, args)
|
|
3987
|
+
elif args.command == "record-linear-publications":
|
|
3988
|
+
result = record_linear_publications(connection, args)
|
|
3738
3989
|
elif args.command == "export-findings":
|
|
3739
3990
|
result = export_findings(connection, args)
|
|
3740
3991
|
elif args.command == "database-info":
|
|
@@ -13,7 +13,6 @@ from workbench.handoff import require_current_continuation
|
|
|
13
13
|
from workbench_constants import PHASES
|
|
14
14
|
from workbench_validation import optional_text, require_uuid, user_text
|
|
15
15
|
|
|
16
|
-
MAX_PREFLIGHT_ISSUES_JSON_BYTES = 64 * 1024
|
|
17
16
|
MAX_PREFLIGHT_ISSUES = 32
|
|
18
17
|
|
|
19
18
|
|
|
@@ -33,10 +32,6 @@ def _preflight_issue_text(value: Any, maximum: int, label: str) -> str:
|
|
|
33
32
|
def preflight_issues_json(value: str | None) -> str | None:
|
|
34
33
|
if value is None:
|
|
35
34
|
return None
|
|
36
|
-
if len(value.encode("utf-8")) > MAX_PREFLIGHT_ISSUES_JSON_BYTES:
|
|
37
|
-
raise SystemExit(
|
|
38
|
-
f"Preflight issues must be no larger than {MAX_PREFLIGHT_ISSUES_JSON_BYTES} bytes."
|
|
39
|
-
)
|
|
40
35
|
try:
|
|
41
36
|
payload = json.loads(value)
|
|
42
37
|
except json.JSONDecodeError as exc:
|
|
@@ -55,6 +55,32 @@ def measured_scan_cost_json(usage: Mapping[str, Any]) -> str:
|
|
|
55
55
|
return json.dumps({"usage": dict(usage)}, separators=(",", ":"), allow_nan=False)
|
|
56
56
|
|
|
57
57
|
|
|
58
|
+
def reconcile_completed_scan_cost(
|
|
59
|
+
connection: sqlite3.Connection,
|
|
60
|
+
scan: sqlite3.Row,
|
|
61
|
+
cost_json: str,
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Persist authoritative SDK cost without discarding measured worker usage."""
|
|
64
|
+
|
|
65
|
+
existing = json.loads(scan["cost_json"]) if scan["cost_json"] is not None else {}
|
|
66
|
+
if isinstance(existing, dict) and "usage" in existing:
|
|
67
|
+
cost_json = json.dumps(
|
|
68
|
+
{**existing, "cost": json.loads(cost_json)},
|
|
69
|
+
separators=(",", ":"),
|
|
70
|
+
allow_nan=False,
|
|
71
|
+
)
|
|
72
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
73
|
+
try:
|
|
74
|
+
connection.execute(
|
|
75
|
+
"UPDATE scans SET cost_json = ? WHERE id = ? AND status = 'complete'",
|
|
76
|
+
(cost_json, scan["id"]),
|
|
77
|
+
)
|
|
78
|
+
connection.commit()
|
|
79
|
+
except BaseException:
|
|
80
|
+
connection.rollback()
|
|
81
|
+
raise
|
|
82
|
+
|
|
83
|
+
|
|
58
84
|
def collect_scan_usage(
|
|
59
85
|
connection: sqlite3.Connection,
|
|
60
86
|
scan: sqlite3.Row,
|
|
@@ -189,10 +215,18 @@ def _codex_state_database() -> Path | None:
|
|
|
189
215
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
190
216
|
|
|
191
217
|
configured_home = os.environ.get("CODEX_HOME", "").strip()
|
|
192
|
-
codex_home =
|
|
218
|
+
codex_home = (
|
|
219
|
+
Path(configured_home).expanduser()
|
|
220
|
+
if configured_home
|
|
221
|
+
else Path.home() / ".codex"
|
|
222
|
+
)
|
|
193
223
|
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "").strip()
|
|
194
224
|
search_roots = [
|
|
195
|
-
*(
|
|
225
|
+
*(
|
|
226
|
+
[Path(configured_sqlite_home).expanduser()]
|
|
227
|
+
if configured_sqlite_home
|
|
228
|
+
else []
|
|
229
|
+
),
|
|
196
230
|
codex_home,
|
|
197
231
|
codex_home / "sqlite",
|
|
198
232
|
]
|
|
@@ -329,7 +363,9 @@ def _require_state_columns(
|
|
|
329
363
|
}
|
|
330
364
|
columns = {str(row["name"]) for row in connection.execute(statements[table])}
|
|
331
365
|
if not required.issubset(columns):
|
|
332
|
-
raise ValueError(
|
|
366
|
+
raise ValueError(
|
|
367
|
+
"Codex state graph does not expose the required thread columns."
|
|
368
|
+
)
|
|
333
369
|
|
|
334
370
|
|
|
335
371
|
def _rollout_path(value: object) -> Path | None:
|
|
@@ -375,7 +411,9 @@ def _read_rollout_usage(
|
|
|
375
411
|
event = json.loads(raw_line)
|
|
376
412
|
except (UnicodeError, ValueError):
|
|
377
413
|
if line_number == 1:
|
|
378
|
-
raise ValueError(
|
|
414
|
+
raise ValueError(
|
|
415
|
+
"The rollout session metadata is unreadable."
|
|
416
|
+
) from None
|
|
379
417
|
if boundary_reached:
|
|
380
418
|
warnings.add("rollout_record_invalid")
|
|
381
419
|
continue
|
|
@@ -418,7 +456,10 @@ def _read_rollout_usage(
|
|
|
418
456
|
warnings.add("thread_outside_scan_window")
|
|
419
457
|
return total, warnings
|
|
420
458
|
boundary_reached = True
|
|
421
|
-
elif
|
|
459
|
+
elif (
|
|
460
|
+
event.get("type") == "event_msg"
|
|
461
|
+
and payload.get("type") == "token_count"
|
|
462
|
+
):
|
|
422
463
|
inherited_usage = _token_snapshot(payload)
|
|
423
464
|
if inherited_usage is not None:
|
|
424
465
|
previous = inherited_usage
|
|
@@ -497,10 +538,25 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
497
538
|
usage = info.get("total_token_usage")
|
|
498
539
|
if not isinstance(usage, dict):
|
|
499
540
|
return None
|
|
541
|
+
cache_write = usage.get(
|
|
542
|
+
"cache_write_input_tokens", usage.get("cache_write_tokens", 0)
|
|
543
|
+
)
|
|
544
|
+
legacy_cache_write = usage.get("cache_write_tokens")
|
|
545
|
+
input_tokens = usage.get("input_tokens")
|
|
546
|
+
cached_input_tokens = usage.get("cached_input_tokens", 0)
|
|
547
|
+
if (
|
|
548
|
+
cache_write == 0
|
|
549
|
+
and type(legacy_cache_write) is int
|
|
550
|
+
and legacy_cache_write > 0
|
|
551
|
+
and type(input_tokens) is int
|
|
552
|
+
and type(cached_input_tokens) is int
|
|
553
|
+
and cached_input_tokens + legacy_cache_write <= input_tokens
|
|
554
|
+
):
|
|
555
|
+
cache_write = legacy_cache_write
|
|
500
556
|
result: dict[str, int] = {}
|
|
501
557
|
for source_key, result_key in TOKEN_FIELDS.items():
|
|
502
558
|
value = (
|
|
503
|
-
|
|
559
|
+
cache_write
|
|
504
560
|
if source_key == "cache_write_input_tokens"
|
|
505
561
|
else usage.get(source_key, 0)
|
|
506
562
|
)
|
|
@@ -511,7 +567,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
511
567
|
):
|
|
512
568
|
return None
|
|
513
569
|
result[result_key] = value
|
|
514
|
-
if
|
|
570
|
+
if (
|
|
571
|
+
result["cachedInputTokens"] + result["cacheWriteInputTokens"]
|
|
572
|
+
> result["inputTokens"]
|
|
573
|
+
):
|
|
515
574
|
return None
|
|
516
575
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
517
576
|
return result
|
|
@@ -540,7 +599,9 @@ def _timestamp(value: object) -> datetime | None:
|
|
|
540
599
|
return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
|
|
541
600
|
|
|
542
601
|
|
|
543
|
-
def _unavailable_usage(
|
|
602
|
+
def _unavailable_usage(
|
|
603
|
+
reason: str, *, warnings: set[str] | None = None
|
|
604
|
+
) -> dict[str, Any]:
|
|
544
605
|
return {
|
|
545
606
|
"coverage": "unavailable",
|
|
546
607
|
"source": "codex_rollout",
|
|
@@ -643,6 +643,46 @@ MIGRATIONS = (
|
|
|
643
643
|
ADD COLUMN max_time_hours REAL NOT NULL DEFAULT 96;
|
|
644
644
|
""",
|
|
645
645
|
),
|
|
646
|
+
(
|
|
647
|
+
29,
|
|
648
|
+
"persist finding publication associations",
|
|
649
|
+
"""
|
|
650
|
+
CREATE TABLE finding_publications (
|
|
651
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
652
|
+
scan_id TEXT NOT NULL REFERENCES scans(id) ON DELETE CASCADE,
|
|
653
|
+
finding_id TEXT NOT NULL REFERENCES findings(id),
|
|
654
|
+
occurrence_id TEXT NOT NULL
|
|
655
|
+
REFERENCES finding_occurrences(id) ON DELETE CASCADE,
|
|
656
|
+
destination_type TEXT NOT NULL,
|
|
657
|
+
team_id TEXT,
|
|
658
|
+
project_id TEXT,
|
|
659
|
+
external_id TEXT NOT NULL,
|
|
660
|
+
external_url TEXT,
|
|
661
|
+
created_at TEXT NOT NULL,
|
|
662
|
+
UNIQUE (occurrence_id, destination_type, team_id, project_id, external_id),
|
|
663
|
+
UNIQUE (destination_type, team_id, project_id, external_id)
|
|
664
|
+
);
|
|
665
|
+
|
|
666
|
+
CREATE INDEX finding_publications_by_scan
|
|
667
|
+
ON finding_publications(scan_id, occurrence_id, id);
|
|
668
|
+
|
|
669
|
+
CREATE INDEX finding_publications_by_finding
|
|
670
|
+
ON finding_publications(finding_id, id);
|
|
671
|
+
""",
|
|
672
|
+
),
|
|
673
|
+
(
|
|
674
|
+
30,
|
|
675
|
+
"preserve team-only finding publication associations",
|
|
676
|
+
"""
|
|
677
|
+
CREATE UNIQUE INDEX finding_publications_team_only_occurrence
|
|
678
|
+
ON finding_publications(occurrence_id, destination_type, team_id, external_id)
|
|
679
|
+
WHERE project_id IS NULL;
|
|
680
|
+
|
|
681
|
+
CREATE UNIQUE INDEX finding_publications_team_only_external_issue
|
|
682
|
+
ON finding_publications(destination_type, team_id, external_id)
|
|
683
|
+
WHERE project_id IS NULL;
|
|
684
|
+
""",
|
|
685
|
+
),
|
|
646
686
|
)
|
|
647
687
|
|
|
648
688
|
|
|
@@ -744,7 +784,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
744
784
|
12: "scan continuation threads",
|
|
745
785
|
}
|
|
746
786
|
model_migration_name = "persist scan model settings"
|
|
747
|
-
warnings_migration_name = "persist scan completion warnings"
|
|
748
787
|
if execution_migrations.get(25) == "dynamic scan execution profiles":
|
|
749
788
|
connection.execute(
|
|
750
789
|
"UPDATE schema_migrations SET name = ? WHERE version = 25 AND name = ?",
|
|
@@ -752,9 +791,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
752
791
|
)
|
|
753
792
|
execution_migrations[25] = model_migration_name
|
|
754
793
|
has_legacy_profile_history = execution_migrations.get(11) == legacy_names[11]
|
|
755
|
-
has_public_warnings_history = (
|
|
756
|
-
execution_migrations.get(25) == warnings_migration_name
|
|
757
|
-
)
|
|
758
794
|
|
|
759
795
|
scan_columns = {
|
|
760
796
|
row["name"] for row in connection.execute("PRAGMA table_info(scans)")
|
|
@@ -765,11 +801,7 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
765
801
|
has_legacy_profile_columns = (
|
|
766
802
|
"execution_model" in scan_columns or "execution_model" in workspace_columns
|
|
767
803
|
)
|
|
768
|
-
if not (
|
|
769
|
-
has_legacy_profile_history
|
|
770
|
-
or has_public_warnings_history
|
|
771
|
-
or has_legacy_profile_columns
|
|
772
|
-
):
|
|
804
|
+
if not (has_legacy_profile_history or has_legacy_profile_columns):
|
|
773
805
|
return
|
|
774
806
|
|
|
775
807
|
if (
|
|
@@ -811,17 +843,6 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
811
843
|
"The Codex Security database has an unsupported execution-profile migration history."
|
|
812
844
|
)
|
|
813
845
|
|
|
814
|
-
if has_public_warnings_history:
|
|
815
|
-
if execution_migrations.get(26) is not None:
|
|
816
|
-
raise SystemExit(
|
|
817
|
-
"The Codex Security database has an unsupported pre-release migration history."
|
|
818
|
-
)
|
|
819
|
-
connection.execute(
|
|
820
|
-
"UPDATE schema_migrations SET version = 26 WHERE version = 25 AND name = ?",
|
|
821
|
-
(warnings_migration_name,),
|
|
822
|
-
)
|
|
823
|
-
execution_migrations.pop(25)
|
|
824
|
-
|
|
825
846
|
if execution_migrations.get(25) not in (None, model_migration_name):
|
|
826
847
|
raise SystemExit(
|
|
827
848
|
"The Codex Security database has an unsupported execution-profile migration history."
|