@openai/codex-security 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -8
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +8 -1
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -57
- package/_bundled_plugin/references/config-preflight.md +1 -1
- package/_bundled_plugin/references/final-report.md +15 -7
- package/_bundled_plugin/references/scan-artifacts.md +11 -9
- package/_bundled_plugin/references/scan-contract.md +3 -1
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +42 -0
- package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +100 -0
- package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +181 -0
- package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +166 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +120 -0
- package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +83 -0
- package/_bundled_plugin/schemas/tools/review-items.schema.json +80 -0
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +572 -0
- package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +18 -0
- package/_bundled_plugin/scripts/deep_scan_config.py +11 -3
- package/_bundled_plugin/scripts/deep_scan_workbench.py +153 -133
- package/_bundled_plugin/scripts/finalize_scan_contract.py +21 -0
- package/_bundled_plugin/scripts/generate_in_scope_files.py +134 -0
- package/_bundled_plugin/scripts/workbench_cli.py +33 -1
- package/_bundled_plugin/scripts/workbench_db.py +127 -117
- package/_bundled_plugin/scripts/workbench_feedback.py +2 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +5 -49
- package/_bundled_plugin/scripts/workbench_progress.py +86 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +19 -1
- package/_bundled_plugin/scripts/workbench_scan_history.py +7 -1
- package/_bundled_plugin/scripts/workbench_scan_start.py +11 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +557 -0
- package/_bundled_plugin/scripts/workbench_schema.py +501 -0
- package/_bundled_plugin/scripts/workbench_validation.py +117 -13
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +4 -4
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +34 -28
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +11 -8
- package/_bundled_plugin/skills/security-scan/SKILL.md +16 -9
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +26 -14
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +1 -1
- package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
- package/_bundled_plugin/skills/validation/SKILL.md +5 -5
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +145 -227
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +53 -68
- package/dist/api.d.ts +29 -7
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +323 -47
- package/dist/api.js.map +1 -1
- package/dist/auth.js +1 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +3 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +458 -40
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +28 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +39 -2
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +7 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +345 -17
- package/dist/cost.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +3 -0
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts +16 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +540 -25
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.d.ts +13 -0
- package/dist/scan-activity.d.ts.map +1 -0
- package/dist/scan-activity.js +319 -0
- package/dist/scan-activity.js.map +1 -0
- package/dist/scan-dashboard.d.ts +47 -0
- package/dist/scan-dashboard.d.ts.map +1 -0
- package/dist/scan-dashboard.js +444 -0
- package/dist/scan-dashboard.js.map +1 -0
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +4 -1
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/dist/worker-progress.d.ts +7 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +68 -0
- package/dist/worker-progress.js.map +1 -1
- package/package.json +3 -3
|
@@ -22,10 +22,15 @@ from workbench_target import (
|
|
|
22
22
|
git_revision,
|
|
23
23
|
worktree_content_digest,
|
|
24
24
|
)
|
|
25
|
-
from workbench_validation import optional_text, require_uuid
|
|
25
|
+
from workbench_validation import optional_text, require_uuid, user_text
|
|
26
26
|
|
|
27
27
|
DEEP_SCAN_WORKER_KINDS = ("setup", "discovery", "dedup")
|
|
28
28
|
DEEP_SCAN_WORKER_STATUSES = ("queued", "running", "succeeded", "failed", "canceled")
|
|
29
|
+
DEEP_SCAN_REPLACEABLE_FAILURE_KINDS = (
|
|
30
|
+
"policy_refusal",
|
|
31
|
+
"transient_error",
|
|
32
|
+
"invalid_discovery_artifacts",
|
|
33
|
+
)
|
|
29
34
|
DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped")
|
|
30
35
|
DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1"
|
|
31
36
|
|
|
@@ -40,6 +45,8 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
40
45
|
begin_deep_scan.add_argument("--user-context")
|
|
41
46
|
begin_deep_scan.add_argument("--scan-root")
|
|
42
47
|
begin_deep_scan.add_argument("--claim-token")
|
|
48
|
+
begin_deep_scan.add_argument("--model")
|
|
49
|
+
begin_deep_scan.add_argument("--reasoning-effort")
|
|
43
50
|
begin_deep_scan.add_argument("--available-parallelism", type=positive_int)
|
|
44
51
|
begin_deep_scan.add_argument("--workflow-version", default=DEEP_SCAN_WORKFLOW_VERSION)
|
|
45
52
|
|
|
@@ -58,6 +65,9 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
58
65
|
upsert_deep_worker.add_argument("--attempt", type=non_negative_int)
|
|
59
66
|
upsert_deep_worker.add_argument("--sdk-thread-id")
|
|
60
67
|
upsert_deep_worker.add_argument("--error-message")
|
|
68
|
+
upsert_deep_worker.add_argument(
|
|
69
|
+
"--replaceable-failure-kind", choices=DEEP_SCAN_REPLACEABLE_FAILURE_KINDS
|
|
70
|
+
)
|
|
61
71
|
|
|
62
72
|
claim_deep_dedup = subparsers.add_parser("claim-deep-scan-dedup")
|
|
63
73
|
claim_deep_dedup.add_argument("--scan-id", required=True)
|
|
@@ -69,15 +79,6 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
69
79
|
commit_deep_dedup = subparsers.add_parser("commit-deep-scan-dedup")
|
|
70
80
|
commit_deep_dedup.add_argument("--scan-id", required=True)
|
|
71
81
|
commit_deep_dedup.add_argument("--worker-id", required=True)
|
|
72
|
-
commit_deep_dedup.add_argument("--canonical-inventory-path", required=True)
|
|
73
|
-
commit_deep_dedup.add_argument("--canonical-finding-report-path", required=True)
|
|
74
|
-
commit_deep_dedup.add_argument("--canonical-candidates-path", required=True)
|
|
75
|
-
commit_deep_dedup.add_argument("--dedupe-report-path", required=True)
|
|
76
|
-
commit_deep_dedup.add_argument("--seed-research-path", required=True)
|
|
77
|
-
commit_deep_dedup.add_argument("--work-ledger-path", required=True)
|
|
78
|
-
commit_deep_dedup.add_argument("--raw-candidates-path", required=True)
|
|
79
|
-
commit_deep_dedup.add_argument("--coverage-ledger-path", required=True)
|
|
80
|
-
commit_deep_dedup.add_argument("--findings-dir", required=True)
|
|
81
82
|
commit_deep_dedup.add_argument("--result-manifest-path", required=True)
|
|
82
83
|
commit_deep_dedup.add_argument("--new-findings-count", type=non_negative_int, required=True)
|
|
83
84
|
|
|
@@ -248,6 +249,22 @@ def deep_scan_path(
|
|
|
248
249
|
return str(resolved)
|
|
249
250
|
|
|
250
251
|
|
|
252
|
+
def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]:
|
|
253
|
+
discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery"
|
|
254
|
+
artifacts = {
|
|
255
|
+
"inScopeFilesPath": discovery_dir / "in_scope_files.txt",
|
|
256
|
+
"candidateLedgerPath": discovery_dir / "candidate_ledger.jsonl",
|
|
257
|
+
}
|
|
258
|
+
labels = {
|
|
259
|
+
"inScopeFilesPath": "Canonical in-scope inventory path",
|
|
260
|
+
"candidateLedgerPath": "Canonical candidate ledger path",
|
|
261
|
+
}
|
|
262
|
+
return {
|
|
263
|
+
name: deep_scan_path(scan, str(path), labels[name], kind="file")
|
|
264
|
+
for name, path in artifacts.items()
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
|
|
251
268
|
def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, Any]:
|
|
252
269
|
run = require_deep_scan_run(connection, scan_id)
|
|
253
270
|
scan = require_scan(connection, run["scan_id"])
|
|
@@ -269,6 +286,14 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
269
286
|
""",
|
|
270
287
|
(run["scan_id"],),
|
|
271
288
|
)
|
|
289
|
+
successful_reducer = connection.execute(
|
|
290
|
+
"""
|
|
291
|
+
SELECT 1 FROM deep_scan_workers
|
|
292
|
+
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
293
|
+
LIMIT 1
|
|
294
|
+
""",
|
|
295
|
+
(run["scan_id"],),
|
|
296
|
+
).fetchone()
|
|
272
297
|
return {
|
|
273
298
|
"scanId": run["scan_id"],
|
|
274
299
|
"targetPath": scan["target_path"],
|
|
@@ -284,24 +309,19 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
284
309
|
"workers": run["workers"],
|
|
285
310
|
"subagents": run["subagents"],
|
|
286
311
|
"stopAfterNoNew": run["stop_after_no_new"],
|
|
312
|
+
"stopAfterConsecutiveErrors": run["stop_after_consecutive_errors"],
|
|
287
313
|
"maxDiscoveryRuns": run["max_discovery_runs"],
|
|
288
314
|
},
|
|
289
315
|
"dispatchedCount": run["discovery_runs_dispatched"],
|
|
290
316
|
"completionSequence": run["completion_sequence"],
|
|
291
317
|
"noNewStreak": run["consecutive_no_new"],
|
|
318
|
+
"consecutiveErrors": run["consecutive_errors"],
|
|
292
319
|
"cancelRequested": bool(run["cancel_requested"]),
|
|
293
|
-
"
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
"dedupeReportPath": run["dedupe_report_path"],
|
|
299
|
-
"seedResearchPath": run["seed_research_path"],
|
|
300
|
-
"workLedgerPath": run["work_ledger_path"],
|
|
301
|
-
"rawCandidatesPath": run["raw_candidates_path"],
|
|
302
|
-
"coverageLedgerPath": run["coverage_ledger_path"],
|
|
303
|
-
"findingsDir": run["findings_dir"],
|
|
304
|
-
},
|
|
320
|
+
"canonicalArtifacts": (
|
|
321
|
+
canonical_discovery_artifacts(scan)
|
|
322
|
+
if successful_reducer is not None and run["canonical_inventory_path"] is None
|
|
323
|
+
else None
|
|
324
|
+
),
|
|
305
325
|
"manifestPath": run["manifest_path"],
|
|
306
326
|
"terminalReason": run["terminal_reason"],
|
|
307
327
|
"error": run["error_message"],
|
|
@@ -324,7 +344,7 @@ def independent_review_progress(
|
|
|
324
344
|
connection: sqlite3.Connection, scan_id: str
|
|
325
345
|
) -> dict[str, int | str] | None:
|
|
326
346
|
run = connection.execute(
|
|
327
|
-
"SELECT completion_sequence, updated_at FROM deep_scan_runs WHERE scan_id = ?",
|
|
347
|
+
"SELECT completion_sequence, phase, updated_at FROM deep_scan_runs WHERE scan_id = ?",
|
|
328
348
|
(scan_id,),
|
|
329
349
|
).fetchone()
|
|
330
350
|
if run is None:
|
|
@@ -342,6 +362,7 @@ def independent_review_progress(
|
|
|
342
362
|
return {
|
|
343
363
|
"active": int(active),
|
|
344
364
|
"completed": int(run["completion_sequence"]),
|
|
365
|
+
"consolidating": run["phase"] == "reducing",
|
|
345
366
|
"updatedAt": str(run["updated_at"]),
|
|
346
367
|
}
|
|
347
368
|
|
|
@@ -403,9 +424,10 @@ def ensure_deep_scan_run(
|
|
|
403
424
|
"""
|
|
404
425
|
INSERT INTO deep_scan_runs (
|
|
405
426
|
scan_id, schema_version, workflow_version, status, phase,
|
|
406
|
-
workers, subagents, stop_after_no_new,
|
|
427
|
+
workers, subagents, stop_after_no_new, stop_after_consecutive_errors,
|
|
428
|
+
max_discovery_runs,
|
|
407
429
|
created_at, updated_at
|
|
408
|
-
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?)
|
|
430
|
+
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?)
|
|
409
431
|
""",
|
|
410
432
|
(
|
|
411
433
|
scan["id"],
|
|
@@ -413,6 +435,7 @@ def ensure_deep_scan_run(
|
|
|
413
435
|
config["workers"],
|
|
414
436
|
config["subagents"],
|
|
415
437
|
config["stopAfterNoNew"],
|
|
438
|
+
config["stopAfterConsecutiveErrors"],
|
|
416
439
|
config["maxDiscoveryRuns"],
|
|
417
440
|
timestamp,
|
|
418
441
|
timestamp,
|
|
@@ -531,6 +554,36 @@ def begin_deep_scan_for_scan(
|
|
|
531
554
|
args: argparse.Namespace,
|
|
532
555
|
) -> dict[str, Any]:
|
|
533
556
|
scan_id = require_uuid(scan_id, "scan-id")
|
|
557
|
+
candidate = require_scan(connection, scan_id)
|
|
558
|
+
workspace = require_workspace(connection, candidate["workspace_id"])
|
|
559
|
+
if (
|
|
560
|
+
candidate["mode"] == "deep"
|
|
561
|
+
and candidate["status"] == "running"
|
|
562
|
+
and candidate["recipe_json"] is not None
|
|
563
|
+
and candidate["handoff_status"] == "delivered"
|
|
564
|
+
and candidate["deep_scan_owner_thread_id"] is None
|
|
565
|
+
and workspace["thread_id"] is None
|
|
566
|
+
):
|
|
567
|
+
require_current_continuation(
|
|
568
|
+
candidate,
|
|
569
|
+
args.claim_token,
|
|
570
|
+
error_message="Deep Scan orchestration is owned by another continuation.",
|
|
571
|
+
)
|
|
572
|
+
timestamp = now()
|
|
573
|
+
with connection:
|
|
574
|
+
claimed_workspace = connection.execute(
|
|
575
|
+
"UPDATE workspaces SET thread_id = ?, updated_at = ? "
|
|
576
|
+
"WHERE id = ? AND thread_id IS NULL",
|
|
577
|
+
(thread_id, timestamp, workspace["id"]),
|
|
578
|
+
)
|
|
579
|
+
claimed_scan = connection.execute(
|
|
580
|
+
"UPDATE scans SET deep_scan_owner_thread_id = ?, updated_at = ? "
|
|
581
|
+
"WHERE id = ? AND deep_scan_owner_thread_id IS NULL "
|
|
582
|
+
"AND handoff_status = 'delivered' AND handoff_claim_token IS ?",
|
|
583
|
+
(thread_id, timestamp, scan_id, candidate["handoff_claim_token"]),
|
|
584
|
+
)
|
|
585
|
+
if claimed_workspace.rowcount != 1 or claimed_scan.rowcount != 1:
|
|
586
|
+
raise SystemExit("A scan can only be orchestrated from its owning Codex thread.")
|
|
534
587
|
scan, _ = require_owned_scan(connection, scan_id, thread_id)
|
|
535
588
|
require_current_continuation(
|
|
536
589
|
scan,
|
|
@@ -539,6 +592,18 @@ def begin_deep_scan_for_scan(
|
|
|
539
592
|
)
|
|
540
593
|
if scan["mode"] != "deep":
|
|
541
594
|
raise SystemExit("Deep Scan orchestration requires a scan in deep mode.")
|
|
595
|
+
model = optional_text(args.model, maximum=200)
|
|
596
|
+
reasoning_effort = optional_text(args.reasoning_effort, maximum=32)
|
|
597
|
+
if model is not None or reasoning_effort is not None:
|
|
598
|
+
connection.execute(
|
|
599
|
+
"""
|
|
600
|
+
UPDATE scans
|
|
601
|
+
SET model = COALESCE(?, model), reasoning_effort = COALESCE(?, reasoning_effort)
|
|
602
|
+
WHERE id = ?
|
|
603
|
+
""",
|
|
604
|
+
(model, reasoning_effort, scan_id),
|
|
605
|
+
)
|
|
606
|
+
connection.commit()
|
|
542
607
|
existing = connection.execute(
|
|
543
608
|
"SELECT scan_id FROM deep_scan_runs WHERE scan_id = ?", (scan_id,)
|
|
544
609
|
).fetchone()
|
|
@@ -650,7 +715,9 @@ def begin_deep_scan_for_target(
|
|
|
650
715
|
if target_root == target or target in target_root.parents:
|
|
651
716
|
raise SystemExit("The scan artifact directory must be outside the selected target.")
|
|
652
717
|
target_root.mkdir(parents=True, exist_ok=True)
|
|
653
|
-
user_context =
|
|
718
|
+
user_context = user_text(args.user_context)
|
|
719
|
+
model = optional_text(args.model, maximum=200)
|
|
720
|
+
reasoning_effort = optional_text(args.reasoning_effort, maximum=32)
|
|
654
721
|
workspace_id = str(uuid.uuid4())
|
|
655
722
|
scan_id = str(uuid.uuid4())
|
|
656
723
|
timestamp = now()
|
|
@@ -685,10 +752,10 @@ def begin_deep_scan_for_target(
|
|
|
685
752
|
INSERT INTO scans (
|
|
686
753
|
id, workspace_id, target_id, target_path, target_revision, target_snapshot_digest,
|
|
687
754
|
target_device, target_inode, scope, mode, user_context,
|
|
688
|
-
deep_scan_owner_thread_id, scan_dir, status, phase,
|
|
689
|
-
started_at, created_at, updated_at
|
|
690
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?,
|
|
691
|
-
'delivered', ?, ?, ?)
|
|
755
|
+
deep_scan_owner_thread_id, scan_dir, model, reasoning_effort, status, phase,
|
|
756
|
+
handoff_status, started_at, created_at, updated_at
|
|
757
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'deep', ?, ?, ?, ?, ?,
|
|
758
|
+
'running', 'preflight', 'delivered', ?, ?, ?)
|
|
692
759
|
""",
|
|
693
760
|
(
|
|
694
761
|
scan_id,
|
|
@@ -703,6 +770,8 @@ def begin_deep_scan_for_target(
|
|
|
703
770
|
user_context,
|
|
704
771
|
thread_id,
|
|
705
772
|
str(scan_dir),
|
|
773
|
+
model,
|
|
774
|
+
reasoning_effort,
|
|
706
775
|
timestamp,
|
|
707
776
|
timestamp,
|
|
708
777
|
timestamp,
|
|
@@ -794,6 +863,18 @@ def upsert_deep_scan_worker(
|
|
|
794
863
|
existing = connection.execute(
|
|
795
864
|
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
796
865
|
).fetchone()
|
|
866
|
+
replaceable_failure_kind = args.replaceable_failure_kind
|
|
867
|
+
if replaceable_failure_kind is not None and (
|
|
868
|
+
args.kind != "discovery"
|
|
869
|
+
or args.status != "canceled"
|
|
870
|
+
or existing is None
|
|
871
|
+
or existing["status"] not in {"running", "canceled"}
|
|
872
|
+
or optional_text(args.error_message, maximum=2400) is None
|
|
873
|
+
):
|
|
874
|
+
raise SystemExit(
|
|
875
|
+
"A replaceable Deep Scan failure requires a running discovery worker, "
|
|
876
|
+
"canceled status, and an error message."
|
|
877
|
+
)
|
|
797
878
|
cleanup_update = (
|
|
798
879
|
existing is not None
|
|
799
880
|
and args.status == "canceled"
|
|
@@ -921,11 +1002,26 @@ def upsert_deep_scan_worker(
|
|
|
921
1002
|
connection.execute(
|
|
922
1003
|
"""
|
|
923
1004
|
UPDATE deep_scan_runs
|
|
924
|
-
SET completion_sequence = ?, phase = 'discovery',
|
|
1005
|
+
SET completion_sequence = ?, phase = 'discovery',
|
|
1006
|
+
consecutive_errors = 0, updated_at = ?
|
|
925
1007
|
WHERE scan_id = ?
|
|
926
1008
|
""",
|
|
927
1009
|
(completion_sequence, timestamp, scan_id),
|
|
928
1010
|
)
|
|
1011
|
+
elif (
|
|
1012
|
+
args.kind == "discovery"
|
|
1013
|
+
and args.status == "canceled"
|
|
1014
|
+
and replaceable_failure_kind is not None
|
|
1015
|
+
and existing["status"] == "running"
|
|
1016
|
+
):
|
|
1017
|
+
connection.execute(
|
|
1018
|
+
"""
|
|
1019
|
+
UPDATE deep_scan_runs
|
|
1020
|
+
SET consecutive_errors = consecutive_errors + 1, updated_at = ?
|
|
1021
|
+
WHERE scan_id = ?
|
|
1022
|
+
""",
|
|
1023
|
+
(timestamp, scan_id),
|
|
1024
|
+
)
|
|
929
1025
|
completed_at = (
|
|
930
1026
|
timestamp
|
|
931
1027
|
if args.status in {"succeeded", "failed", "canceled"}
|
|
@@ -1046,7 +1142,15 @@ def claim_deep_scan_dedup(
|
|
|
1046
1142
|
).fetchone()
|
|
1047
1143
|
is None
|
|
1048
1144
|
)
|
|
1049
|
-
|
|
1145
|
+
successful_reducer = connection.execute(
|
|
1146
|
+
"""
|
|
1147
|
+
SELECT 1 FROM deep_scan_workers
|
|
1148
|
+
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
1149
|
+
LIMIT 1
|
|
1150
|
+
""",
|
|
1151
|
+
(scan_id,),
|
|
1152
|
+
).fetchone()
|
|
1153
|
+
minimum_inputs = 1 if successful_reducer is not None or hard_cap_singleton else 2
|
|
1050
1154
|
if len(input_ids) < minimum_inputs:
|
|
1051
1155
|
raise SystemExit(
|
|
1052
1156
|
"The first Deep Scan dedup requires two buffered discovery results."
|
|
@@ -1084,6 +1188,14 @@ def claim_deep_scan_dedup(
|
|
|
1084
1188
|
"UPDATE deep_scan_runs SET phase = 'reducing', updated_at = ? WHERE scan_id = ?",
|
|
1085
1189
|
(timestamp, scan_id),
|
|
1086
1190
|
)
|
|
1191
|
+
connection.execute(
|
|
1192
|
+
"""
|
|
1193
|
+
UPDATE scan_progress
|
|
1194
|
+
SET deep_review_pass = COALESCE(deep_review_pass, 0) + 1, updated_at = ?
|
|
1195
|
+
WHERE scan_id = ?
|
|
1196
|
+
""",
|
|
1197
|
+
(timestamp, scan_id),
|
|
1198
|
+
)
|
|
1087
1199
|
connection.commit()
|
|
1088
1200
|
except BaseException:
|
|
1089
1201
|
connection.rollback()
|
|
@@ -1109,75 +1221,7 @@ def commit_deep_scan_dedup(
|
|
|
1109
1221
|
require_running_deep_scan(connection, scan_id)
|
|
1110
1222
|
if worker["status"] not in {"queued", "running"}:
|
|
1111
1223
|
raise SystemExit("Only an active dedup worker can commit a result.")
|
|
1112
|
-
|
|
1113
|
-
scan,
|
|
1114
|
-
args.canonical_inventory_path,
|
|
1115
|
-
"Canonical inventory path",
|
|
1116
|
-
kind="file",
|
|
1117
|
-
)
|
|
1118
|
-
canonical_finding_report_path = deep_scan_path(
|
|
1119
|
-
scan,
|
|
1120
|
-
args.canonical_finding_report_path,
|
|
1121
|
-
"Canonical finding report path",
|
|
1122
|
-
kind="file",
|
|
1123
|
-
)
|
|
1124
|
-
canonical_candidates_path = deep_scan_path(
|
|
1125
|
-
scan,
|
|
1126
|
-
args.canonical_candidates_path,
|
|
1127
|
-
"Canonical candidates path",
|
|
1128
|
-
kind="file",
|
|
1129
|
-
)
|
|
1130
|
-
dedupe_report_path = deep_scan_path(
|
|
1131
|
-
scan, args.dedupe_report_path, "Canonical dedupe report path", kind="file"
|
|
1132
|
-
)
|
|
1133
|
-
seed_research_path = deep_scan_path(
|
|
1134
|
-
scan, args.seed_research_path, "Canonical seed research path", kind="file"
|
|
1135
|
-
)
|
|
1136
|
-
work_ledger_path = deep_scan_path(
|
|
1137
|
-
scan, args.work_ledger_path, "Canonical work ledger path", kind="file"
|
|
1138
|
-
)
|
|
1139
|
-
raw_candidates_path = deep_scan_path(
|
|
1140
|
-
scan, args.raw_candidates_path, "Canonical raw candidates path", kind="file"
|
|
1141
|
-
)
|
|
1142
|
-
coverage_ledger_path = deep_scan_path(
|
|
1143
|
-
scan, args.coverage_ledger_path, "Canonical coverage ledger path", kind="file"
|
|
1144
|
-
)
|
|
1145
|
-
findings_dir = deep_scan_path(
|
|
1146
|
-
scan, args.findings_dir, "Canonical findings directory", kind="directory"
|
|
1147
|
-
)
|
|
1148
|
-
persisted_canonical_paths = {
|
|
1149
|
-
"canonical inventory": run["canonical_inventory_path"],
|
|
1150
|
-
"canonical finding report": run["canonical_finding_report_path"],
|
|
1151
|
-
"canonical candidates": run["canonical_candidates_path"],
|
|
1152
|
-
"dedupe report": run["dedupe_report_path"],
|
|
1153
|
-
"seed research": run["seed_research_path"],
|
|
1154
|
-
"work ledger": run["work_ledger_path"],
|
|
1155
|
-
"raw candidates": run["raw_candidates_path"],
|
|
1156
|
-
"coverage ledger": run["coverage_ledger_path"],
|
|
1157
|
-
"findings directory": run["findings_dir"],
|
|
1158
|
-
}
|
|
1159
|
-
submitted_canonical_paths = {
|
|
1160
|
-
"canonical inventory": canonical_inventory_path,
|
|
1161
|
-
"canonical finding report": canonical_finding_report_path,
|
|
1162
|
-
"canonical candidates": canonical_candidates_path,
|
|
1163
|
-
"dedupe report": dedupe_report_path,
|
|
1164
|
-
"seed research": seed_research_path,
|
|
1165
|
-
"work ledger": work_ledger_path,
|
|
1166
|
-
"raw candidates": raw_candidates_path,
|
|
1167
|
-
"coverage ledger": coverage_ledger_path,
|
|
1168
|
-
"findings directory": findings_dir,
|
|
1169
|
-
}
|
|
1170
|
-
if run["canonical_inventory_path"] is not None:
|
|
1171
|
-
changed = [
|
|
1172
|
-
label
|
|
1173
|
-
for label, persisted in persisted_canonical_paths.items()
|
|
1174
|
-
if submitted_canonical_paths[label] != persisted
|
|
1175
|
-
]
|
|
1176
|
-
if changed:
|
|
1177
|
-
raise SystemExit(
|
|
1178
|
-
"Deep Scan canonical artifact paths are immutable after the first dedup: "
|
|
1179
|
-
f"{', '.join(changed)}."
|
|
1180
|
-
)
|
|
1224
|
+
canonical_discovery_artifacts(scan)
|
|
1181
1225
|
result_manifest_path = deep_scan_path(
|
|
1182
1226
|
scan,
|
|
1183
1227
|
args.result_manifest_path,
|
|
@@ -1226,27 +1270,10 @@ def commit_deep_scan_dedup(
|
|
|
1226
1270
|
connection.execute(
|
|
1227
1271
|
"""
|
|
1228
1272
|
UPDATE deep_scan_runs
|
|
1229
|
-
SET phase = 'discovery', consecutive_no_new = ?,
|
|
1230
|
-
canonical_inventory_path = ?, canonical_finding_report_path = ?,
|
|
1231
|
-
canonical_candidates_path = ?, dedupe_report_path = ?,
|
|
1232
|
-
seed_research_path = ?, work_ledger_path = ?, raw_candidates_path = ?,
|
|
1233
|
-
coverage_ledger_path = ?, findings_dir = ?, updated_at = ?
|
|
1273
|
+
SET phase = 'discovery', consecutive_no_new = ?, updated_at = ?
|
|
1234
1274
|
WHERE scan_id = ?
|
|
1235
1275
|
""",
|
|
1236
|
-
(
|
|
1237
|
-
no_new_streak,
|
|
1238
|
-
canonical_inventory_path,
|
|
1239
|
-
canonical_finding_report_path,
|
|
1240
|
-
canonical_candidates_path,
|
|
1241
|
-
dedupe_report_path,
|
|
1242
|
-
seed_research_path,
|
|
1243
|
-
work_ledger_path,
|
|
1244
|
-
raw_candidates_path,
|
|
1245
|
-
coverage_ledger_path,
|
|
1246
|
-
findings_dir,
|
|
1247
|
-
timestamp,
|
|
1248
|
-
scan_id,
|
|
1249
|
-
),
|
|
1276
|
+
(no_new_streak, timestamp, scan_id),
|
|
1250
1277
|
)
|
|
1251
1278
|
connection.commit()
|
|
1252
1279
|
except BaseException:
|
|
@@ -1318,19 +1345,12 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1318
1345
|
raise SystemExit(
|
|
1319
1346
|
"Deep Scan cannot finish capped before reaching its configured maximum."
|
|
1320
1347
|
)
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
"work_ledger_path",
|
|
1328
|
-
"raw_candidates_path",
|
|
1329
|
-
"coverage_ledger_path",
|
|
1330
|
-
"findings_dir",
|
|
1331
|
-
)
|
|
1332
|
-
if any(run[column] is None for column in canonical_columns):
|
|
1333
|
-
raise SystemExit("Deep Scan cannot finish without canonical discovery artifacts.")
|
|
1348
|
+
try:
|
|
1349
|
+
canonical_discovery_artifacts(scan)
|
|
1350
|
+
except SystemExit as exc:
|
|
1351
|
+
raise SystemExit(
|
|
1352
|
+
f"Deep Scan cannot finish without canonical discovery artifacts: {exc}"
|
|
1353
|
+
) from exc
|
|
1334
1354
|
successful_reducer = connection.execute(
|
|
1335
1355
|
"""
|
|
1336
1356
|
SELECT 1 FROM deep_scan_workers
|
|
@@ -836,6 +836,27 @@ def _recover_unsealed_findings(
|
|
|
836
836
|
parent[field] = normalized
|
|
837
837
|
normalized_fields.append(label)
|
|
838
838
|
|
|
839
|
+
severity = finding.get("severity")
|
|
840
|
+
if isinstance(severity, dict):
|
|
841
|
+
change_conditions = severity.get("changeConditions")
|
|
842
|
+
if (
|
|
843
|
+
isinstance(change_conditions, list)
|
|
844
|
+
and change_conditions
|
|
845
|
+
and all(
|
|
846
|
+
isinstance(condition, str) and condition.strip()
|
|
847
|
+
for condition in change_conditions
|
|
848
|
+
)
|
|
849
|
+
):
|
|
850
|
+
for condition_index, condition in enumerate(change_conditions):
|
|
851
|
+
_require_safe_json_string(
|
|
852
|
+
condition,
|
|
853
|
+
f"{context}.severity.changeConditions[{condition_index}]",
|
|
854
|
+
)
|
|
855
|
+
severity["changeConditions"] = " ".join(
|
|
856
|
+
condition.strip() for condition in change_conditions
|
|
857
|
+
)
|
|
858
|
+
normalized_fields.append("severity change conditions")
|
|
859
|
+
|
|
839
860
|
_populate_unsealed_finding_identities(
|
|
840
861
|
manifest,
|
|
841
862
|
{"scanId": scan_id, "findings": [finding]},
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Generate the shared, deterministically ordered security-scan file inventory."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class InventoryError(ValueError):
|
|
14
|
+
"""Raised when the repository, scope, or inventory cannot be used safely."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def resolve_repository(value: str) -> Path:
|
|
18
|
+
"""Resolve the repository once so every scope is bound to its real root."""
|
|
19
|
+
try:
|
|
20
|
+
repository = Path(value).expanduser().resolve(strict=True)
|
|
21
|
+
except (OSError, ValueError) as error:
|
|
22
|
+
raise InventoryError(f"--repo: cannot resolve repository: {value}") from error
|
|
23
|
+
if not repository.is_dir():
|
|
24
|
+
raise InventoryError(f"--repo: expected a directory: {repository}")
|
|
25
|
+
return repository
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def resolve_scope(repository: Path, value: str) -> str:
|
|
29
|
+
"""Preserve ripgrep's relative path spelling while rejecting escaped scopes."""
|
|
30
|
+
if not value or "\0" in value:
|
|
31
|
+
raise InventoryError("--scope: expected a non-empty file or directory")
|
|
32
|
+
|
|
33
|
+
requested = Path(value).expanduser()
|
|
34
|
+
scope = requested if requested.is_absolute() else repository / requested
|
|
35
|
+
try:
|
|
36
|
+
resolved = scope.resolve(strict=True)
|
|
37
|
+
except (OSError, ValueError) as error:
|
|
38
|
+
raise InventoryError(f"--scope: path does not exist: {value}") from error
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
relative = resolved.relative_to(repository)
|
|
42
|
+
except ValueError as error:
|
|
43
|
+
raise InventoryError(f"--scope: path must remain inside --repo: {value}") from error
|
|
44
|
+
|
|
45
|
+
if not resolved.is_dir() and not resolved.is_file():
|
|
46
|
+
raise InventoryError(f"--scope: expected a file or directory: {value}")
|
|
47
|
+
|
|
48
|
+
if requested.is_absolute():
|
|
49
|
+
return relative.as_posix() if relative.parts else "."
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_output(value: str) -> Path:
|
|
54
|
+
"""Reject direct symlink outputs without constraining the artifact root."""
|
|
55
|
+
if not value or "\0" in value:
|
|
56
|
+
raise InventoryError("--out: expected an inventory file path")
|
|
57
|
+
requested = Path(value).expanduser()
|
|
58
|
+
if requested.is_symlink():
|
|
59
|
+
raise InventoryError("--out: refusing to replace a symbolic link")
|
|
60
|
+
try:
|
|
61
|
+
output = requested.resolve(strict=False)
|
|
62
|
+
except (OSError, ValueError) as error:
|
|
63
|
+
raise InventoryError(f"--out: cannot resolve inventory path: {value}") from error
|
|
64
|
+
if output.exists() and not output.is_file():
|
|
65
|
+
raise InventoryError(f"--out: expected a regular file path: {output}")
|
|
66
|
+
return output
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
70
|
+
"""Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``."""
|
|
71
|
+
command = ["rg", "--files", "--hidden", "--no-ignore", "--glob", "!.git/**", "--", scope]
|
|
72
|
+
with tempfile.TemporaryFile(mode="w+b") as inventory:
|
|
73
|
+
try:
|
|
74
|
+
result = subprocess.run(
|
|
75
|
+
command,
|
|
76
|
+
cwd=repository,
|
|
77
|
+
stdout=inventory,
|
|
78
|
+
stderr=subprocess.PIPE,
|
|
79
|
+
check=False,
|
|
80
|
+
)
|
|
81
|
+
except OSError as error:
|
|
82
|
+
raise InventoryError(f"could not run ripgrep: {error}") from error
|
|
83
|
+
|
|
84
|
+
if result.returncode not in (0, 1):
|
|
85
|
+
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
86
|
+
message = f"ripgrep exited with status {result.returncode}"
|
|
87
|
+
if detail:
|
|
88
|
+
message = f"{message}: {detail}"
|
|
89
|
+
raise InventoryError(message)
|
|
90
|
+
|
|
91
|
+
inventory.seek(0)
|
|
92
|
+
rows = sorted(inventory)
|
|
93
|
+
|
|
94
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
temporary: Path | None = None
|
|
96
|
+
try:
|
|
97
|
+
with tempfile.NamedTemporaryFile(
|
|
98
|
+
mode="wb",
|
|
99
|
+
dir=output.parent,
|
|
100
|
+
prefix=f".{output.name}.",
|
|
101
|
+
suffix=".tmp",
|
|
102
|
+
delete=False,
|
|
103
|
+
) as handle:
|
|
104
|
+
temporary = Path(handle.name)
|
|
105
|
+
handle.writelines(rows)
|
|
106
|
+
temporary.replace(output)
|
|
107
|
+
finally:
|
|
108
|
+
if temporary is not None:
|
|
109
|
+
temporary.unlink(missing_ok=True)
|
|
110
|
+
|
|
111
|
+
return len(rows)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def main() -> None:
|
|
115
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
116
|
+
parser.add_argument("--repo", required=True, help="Repository root.")
|
|
117
|
+
parser.add_argument("--scope", required=True, help="File or directory within the repository.")
|
|
118
|
+
parser.add_argument("--out", required=True, help="Destination for the file inventory.")
|
|
119
|
+
args = parser.parse_args()
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
repository = resolve_repository(args.repo)
|
|
123
|
+
scope = resolve_scope(repository, args.scope)
|
|
124
|
+
output = resolve_output(args.out)
|
|
125
|
+
count = generate_in_scope_files(repository, scope, output)
|
|
126
|
+
except (OSError, ValueError) as error:
|
|
127
|
+
print(f"generate_in_scope_files: {error}", file=sys.stderr)
|
|
128
|
+
raise SystemExit(2) from error
|
|
129
|
+
|
|
130
|
+
print(f"Recorded {count} in-scope files.")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__":
|
|
134
|
+
main()
|