@openai/codex-security 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/_bundled_plugin/.mcp.json +16 -0
- package/_bundled_plugin/mcp/server.mjs +1 -1
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +2 -2
- package/_bundled_plugin/references/config-preflight.md +6 -6
- package/_bundled_plugin/references/desktop-config-preflight.md +11 -0
- package/_bundled_plugin/references/final-report.md +10 -14
- package/_bundled_plugin/references/scan-artifacts.md +17 -17
- package/_bundled_plugin/references/shared-hard-rules.md +2 -2
- package/_bundled_plugin/scripts/deep_scan_workbench.py +422 -48
- package/_bundled_plugin/scripts/generate_rank_input.py +91 -0
- package/_bundled_plugin/scripts/workbench_cli.py +1 -50
- package/_bundled_plugin/scripts/workbench_constants.py +0 -2
- package/_bundled_plugin/scripts/workbench_db.py +7 -289
- package/_bundled_plugin/scripts/workbench_progress.py +11 -0
- package/_bundled_plugin/scripts/workbench_scan_history.py +0 -54
- package/_bundled_plugin/scripts/workbench_target.py +2 -6
- package/_bundled_plugin/scripts/workbench_validation.py +0 -211
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +10 -10
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +25 -43
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +2 -4
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +11 -23
- package/_bundled_plugin/skills/security-scan/SKILL.md +89 -26
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +25 -0
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +11 -38
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +5 -6
- package/_bundled_plugin/skills/triage-finding/SKILL.md +6 -14
- package/_bundled_plugin/skills/validation/SKILL.md +9 -9
- package/dist/api.d.ts +6 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +111 -31
- package/dist/api.js.map +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +77 -17
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +18 -10
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +4 -1
- package/dist/cost.js.map +1 -1
- package/dist/multiscan.d.ts +5 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +206 -26
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts +13 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +122 -92
- package/dist/runtime.js.map +1 -1
- package/package.json +2 -2
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
|
@@ -3,12 +3,14 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
|
+
import json
|
|
6
7
|
import os
|
|
7
8
|
import sqlite3
|
|
8
9
|
import sys
|
|
9
10
|
import tempfile
|
|
10
11
|
import uuid
|
|
11
12
|
from dataclasses import dataclass
|
|
13
|
+
from datetime import datetime, timedelta
|
|
12
14
|
from pathlib import Path
|
|
13
15
|
from typing import Any, Callable
|
|
14
16
|
|
|
@@ -33,6 +35,8 @@ DEEP_SCAN_REPLACEABLE_FAILURE_KINDS = (
|
|
|
33
35
|
)
|
|
34
36
|
DEEP_SCAN_TERMINAL_REASONS = ("saturated", "capped")
|
|
35
37
|
DEEP_SCAN_WORKFLOW_VERSION = "deep-security-scan/v1"
|
|
38
|
+
DEEP_SCAN_COORDINATOR_LEASE_SECONDS = 30
|
|
39
|
+
DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS = 120
|
|
36
40
|
|
|
37
41
|
|
|
38
42
|
def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) -> None:
|
|
@@ -54,6 +58,12 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
54
58
|
get_deep_scan.add_argument("--scan-id", required=True)
|
|
55
59
|
get_deep_scan.add_argument("--thread-id", required=True)
|
|
56
60
|
|
|
61
|
+
claim_coordinator = subparsers.add_parser("claim-deep-scan-coordinator")
|
|
62
|
+
claim_coordinator.add_argument("--scan-id", required=True)
|
|
63
|
+
claim_coordinator.add_argument("--thread-id", required=True)
|
|
64
|
+
claim_coordinator.add_argument("--claim-token")
|
|
65
|
+
claim_coordinator.add_argument("--coordinator-generation", type=positive_int)
|
|
66
|
+
|
|
57
67
|
upsert_deep_worker = subparsers.add_parser("upsert-deep-scan-worker")
|
|
58
68
|
upsert_deep_worker.add_argument("--scan-id", required=True)
|
|
59
69
|
upsert_deep_worker.add_argument("--worker-id", required=True)
|
|
@@ -68,6 +78,7 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
68
78
|
upsert_deep_worker.add_argument(
|
|
69
79
|
"--replaceable-failure-kind", choices=DEEP_SCAN_REPLACEABLE_FAILURE_KINDS
|
|
70
80
|
)
|
|
81
|
+
upsert_deep_worker.add_argument("--coordinator-generation", type=positive_int)
|
|
71
82
|
|
|
72
83
|
claim_deep_dedup = subparsers.add_parser("claim-deep-scan-dedup")
|
|
73
84
|
claim_deep_dedup.add_argument("--scan-id", required=True)
|
|
@@ -75,12 +86,15 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
75
86
|
claim_deep_dedup.add_argument("--prompt-path", required=True)
|
|
76
87
|
claim_deep_dedup.add_argument("--artifact-dir", required=True)
|
|
77
88
|
claim_deep_dedup.add_argument("--input-worker-id", action="append", required=True)
|
|
89
|
+
claim_deep_dedup.add_argument("--coordinator-generation", type=positive_int)
|
|
78
90
|
|
|
79
91
|
commit_deep_dedup = subparsers.add_parser("commit-deep-scan-dedup")
|
|
80
92
|
commit_deep_dedup.add_argument("--scan-id", required=True)
|
|
81
93
|
commit_deep_dedup.add_argument("--worker-id", required=True)
|
|
82
94
|
commit_deep_dedup.add_argument("--result-manifest-path", required=True)
|
|
95
|
+
commit_deep_dedup.add_argument("--candidate-ledger-path")
|
|
83
96
|
commit_deep_dedup.add_argument("--new-findings-count", type=non_negative_int, required=True)
|
|
97
|
+
commit_deep_dedup.add_argument("--coordinator-generation", type=positive_int)
|
|
84
98
|
|
|
85
99
|
finish_deep_scan = subparsers.add_parser("finish-deep-scan")
|
|
86
100
|
finish_deep_scan.add_argument("--scan-id", required=True)
|
|
@@ -88,15 +102,19 @@ def register_subcommands(subparsers: Any, positive_int: Callable[[str], int]) ->
|
|
|
88
102
|
"--terminal-reason", choices=DEEP_SCAN_TERMINAL_REASONS, required=True
|
|
89
103
|
)
|
|
90
104
|
finish_deep_scan.add_argument("--manifest-path", required=True)
|
|
105
|
+
finish_deep_scan.add_argument("--staged-manifest-path")
|
|
91
106
|
finish_deep_scan.add_argument("--omitted-worker-id", action="append", default=[])
|
|
107
|
+
finish_deep_scan.add_argument("--coordinator-generation", type=positive_int)
|
|
92
108
|
|
|
93
109
|
fail_deep_scan = subparsers.add_parser("fail-deep-scan")
|
|
94
110
|
fail_deep_scan.add_argument("--scan-id", required=True)
|
|
95
111
|
fail_deep_scan.add_argument("--message", required=True)
|
|
96
112
|
fail_deep_scan.add_argument("--manifest-path")
|
|
113
|
+
fail_deep_scan.add_argument("--staged-manifest-path")
|
|
97
114
|
fail_deep_scan.add_argument(
|
|
98
115
|
"--deep-status", choices=("failed", "interrupted"), default="failed"
|
|
99
116
|
)
|
|
117
|
+
fail_deep_scan.add_argument("--coordinator-generation", type=positive_int)
|
|
100
118
|
|
|
101
119
|
|
|
102
120
|
def non_negative_int(value: str) -> int:
|
|
@@ -120,6 +138,7 @@ class DeepScanDependencies:
|
|
|
120
138
|
require_canonical_scan_directory: Callable[[Path], Path]
|
|
121
139
|
safe_segment: Callable[[str], str]
|
|
122
140
|
compact_timestamp: Callable[[], str]
|
|
141
|
+
scan_completion_lock: Callable[[str], Any]
|
|
123
142
|
|
|
124
143
|
|
|
125
144
|
_dependencies: DeepScanDependencies | None = None
|
|
@@ -184,6 +203,10 @@ def compact_timestamp() -> str:
|
|
|
184
203
|
return dependencies().compact_timestamp()
|
|
185
204
|
|
|
186
205
|
|
|
206
|
+
def scan_completion_lock(scan_id: str) -> Any:
|
|
207
|
+
return dependencies().scan_completion_lock(scan_id)
|
|
208
|
+
|
|
209
|
+
|
|
187
210
|
def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlite3.Row:
|
|
188
211
|
scan_id = require_uuid(scan_id, "scan-id")
|
|
189
212
|
row = connection.execute(
|
|
@@ -249,6 +272,50 @@ def deep_scan_path(
|
|
|
249
272
|
return str(resolved)
|
|
250
273
|
|
|
251
274
|
|
|
275
|
+
def deep_scan_output_path(scan: sqlite3.Row, value: str, label: str) -> str:
|
|
276
|
+
supplied = Path(value).expanduser()
|
|
277
|
+
if not supplied.is_absolute():
|
|
278
|
+
raise SystemExit(f"{label} must be an absolute path inside the scan directory.")
|
|
279
|
+
if supplied.exists():
|
|
280
|
+
return deep_scan_path(scan, str(supplied), label, kind="file")
|
|
281
|
+
parent = Path(deep_scan_path(scan, str(supplied.parent), label, kind="directory"))
|
|
282
|
+
output = parent / supplied.name
|
|
283
|
+
if os.path.normcase(output) != os.path.normcase(supplied.absolute()):
|
|
284
|
+
raise SystemExit(f"{label} must be a canonical non-symlink path.")
|
|
285
|
+
return str(output)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def promote_staged_file(staged_path: str, output_path: str) -> tuple[Path, Path, Path | None]:
|
|
289
|
+
staged = Path(staged_path)
|
|
290
|
+
output = Path(output_path)
|
|
291
|
+
if staged == output:
|
|
292
|
+
raise SystemExit("A staged Deep Scan artifact must not be its published output path.")
|
|
293
|
+
backup = output.with_name(f".{output.name}.{uuid.uuid4()}.backup") if output.exists() else None
|
|
294
|
+
if backup is not None:
|
|
295
|
+
os.replace(output, backup)
|
|
296
|
+
try:
|
|
297
|
+
os.replace(staged, output)
|
|
298
|
+
except BaseException:
|
|
299
|
+
if backup is not None:
|
|
300
|
+
os.replace(backup, output)
|
|
301
|
+
raise
|
|
302
|
+
return staged, output, backup
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def rollback_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
306
|
+
staged, output, backup = promotion
|
|
307
|
+
if output.exists():
|
|
308
|
+
os.replace(output, staged)
|
|
309
|
+
if backup is not None:
|
|
310
|
+
os.replace(backup, output)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def finish_staged_file(promotion: tuple[Path, Path, Path | None]) -> None:
|
|
314
|
+
backup = promotion[2]
|
|
315
|
+
if backup is not None:
|
|
316
|
+
backup.unlink(missing_ok=True)
|
|
317
|
+
|
|
318
|
+
|
|
252
319
|
def canonical_discovery_artifacts(scan: sqlite3.Row) -> dict[str, str]:
|
|
253
320
|
discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery"
|
|
254
321
|
artifacts = {
|
|
@@ -521,32 +588,6 @@ def terminal_deep_scan_for_target_snapshot(
|
|
|
521
588
|
).fetchone()
|
|
522
589
|
|
|
523
590
|
|
|
524
|
-
def pending_deep_workspace_for_target(
|
|
525
|
-
connection: sqlite3.Connection, thread_id: str, target_path: str, scope: str
|
|
526
|
-
) -> sqlite3.Row | None:
|
|
527
|
-
return connection.execute(
|
|
528
|
-
"""
|
|
529
|
-
SELECT *
|
|
530
|
-
FROM workspaces
|
|
531
|
-
WHERE thread_id = ?
|
|
532
|
-
AND target_path = ?
|
|
533
|
-
AND default_scope = ?
|
|
534
|
-
AND default_mode = 'deep'
|
|
535
|
-
AND active_scan_id IS NULL
|
|
536
|
-
ORDER BY updated_at DESC, created_at DESC, id
|
|
537
|
-
LIMIT 1
|
|
538
|
-
""",
|
|
539
|
-
(thread_id, target_path, scope),
|
|
540
|
-
).fetchone()
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
def setup_ui_opt_out_enabled(connection: sqlite3.Connection) -> bool:
|
|
544
|
-
row = connection.execute(
|
|
545
|
-
"SELECT skip_setup_ui FROM setup_preferences WHERE singleton = 1"
|
|
546
|
-
).fetchone()
|
|
547
|
-
return row is not None and bool(row["skip_setup_ui"])
|
|
548
|
-
|
|
549
|
-
|
|
550
591
|
def begin_deep_scan_for_scan(
|
|
551
592
|
connection: sqlite3.Connection,
|
|
552
593
|
scan_id: str,
|
|
@@ -670,14 +711,6 @@ def begin_deep_scan_for_target(
|
|
|
670
711
|
existing["id"],
|
|
671
712
|
start_disposition="joined" if existing_run is not None else "created",
|
|
672
713
|
)
|
|
673
|
-
pending_workspace = pending_deep_workspace_for_target(
|
|
674
|
-
connection, thread_id, target_path, scope
|
|
675
|
-
)
|
|
676
|
-
if pending_workspace is not None and not setup_ui_opt_out_enabled(connection):
|
|
677
|
-
raise SystemExit(
|
|
678
|
-
"A matching Codex Security setup workspace is waiting for Start scan. "
|
|
679
|
-
"Finish that setup and retry with its scanId."
|
|
680
|
-
)
|
|
681
714
|
current_target = require_remediation_target(target_path)
|
|
682
715
|
current_metadata = current_target.stat()
|
|
683
716
|
if (current_metadata.st_dev, current_metadata.st_ino) != (
|
|
@@ -817,6 +850,224 @@ def get_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> d
|
|
|
817
850
|
return deep_scan_result(connection, scan["id"])
|
|
818
851
|
|
|
819
852
|
|
|
853
|
+
def coordinator_lease_is_live(
|
|
854
|
+
connection: sqlite3.Connection,
|
|
855
|
+
run: sqlite3.Row,
|
|
856
|
+
scan: sqlite3.Row,
|
|
857
|
+
timestamp: str,
|
|
858
|
+
) -> bool:
|
|
859
|
+
if run["coordinator_generation"] == 1:
|
|
860
|
+
active_worker = connection.execute(
|
|
861
|
+
"""
|
|
862
|
+
SELECT 1 FROM deep_scan_workers
|
|
863
|
+
WHERE scan_id = ? AND status IN ('queued', 'running')
|
|
864
|
+
LIMIT 1
|
|
865
|
+
""",
|
|
866
|
+
(run["scan_id"],),
|
|
867
|
+
).fetchone()
|
|
868
|
+
return active_worker is not None and datetime.fromisoformat(
|
|
869
|
+
str(run["updated_at"])
|
|
870
|
+
) > datetime.fromisoformat(timestamp) - timedelta(
|
|
871
|
+
seconds=DEEP_SCAN_LEGACY_COORDINATOR_GRACE_SECONDS
|
|
872
|
+
)
|
|
873
|
+
heartbeat_time = datetime.fromisoformat(str(run["updated_at"]))
|
|
874
|
+
heartbeat_path = (
|
|
875
|
+
Path(scan["scan_dir"])
|
|
876
|
+
/ "artifacts"
|
|
877
|
+
/ "deep_discovery"
|
|
878
|
+
/ f"coordinator-heartbeat-{run['coordinator_generation']}.json"
|
|
879
|
+
)
|
|
880
|
+
try:
|
|
881
|
+
heartbeat = json.loads(heartbeat_path.read_text(encoding="utf-8"))
|
|
882
|
+
if heartbeat["coordinatorGeneration"] == run["coordinator_generation"]:
|
|
883
|
+
heartbeat_time = max(heartbeat_time, datetime.fromisoformat(heartbeat["updatedAt"]))
|
|
884
|
+
except (OSError, KeyError, TypeError, ValueError):
|
|
885
|
+
pass
|
|
886
|
+
current_time = datetime.fromisoformat(timestamp)
|
|
887
|
+
return heartbeat_time > current_time - timedelta(seconds=DEEP_SCAN_COORDINATOR_LEASE_SECONDS)
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def require_current_coordinator(run: sqlite3.Row, args: argparse.Namespace) -> None:
|
|
891
|
+
generation = getattr(args, "coordinator_generation", None)
|
|
892
|
+
if run["coordinator_generation"] == 1:
|
|
893
|
+
if generation is not None:
|
|
894
|
+
raise SystemExit("Deep Scan coordinator lease has not been claimed.")
|
|
895
|
+
return
|
|
896
|
+
if generation is None:
|
|
897
|
+
raise SystemExit("Deep Scan mutation requires the current coordinator lease.")
|
|
898
|
+
if generation != run["coordinator_generation"]:
|
|
899
|
+
raise SystemExit("Deep Scan coordinator lease belongs to a newer generation.")
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def claim_deep_scan_coordinator(
|
|
903
|
+
connection: sqlite3.Connection, args: argparse.Namespace
|
|
904
|
+
) -> dict[str, Any]:
|
|
905
|
+
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
906
|
+
with scan_completion_lock(scan_id):
|
|
907
|
+
return claim_deep_scan_coordinator_locked(connection, args, scan_id)
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def claim_deep_scan_coordinator_locked(
|
|
911
|
+
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
912
|
+
) -> dict[str, Any]:
|
|
913
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
914
|
+
try:
|
|
915
|
+
scan, _ = require_owned_scan(connection, scan_id, args.thread_id)
|
|
916
|
+
require_current_continuation(
|
|
917
|
+
scan,
|
|
918
|
+
args.claim_token,
|
|
919
|
+
error_message="Deep Scan orchestration is owned by another continuation.",
|
|
920
|
+
)
|
|
921
|
+
run, _ = require_running_deep_scan(connection, scan_id)
|
|
922
|
+
timestamp = now()
|
|
923
|
+
if args.coordinator_generation is not None:
|
|
924
|
+
require_current_coordinator(run, args)
|
|
925
|
+
disposition = "claimed"
|
|
926
|
+
elif coordinator_lease_is_live(connection, run, scan, timestamp):
|
|
927
|
+
connection.commit()
|
|
928
|
+
return {
|
|
929
|
+
**deep_scan_result(connection, scan_id),
|
|
930
|
+
"coordinatorDisposition": "observing",
|
|
931
|
+
}
|
|
932
|
+
else:
|
|
933
|
+
adopted = run["coordinator_generation"] > 1 or run["phase"] != "setup"
|
|
934
|
+
if adopted:
|
|
935
|
+
recover_expired_coordinator(connection, run, timestamp)
|
|
936
|
+
disposition = "adopted" if adopted else "claimed"
|
|
937
|
+
|
|
938
|
+
connection.execute(
|
|
939
|
+
"""
|
|
940
|
+
UPDATE deep_scan_runs
|
|
941
|
+
SET coordinator_generation = coordinator_generation + ?, updated_at = ?
|
|
942
|
+
WHERE scan_id = ? AND status = 'running'
|
|
943
|
+
""",
|
|
944
|
+
(int(args.coordinator_generation != run["coordinator_generation"]), timestamp, scan_id),
|
|
945
|
+
)
|
|
946
|
+
connection.commit()
|
|
947
|
+
except BaseException:
|
|
948
|
+
connection.rollback()
|
|
949
|
+
raise
|
|
950
|
+
return {
|
|
951
|
+
**deep_scan_result(connection, scan_id),
|
|
952
|
+
"coordinatorDisposition": disposition,
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def recover_expired_coordinator(
|
|
957
|
+
connection: sqlite3.Connection, run: sqlite3.Row, timestamp: str
|
|
958
|
+
) -> None:
|
|
959
|
+
scan_id = run["scan_id"]
|
|
960
|
+
recover_candidate_ledger_publication(connection, scan_id)
|
|
961
|
+
legacy_generation = int(run["coordinator_generation"] == 1)
|
|
962
|
+
interrupted_discoveries = int(
|
|
963
|
+
connection.execute(
|
|
964
|
+
"""
|
|
965
|
+
SELECT COUNT(*)
|
|
966
|
+
FROM deep_scan_workers
|
|
967
|
+
WHERE scan_id = ? AND kind = 'discovery'
|
|
968
|
+
AND (
|
|
969
|
+
status IN ('queued', 'running')
|
|
970
|
+
OR (
|
|
971
|
+
status = 'canceled'
|
|
972
|
+
AND (
|
|
973
|
+
error_message LIKE 'coordinator_shutdown:%'
|
|
974
|
+
OR (? = 1 AND error_message IS NULL)
|
|
975
|
+
)
|
|
976
|
+
)
|
|
977
|
+
)
|
|
978
|
+
""",
|
|
979
|
+
(scan_id, legacy_generation),
|
|
980
|
+
).fetchone()[0]
|
|
981
|
+
)
|
|
982
|
+
connection.execute(
|
|
983
|
+
"""
|
|
984
|
+
UPDATE deep_scan_workers
|
|
985
|
+
SET merge_state = 'buffered', updated_at = ?
|
|
986
|
+
WHERE scan_id = ? AND merge_state = 'merging'
|
|
987
|
+
AND id IN (
|
|
988
|
+
SELECT inputs.discovery_worker_id
|
|
989
|
+
FROM deep_scan_dedup_inputs AS inputs
|
|
990
|
+
JOIN deep_scan_workers AS reducers ON reducers.id = inputs.dedup_worker_id
|
|
991
|
+
WHERE reducers.scan_id = ?
|
|
992
|
+
AND reducers.kind = 'dedup'
|
|
993
|
+
AND (
|
|
994
|
+
reducers.status IN ('queued', 'running', 'failed')
|
|
995
|
+
OR (
|
|
996
|
+
reducers.status = 'canceled'
|
|
997
|
+
AND (
|
|
998
|
+
reducers.error_message LIKE 'coordinator_shutdown:%'
|
|
999
|
+
OR (? = 1 AND reducers.error_message IS NULL)
|
|
1000
|
+
)
|
|
1001
|
+
)
|
|
1002
|
+
)
|
|
1003
|
+
)
|
|
1004
|
+
""",
|
|
1005
|
+
(timestamp, scan_id, scan_id, legacy_generation),
|
|
1006
|
+
)
|
|
1007
|
+
cancel_active_workers(connection, scan_id, timestamp)
|
|
1008
|
+
connection.execute(
|
|
1009
|
+
"""
|
|
1010
|
+
UPDATE deep_scan_workers
|
|
1011
|
+
SET error_message = 'coordinator_shutdown_recovered: replacement attempt required',
|
|
1012
|
+
updated_at = ?
|
|
1013
|
+
WHERE scan_id = ? AND status = 'canceled'
|
|
1014
|
+
AND (
|
|
1015
|
+
error_message LIKE 'coordinator_shutdown:%'
|
|
1016
|
+
OR (? = 1 AND error_message IS NULL)
|
|
1017
|
+
)
|
|
1018
|
+
""",
|
|
1019
|
+
(timestamp, scan_id, legacy_generation),
|
|
1020
|
+
)
|
|
1021
|
+
connection.execute(
|
|
1022
|
+
"""
|
|
1023
|
+
UPDATE deep_scan_runs
|
|
1024
|
+
SET discovery_runs_dispatched = discovery_runs_dispatched - ?,
|
|
1025
|
+
phase = CASE WHEN phase = 'setup' THEN 'setup' ELSE 'discovery' END,
|
|
1026
|
+
updated_at = ?
|
|
1027
|
+
WHERE scan_id = ?
|
|
1028
|
+
""",
|
|
1029
|
+
(interrupted_discoveries, timestamp, scan_id),
|
|
1030
|
+
)
|
|
1031
|
+
|
|
1032
|
+
|
|
1033
|
+
def recover_candidate_ledger_publication(connection: sqlite3.Connection, scan_id: str) -> None:
|
|
1034
|
+
scan = require_scan(connection, scan_id)
|
|
1035
|
+
ledger = Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "candidate_ledger.jsonl"
|
|
1036
|
+
backups = sorted(
|
|
1037
|
+
ledger.parent.glob(f".{ledger.name}.*.backup"),
|
|
1038
|
+
key=lambda backup: backup.stat().st_mtime_ns,
|
|
1039
|
+
reverse=True,
|
|
1040
|
+
)
|
|
1041
|
+
if not ledger.exists() and not backups:
|
|
1042
|
+
return
|
|
1043
|
+
reducers = connection.execute(
|
|
1044
|
+
"""
|
|
1045
|
+
SELECT status, artifact_dir
|
|
1046
|
+
FROM deep_scan_workers
|
|
1047
|
+
WHERE scan_id = ? AND kind = 'dedup'
|
|
1048
|
+
AND status IN ('queued', 'running', 'succeeded')
|
|
1049
|
+
ORDER BY updated_at DESC
|
|
1050
|
+
""",
|
|
1051
|
+
(scan_id,),
|
|
1052
|
+
)
|
|
1053
|
+
for reducer in reducers:
|
|
1054
|
+
snapshot = Path(reducer["artifact_dir"]) / "canonical" / ledger.name
|
|
1055
|
+
if not snapshot.exists():
|
|
1056
|
+
continue
|
|
1057
|
+
published = ledger.exists() and ledger.samefile(snapshot)
|
|
1058
|
+
interrupted = reducer["status"] != "succeeded"
|
|
1059
|
+
if not published and not (interrupted and backups and not ledger.exists()):
|
|
1060
|
+
continue
|
|
1061
|
+
if interrupted:
|
|
1062
|
+
if backups:
|
|
1063
|
+
os.replace(backups.pop(0), ledger)
|
|
1064
|
+
else:
|
|
1065
|
+
ledger.unlink(missing_ok=True)
|
|
1066
|
+
for backup in backups:
|
|
1067
|
+
backup.unlink(missing_ok=True)
|
|
1068
|
+
return
|
|
1069
|
+
|
|
1070
|
+
|
|
820
1071
|
def require_deep_scan_worker(connection: sqlite3.Connection, worker_id: str) -> sqlite3.Row:
|
|
821
1072
|
worker_id = require_uuid(worker_id, "worker-id")
|
|
822
1073
|
row = connection.execute(
|
|
@@ -859,6 +1110,7 @@ def upsert_deep_scan_worker(
|
|
|
859
1110
|
connection.execute("BEGIN IMMEDIATE")
|
|
860
1111
|
try:
|
|
861
1112
|
run = require_deep_scan_run(connection, scan_id)
|
|
1113
|
+
require_current_coordinator(run, args)
|
|
862
1114
|
scan = require_scan(connection, scan_id)
|
|
863
1115
|
existing = connection.execute(
|
|
864
1116
|
"SELECT * FROM deep_scan_workers WHERE id = ?", (worker_id,)
|
|
@@ -1073,6 +1325,7 @@ def claim_deep_scan_dedup(
|
|
|
1073
1325
|
connection.execute("BEGIN IMMEDIATE")
|
|
1074
1326
|
try:
|
|
1075
1327
|
run, scan = require_running_deep_scan(connection, scan_id)
|
|
1328
|
+
require_current_coordinator(run, args)
|
|
1076
1329
|
prompt_path = deep_scan_path(scan, args.prompt_path, "Dedup prompt path", kind="file")
|
|
1077
1330
|
artifact_dir = deep_scan_path(
|
|
1078
1331
|
scan, args.artifact_dir, "Dedup artifact directory", kind="directory"
|
|
@@ -1207,10 +1460,20 @@ def commit_deep_scan_dedup(
|
|
|
1207
1460
|
connection: sqlite3.Connection, args: argparse.Namespace
|
|
1208
1461
|
) -> dict[str, Any]:
|
|
1209
1462
|
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1463
|
+
with scan_completion_lock(scan_id):
|
|
1464
|
+
return commit_deep_scan_dedup_locked(connection, args, scan_id)
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def commit_deep_scan_dedup_locked(
|
|
1468
|
+
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1469
|
+
) -> dict[str, Any]:
|
|
1210
1470
|
worker_id = require_uuid(args.worker_id, "worker-id")
|
|
1471
|
+
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1472
|
+
publication_copy: Path | None = None
|
|
1211
1473
|
connection.execute("BEGIN IMMEDIATE")
|
|
1212
1474
|
try:
|
|
1213
1475
|
run = require_deep_scan_run(connection, scan_id)
|
|
1476
|
+
require_current_coordinator(run, args)
|
|
1214
1477
|
scan = require_scan(connection, scan_id)
|
|
1215
1478
|
worker = require_deep_scan_worker(connection, worker_id)
|
|
1216
1479
|
if worker["scan_id"] != scan_id or worker["kind"] != "dedup":
|
|
@@ -1221,7 +1484,29 @@ def commit_deep_scan_dedup(
|
|
|
1221
1484
|
require_running_deep_scan(connection, scan_id)
|
|
1222
1485
|
if worker["status"] not in {"queued", "running"}:
|
|
1223
1486
|
raise SystemExit("Only an active dedup worker can commit a result.")
|
|
1224
|
-
|
|
1487
|
+
if args.candidate_ledger_path:
|
|
1488
|
+
candidate_ledger_path = deep_scan_path(
|
|
1489
|
+
scan,
|
|
1490
|
+
args.candidate_ledger_path,
|
|
1491
|
+
"Staged candidate ledger path",
|
|
1492
|
+
kind="file",
|
|
1493
|
+
)
|
|
1494
|
+
discovery_dir = Path(scan["scan_dir"]) / "artifacts" / "02_discovery"
|
|
1495
|
+
deep_scan_path(
|
|
1496
|
+
scan,
|
|
1497
|
+
str(discovery_dir / "in_scope_files.txt"),
|
|
1498
|
+
"Canonical in-scope inventory path",
|
|
1499
|
+
kind="file",
|
|
1500
|
+
)
|
|
1501
|
+
canonical_candidate_ledger_path = deep_scan_output_path(
|
|
1502
|
+
scan,
|
|
1503
|
+
str(discovery_dir / "candidate_ledger.jsonl"),
|
|
1504
|
+
"Canonical candidate ledger path",
|
|
1505
|
+
)
|
|
1506
|
+
else:
|
|
1507
|
+
canonical_discovery_artifacts(scan)
|
|
1508
|
+
candidate_ledger_path = None
|
|
1509
|
+
canonical_candidate_ledger_path = None
|
|
1225
1510
|
result_manifest_path = deep_scan_path(
|
|
1226
1511
|
scan,
|
|
1227
1512
|
args.result_manifest_path,
|
|
@@ -1242,6 +1527,16 @@ def commit_deep_scan_dedup(
|
|
|
1242
1527
|
)
|
|
1243
1528
|
if not inputs or any(row["merge_state"] != "merging" for row in inputs):
|
|
1244
1529
|
raise SystemExit("Dedup inputs are not in the claimed merging state.")
|
|
1530
|
+
if candidate_ledger_path and canonical_candidate_ledger_path:
|
|
1531
|
+
canonical_path = Path(canonical_candidate_ledger_path)
|
|
1532
|
+
publication_copy = canonical_path.with_name(
|
|
1533
|
+
f".{canonical_path.name}.{uuid.uuid4()}.publish"
|
|
1534
|
+
)
|
|
1535
|
+
os.link(candidate_ledger_path, publication_copy)
|
|
1536
|
+
promotion = promote_staged_file(
|
|
1537
|
+
str(publication_copy),
|
|
1538
|
+
canonical_candidate_ledger_path,
|
|
1539
|
+
)
|
|
1245
1540
|
timestamp = now()
|
|
1246
1541
|
connection.execute(
|
|
1247
1542
|
"""
|
|
@@ -1278,23 +1573,44 @@ def commit_deep_scan_dedup(
|
|
|
1278
1573
|
connection.commit()
|
|
1279
1574
|
except BaseException:
|
|
1280
1575
|
connection.rollback()
|
|
1576
|
+
if promotion is not None:
|
|
1577
|
+
rollback_staged_file(promotion)
|
|
1578
|
+
if publication_copy is not None:
|
|
1579
|
+
publication_copy.unlink(missing_ok=True)
|
|
1281
1580
|
raise
|
|
1581
|
+
if promotion is not None:
|
|
1582
|
+
finish_staged_file(promotion)
|
|
1583
|
+
if publication_copy is not None:
|
|
1584
|
+
publication_copy.unlink(missing_ok=True)
|
|
1282
1585
|
return deep_scan_result(connection, scan_id)
|
|
1283
1586
|
|
|
1284
1587
|
|
|
1285
1588
|
def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
1286
1589
|
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1590
|
+
with scan_completion_lock(scan_id):
|
|
1591
|
+
return finish_deep_scan_locked(connection, args, scan_id)
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def finish_deep_scan_locked(
|
|
1595
|
+
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1596
|
+
) -> dict[str, Any]:
|
|
1287
1597
|
omitted_worker_ids = [
|
|
1288
1598
|
require_uuid(value, "omitted-worker-id") for value in args.omitted_worker_id
|
|
1289
1599
|
]
|
|
1290
1600
|
if len(set(omitted_worker_ids)) != len(omitted_worker_ids):
|
|
1291
1601
|
raise SystemExit("Omitted Deep Scan worker IDs must be unique.")
|
|
1602
|
+
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1292
1603
|
connection.execute("BEGIN IMMEDIATE")
|
|
1293
1604
|
try:
|
|
1294
1605
|
run = require_deep_scan_run(connection, scan_id)
|
|
1606
|
+
require_current_coordinator(run, args)
|
|
1295
1607
|
scan = require_scan(connection, scan_id)
|
|
1296
|
-
manifest_path =
|
|
1297
|
-
scan, args.manifest_path, "Deep Scan coordinator manifest path"
|
|
1608
|
+
manifest_path = (
|
|
1609
|
+
deep_scan_output_path(scan, args.manifest_path, "Deep Scan coordinator manifest path")
|
|
1610
|
+
if args.staged_manifest_path
|
|
1611
|
+
else deep_scan_path(
|
|
1612
|
+
scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file"
|
|
1613
|
+
)
|
|
1298
1614
|
)
|
|
1299
1615
|
buffered_worker_ids = [
|
|
1300
1616
|
row["id"]
|
|
@@ -1363,8 +1679,31 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1363
1679
|
raise SystemExit("Deep Scan cannot finish without a successful dedup worker.")
|
|
1364
1680
|
failed_worker = connection.execute(
|
|
1365
1681
|
"""
|
|
1366
|
-
SELECT 1 FROM deep_scan_workers
|
|
1367
|
-
WHERE scan_id = ? AND status = 'failed'
|
|
1682
|
+
SELECT 1 FROM deep_scan_workers AS failed
|
|
1683
|
+
WHERE failed.scan_id = ? AND failed.status = 'failed'
|
|
1684
|
+
AND (
|
|
1685
|
+
failed.kind != 'dedup'
|
|
1686
|
+
OR NOT EXISTS (
|
|
1687
|
+
SELECT 1 FROM deep_scan_dedup_inputs AS failed_inputs
|
|
1688
|
+
WHERE failed_inputs.dedup_worker_id = failed.id
|
|
1689
|
+
)
|
|
1690
|
+
OR EXISTS (
|
|
1691
|
+
SELECT 1 FROM deep_scan_dedup_inputs AS failed_inputs
|
|
1692
|
+
WHERE failed_inputs.dedup_worker_id = failed.id
|
|
1693
|
+
AND NOT EXISTS (
|
|
1694
|
+
SELECT 1
|
|
1695
|
+
FROM deep_scan_dedup_inputs AS replacement_inputs
|
|
1696
|
+
JOIN deep_scan_workers AS replacement
|
|
1697
|
+
ON replacement.scan_id = replacement_inputs.scan_id
|
|
1698
|
+
AND replacement.id = replacement_inputs.dedup_worker_id
|
|
1699
|
+
WHERE replacement_inputs.scan_id = failed.scan_id
|
|
1700
|
+
AND replacement_inputs.discovery_worker_id =
|
|
1701
|
+
failed_inputs.discovery_worker_id
|
|
1702
|
+
AND replacement.kind = 'dedup'
|
|
1703
|
+
AND replacement.status = 'succeeded'
|
|
1704
|
+
)
|
|
1705
|
+
)
|
|
1706
|
+
)
|
|
1368
1707
|
LIMIT 1
|
|
1369
1708
|
""",
|
|
1370
1709
|
(scan_id,),
|
|
@@ -1402,6 +1741,14 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1402
1741
|
"Deep Scan saturated completion must exactly identify all buffered discovery "
|
|
1403
1742
|
"workers with --omitted-worker-id."
|
|
1404
1743
|
)
|
|
1744
|
+
if args.staged_manifest_path:
|
|
1745
|
+
staged_manifest_path = deep_scan_path(
|
|
1746
|
+
scan,
|
|
1747
|
+
args.staged_manifest_path,
|
|
1748
|
+
"Staged Deep Scan coordinator manifest path",
|
|
1749
|
+
kind="file",
|
|
1750
|
+
)
|
|
1751
|
+
promotion = promote_staged_file(staged_manifest_path, manifest_path)
|
|
1405
1752
|
timestamp = now()
|
|
1406
1753
|
connection.execute(
|
|
1407
1754
|
"""
|
|
@@ -1416,29 +1763,44 @@ def finish_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -
|
|
|
1416
1763
|
connection.commit()
|
|
1417
1764
|
except BaseException:
|
|
1418
1765
|
connection.rollback()
|
|
1766
|
+
if promotion is not None:
|
|
1767
|
+
rollback_staged_file(promotion)
|
|
1419
1768
|
raise
|
|
1769
|
+
if promotion is not None:
|
|
1770
|
+
finish_staged_file(promotion)
|
|
1420
1771
|
return deep_scan_result(connection, scan_id)
|
|
1421
1772
|
|
|
1422
1773
|
|
|
1423
1774
|
def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
1424
1775
|
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1776
|
+
with scan_completion_lock(scan_id):
|
|
1777
|
+
return fail_deep_scan_locked(connection, args, scan_id)
|
|
1778
|
+
|
|
1779
|
+
|
|
1780
|
+
def fail_deep_scan_locked(
|
|
1781
|
+
connection: sqlite3.Connection, args: argparse.Namespace, scan_id: str
|
|
1782
|
+
) -> dict[str, Any]:
|
|
1425
1783
|
message = optional_text(args.message, maximum=2400)
|
|
1426
1784
|
if message is None:
|
|
1427
1785
|
raise SystemExit("message is required.")
|
|
1786
|
+
promotion: tuple[Path, Path, Path | None] | None = None
|
|
1428
1787
|
connection.execute("BEGIN IMMEDIATE")
|
|
1429
1788
|
try:
|
|
1430
1789
|
run = require_deep_scan_run(connection, scan_id)
|
|
1790
|
+
require_current_coordinator(run, args)
|
|
1431
1791
|
scan = require_scan(connection, scan_id)
|
|
1432
|
-
manifest_path =
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
args.manifest_path,
|
|
1436
|
-
|
|
1437
|
-
|
|
1792
|
+
manifest_path = None
|
|
1793
|
+
if args.manifest_path:
|
|
1794
|
+
manifest_path = (
|
|
1795
|
+
deep_scan_output_path(scan, args.manifest_path, "Deep Scan failure manifest path")
|
|
1796
|
+
if args.staged_manifest_path
|
|
1797
|
+
else deep_scan_path(
|
|
1798
|
+
scan,
|
|
1799
|
+
args.manifest_path,
|
|
1800
|
+
"Deep Scan failure manifest path",
|
|
1801
|
+
kind="file",
|
|
1802
|
+
)
|
|
1438
1803
|
)
|
|
1439
|
-
if args.manifest_path
|
|
1440
|
-
else None
|
|
1441
|
-
)
|
|
1442
1804
|
if run["status"] in {"failed", "interrupted"} or scan["status"] == "failed":
|
|
1443
1805
|
if (
|
|
1444
1806
|
run["status"] == args.deep_status
|
|
@@ -1465,6 +1827,14 @@ def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) ->
|
|
|
1465
1827
|
raise SystemExit("Only a running Deep Scan can be failed or interrupted.")
|
|
1466
1828
|
if run["manifest_path"] not in {None, manifest_path}:
|
|
1467
1829
|
raise SystemExit("Deep Scan coordinator manifest path is immutable.")
|
|
1830
|
+
if args.staged_manifest_path and manifest_path:
|
|
1831
|
+
staged_manifest_path = deep_scan_path(
|
|
1832
|
+
scan,
|
|
1833
|
+
args.staged_manifest_path,
|
|
1834
|
+
"Staged Deep Scan failure manifest path",
|
|
1835
|
+
kind="file",
|
|
1836
|
+
)
|
|
1837
|
+
promotion = promote_staged_file(staged_manifest_path, manifest_path)
|
|
1468
1838
|
timestamp = now()
|
|
1469
1839
|
connection.execute(
|
|
1470
1840
|
"""
|
|
@@ -1493,7 +1863,11 @@ def fail_deep_scan(connection: sqlite3.Connection, args: argparse.Namespace) ->
|
|
|
1493
1863
|
connection.commit()
|
|
1494
1864
|
except BaseException:
|
|
1495
1865
|
connection.rollback()
|
|
1866
|
+
if promotion is not None:
|
|
1867
|
+
rollback_staged_file(promotion)
|
|
1496
1868
|
raise
|
|
1869
|
+
if promotion is not None:
|
|
1870
|
+
finish_staged_file(promotion)
|
|
1497
1871
|
return deep_scan_result(connection, scan_id)
|
|
1498
1872
|
|
|
1499
1873
|
|