@openai/codex-security 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -13
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +8 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +6 -0
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -57
- package/_bundled_plugin/references/config-preflight.md +1 -1
- package/_bundled_plugin/references/final-report.md +15 -7
- package/_bundled_plugin/references/finding-detail-fields.md +10 -2
- package/_bundled_plugin/references/scan-artifacts.md +11 -9
- package/_bundled_plugin/references/scan-contract.md +3 -1
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +42 -0
- package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +100 -0
- package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +181 -0
- package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +166 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +120 -0
- package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +83 -0
- package/_bundled_plugin/schemas/tools/review-items.schema.json +80 -0
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +572 -0
- package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +18 -0
- package/_bundled_plugin/scripts/deep_scan_config.py +11 -3
- package/_bundled_plugin/scripts/deep_scan_workbench.py +153 -133
- package/_bundled_plugin/scripts/finalize_scan_contract.py +39 -0
- package/_bundled_plugin/scripts/generate_in_scope_files.py +134 -0
- package/_bundled_plugin/scripts/workbench_cli.py +33 -1
- package/_bundled_plugin/scripts/workbench_db.py +181 -118
- package/_bundled_plugin/scripts/workbench_feedback.py +2 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +5 -49
- package/_bundled_plugin/scripts/workbench_progress.py +86 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +19 -1
- package/_bundled_plugin/scripts/workbench_scan_history.py +7 -1
- package/_bundled_plugin/scripts/workbench_scan_start.py +11 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +557 -0
- package/_bundled_plugin/scripts/workbench_schema.py +501 -0
- package/_bundled_plugin/scripts/workbench_target.py +14 -1
- package/_bundled_plugin/scripts/workbench_validation.py +117 -13
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +4 -4
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +34 -28
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +1 -1
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +11 -8
- package/_bundled_plugin/skills/security-scan/SKILL.md +16 -9
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +26 -14
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +1 -1
- package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
- package/_bundled_plugin/skills/validation/SKILL.md +5 -5
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +145 -227
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +53 -68
- package/dist/api.d.ts +30 -8
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +438 -66
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +262 -44
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +4 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +713 -148
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +28 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +39 -2
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +14 -0
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +7 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +397 -26
- package/dist/cost.js.map +1 -1
- package/dist/errors.d.ts +2 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +48 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +5 -7
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts +40 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +742 -46
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.d.ts +13 -0
- package/dist/scan-activity.d.ts.map +1 -0
- package/dist/scan-activity.js +319 -0
- package/dist/scan-activity.js.map +1 -0
- package/dist/scan-dashboard.d.ts +47 -0
- package/dist/scan-dashboard.d.ts.map +1 -0
- package/dist/scan-dashboard.js +444 -0
- package/dist/scan-dashboard.js.map +1 -0
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +4 -1
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/dist/worker-progress.d.ts +7 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +68 -0
- package/dist/worker-progress.js.map +1 -1
- package/package.json +3 -3
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
4
|
import sqlite3
|
|
5
|
+
from collections.abc import Callable
|
|
5
6
|
|
|
6
7
|
MIGRATIONS = (
|
|
7
8
|
(
|
|
@@ -604,16 +605,267 @@ MIGRATIONS = (
|
|
|
604
605
|
),
|
|
605
606
|
(
|
|
606
607
|
25,
|
|
608
|
+
"persist scan model settings",
|
|
609
|
+
"""
|
|
610
|
+
ALTER TABLE scans ADD COLUMN model TEXT;
|
|
611
|
+
ALTER TABLE scans ADD COLUMN reasoning_effort TEXT;
|
|
612
|
+
""",
|
|
613
|
+
),
|
|
614
|
+
(
|
|
615
|
+
26,
|
|
607
616
|
"persist scan completion warnings",
|
|
608
617
|
"""
|
|
609
618
|
ALTER TABLE scans
|
|
610
619
|
ADD COLUMN completion_warnings_json TEXT NOT NULL DEFAULT '[]';
|
|
611
620
|
""",
|
|
612
621
|
),
|
|
622
|
+
(
|
|
623
|
+
27,
|
|
624
|
+
"persist deep scan consecutive discovery failures",
|
|
625
|
+
"""
|
|
626
|
+
ALTER TABLE deep_scan_runs
|
|
627
|
+
ADD COLUMN stop_after_consecutive_errors INTEGER NOT NULL DEFAULT 1
|
|
628
|
+
CHECK (stop_after_consecutive_errors >= 1);
|
|
629
|
+
|
|
630
|
+
ALTER TABLE deep_scan_runs
|
|
631
|
+
ADD COLUMN consecutive_errors INTEGER NOT NULL DEFAULT 0
|
|
632
|
+
CHECK (consecutive_errors >= 0);
|
|
633
|
+
|
|
634
|
+
UPDATE deep_scan_runs
|
|
635
|
+
SET stop_after_consecutive_errors = stop_after_no_new;
|
|
636
|
+
""",
|
|
637
|
+
),
|
|
613
638
|
)
|
|
614
639
|
|
|
615
640
|
|
|
641
|
+
def apply_migrations(
|
|
642
|
+
connection: sqlite3.Connection,
|
|
643
|
+
migrations: tuple[tuple[int, str, str], ...],
|
|
644
|
+
now: Callable[[], str],
|
|
645
|
+
backfill_security_targets: Callable[[sqlite3.Connection], None],
|
|
646
|
+
) -> None:
|
|
647
|
+
connection.commit()
|
|
648
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
649
|
+
try:
|
|
650
|
+
connection.execute(
|
|
651
|
+
"""
|
|
652
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
653
|
+
version INTEGER PRIMARY KEY,
|
|
654
|
+
name TEXT NOT NULL,
|
|
655
|
+
applied_at TEXT NOT NULL
|
|
656
|
+
)
|
|
657
|
+
"""
|
|
658
|
+
)
|
|
659
|
+
normalize_pre_release_migrations(connection, now())
|
|
660
|
+
applied = {
|
|
661
|
+
row["version"] for row in connection.execute("SELECT version FROM schema_migrations")
|
|
662
|
+
}
|
|
663
|
+
should_backfill_targets = False
|
|
664
|
+
for version, name, sql in migrations:
|
|
665
|
+
if version in applied:
|
|
666
|
+
if version == 2:
|
|
667
|
+
add_column_if_missing(
|
|
668
|
+
connection, "workspaces", "capability_preflight_json", "TEXT"
|
|
669
|
+
)
|
|
670
|
+
elif version == 6:
|
|
671
|
+
repair_thread_scoped_workspaces_migration(connection)
|
|
672
|
+
elif version == 11:
|
|
673
|
+
repair_deep_scan_migration(connection)
|
|
674
|
+
elif version == 12:
|
|
675
|
+
add_column_if_missing(connection, "scans", "continuation_thread_id", "TEXT")
|
|
676
|
+
elif version == 13:
|
|
677
|
+
add_column_if_missing(
|
|
678
|
+
connection,
|
|
679
|
+
"scan_progress",
|
|
680
|
+
"scope_file_count",
|
|
681
|
+
"INTEGER CHECK (scope_file_count >= 0)",
|
|
682
|
+
)
|
|
683
|
+
elif version == 16:
|
|
684
|
+
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
685
|
+
elif version == 26:
|
|
686
|
+
add_column_if_missing(
|
|
687
|
+
connection,
|
|
688
|
+
"scans",
|
|
689
|
+
"completion_warnings_json",
|
|
690
|
+
"TEXT NOT NULL DEFAULT '[]'",
|
|
691
|
+
)
|
|
692
|
+
continue
|
|
693
|
+
if version == 6:
|
|
694
|
+
repair_thread_scoped_workspaces_migration(connection)
|
|
695
|
+
elif version == 16:
|
|
696
|
+
should_backfill_targets = repair_stable_targets_migration(connection)
|
|
697
|
+
else:
|
|
698
|
+
for statement in sql_statements(sql):
|
|
699
|
+
connection.execute(statement)
|
|
700
|
+
connection.execute(
|
|
701
|
+
"INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
|
|
702
|
+
(version, name, now()),
|
|
703
|
+
)
|
|
704
|
+
if 27 in applied:
|
|
705
|
+
repair_deep_scan_failure_counter_migration(connection)
|
|
706
|
+
if should_backfill_targets:
|
|
707
|
+
backfill_security_targets(connection)
|
|
708
|
+
connection.commit()
|
|
709
|
+
except BaseException:
|
|
710
|
+
connection.rollback()
|
|
711
|
+
raise
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def normalize_pre_release_execution_profile_migrations(
|
|
715
|
+
connection: sqlite3.Connection, timestamp: str
|
|
716
|
+
) -> None:
|
|
717
|
+
execution_migrations = {
|
|
718
|
+
row["version"]: row["name"]
|
|
719
|
+
for row in connection.execute(
|
|
720
|
+
"SELECT version, name FROM schema_migrations WHERE version IN (11, 12, 25, 26)"
|
|
721
|
+
)
|
|
722
|
+
}
|
|
723
|
+
legacy_names = {
|
|
724
|
+
11: "scan execution profiles",
|
|
725
|
+
12: "dynamic scan execution profiles",
|
|
726
|
+
}
|
|
727
|
+
released_names = {
|
|
728
|
+
11: "deep scan orchestration state",
|
|
729
|
+
12: "scan continuation threads",
|
|
730
|
+
}
|
|
731
|
+
model_migration_name = "persist scan model settings"
|
|
732
|
+
warnings_migration_name = "persist scan completion warnings"
|
|
733
|
+
if execution_migrations.get(25) == "dynamic scan execution profiles":
|
|
734
|
+
connection.execute(
|
|
735
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 25 AND name = ?",
|
|
736
|
+
(model_migration_name, "dynamic scan execution profiles"),
|
|
737
|
+
)
|
|
738
|
+
execution_migrations[25] = model_migration_name
|
|
739
|
+
has_legacy_profile_history = execution_migrations.get(11) == legacy_names[11]
|
|
740
|
+
has_public_warnings_history = (
|
|
741
|
+
execution_migrations.get(25) == warnings_migration_name
|
|
742
|
+
)
|
|
743
|
+
|
|
744
|
+
scan_columns = {
|
|
745
|
+
row["name"] for row in connection.execute("PRAGMA table_info(scans)")
|
|
746
|
+
}
|
|
747
|
+
workspace_columns = {
|
|
748
|
+
row["name"] for row in connection.execute("PRAGMA table_info(workspaces)")
|
|
749
|
+
}
|
|
750
|
+
has_legacy_profile_columns = (
|
|
751
|
+
"execution_model" in scan_columns or "execution_model" in workspace_columns
|
|
752
|
+
)
|
|
753
|
+
if not (
|
|
754
|
+
has_legacy_profile_history
|
|
755
|
+
or has_public_warnings_history
|
|
756
|
+
or has_legacy_profile_columns
|
|
757
|
+
):
|
|
758
|
+
return
|
|
759
|
+
|
|
760
|
+
if (
|
|
761
|
+
any(
|
|
762
|
+
execution_migrations.get(version)
|
|
763
|
+
not in (None, released_names[version], legacy_names[version])
|
|
764
|
+
for version in (11, 12)
|
|
765
|
+
)
|
|
766
|
+
or (
|
|
767
|
+
has_legacy_profile_history
|
|
768
|
+
and execution_migrations.get(12)
|
|
769
|
+
not in (None, legacy_names[12], released_names[12])
|
|
770
|
+
)
|
|
771
|
+
or (
|
|
772
|
+
not has_legacy_profile_history
|
|
773
|
+
and execution_migrations.get(12) == legacy_names[12]
|
|
774
|
+
)
|
|
775
|
+
):
|
|
776
|
+
raise SystemExit(
|
|
777
|
+
"The Codex Security database has an unsupported execution-profile migration history."
|
|
778
|
+
)
|
|
779
|
+
|
|
780
|
+
expected_legacy_columns = {"execution_model", "reasoning_effort"}
|
|
781
|
+
renamed_legacy_columns = {
|
|
782
|
+
"legacy_execution_model",
|
|
783
|
+
"legacy_reasoning_effort",
|
|
784
|
+
}
|
|
785
|
+
if has_legacy_profile_columns and (
|
|
786
|
+
not expected_legacy_columns.issubset(scan_columns)
|
|
787
|
+
or not expected_legacy_columns.issubset(workspace_columns)
|
|
788
|
+
or renamed_legacy_columns.intersection(scan_columns)
|
|
789
|
+
or renamed_legacy_columns.intersection(workspace_columns)
|
|
790
|
+
):
|
|
791
|
+
raise SystemExit(
|
|
792
|
+
"The Codex Security database has an unsupported execution-profile migration history."
|
|
793
|
+
)
|
|
794
|
+
if has_legacy_profile_history and not has_legacy_profile_columns:
|
|
795
|
+
raise SystemExit(
|
|
796
|
+
"The Codex Security database has an unsupported execution-profile migration history."
|
|
797
|
+
)
|
|
798
|
+
|
|
799
|
+
if has_public_warnings_history:
|
|
800
|
+
if execution_migrations.get(26) is not None:
|
|
801
|
+
raise SystemExit(
|
|
802
|
+
"The Codex Security database has an unsupported pre-release migration history."
|
|
803
|
+
)
|
|
804
|
+
connection.execute(
|
|
805
|
+
"UPDATE schema_migrations SET version = 26 WHERE version = 25 AND name = ?",
|
|
806
|
+
(warnings_migration_name,),
|
|
807
|
+
)
|
|
808
|
+
execution_migrations.pop(25)
|
|
809
|
+
|
|
810
|
+
if execution_migrations.get(25) not in (None, model_migration_name):
|
|
811
|
+
raise SystemExit(
|
|
812
|
+
"The Codex Security database has an unsupported execution-profile migration history."
|
|
813
|
+
)
|
|
814
|
+
|
|
815
|
+
# Keep the historical values and constraints for recovery while moving
|
|
816
|
+
# them out of the namespace used by the current independent scan settings.
|
|
817
|
+
for table in ("workspaces", "scans") if has_legacy_profile_columns else ():
|
|
818
|
+
connection.execute(
|
|
819
|
+
f"ALTER TABLE {table} RENAME COLUMN execution_model TO legacy_execution_model"
|
|
820
|
+
)
|
|
821
|
+
connection.execute(
|
|
822
|
+
f"ALTER TABLE {table} RENAME COLUMN reasoning_effort TO legacy_reasoning_effort"
|
|
823
|
+
)
|
|
824
|
+
add_column_if_missing(connection, "scans", "model", "TEXT")
|
|
825
|
+
add_column_if_missing(connection, "scans", "reasoning_effort", "TEXT")
|
|
826
|
+
if has_legacy_profile_columns:
|
|
827
|
+
connection.execute(
|
|
828
|
+
"""
|
|
829
|
+
UPDATE scans
|
|
830
|
+
SET model = COALESCE(model, legacy_execution_model),
|
|
831
|
+
reasoning_effort = legacy_reasoning_effort
|
|
832
|
+
WHERE legacy_execution_model IS NOT NULL
|
|
833
|
+
OR legacy_reasoning_effort IS NOT NULL
|
|
834
|
+
"""
|
|
835
|
+
)
|
|
836
|
+
if has_legacy_profile_history:
|
|
837
|
+
for version, name in legacy_names.items():
|
|
838
|
+
connection.execute(
|
|
839
|
+
"DELETE FROM schema_migrations WHERE version = ? AND name = ?",
|
|
840
|
+
(version, name),
|
|
841
|
+
)
|
|
842
|
+
if execution_migrations.get(25) is None:
|
|
843
|
+
connection.execute(
|
|
844
|
+
"INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?)",
|
|
845
|
+
(25, model_migration_name, timestamp),
|
|
846
|
+
)
|
|
847
|
+
|
|
848
|
+
|
|
616
849
|
def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: str) -> None:
|
|
850
|
+
completion_warning_migration = connection.execute(
|
|
851
|
+
"SELECT name FROM schema_migrations WHERE version = 25"
|
|
852
|
+
).fetchone()
|
|
853
|
+
if (
|
|
854
|
+
completion_warning_migration is not None
|
|
855
|
+
and completion_warning_migration["name"] == "persist scan completion warnings"
|
|
856
|
+
):
|
|
857
|
+
if (
|
|
858
|
+
connection.execute("SELECT 1 FROM schema_migrations WHERE version = 26").fetchone()
|
|
859
|
+
is not None
|
|
860
|
+
):
|
|
861
|
+
raise SystemExit(
|
|
862
|
+
"The Codex Security database has an unsupported pre-release migration history."
|
|
863
|
+
)
|
|
864
|
+
connection.execute(
|
|
865
|
+
"UPDATE schema_migrations SET version = 26 WHERE version = 25 AND name = ?",
|
|
866
|
+
("persist scan completion warnings",),
|
|
867
|
+
)
|
|
868
|
+
|
|
617
869
|
phase_progress_migration = connection.execute(
|
|
618
870
|
"SELECT name FROM schema_migrations WHERE version = 12"
|
|
619
871
|
).fetchone()
|
|
@@ -633,6 +885,8 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp:
|
|
|
633
885
|
("phase-specific scan progress",),
|
|
634
886
|
)
|
|
635
887
|
|
|
888
|
+
normalize_pre_release_execution_profile_migrations(connection, timestamp)
|
|
889
|
+
|
|
636
890
|
preflight_progress_migration = connection.execute(
|
|
637
891
|
"SELECT name FROM schema_migrations WHERE version = 13"
|
|
638
892
|
).fetchone()
|
|
@@ -652,6 +906,132 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp:
|
|
|
652
906
|
("current scan preflight state",),
|
|
653
907
|
)
|
|
654
908
|
|
|
909
|
+
delivered_claim_migration = connection.execute(
|
|
910
|
+
"SELECT name FROM schema_migrations WHERE version = 18"
|
|
911
|
+
).fetchone()
|
|
912
|
+
if (
|
|
913
|
+
delivered_claim_migration is not None
|
|
914
|
+
and delivered_claim_migration["name"] == "scan target summaries"
|
|
915
|
+
):
|
|
916
|
+
connection.execute(
|
|
917
|
+
"UPDATE scans SET handoff_claimed_at = NULL, handoff_claim_token = NULL "
|
|
918
|
+
"WHERE handoff_status = 'delivered'"
|
|
919
|
+
)
|
|
920
|
+
connection.execute(
|
|
921
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 18 AND name = ?",
|
|
922
|
+
("clear legacy delivered handoff claims", "scan target summaries"),
|
|
923
|
+
)
|
|
924
|
+
|
|
925
|
+
setup_preferences_migration = connection.execute(
|
|
926
|
+
"SELECT name FROM schema_migrations WHERE version = 19"
|
|
927
|
+
).fetchone()
|
|
928
|
+
legacy_setup_preferences_migrations = {
|
|
929
|
+
"structured scan guidance context",
|
|
930
|
+
"idempotent scan lifecycle requests",
|
|
931
|
+
}
|
|
932
|
+
if (
|
|
933
|
+
setup_preferences_migration is not None
|
|
934
|
+
and setup_preferences_migration["name"] in legacy_setup_preferences_migrations
|
|
935
|
+
):
|
|
936
|
+
migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 19)
|
|
937
|
+
for statement in sql_statements(migration_sql):
|
|
938
|
+
connection.execute(statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1))
|
|
939
|
+
connection.execute(
|
|
940
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 19 AND name = ?",
|
|
941
|
+
("persist setup workspace preference", setup_preferences_migration["name"]),
|
|
942
|
+
)
|
|
943
|
+
|
|
944
|
+
phase_progress_migration = connection.execute(
|
|
945
|
+
"SELECT name FROM schema_migrations WHERE version = 20"
|
|
946
|
+
).fetchone()
|
|
947
|
+
legacy_phase_progress_migrations = {
|
|
948
|
+
"retain superseded scan lifecycle requests",
|
|
949
|
+
"threat model publication receipts",
|
|
950
|
+
}
|
|
951
|
+
if (
|
|
952
|
+
phase_progress_migration is not None
|
|
953
|
+
and phase_progress_migration["name"] in legacy_phase_progress_migrations
|
|
954
|
+
):
|
|
955
|
+
add_column_if_missing(
|
|
956
|
+
connection,
|
|
957
|
+
"scan_progress",
|
|
958
|
+
"phase_items_total",
|
|
959
|
+
"INTEGER NOT NULL DEFAULT 0 CHECK (phase_items_total >= 0)",
|
|
960
|
+
)
|
|
961
|
+
add_column_if_missing(
|
|
962
|
+
connection,
|
|
963
|
+
"scan_progress",
|
|
964
|
+
"phase_items_completed",
|
|
965
|
+
"INTEGER NOT NULL DEFAULT 0 "
|
|
966
|
+
"CHECK (phase_items_completed >= 0 AND phase_items_completed <= phase_items_total)",
|
|
967
|
+
)
|
|
968
|
+
add_column_if_missing(
|
|
969
|
+
connection,
|
|
970
|
+
"scan_progress",
|
|
971
|
+
"phase_progress_unit",
|
|
972
|
+
"TEXT CHECK (phase_progress_unit IS NULL OR phase_progress_unit IN ("
|
|
973
|
+
"'checks', 'threat_surfaces', 'review_receipts', 'candidate_findings', "
|
|
974
|
+
"'validated_findings', 'report_artifacts'))",
|
|
975
|
+
)
|
|
976
|
+
connection.execute(
|
|
977
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 20 AND name = ?",
|
|
978
|
+
("phase-specific scan progress", phase_progress_migration["name"]),
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
preflight_progress_migration = connection.execute(
|
|
982
|
+
"SELECT name FROM schema_migrations WHERE version = 21"
|
|
983
|
+
).fetchone()
|
|
984
|
+
legacy_preflight_progress_migrations = {
|
|
985
|
+
"scan progress projection and activity",
|
|
986
|
+
"deep coordinator manifest receipts",
|
|
987
|
+
}
|
|
988
|
+
if (
|
|
989
|
+
preflight_progress_migration is not None
|
|
990
|
+
and preflight_progress_migration["name"] in legacy_preflight_progress_migrations
|
|
991
|
+
):
|
|
992
|
+
add_column_if_missing(
|
|
993
|
+
connection,
|
|
994
|
+
"scan_progress",
|
|
995
|
+
"preflight_issues_json",
|
|
996
|
+
"TEXT NOT NULL DEFAULT '[]'",
|
|
997
|
+
)
|
|
998
|
+
add_column_if_missing(
|
|
999
|
+
connection,
|
|
1000
|
+
"scan_progress",
|
|
1001
|
+
"preflight_checks_total",
|
|
1002
|
+
"INTEGER NOT NULL DEFAULT 0 CHECK (preflight_checks_total >= 0)",
|
|
1003
|
+
)
|
|
1004
|
+
add_column_if_missing(
|
|
1005
|
+
connection,
|
|
1006
|
+
"scan_progress",
|
|
1007
|
+
"preflight_checks_completed",
|
|
1008
|
+
"INTEGER NOT NULL DEFAULT 0 CHECK (preflight_checks_completed >= 0 "
|
|
1009
|
+
"AND preflight_checks_completed <= preflight_checks_total)",
|
|
1010
|
+
)
|
|
1011
|
+
connection.execute(
|
|
1012
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 21 AND name = ?",
|
|
1013
|
+
("current scan preflight state", preflight_progress_migration["name"]),
|
|
1014
|
+
)
|
|
1015
|
+
|
|
1016
|
+
scan_recipe_migration = connection.execute(
|
|
1017
|
+
"SELECT name FROM schema_migrations WHERE version = 22"
|
|
1018
|
+
).fetchone()
|
|
1019
|
+
if (
|
|
1020
|
+
scan_recipe_migration is not None
|
|
1021
|
+
and scan_recipe_migration["name"] == "dynamic scan execution profiles"
|
|
1022
|
+
):
|
|
1023
|
+
add_column_if_missing(connection, "scans", "recipe_json", "TEXT")
|
|
1024
|
+
add_column_if_missing(
|
|
1025
|
+
connection,
|
|
1026
|
+
"scans",
|
|
1027
|
+
"parent_scan_id",
|
|
1028
|
+
"TEXT REFERENCES scans(id) ON DELETE SET NULL",
|
|
1029
|
+
)
|
|
1030
|
+
connection.execute(
|
|
1031
|
+
"UPDATE schema_migrations SET name = ? WHERE version = 22 AND name = ?",
|
|
1032
|
+
("replayable scan launch recipes", "dynamic scan execution profiles"),
|
|
1033
|
+
)
|
|
1034
|
+
|
|
655
1035
|
migration = connection.execute(
|
|
656
1036
|
"SELECT name FROM schema_migrations WHERE version = 2"
|
|
657
1037
|
).fetchone()
|
|
@@ -693,6 +1073,127 @@ def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp:
|
|
|
693
1073
|
)
|
|
694
1074
|
|
|
695
1075
|
|
|
1076
|
+
def repair_deep_scan_migration(connection: sqlite3.Connection) -> None:
|
|
1077
|
+
scan_columns = {row["name"] for row in connection.execute("PRAGMA table_info(scans)")}
|
|
1078
|
+
owner_column_missing = "deep_scan_owner_thread_id" not in scan_columns
|
|
1079
|
+
expected_objects = {
|
|
1080
|
+
"scans_one_running_deep_per_owner_target",
|
|
1081
|
+
"deep_scan_runs",
|
|
1082
|
+
"deep_scan_workers",
|
|
1083
|
+
"deep_scan_workers_completion_sequence",
|
|
1084
|
+
"deep_scan_workers_by_scan_status",
|
|
1085
|
+
"deep_scan_dedup_inputs",
|
|
1086
|
+
}
|
|
1087
|
+
existing_objects = {
|
|
1088
|
+
row["name"]
|
|
1089
|
+
for row in connection.execute(
|
|
1090
|
+
"SELECT name FROM sqlite_master WHERE name LIKE 'deep_scan_%' "
|
|
1091
|
+
"OR name = 'scans_one_running_deep_per_owner_target'"
|
|
1092
|
+
)
|
|
1093
|
+
}
|
|
1094
|
+
if not owner_column_missing and expected_objects <= existing_objects:
|
|
1095
|
+
return
|
|
1096
|
+
|
|
1097
|
+
if owner_column_missing:
|
|
1098
|
+
add_column_if_missing(connection, "scans", "deep_scan_owner_thread_id", "TEXT")
|
|
1099
|
+
migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 11)
|
|
1100
|
+
for statement in sql_statements(migration_sql):
|
|
1101
|
+
if statement.startswith("ALTER TABLE scans"):
|
|
1102
|
+
continue
|
|
1103
|
+
if statement.startswith("UPDATE scans") and not owner_column_missing:
|
|
1104
|
+
continue
|
|
1105
|
+
for prefix in ("CREATE UNIQUE INDEX ", "CREATE INDEX ", "CREATE TABLE "):
|
|
1106
|
+
if statement.startswith(prefix):
|
|
1107
|
+
statement = statement.replace(prefix, f"{prefix}IF NOT EXISTS ", 1)
|
|
1108
|
+
break
|
|
1109
|
+
connection.execute(statement)
|
|
1110
|
+
if statement.startswith("UPDATE scans") and "continuation_thread_id" in scan_columns:
|
|
1111
|
+
connection.execute(
|
|
1112
|
+
"UPDATE scans SET deep_scan_owner_thread_id = continuation_thread_id "
|
|
1113
|
+
"WHERE mode = 'deep' AND status = 'running' "
|
|
1114
|
+
"AND continuation_thread_id IS NOT NULL"
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def repair_deep_scan_failure_counter_migration(connection: sqlite3.Connection) -> None:
|
|
1119
|
+
columns = {row["name"] for row in connection.execute("PRAGMA table_info(deep_scan_runs)")}
|
|
1120
|
+
threshold_missing = "stop_after_consecutive_errors" not in columns
|
|
1121
|
+
add_column_if_missing(
|
|
1122
|
+
connection,
|
|
1123
|
+
"deep_scan_runs",
|
|
1124
|
+
"stop_after_consecutive_errors",
|
|
1125
|
+
"INTEGER NOT NULL DEFAULT 1 CHECK (stop_after_consecutive_errors >= 1)",
|
|
1126
|
+
)
|
|
1127
|
+
if threshold_missing:
|
|
1128
|
+
connection.execute(
|
|
1129
|
+
"UPDATE deep_scan_runs SET stop_after_consecutive_errors = stop_after_no_new"
|
|
1130
|
+
)
|
|
1131
|
+
add_column_if_missing(
|
|
1132
|
+
connection,
|
|
1133
|
+
"deep_scan_runs",
|
|
1134
|
+
"consecutive_errors",
|
|
1135
|
+
"INTEGER NOT NULL DEFAULT 0 CHECK (consecutive_errors >= 0)",
|
|
1136
|
+
)
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
def repair_thread_scoped_workspaces_migration(connection: sqlite3.Connection) -> None:
|
|
1140
|
+
add_column_if_missing(connection, "workspaces", "thread_id", "TEXT")
|
|
1141
|
+
connection.execute(
|
|
1142
|
+
"CREATE INDEX IF NOT EXISTS workspaces_by_thread_and_updated_at "
|
|
1143
|
+
"ON workspaces(thread_id, updated_at DESC)"
|
|
1144
|
+
)
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def repair_stable_targets_migration(connection: sqlite3.Connection) -> bool:
|
|
1148
|
+
workspace_columns = {row["name"] for row in connection.execute("PRAGMA table_info(workspaces)")}
|
|
1149
|
+
scan_columns = {row["name"] for row in connection.execute("PRAGMA table_info(scans)")}
|
|
1150
|
+
existing_objects = {
|
|
1151
|
+
row["name"]
|
|
1152
|
+
for row in connection.execute(
|
|
1153
|
+
"SELECT name FROM sqlite_master WHERE name IN ('security_targets', 'scans_by_target')"
|
|
1154
|
+
)
|
|
1155
|
+
}
|
|
1156
|
+
if (
|
|
1157
|
+
"target_id" in workspace_columns
|
|
1158
|
+
and "target_id" in scan_columns
|
|
1159
|
+
and existing_objects == {"security_targets", "scans_by_target"}
|
|
1160
|
+
):
|
|
1161
|
+
return False
|
|
1162
|
+
|
|
1163
|
+
migration_sql = next(sql for version, _, sql in MIGRATIONS if version == 16)
|
|
1164
|
+
for statement in sql_statements(migration_sql):
|
|
1165
|
+
if statement.startswith("ALTER TABLE workspaces"):
|
|
1166
|
+
add_column_if_missing(
|
|
1167
|
+
connection,
|
|
1168
|
+
"workspaces",
|
|
1169
|
+
"target_id",
|
|
1170
|
+
"TEXT REFERENCES security_targets(id)",
|
|
1171
|
+
)
|
|
1172
|
+
continue
|
|
1173
|
+
if statement.startswith("ALTER TABLE scans"):
|
|
1174
|
+
add_column_if_missing(
|
|
1175
|
+
connection,
|
|
1176
|
+
"scans",
|
|
1177
|
+
"target_id",
|
|
1178
|
+
"TEXT REFERENCES security_targets(id)",
|
|
1179
|
+
)
|
|
1180
|
+
continue
|
|
1181
|
+
statement = statement.replace("CREATE TABLE ", "CREATE TABLE IF NOT EXISTS ", 1)
|
|
1182
|
+
statement = statement.replace("CREATE INDEX ", "CREATE INDEX IF NOT EXISTS ", 1)
|
|
1183
|
+
connection.execute(statement)
|
|
1184
|
+
connection.execute(
|
|
1185
|
+
"""
|
|
1186
|
+
UPDATE scans
|
|
1187
|
+
SET target_id = NULL
|
|
1188
|
+
WHERE target_id IS NOT NULL
|
|
1189
|
+
AND NOT EXISTS (
|
|
1190
|
+
SELECT 1 FROM security_targets WHERE security_targets.id = scans.target_id
|
|
1191
|
+
)
|
|
1192
|
+
"""
|
|
1193
|
+
)
|
|
1194
|
+
return True
|
|
1195
|
+
|
|
1196
|
+
|
|
696
1197
|
def add_column_if_missing(
|
|
697
1198
|
connection: sqlite3.Connection, table: str, column: str, definition: str
|
|
698
1199
|
) -> None:
|
|
@@ -143,6 +143,13 @@ def worktree_content_digest_for_context(
|
|
|
143
143
|
b"untracked-content",
|
|
144
144
|
os.fsencode(os.readlink(path)),
|
|
145
145
|
)
|
|
146
|
+
elif stat.S_ISDIR(metadata.st_mode):
|
|
147
|
+
update_digest_field(digest, b"untracked-kind", b"directory")
|
|
148
|
+
update_digest_field(
|
|
149
|
+
digest,
|
|
150
|
+
b"untracked-content",
|
|
151
|
+
directory_content_digest(path.resolve()).encode(),
|
|
152
|
+
)
|
|
146
153
|
elif stat.S_ISREG(metadata.st_mode):
|
|
147
154
|
content_digest = hashlib.sha256()
|
|
148
155
|
content_size = 0
|
|
@@ -419,7 +426,13 @@ def copy_git_worktree_files(source: Path, destination: Path, excluded: tuple[Pat
|
|
|
419
426
|
destination_path.symlink_to(os.readlink(source_path))
|
|
420
427
|
elif stat.S_ISREG(metadata.st_mode):
|
|
421
428
|
shutil.copy2(source_path, destination_path, follow_symlinks=False)
|
|
422
|
-
elif
|
|
429
|
+
elif stat.S_ISDIR(metadata.st_mode):
|
|
430
|
+
nested_git_dir = git_output(source_path, "rev-parse", "--absolute-git-dir")
|
|
431
|
+
if nested_git_dir is None:
|
|
432
|
+
raise SystemExit(f"Could not inspect nested Git working tree: {relative}")
|
|
433
|
+
copy_git_worktree_files(source_path, destination_path, excluded)
|
|
434
|
+
(destination_path / ".git").write_text(f"gitdir: {nested_git_dir}\n")
|
|
435
|
+
else:
|
|
423
436
|
raise SystemExit(f"Unsupported Git working-tree file type: {relative}")
|
|
424
437
|
copied_target = destination if pathspec == "." else destination / pathspec
|
|
425
438
|
copied_target.mkdir(parents=True, exist_ok=True)
|