local-triage 0.0.0-stage → 0.10.0-dev.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +90 -2
- package/THIRD-PARTY-NOTICES.md +48 -0
- package/app/__init__.py +3 -0
- package/app/affiliation.py +137 -0
- package/app/ask.py +300 -0
- package/app/build_info.py +98 -0
- package/app/capec.py +168 -0
- package/app/capec_view.py +27 -0
- package/app/codecmp.py +59 -0
- package/app/config.py +411 -0
- package/app/context.py +1451 -0
- package/app/cvss.py +357 -0
- package/app/db.py +6234 -0
- package/app/defect.py +670 -0
- package/app/defect_check.py +323 -0
- package/app/defect_drafts.py +612 -0
- package/app/defect_dto.py +149 -0
- package/app/defect_fields.py +248 -0
- package/app/defect_form.py +124 -0
- package/app/defect_links.py +107 -0
- package/app/defect_local_text.py +103 -0
- package/app/defect_references.py +164 -0
- package/app/defect_where.py +147 -0
- package/app/ecosystem.py +51 -0
- package/app/executors.py +660 -0
- package/app/feeds.py +1231 -0
- package/app/guard.py +164 -0
- package/app/hub_block_builder.py +200 -0
- package/app/hub_block_version.py +220 -0
- package/app/hub_defects.py +4469 -0
- package/app/hub_effects.py +421 -0
- package/app/hub_entity_versions.py +108 -0
- package/app/hub_import.py +2054 -0
- package/app/hub_onboarding.py +114 -0
- package/app/hub_ops.py +330 -0
- package/app/hub_pull.py +1195 -0
- package/app/hub_state.py +255 -0
- package/app/hub_status.py +237 -0
- package/app/hub_sync.py +619 -0
- package/app/hub_sync_api.py +241 -0
- package/app/hub_sync_compare.py +1054 -0
- package/app/hub_sync_contract.py +75 -0
- package/app/hub_template_model.py +74 -0
- package/app/hub_text_scale.py +159 -0
- package/app/journal.py +250 -0
- package/app/llm.py +850 -0
- package/app/llm_chat.py +596 -0
- package/app/llm_http.py +319 -0
- package/app/llm_messages.py +401 -0
- package/app/llm_net.py +764 -0
- package/app/llm_ollama.py +571 -0
- package/app/lockfile.py +505 -0
- package/app/logsafe.py +54 -0
- package/app/main.py +367 -0
- package/app/mask_display.py +132 -0
- package/app/masking.py +814 -0
- package/app/metrics.py +194 -0
- package/app/model_traffic.py +519 -0
- package/app/models.py +483 -0
- package/app/navigation.py +121 -0
- package/app/osv.py +279 -0
- package/app/package_identity.py +189 -0
- package/app/package_transition.py +824 -0
- package/app/profile_gate.py +71 -0
- package/app/prompts.py +1108 -0
- package/app/provider_keys.py +189 -0
- package/app/providers.py +511 -0
- package/app/routes/__init__.py +10 -0
- package/app/routes/api.py +153 -0
- package/app/routes/capec.py +186 -0
- package/app/routes/common.py +407 -0
- package/app/routes/components.py +1499 -0
- package/app/routes/cwe.py +168 -0
- package/app/routes/decision_selected.py +153 -0
- package/app/routes/defect_add.py +223 -0
- package/app/routes/defect_card_routes.py +363 -0
- package/app/routes/defect_selected.py +342 -0
- package/app/routes/feeds.py +155 -0
- package/app/routes/findings.py +2736 -0
- package/app/routes/hub.py +402 -0
- package/app/routes/hub_bulk.py +288 -0
- package/app/routes/hub_defect_routes.py +1945 -0
- package/app/routes/hub_status_routes.py +1132 -0
- package/app/routes/hub_sync_routes.py +463 -0
- package/app/routes/journal.py +132 -0
- package/app/routes/projects.py +1601 -0
- package/app/routes/providers.py +336 -0
- package/app/routes/references.py +26 -0
- package/app/routes/runs.py +219 -0
- package/app/routes/sca.py +2722 -0
- package/app/routes/settings.py +163 -0
- package/app/run_errors.py +151 -0
- package/app/sca.py +1662 -0
- package/app/sca_contract.py +1289 -0
- package/app/sca_research.py +1089 -0
- package/app/sca_research_net.py +1007 -0
- package/app/sca_text.py +965 -0
- package/app/serve.py +70 -0
- package/app/settings_store.py +262 -0
- package/app/static/app.css +2802 -0
- package/app/static/app.js +3772 -0
- package/app/static/capec-license.txt +9 -0
- package/app/static/favicon.svg +1 -0
- package/app/static/fonts/Golos-Text_DemiBold.woff2 +0 -0
- package/app/static/fonts/Golos-Text_Regular.woff2 +0 -0
- package/app/static/fonts/OFL.txt +90 -0
- package/app/static/navigation.js +234 -0
- package/app/static/observability.js +121 -0
- package/app/static/theme.js +49 -0
- package/app/storage/__init__.py +16 -0
- package/app/storage/local_state.py +246 -0
- package/app/storage/migrate.py +110 -0
- package/app/storage/migrations/postgresql/0001_baseline.sql +896 -0
- package/app/storage/migrations/postgresql/0002_feeds_and_secrets.sql +55 -0
- package/app/storage/migrations/postgresql/0003_project_mask_display.sql +5 -0
- package/app/storage/migrations/postgresql/0004_package_identity.sql +36 -0
- package/app/storage/postgresql.py +573 -0
- package/app/storage/schema.py +619 -0
- package/app/storage/sql.py +57 -0
- package/app/storage/tools.py +254 -0
- package/app/telemetry.py +307 -0
- package/app/templates/base.html +91 -0
- package/app/templates/capec/context.html +3 -0
- package/app/templates/capec/detail.html +27 -0
- package/app/templates/capec/index.html +128 -0
- package/app/templates/components/_archive.html +11 -0
- package/app/templates/components/archived.html +19 -0
- package/app/templates/components/batch.html +35 -0
- package/app/templates/components/detail.html +332 -0
- package/app/templates/components/index.html +338 -0
- package/app/templates/components/research_preview.html +80 -0
- package/app/templates/components/research_run.html +54 -0
- package/app/templates/cwe/bdu.html +56 -0
- package/app/templates/cwe/detail.html +233 -0
- package/app/templates/cwe/index.html +97 -0
- package/app/templates/error.html +8 -0
- package/app/templates/feeds/index.html +65 -0
- package/app/templates/findings/card.html +639 -0
- package/app/templates/findings/decision_selected.html +41 -0
- package/app/templates/findings/defect_add.html +46 -0
- package/app/templates/findings/defect_selected.html +47 -0
- package/app/templates/findings/list.html +153 -0
- package/app/templates/findings/send_selected.html +53 -0
- package/app/templates/findings/trace.html +72 -0
- package/app/templates/journal/detail.html +19 -0
- package/app/templates/journal/index.html +19 -0
- package/app/templates/macros.html +440 -0
- package/app/templates/navigation-fields.html +1 -0
- package/app/templates/navigation.html +11 -0
- package/app/templates/projects/_identity_summary.html +42 -0
- package/app/templates/projects/defect_card.html +153 -0
- package/app/templates/projects/defect_create_preview.html +17 -0
- package/app/templates/projects/defect_fields_diff.html +6 -0
- package/app/templates/projects/defect_fields_form.html +40 -0
- package/app/templates/projects/defect_references.html +17 -0
- package/app/templates/projects/defect_sync_preview.html +28 -0
- package/app/templates/projects/defect_update_actions.html +6 -0
- package/app/templates/projects/defect_update_preview.html +24 -0
- package/app/templates/projects/defects.html +126 -0
- package/app/templates/projects/delete.html +31 -0
- package/app/templates/projects/external.html +61 -0
- package/app/templates/projects/folders.html +64 -0
- package/app/templates/projects/hub_accepted.html +10 -0
- package/app/templates/projects/hub_assignment.html +50 -0
- package/app/templates/projects/hub_pending.html +20 -0
- package/app/templates/projects/hub_pending_list.html +17 -0
- package/app/templates/projects/hub_state_note.html +23 -0
- package/app/templates/projects/hub_step_note.html +5 -0
- package/app/templates/projects/hub_sync.html +105 -0
- package/app/templates/projects/hub_sync_confirm.html +32 -0
- package/app/templates/projects/hub_sync_worker.html +5 -0
- package/app/templates/projects/hub_text_rules.html +64 -0
- package/app/templates/projects/identity_report.html +74 -0
- package/app/templates/projects/list.html +90 -0
- package/app/templates/projects/project.html +433 -0
- package/app/templates/reference-missing.html +14 -0
- package/app/templates/runs/run.html +166 -0
- package/app/templates/sca/card.html +476 -0
- package/app/templates/sca/index.html +321 -0
- package/app/templates/settings/hub_url.html +31 -0
- package/app/templates/settings/index.html +216 -0
- package/app/templates/settings/provider.html +229 -0
- package/app/templates/settings/providers.html +37 -0
- package/app/templates/worklists.html +52 -0
- package/app/textsafe.py +117 -0
- package/app/timing.py +245 -0
- package/app/triage.py +1743 -0
- package/app/view.py +430 -0
- package/app/worklists.py +204 -0
- package/examples/demo/demo-hub-report-v2.xlsx +0 -0
- package/examples/demo/demo-hub-report.xlsx +0 -0
- package/examples/demo/repos/billing-core/pom.xml +30 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/Invoice.java +9 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceRepository.java +51 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceStatus.java +11 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/config/DatabaseSettings.java +18 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/payment/PaymentTokenService.java +35 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/web/InvoiceExportController.java +57 -0
- package/examples/demo/repos/catalog-api/catalog_api/__init__.py +1 -0
- package/examples/demo/repos/catalog-api/catalog_api/products.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/tasks.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/views.py +17 -0
- package/examples/demo/repos/catalog-api/requirements.txt +3 -0
- package/npm/build.json +5 -0
- package/npm/cli.js +346 -0
- package/npm/files.manifest +224 -0
- package/npm/lib/binding.js +64 -0
- package/npm/lib/cluster.js +330 -0
- package/npm/lib/env.js +88 -0
- package/npm/lib/fsx.js +125 -0
- package/npm/lib/http.js +53 -0
- package/npm/lib/metrics.js +73 -0
- package/npm/lib/paths.js +150 -0
- package/npm/lib/pg.js +351 -0
- package/npm/lib/proc.js +101 -0
- package/npm/lib/python.js +133 -0
- package/npm/lib/stand.js +46 -0
- package/npm/lib/supervisor.js +1141 -0
- package/npm/lib/version.js +21 -0
- package/npm/requirements.txt +476 -0
- package/npm-shrinkwrap.json +111 -0
- package/package.json +37 -4
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Перенос файлов состояния в PostgreSQL (ADR-0133): справочники feeds.db, ключ маскирования, ключ исполнителя, ключи профилей моделей.
|
|
2
|
+
|
|
3
|
+
Файлы только читаются и не удаляются: их удаляет владелец. Всё пишется одной транзакцией вместе с отметкой о переносе; после отметки
|
|
4
|
+
приложение файлы не читает. Значения ключей ни в ответе, ни в журнале не появляются.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import sqlite3
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
from urllib.parse import quote
|
|
16
|
+
|
|
17
|
+
import psycopg
|
|
18
|
+
|
|
19
|
+
MARKER = "local_state_imported"
|
|
20
|
+
MASK_SECRET = "mask_key"
|
|
21
|
+
WORKER_SECRET = "hub_worker_key"
|
|
22
|
+
PROVIDER_SECRET = "provider:{}"
|
|
23
|
+
MASK_FILE = "mask.key"
|
|
24
|
+
WORKER_FILE = "hub-worker.key"
|
|
25
|
+
PROVIDERS_DIR = "providers"
|
|
26
|
+
FEEDS_FILE = "feeds.db"
|
|
27
|
+
TABLES = ("feed_meta", "bdu", "bdu_cve", "cwe_ru", "cwe", "cwe_refs", "cwe_links", "cwe_index", "cwe_capec",
|
|
28
|
+
"capec", "capec_index", "capec_refs", "capec_links", "capec_cwe", "capec_meta")
|
|
29
|
+
_DIGEST_SIZE = 2 ** 256
|
|
30
|
+
_LOCK = 7133001 # то же, что feeds.IMPORT_LOCK: загрузка справочников и перенос не идут вместе
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class LocalStateError(RuntimeError):
|
|
34
|
+
"""Перенос остановлен, база не изменилась."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def legacy_files(data_dir: Path) -> list[str]:
|
|
38
|
+
"""Какие файлы состояния лежат в папке данных."""
|
|
39
|
+
root = Path(data_dir)
|
|
40
|
+
found = [name for name in (FEEDS_FILE, MASK_FILE, WORKER_FILE) if (root / name).is_file()]
|
|
41
|
+
return found + [f"{PROVIDERS_DIR}/{p.name}" for p in sorted((root / PROVIDERS_DIR).glob("*.key"))]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def pending(data_dir: Path, conninfo: str = "") -> bool:
|
|
45
|
+
"""Перенос нужен: файлы состояния есть, отметки нет. Им же start.ps1 находит прерванный перенос."""
|
|
46
|
+
if not legacy_files(data_dir):
|
|
47
|
+
return False
|
|
48
|
+
with psycopg.connect(conninfo, autocommit=True) as conn:
|
|
49
|
+
return conn.execute("SELECT 1 FROM app_state.secrets WHERE name = %s", (MARKER,)).fetchone() is None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def require_imported(db: Any, data_dir: Path) -> None:
|
|
53
|
+
"""Запуск на базе без отметки при файлах состояния в папке данных – отказ: новый ключ маскирования на существующей базе нельзя
|
|
54
|
+
создавать молча. После отметки файлы не читаются совсем."""
|
|
55
|
+
if db.secret(MARKER) is not None:
|
|
56
|
+
return
|
|
57
|
+
files = legacy_files(data_dir)
|
|
58
|
+
if files:
|
|
59
|
+
from app.config import stand_mode
|
|
60
|
+
how = (" Это служебная папка другой установки: запустите стенд с новой служебной папкой (ключ --home)."
|
|
61
|
+
if stand_mode() == "npm" else
|
|
62
|
+
" Выполните .\\start.ps1 -Migrate или python -m app.storage.tools local-state-import.")
|
|
63
|
+
raise RuntimeError("В папке данных есть файлы состояния, а в базу они не перенесены: " + ", ".join(files)
|
|
64
|
+
+ "." + how)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _file_keys(data_dir: Path) -> dict[str, str]:
|
|
68
|
+
"""Ключи из файлов: {имя в базе: значение}. Повреждённый файл – остановка: заменить его молча нельзя."""
|
|
69
|
+
from app.provider_keys import parse_entry
|
|
70
|
+
root, keys = Path(data_dir), {}
|
|
71
|
+
mask = root / MASK_FILE
|
|
72
|
+
if mask.is_file():
|
|
73
|
+
raw = mask.read_bytes()
|
|
74
|
+
if len(raw) != 32:
|
|
75
|
+
raise LocalStateError(f"Файл {MASK_FILE} повреждён: длина не 32 байта. База не менялась.")
|
|
76
|
+
keys[MASK_SECRET] = raw.hex()
|
|
77
|
+
worker = root / WORKER_FILE
|
|
78
|
+
if worker.is_file():
|
|
79
|
+
value = worker.read_text(encoding="ascii", errors="replace").strip()
|
|
80
|
+
if len(value) != 64 or any(ch not in "0123456789abcdef" for ch in value):
|
|
81
|
+
raise LocalStateError(f"Файл {WORKER_FILE} повреждён. База не менялась.")
|
|
82
|
+
keys[WORKER_SECRET] = value
|
|
83
|
+
for path in sorted((root / PROVIDERS_DIR).glob("*.key")):
|
|
84
|
+
if not path.stem.isdigit():
|
|
85
|
+
raise LocalStateError(f"Неожиданный файл ключа: {path.name}. База не менялась.")
|
|
86
|
+
entry = parse_entry(path.read_bytes())
|
|
87
|
+
if entry is None:
|
|
88
|
+
raise LocalStateError(f"Файл ключа профиля {path.name} повреждён. База не менялась.")
|
|
89
|
+
keys[PROVIDER_SECRET.format(int(path.stem))] = json.dumps(entry, ensure_ascii=True, sort_keys=True)
|
|
90
|
+
return keys
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _open_source(data_dir: Path) -> sqlite3.Connection | None:
|
|
94
|
+
path = Path(data_dir) / FEEDS_FILE
|
|
95
|
+
if not path.is_file():
|
|
96
|
+
return None
|
|
97
|
+
wal = Path(f"{path}-wal")
|
|
98
|
+
live = wal.is_file() and wal.stat().st_size > 0
|
|
99
|
+
# журнал пуст – файл открывается как неизменяемый: том Docker Desktop не даёт блокировок SQLite
|
|
100
|
+
return sqlite3.connect(f"file:{quote(path.resolve().as_posix())}?mode=ro" + ("" if live else "&immutable=1"), uri=True)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _row_hash(row: Any) -> int:
|
|
104
|
+
"""Хеш строки без учёта типа драйвера: числа и строки сравниваются по значению."""
|
|
105
|
+
text = json.dumps([repr(v) if isinstance(v, float) else v for v in row], ensure_ascii=False)
|
|
106
|
+
return int.from_bytes(hashlib.sha256(text.encode("utf-8")).digest(), "big")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _source_columns(source: sqlite3.Connection, table: str) -> list[str]:
|
|
110
|
+
return [row[1] for row in source.execute(f"PRAGMA table_info({table})")]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _source_digest(source: sqlite3.Connection, table: str, columns: list[str]) -> tuple[int, int]:
|
|
114
|
+
count, total = 0, 0
|
|
115
|
+
for row in source.execute(f"SELECT {', '.join(columns)} FROM {table}"):
|
|
116
|
+
count += 1
|
|
117
|
+
total = (total + _row_hash(row)) % _DIGEST_SIZE
|
|
118
|
+
return count, total
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _target_digest(conn: psycopg.Connection, table: str, columns: list[str]) -> tuple[int, int]:
|
|
122
|
+
count, total = 0, 0
|
|
123
|
+
with conn.cursor(name=f"digest_{table}") as cursor:
|
|
124
|
+
cursor.execute(f"SELECT {', '.join(columns)} FROM feeds.{table}")
|
|
125
|
+
for row in cursor:
|
|
126
|
+
count += 1
|
|
127
|
+
total = (total + _row_hash(row)) % _DIGEST_SIZE
|
|
128
|
+
return count, total
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _copy_feeds(conn: psycopg.Connection, source: sqlite3.Connection) -> dict[str, int]:
|
|
132
|
+
"""Строки feeds.db – в схему feeds. Столбец, которого в базе нет, – остановка."""
|
|
133
|
+
copied: dict[str, int] = {}
|
|
134
|
+
for table in TABLES:
|
|
135
|
+
columns = _source_columns(source, table)
|
|
136
|
+
if not columns: # таблицы нет: каталог этого вида не загружался прежней версией
|
|
137
|
+
copied[table] = 0
|
|
138
|
+
continue
|
|
139
|
+
known = {row[0] for row in conn.execute(
|
|
140
|
+
"SELECT column_name FROM information_schema.columns WHERE table_schema = 'feeds' AND table_name = %s", (table,))}
|
|
141
|
+
extra = sorted(set(columns) - known)
|
|
142
|
+
if extra:
|
|
143
|
+
raise LocalStateError(f"В {FEEDS_FILE}, таблица {table}, есть неизвестные столбцы: {extra}. База не менялась.")
|
|
144
|
+
count = 0
|
|
145
|
+
with conn.cursor() as cursor, cursor.copy(f"COPY feeds.{table} ({', '.join(columns)}) FROM STDIN") as copy:
|
|
146
|
+
for row in source.execute(f"SELECT {', '.join(columns)} FROM {table}"):
|
|
147
|
+
for column, value in zip(columns, row, strict=True):
|
|
148
|
+
if isinstance(value, str) and "\x00" in value: # в PostgreSQL такого значения нет, изменить его молча нельзя
|
|
149
|
+
raise LocalStateError(f"В {FEEDS_FILE}, таблица {table}, столбец {column}, есть знак NUL. База не менялась.")
|
|
150
|
+
copy.write_row(row)
|
|
151
|
+
count += 1
|
|
152
|
+
copied[table] = count
|
|
153
|
+
return copied
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _fill_cwe_capec(conn: psycopg.Connection, source: sqlite3.Connection) -> None:
|
|
157
|
+
"""Файл прежней версии без таблицы пар CWE и CAPEC: пары строятся из записей каталога CWE, как делала прежняя версия при открытии."""
|
|
158
|
+
if _source_columns(source, "cwe_capec"):
|
|
159
|
+
return
|
|
160
|
+
from app.capec import identifier
|
|
161
|
+
rows = conn.execute("SELECT id, data_json FROM feeds.cwe").fetchall()
|
|
162
|
+
with conn.cursor() as cursor:
|
|
163
|
+
for ident, raw in rows:
|
|
164
|
+
for value in json.loads(raw).get("attack_patterns", []):
|
|
165
|
+
if identifier(value):
|
|
166
|
+
cursor.execute("INSERT INTO feeds.cwe_capec(cwe, id) VALUES (%s, %s) ON CONFLICT DO NOTHING", (ident, identifier(value)))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _compare_feeds(conn: psycopg.Connection, source: sqlite3.Connection) -> dict[str, dict]:
|
|
170
|
+
report: dict[str, dict] = {}
|
|
171
|
+
for table in TABLES:
|
|
172
|
+
columns = _source_columns(source, table)
|
|
173
|
+
if not columns:
|
|
174
|
+
report[table] = {"source": 0, "target": int(conn.execute(f"SELECT COUNT(*) FROM feeds.{table}").fetchone()[0]), "same": True}
|
|
175
|
+
continue
|
|
176
|
+
source_count, source_sum = _source_digest(source, table, columns)
|
|
177
|
+
target_count, target_sum = _target_digest(conn, table, columns)
|
|
178
|
+
report[table] = {"source": source_count, "target": target_count, "same": source_count == target_count and source_sum == target_sum}
|
|
179
|
+
return report
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _key_names(keys: dict[str, str]) -> list[str]:
|
|
183
|
+
"""Имена без номеров профилей не нужны: это служебные имена, значений в них нет."""
|
|
184
|
+
return sorted(keys)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def import_all(data_dir: Path, conninfo: str = "") -> dict:
|
|
188
|
+
"""Перенос. Повторный запуск после отметки ничего не делает. Разные ключ в файле и в базе, справочники в базе без отметки – остановка."""
|
|
189
|
+
with psycopg.connect(conninfo, autocommit=True) as conn:
|
|
190
|
+
if not conn.execute("SELECT to_regclass('app_state.secrets')").fetchone()[0]: # пустая база: схема – как при первом запуске
|
|
191
|
+
from app.db import Database
|
|
192
|
+
Database(conninfo or None).close()
|
|
193
|
+
if not conn.execute("SELECT pg_try_advisory_lock(%s)", (_LOCK,)).fetchone()[0]:
|
|
194
|
+
raise LocalStateError("Идёт загрузка справочников: остановите приложение и повторите перенос.")
|
|
195
|
+
done = conn.execute("SELECT value FROM app_state.secrets WHERE name = %s", (MARKER,)).fetchone()
|
|
196
|
+
if done:
|
|
197
|
+
return {"imported": False, "already": done[0]}
|
|
198
|
+
keys = _file_keys(data_dir)
|
|
199
|
+
stored = {row[0]: row[1] for row in conn.execute("SELECT name, value FROM app_state.secrets")}
|
|
200
|
+
different = sorted(name for name, value in keys.items() if name in stored and stored[name] != value)
|
|
201
|
+
if different:
|
|
202
|
+
raise LocalStateError(f"Ключи в файлах и в базе различаются: {different}. Значения не показаны. База не менялась.")
|
|
203
|
+
source = _open_source(data_dir)
|
|
204
|
+
try:
|
|
205
|
+
if source is not None and conn.execute("SELECT 1 FROM feeds.feed_meta LIMIT 1").fetchone():
|
|
206
|
+
raise LocalStateError("В базе уже есть справочники, а отметки переноса нет: перезаписывать их нельзя. База не менялась.")
|
|
207
|
+
now = datetime.now(UTC).isoformat(timespec="seconds")
|
|
208
|
+
report: dict[str, dict] = {}
|
|
209
|
+
with conn.transaction():
|
|
210
|
+
for name, value in keys.items():
|
|
211
|
+
conn.execute("INSERT INTO app_state.secrets(name, value, updated_at) VALUES (%s, %s, %s) ON CONFLICT (name) DO NOTHING",
|
|
212
|
+
(name, value, now))
|
|
213
|
+
if source is not None:
|
|
214
|
+
_copy_feeds(conn, source)
|
|
215
|
+
_fill_cwe_capec(conn, source)
|
|
216
|
+
report = _compare_feeds(conn, source)
|
|
217
|
+
wrong = sorted(table for table, item in report.items() if not item["same"])
|
|
218
|
+
if wrong:
|
|
219
|
+
raise LocalStateError(f"После переноса не совпали таблицы: {wrong}. Перенос отменён, база не менялась.")
|
|
220
|
+
conn.execute("INSERT INTO app_state.secrets(name, value, updated_at) VALUES (%s, %s, %s)", (MARKER, now, now))
|
|
221
|
+
finally:
|
|
222
|
+
if source is not None:
|
|
223
|
+
source.close()
|
|
224
|
+
return {"imported": True, "at": now, "keys": _key_names(keys), "feeds": report}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def verify(data_dir: Path, conninfo: str = "") -> dict:
|
|
228
|
+
"""Сверка без записи: отметка, ключи и справочники с файлами, какие из них ещё есть. Файлов нет – проверяется наличие в базе."""
|
|
229
|
+
with psycopg.connect(conninfo, autocommit=True) as conn:
|
|
230
|
+
with conn.transaction():
|
|
231
|
+
conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
|
232
|
+
stored = {row[0]: row[1] for row in conn.execute("SELECT name, value FROM app_state.secrets")}
|
|
233
|
+
keys = _file_keys(data_dir)
|
|
234
|
+
key_report = {name: ("same" if stored.get(name) == value else "absent" if name not in stored else "different")
|
|
235
|
+
for name, value in keys.items()}
|
|
236
|
+
source = _open_source(data_dir)
|
|
237
|
+
try:
|
|
238
|
+
feeds = _compare_feeds(conn, source) if source is not None else {
|
|
239
|
+
table: {"target": int(conn.execute(f"SELECT COUNT(*) FROM feeds.{table}").fetchone()[0])} for table in TABLES}
|
|
240
|
+
finally:
|
|
241
|
+
if source is not None:
|
|
242
|
+
source.close()
|
|
243
|
+
in_base = sorted(name for name in stored if name != MARKER)
|
|
244
|
+
ok = MARKER in stored and all(value == "same" for value in key_report.values()) and all(item.get("same", True) for item in feeds.values())
|
|
245
|
+
return {"ok": ok, "imported_at": stored.get(MARKER), "keys_in_base": in_base, "keys_with_files": key_report,
|
|
246
|
+
"feeds_compared_with_file": source is not None, "feeds": feeds}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Схема PostgreSQL: базовая схема и технические миграции (ADR-0128, раздел 7).
|
|
2
|
+
|
|
3
|
+
Базовая схема 33 не меняется: новая база – она плюс все миграции, существующая получает миграции командой обслуживания после копии
|
|
4
|
+
(start.ps1). Других изменений схемы при открытии базы нет.
|
|
5
|
+
Каждый применённый файл – строка в storage_migrations с контрольной суммой; изменённый после применения файл останавливает работу.
|
|
6
|
+
Файл миграции: NNNN_имя.sql, первая строка – «-- schema: N»; выполняется одной транзакцией.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations" / "postgresql"
|
|
18
|
+
BASELINE = MIGRATIONS_DIR / "0001_baseline.sql"
|
|
19
|
+
BASELINE_SCHEMA = 33
|
|
20
|
+
_FILE = re.compile(r"^(\d{4})_[^/\\]+\.sql$")
|
|
21
|
+
_SCHEMA_LINE = re.compile(r"^--\s*schema:\s*(\d+)\s*$")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MigrationError(RuntimeError):
|
|
25
|
+
"""Файлы миграций и база не согласованы: работа останавливается, база не меняется."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Migration:
|
|
30
|
+
number: int
|
|
31
|
+
path: Path
|
|
32
|
+
schema: int
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def name(self) -> str:
|
|
36
|
+
return self.path.name
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def checksum(path: Path) -> str:
|
|
40
|
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def available() -> list[Migration]:
|
|
44
|
+
"""Файлы миграций по порядку, без базовой схемы. Номера идут без пропусков, схема растёт."""
|
|
45
|
+
found = []
|
|
46
|
+
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
|
47
|
+
match = _FILE.match(path.name)
|
|
48
|
+
if not match:
|
|
49
|
+
raise MigrationError(f"Имя файла миграции не по правилу NNNN_имя.sql: {path.name}")
|
|
50
|
+
number = int(match.group(1))
|
|
51
|
+
if number == 1:
|
|
52
|
+
continue
|
|
53
|
+
with path.open(encoding="utf-8") as handle:
|
|
54
|
+
line = _SCHEMA_LINE.match(handle.readline().strip())
|
|
55
|
+
if not line:
|
|
56
|
+
raise MigrationError(f"Первая строка миграции – «-- schema: N»: {path.name}")
|
|
57
|
+
found.append(Migration(number, path, int(line.group(1))))
|
|
58
|
+
for expected, item in enumerate(found, start=2):
|
|
59
|
+
if item.number != expected:
|
|
60
|
+
raise MigrationError(f"Пропущен номер миграции {expected:04d}.")
|
|
61
|
+
if item.schema != BASELINE_SCHEMA + expected - 1:
|
|
62
|
+
raise MigrationError(f"{item.name}: схема {item.schema}, а по порядку должна быть {BASELINE_SCHEMA + expected - 1}.")
|
|
63
|
+
return found
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _applied(conn) -> dict[int, str]:
|
|
67
|
+
return {int(row[0]): row[1] for row in conn.execute("SELECT number, checksum FROM storage_migrations")}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def pending(conn) -> list[Migration]:
|
|
71
|
+
"""Ещё не применённые миграции. Применённая и изменённая после этого миграция – отказ."""
|
|
72
|
+
applied = _applied(conn)
|
|
73
|
+
files = available()
|
|
74
|
+
for item in files:
|
|
75
|
+
if item.number in applied and applied[item.number] != checksum(item.path):
|
|
76
|
+
raise MigrationError(f"Файл уже применённой миграции изменён: {item.name}.")
|
|
77
|
+
known = {item.number for item in files} | {1}
|
|
78
|
+
unknown = sorted(set(applied) - known)
|
|
79
|
+
if unknown:
|
|
80
|
+
raise MigrationError(f"База новее этой сборки: применены миграции {unknown}, файлов которых нет.")
|
|
81
|
+
return [item for item in files if item.number not in applied]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def apply_baseline(conn, mark: str = "?") -> None:
|
|
85
|
+
"""Базовая схема и запись о ней: внутри открытой транзакции записи вызывающего. mark – маркер параметра его соединения."""
|
|
86
|
+
text = BASELINE.read_text(encoding="utf-8")
|
|
87
|
+
conn.execute(text) # без параметров: драйвер отправляет весь файл как есть
|
|
88
|
+
conn.execute(f"INSERT INTO storage_migrations(number, schema_version, checksum, applied_at) VALUES ({mark}, {mark}, {mark}, {mark})",
|
|
89
|
+
(1, BASELINE_SCHEMA, checksum(BASELINE), datetime.now(UTC).isoformat(timespec="microseconds")))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def apply_one(conn, item: Migration, mark: str = "%s") -> None:
|
|
93
|
+
"""Миграция, запись о ней и версия схемы – внутри открытой транзакции вызывающего."""
|
|
94
|
+
ts = datetime.now(UTC).isoformat(timespec="seconds")
|
|
95
|
+
conn.execute(item.path.read_text(encoding="utf-8"))
|
|
96
|
+
conn.execute(f"INSERT INTO storage_migrations(number, schema_version, checksum, applied_at) VALUES ({mark}, {mark}, {mark}, {mark})",
|
|
97
|
+
(item.number, item.schema, checksum(item.path), ts))
|
|
98
|
+
conn.execute(f"UPDATE meta SET value = {mark} WHERE key = 'schema_version'", (str(item.schema),))
|
|
99
|
+
conn.execute(f"INSERT INTO schema_version(version, applied_at, note) VALUES ({mark}, {mark}, {mark})", (item.schema, ts, item.name))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def apply_pending(conn) -> list[str]:
|
|
103
|
+
"""Применить миграции: каждая – своя транзакция. Сбой откатывает только эту миграцию. Соединение в режиме autocommit; копию
|
|
104
|
+
перед запуском снимает вызывающий (start.ps1)."""
|
|
105
|
+
done = []
|
|
106
|
+
for item in pending(conn):
|
|
107
|
+
with conn.transaction():
|
|
108
|
+
apply_one(conn, item)
|
|
109
|
+
done.append(item.name)
|
|
110
|
+
return done
|