dsh-plugin-opc 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +112 -20
- package/lib/index.js +168 -29
- package/package.json +1 -1
- package/runtime/app_delivery/SKILL.md +1 -1
- package/runtime/app_delivery_migration.py +213 -353
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Step 1 & Step 2 helpers for OPC App Delivery:
|
|
2
2
|
- Dockerfile template fetching from platform-opc (deploy-opc database)
|
|
3
|
-
- Local SQLite / local DB (PostgreSQL, MySQL, SQLServer) / local Redis inspection and
|
|
3
|
+
- Local SQLite / local DB (PostgreSQL, MySQL, SQLServer) / local Redis inspection and persistent storage adaptation
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
from __future__ import annotations
|
|
@@ -8,9 +8,12 @@ from __future__ import annotations
|
|
|
8
8
|
import ast
|
|
9
9
|
import base64
|
|
10
10
|
import json
|
|
11
|
+
import os
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
import re
|
|
14
|
+
import shutil
|
|
13
15
|
import sqlite3
|
|
16
|
+
import tempfile
|
|
14
17
|
import urllib.error
|
|
15
18
|
import urllib.parse
|
|
16
19
|
import urllib.request
|
|
@@ -112,7 +115,6 @@ def inspect_local_file_storage(
|
|
|
112
115
|
source: Path,
|
|
113
116
|
slug: str = "",
|
|
114
117
|
workdir: str = "/app",
|
|
115
|
-
has_migrations: bool = False,
|
|
116
118
|
) -> dict:
|
|
117
119
|
"""Detect local JSON/file data directories, upload folders, report/backup folders, and seed files
|
|
118
120
|
that require external persistent volume mounting (Docker Volume / Host Bind / S3-OSS).
|
|
@@ -143,7 +145,7 @@ def inspect_local_file_storage(
|
|
|
143
145
|
ext = p.suffix.lower()
|
|
144
146
|
if ext in _SOURCE_CODE_EXTENSIONS:
|
|
145
147
|
dir_has_code.add(top)
|
|
146
|
-
|
|
148
|
+
else:
|
|
147
149
|
dir_seed_files.setdefault(top, []).append(p.relative_to(source).as_posix())
|
|
148
150
|
|
|
149
151
|
# 2. Check second-level persistent directories (e.g. reimburse-web/server/data)
|
|
@@ -159,7 +161,7 @@ def inspect_local_file_storage(
|
|
|
159
161
|
ext = p.suffix.lower()
|
|
160
162
|
if ext in _SOURCE_CODE_EXTENSIONS:
|
|
161
163
|
dir_has_code.add(rel_dir)
|
|
162
|
-
|
|
164
|
+
else:
|
|
163
165
|
dir_seed_files.setdefault(rel_dir, []).append(p.relative_to(source).as_posix())
|
|
164
166
|
|
|
165
167
|
# 3. Scan source code for runtime mutable directories (e.g. join(root, "uploads"), join(root, "reports"))
|
|
@@ -352,75 +354,6 @@ def extract_sqlite_tables(sqlite_file: Path, max_rows_per_table: int | None = No
|
|
|
352
354
|
return tables
|
|
353
355
|
|
|
354
356
|
|
|
355
|
-
def migrate_local_data_to_platform(
|
|
356
|
-
*,
|
|
357
|
-
platform_url: str,
|
|
358
|
-
slug: str,
|
|
359
|
-
source_dir: Path,
|
|
360
|
-
source_type: str = "sqlite",
|
|
361
|
-
sqlite_path: str = "",
|
|
362
|
-
target_type: str = "postgres",
|
|
363
|
-
target_resource_id: str = "",
|
|
364
|
-
tables_override: list | None = None,
|
|
365
|
-
redis_keys: list | None = None,
|
|
366
|
-
) -> dict:
|
|
367
|
-
"""Extract schema & 100% of data from local SQLite (or provided tables/redis_keys) and upload to platform-opc POST /api/v1/apps/:slug/migrate-data."""
|
|
368
|
-
tables = tables_override or []
|
|
369
|
-
if not tables and source_type.lower() == "sqlite":
|
|
370
|
-
if sqlite_path.strip():
|
|
371
|
-
db_file = delivery.relative(source_dir, sqlite_path.strip())
|
|
372
|
-
if db_file.is_file() and db_file.stat().st_size > 0:
|
|
373
|
-
tables = extract_sqlite_tables(db_file, max_rows_per_table=None)
|
|
374
|
-
if not tables:
|
|
375
|
-
found = [
|
|
376
|
-
p for p in sorted(source_dir.rglob("*"))
|
|
377
|
-
if p.is_file() and p.suffix.lower() in {".sqlite", ".sqlite3", ".db"} and p.stat().st_size > 0
|
|
378
|
-
]
|
|
379
|
-
if not found:
|
|
380
|
-
raise ValueError("未在源码目录中找到非空 SQLite 数据库文件")
|
|
381
|
-
# Sort non-empty SQLite files by size descending and merge all non-empty tables
|
|
382
|
-
found.sort(key=lambda p: p.stat().st_size, reverse=True)
|
|
383
|
-
seen_tables = set()
|
|
384
|
-
for db_file in found:
|
|
385
|
-
try:
|
|
386
|
-
extracted = extract_sqlite_tables(db_file, max_rows_per_table=None)
|
|
387
|
-
except Exception:
|
|
388
|
-
continue
|
|
389
|
-
for tbl in extracted:
|
|
390
|
-
if tbl["name"] not in seen_tables:
|
|
391
|
-
seen_tables.add(tbl["name"])
|
|
392
|
-
tables.append(tbl)
|
|
393
|
-
|
|
394
|
-
payload = {
|
|
395
|
-
"source_type": source_type.lower().strip(),
|
|
396
|
-
"target_type": target_type.lower().strip() or "postgres",
|
|
397
|
-
"target_resource_id": target_resource_id.strip(),
|
|
398
|
-
"tables": tables,
|
|
399
|
-
"redis_keys": redis_keys or [],
|
|
400
|
-
}
|
|
401
|
-
base = (platform_url or "https://shtokendance.benlai.com").rstrip("/")
|
|
402
|
-
url = f"{base}/api/v1/apps/{urllib.parse.quote(slug)}/migrate-data"
|
|
403
|
-
body_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
404
|
-
req = urllib.request.Request(
|
|
405
|
-
url,
|
|
406
|
-
data=body_bytes,
|
|
407
|
-
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
|
408
|
-
method="POST",
|
|
409
|
-
)
|
|
410
|
-
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
411
|
-
try:
|
|
412
|
-
with opener.open(req, timeout=600) as resp:
|
|
413
|
-
resp_json = json.loads(resp.read().decode("utf-8"))
|
|
414
|
-
if resp_json.get("code") == 0 and resp_json.get("data"):
|
|
415
|
-
return resp_json["data"]
|
|
416
|
-
return resp_json
|
|
417
|
-
except urllib.error.HTTPError as exc:
|
|
418
|
-
detail = exc.read().decode("utf-8", errors="replace")
|
|
419
|
-
raise ValueError(f"平台数据迁移接口返回错误 ({exc.code}): {detail}") from exc
|
|
420
|
-
except Exception as exc:
|
|
421
|
-
raise ValueError(f"调用平台数据迁移接口失败: {exc}") from exc
|
|
422
|
-
|
|
423
|
-
|
|
424
357
|
def _join_route_paths(*parts: str) -> str:
|
|
425
358
|
cleaned = []
|
|
426
359
|
for p in parts:
|
|
@@ -1784,6 +1717,8 @@ def _normalize_resource_type_group(raw_type: str) -> str:
|
|
|
1784
1717
|
return "postgres"
|
|
1785
1718
|
if t == "mssql":
|
|
1786
1719
|
return "sqlserver"
|
|
1720
|
+
if t in {"local-directory", "aliyun-oss-volume"}:
|
|
1721
|
+
return "application-volume"
|
|
1787
1722
|
return t
|
|
1788
1723
|
|
|
1789
1724
|
|
|
@@ -1856,258 +1791,97 @@ def select_and_filter_platform_resources(resources: list[dict] | None, slug: str
|
|
|
1856
1791
|
return result
|
|
1857
1792
|
|
|
1858
1793
|
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
def __getitem__(self, key):
|
|
1868
|
-
return self._vals[key] if isinstance(key, (int, slice)) else super().__getitem__(key)
|
|
1869
|
-
def _opc_translate_sql(sql: str) -> str:
|
|
1870
|
-
s = sql.strip()
|
|
1871
|
-
u = s.upper()
|
|
1872
|
-
if u.startswith("BEGIN IMMEDIATE") or u.startswith("BEGIN EXCLUSIVE") or u == "BEGIN":
|
|
1873
|
-
return "SELECT 1"
|
|
1874
|
-
if _opc_re.match(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", s):
|
|
1875
|
-
s = _opc_re.sub(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", "INSERT INTO", s, count=1).rstrip().rstrip(";") + " ON CONFLICT DO NOTHING"
|
|
1876
|
-
out, in_sq, in_dq, i = [], False, False, 0
|
|
1877
|
-
while i < len(s):
|
|
1878
|
-
ch = s[i]
|
|
1879
|
-
if ch == "'" and not in_dq:
|
|
1880
|
-
in_sq = not in_sq
|
|
1881
|
-
out.append(ch)
|
|
1882
|
-
elif ch == '"' and not in_sq:
|
|
1883
|
-
in_dq = not in_dq
|
|
1884
|
-
out.append(ch)
|
|
1885
|
-
elif ch == "?" and not in_sq and not in_dq:
|
|
1886
|
-
out.append("%s")
|
|
1887
|
-
elif ch == "%" and not in_sq and not in_dq and (i + 1 >= len(s) or s[i + 1] != "s"):
|
|
1888
|
-
out.append("%%")
|
|
1889
|
-
else:
|
|
1890
|
-
out.append(ch)
|
|
1891
|
-
i += 1
|
|
1892
|
-
return "".join(out)
|
|
1893
|
-
class _OpcPgCursor:
|
|
1894
|
-
def __init__(self, rows=None, rowcount=0, lastrowid=None):
|
|
1895
|
-
self._rows = list(rows or [])
|
|
1896
|
-
self._idx = 0
|
|
1897
|
-
self.rowcount = rowcount
|
|
1898
|
-
self.lastrowid = lastrowid
|
|
1899
|
-
def fetchone(self):
|
|
1900
|
-
if self._idx >= len(self._rows):
|
|
1901
|
-
return None
|
|
1902
|
-
r = self._rows[self._idx]
|
|
1903
|
-
self._idx += 1
|
|
1904
|
-
return r
|
|
1905
|
-
def fetchall(self):
|
|
1906
|
-
r = self._rows[self._idx:]
|
|
1907
|
-
self._idx = len(self._rows)
|
|
1908
|
-
return r
|
|
1909
|
-
def __iter__(self):
|
|
1910
|
-
return iter(self._rows)
|
|
1911
|
-
class _OpcPgConn:
|
|
1912
|
-
def __init__(self, dsn: str):
|
|
1913
|
-
self._conn = _opc_pg.connect(dsn)
|
|
1914
|
-
self.row_factory = None
|
|
1915
|
-
def execute(self, sql: str, params=()):
|
|
1916
|
-
s = sql.strip()
|
|
1917
|
-
u = s.upper()
|
|
1918
|
-
if u.startswith("PRAGMA"):
|
|
1919
|
-
m = _opc_re.match(r'(?is)^PRAGMA\\s+table_info\\(\\s*["\\\']?([A-Za-z0-9_]+)["\\\']?\\s*\\)', s)
|
|
1920
|
-
if m and m.group(1).lower() != "invoice_registry":
|
|
1921
|
-
tbl = m.group(1).lower()
|
|
1922
|
-
with self._conn.cursor() as cur:
|
|
1923
|
-
cur.execute("SELECT ordinal_position - 1 AS cid, column_name AS name, data_type AS type, CASE WHEN is_nullable = 'NO' THEN 1 ELSE 0 END AS notnull, column_default AS dflt_value, 0 AS pk FROM information_schema.columns WHERE table_schema = 'public' AND table_name = %s ORDER BY ordinal_position", (tbl,))
|
|
1924
|
-
cols = [d[0] for d in cur.description]
|
|
1925
|
-
return _OpcPgCursor([_OpcPgRow(cols, r) for r in cur.fetchall()])
|
|
1926
|
-
return _OpcPgCursor([])
|
|
1927
|
-
if "SQLITE_MASTER" in u:
|
|
1928
|
-
return _OpcPgCursor([])
|
|
1929
|
-
pg_sql = _opc_translate_sql(s)
|
|
1930
|
-
with self._conn.cursor() as cur:
|
|
1931
|
-
cur.execute(pg_sql, tuple(params) if params else None)
|
|
1932
|
-
if cur.description:
|
|
1933
|
-
cols = [d[0] for d in cur.description]
|
|
1934
|
-
rows = [_OpcPgRow(cols, r) for r in cur.fetchall()]
|
|
1935
|
-
return _OpcPgCursor(rows, cur.rowcount)
|
|
1936
|
-
last_id = None
|
|
1937
|
-
if u.startswith("INSERT"):
|
|
1938
|
-
try:
|
|
1939
|
-
cur.execute("SAVEPOINT _opc_lv; SELECT LASTVAL()")
|
|
1940
|
-
r = cur.fetchone()
|
|
1941
|
-
last_id = r[0] if r else None
|
|
1942
|
-
cur.execute("RELEASE SAVEPOINT _opc_lv")
|
|
1943
|
-
except Exception:
|
|
1944
|
-
try:
|
|
1945
|
-
cur.execute("ROLLBACK TO SAVEPOINT _opc_lv")
|
|
1946
|
-
except Exception:
|
|
1947
|
-
pass
|
|
1948
|
-
return _OpcPgCursor([], cur.rowcount, last_id)
|
|
1949
|
-
def executemany(self, sql: str, seq_of_params):
|
|
1950
|
-
pg_sql = _opc_translate_sql(sql)
|
|
1951
|
-
with self._conn.cursor() as cur:
|
|
1952
|
-
for p in seq_of_params:
|
|
1953
|
-
cur.execute(pg_sql, tuple(p))
|
|
1954
|
-
return _OpcPgCursor([], cur.rowcount)
|
|
1955
|
-
def executescript(self, script: str):
|
|
1956
|
-
return _OpcPgCursor([])
|
|
1957
|
-
def cursor(self):
|
|
1958
|
-
return self
|
|
1959
|
-
def commit(self):
|
|
1960
|
-
self._conn.commit()
|
|
1961
|
-
def rollback(self):
|
|
1962
|
-
self._conn.rollback()
|
|
1963
|
-
def close(self):
|
|
1964
|
-
self._conn.close()
|
|
1965
|
-
def __enter__(self):
|
|
1966
|
-
return self
|
|
1967
|
-
def __exit__(self, exc_type, exc, tb):
|
|
1968
|
-
if exc_type is None:
|
|
1969
|
-
self.commit()
|
|
1970
|
-
else:
|
|
1971
|
-
self.rollback()
|
|
1972
|
-
sqlite3.connect = lambda *a, **kw: _OpcPgConn(_opc_os.environ["DATABASE_URL"])
|
|
1973
|
-
# --- End OPC Platform PostgreSQL Compatibility Shim ---
|
|
1974
|
-
'''
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
def _auto_adapt_sqlite_to_postgres_changes(source: Path, slug: str, data: dict, proposal: dict) -> None:
|
|
1978
|
-
"""When the user has migrated local SQLite to platform PostgreSQL (`data['migrations']`),
|
|
1979
|
-
guarantee that:
|
|
1980
|
-
1. All SQLite database connection modules in Python (e.g. `db.py`) are adapted to connect via `DATABASE_URL` (`psycopg2`).
|
|
1981
|
-
2. `requirements.txt` includes `psycopg2-binary`.
|
|
1982
|
-
3. `contract.configs` declares `DATABASE_URL` and `contract.resources` includes the migrated database resource.
|
|
1983
|
-
"""
|
|
1984
|
-
migrations = [m for m in (data.get("migrations") or []) if isinstance(m, dict)]
|
|
1985
|
-
pg_migs = [m for m in migrations if str(m.get("target_type", "postgres")).lower() in {"postgres", "postgresql"}]
|
|
1986
|
-
if not pg_migs:
|
|
1987
|
-
return
|
|
1988
|
-
|
|
1989
|
-
latest_mig = pg_migs[-1]
|
|
1990
|
-
res_id = str(latest_mig.get("resource_id") or f"db-{slug}")
|
|
1991
|
-
db_name = str(latest_mig.get("database_name") or f"app_{slug.replace('-', '_')}")
|
|
1992
|
-
|
|
1993
|
-
contract = proposal.setdefault("contract", {})
|
|
1994
|
-
changes: list[dict] = proposal.setdefault("changes", [])
|
|
1995
|
-
changes_by_path: dict[str, dict] = {c.get("path", ""): c for c in changes if isinstance(c, dict) and c.get("path")}
|
|
1996
|
-
used_ids: set[str] = {str(c.get("id", "")) for c in changes if isinstance(c, dict)}
|
|
1997
|
-
|
|
1998
|
-
def _unique_change_id(base_id: str) -> str:
|
|
1999
|
-
cid = base_id
|
|
2000
|
-
idx = 2
|
|
2001
|
-
while cid in used_ids:
|
|
2002
|
-
cid = f"{base_id}-{idx}"
|
|
2003
|
-
idx += 1
|
|
2004
|
-
used_ids.add(cid)
|
|
2005
|
-
return cid
|
|
2006
|
-
|
|
2007
|
-
# 1. Find Python files that call sqlite3.connect(...)
|
|
2008
|
-
sqlite_py_files: list[Path] = []
|
|
2009
|
-
for p in sorted(source.rglob("*.py")):
|
|
2010
|
-
if not p.is_file() or p.is_symlink():
|
|
1794
|
+
def _strip_opc_seed_block(df_text: str) -> str:
|
|
1795
|
+
lines = df_text.splitlines()
|
|
1796
|
+
out: list[str] = []
|
|
1797
|
+
in_cont = False
|
|
1798
|
+
for line in lines:
|
|
1799
|
+
trimmed = line.strip()
|
|
1800
|
+
if in_cont:
|
|
1801
|
+
in_cont = trimmed.endswith("\\")
|
|
2011
1802
|
continue
|
|
2012
|
-
|
|
2013
|
-
if any(part in delivery.EXCLUDED or part in {"tests", "test"} for part in rel_parts):
|
|
1803
|
+
if trimmed.startswith("# [OPC Persistent Storage]") or trimmed.startswith("# OPC-PERSISTENT-STORAGE-"):
|
|
2014
1804
|
continue
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
except Exception:
|
|
1805
|
+
if ".opc-seed" in trimmed or "opc-storage-init.sh" in trimmed:
|
|
1806
|
+
in_cont = trimmed.endswith("\\")
|
|
2018
1807
|
continue
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
1808
|
+
out.append(line)
|
|
1809
|
+
return "\n".join(out) + ("\n" if df_text.endswith("\n") else "")
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
_ADDITIVE_SQLITE_SYNC_PY = """import os, sys, sqlite3
|
|
1813
|
+
seed_root, dst_root = sys.argv[1], sys.argv[2]
|
|
1814
|
+
if not os.path.isdir(seed_root) or not os.path.isdir(dst_root):
|
|
1815
|
+
sys.exit(0)
|
|
1816
|
+
def q(ident):
|
|
1817
|
+
return '"' + str(ident).replace('"', '""') + '"'
|
|
1818
|
+
for root, dirs, files in os.walk(seed_root):
|
|
1819
|
+
dirs[:] = [d for d in dirs if d != '.opc-backups']
|
|
1820
|
+
rel_dir = os.path.relpath(root, seed_root)
|
|
1821
|
+
for fn in files:
|
|
1822
|
+
if not fn.lower().endswith(('.db', '.sqlite', '.sqlite3')):
|
|
2030
1823
|
continue
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
lambda m: m.group(1) + _PG_SQLITE3_COMPAT_SHIM,
|
|
2035
|
-
current_content,
|
|
2036
|
-
count=1,
|
|
2037
|
-
)
|
|
2038
|
-
if n_subs == 0:
|
|
1824
|
+
sp = os.path.join(root, fn)
|
|
1825
|
+
dp = os.path.join(dst_root, fn) if rel_dir == '.' else os.path.join(dst_root, rel_dir, fn)
|
|
1826
|
+
if not os.path.isfile(sp) or not os.path.isfile(dp):
|
|
2039
1827
|
continue
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
c_resources.append({
|
|
2098
|
-
"id": res_id,
|
|
2099
|
-
"type": "postgres",
|
|
2100
|
-
"purpose": f"平台自动迁移创建的专属 PostgreSQL 数据库 ({db_name})",
|
|
2101
|
-
})
|
|
2102
|
-
|
|
2103
|
-
# 5. Ensure build_context covers any newly added/modified files
|
|
2104
|
-
build_ctx: list[str] = contract.setdefault("build_context", [])
|
|
2105
|
-
if isinstance(build_ctx, list) and build_ctx and "." not in build_ctx:
|
|
2106
|
-
for ch_path in changes_by_path:
|
|
2107
|
-
covered = any(ch_path == entry or ch_path.startswith(entry.rstrip("/") + "/") for entry in build_ctx if isinstance(entry, str))
|
|
2108
|
-
if not covered:
|
|
2109
|
-
top_dir = ch_path.split("/")[0]
|
|
2110
|
-
build_ctx.append(top_dir if "/" in ch_path else ch_path)
|
|
1828
|
+
try:
|
|
1829
|
+
with open(sp, 'rb') as f:
|
|
1830
|
+
if f.read(16) != b'SQLite format 3\\x00':
|
|
1831
|
+
continue
|
|
1832
|
+
sc = sqlite3.connect('file:' + sp + '?mode=ro', uri=True)
|
|
1833
|
+
dc = sqlite3.connect(dp, timeout=10.0)
|
|
1834
|
+
dc.execute('PRAGMA busy_timeout=10000')
|
|
1835
|
+
existing_tables = {r[0] for r in dc.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
1836
|
+
for tbl, ddl in sc.execute("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL").fetchall():
|
|
1837
|
+
try:
|
|
1838
|
+
if tbl not in existing_tables:
|
|
1839
|
+
dc.execute(ddl)
|
|
1840
|
+
rows = sc.execute('SELECT * FROM ' + q(tbl)).fetchall()
|
|
1841
|
+
if rows:
|
|
1842
|
+
ph = ','.join('?' for _ in rows[0])
|
|
1843
|
+
dc.executemany('INSERT OR IGNORE INTO ' + q(tbl) + ' VALUES (' + ph + ')', rows)
|
|
1844
|
+
else:
|
|
1845
|
+
dst_cols = {r[1] for r in dc.execute('PRAGMA table_info(' + q(tbl) + ')').fetchall()}
|
|
1846
|
+
for cid, cname, ctype, notnull, dflt, pk in sc.execute('PRAGMA table_info(' + q(tbl) + ')').fetchall():
|
|
1847
|
+
if cname in dst_cols or pk:
|
|
1848
|
+
continue
|
|
1849
|
+
t = (ctype or 'TEXT').strip() or 'TEXT'
|
|
1850
|
+
tu = t.upper()
|
|
1851
|
+
if 'INT' in tu or 'NUM' in tu or 'REAL' in tu or 'FLOA' in tu or 'DOUB' in tu:
|
|
1852
|
+
fallback_dflt = '0'
|
|
1853
|
+
elif 'BLOB' in tu:
|
|
1854
|
+
fallback_dflt = "X''"
|
|
1855
|
+
else:
|
|
1856
|
+
fallback_dflt = "''"
|
|
1857
|
+
clause = q(cname) + ' ' + t
|
|
1858
|
+
if dflt is not None:
|
|
1859
|
+
clause += ' DEFAULT ' + str(dflt)
|
|
1860
|
+
elif notnull:
|
|
1861
|
+
clause += ' NOT NULL DEFAULT ' + fallback_dflt
|
|
1862
|
+
try:
|
|
1863
|
+
dc.execute('ALTER TABLE ' + q(tbl) + ' ADD COLUMN ' + clause)
|
|
1864
|
+
except Exception:
|
|
1865
|
+
try:
|
|
1866
|
+
dc.execute('ALTER TABLE ' + q(tbl) + ' ADD COLUMN ' + q(cname) + ' ' + t + ' DEFAULT ' + fallback_dflt)
|
|
1867
|
+
except Exception:
|
|
1868
|
+
pass
|
|
1869
|
+
except Exception:
|
|
1870
|
+
pass
|
|
1871
|
+
for idx_name, idx_sql in sc.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND sql IS NOT NULL"):
|
|
1872
|
+
try:
|
|
1873
|
+
stmt = idx_sql
|
|
1874
|
+
if 'IF NOT EXISTS' not in stmt.upper():
|
|
1875
|
+
stmt = stmt.replace('CREATE UNIQUE INDEX ', 'CREATE UNIQUE INDEX IF NOT EXISTS ', 1).replace('CREATE INDEX ', 'CREATE INDEX IF NOT EXISTS ', 1)
|
|
1876
|
+
dc.execute(stmt)
|
|
1877
|
+
except Exception:
|
|
1878
|
+
pass
|
|
1879
|
+
dc.commit()
|
|
1880
|
+
sc.close()
|
|
1881
|
+
dc.close()
|
|
1882
|
+
except Exception:
|
|
1883
|
+
pass
|
|
1884
|
+
"""
|
|
2111
1885
|
|
|
2112
1886
|
|
|
2113
1887
|
def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, proposal: dict) -> list[dict]:
|
|
@@ -2142,7 +1916,7 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
|
|
|
2142
1916
|
workdir = workdirs[-1].rstrip("/") or "/app"
|
|
2143
1917
|
|
|
2144
1918
|
# 2. Deterministically inspect local file storage & merge with any AI-declared mounts
|
|
2145
|
-
scanned_storage = inspect_local_file_storage(source, slug=slug, workdir=workdir
|
|
1919
|
+
scanned_storage = inspect_local_file_storage(source, slug=slug, workdir=workdir)
|
|
2146
1920
|
mounts_by_cpath: dict[str, dict] = {}
|
|
2147
1921
|
for item in scanned_storage.get("storage_mounts") or []:
|
|
2148
1922
|
if isinstance(item, dict) and item.get("container_path"):
|
|
@@ -2190,16 +1964,98 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
|
|
|
2190
1964
|
build_ctx.append(top_dir)
|
|
2191
1965
|
|
|
2192
1966
|
# 4. Seed the actual volume destinations without replacing a custom startup.
|
|
2193
|
-
|
|
2194
|
-
|
|
1967
|
+
# If Dockerfile already contains an OPC seed block from v1, update it when mounts change.
|
|
1968
|
+
stripped_df = _strip_opc_seed_block(df_text)
|
|
1969
|
+
stripped_lines = stripped_df.splitlines()
|
|
1970
|
+
stripped_stage_start = next(
|
|
1971
|
+
(index for index in range(len(stripped_lines) - 1, -1, -1) if re.match(r"\s*FROM\s+", stripped_lines[index], re.I)),
|
|
1972
|
+
0,
|
|
1973
|
+
)
|
|
1974
|
+
stripped_final_stage = "\n".join(stripped_lines[stripped_stage_start:])
|
|
1975
|
+
has_custom_entrypoint = bool(re.search(r"(?im)^\s*ENTRYPOINT\s+", stripped_final_stage))
|
|
1976
|
+
already_has_opc_seed = ".opc-seed" in final_stage or "opc-storage-init.sh" in final_stage
|
|
1977
|
+
all_mounts_covered = all(m["container_path"] in final_stage for m in mounts)
|
|
1978
|
+
|
|
1979
|
+
if df_text and not has_custom_entrypoint and (not already_has_opc_seed or not all_mounts_covered):
|
|
2195
1980
|
seed_dir = f"{workdir}/.opc-seed"
|
|
2196
|
-
|
|
2197
|
-
|
|
1981
|
+
safe_paths = [
|
|
1982
|
+
path for path in (m["container_path"] for m in mounts)
|
|
2198
1983
|
if re.fullmatch(r"/[A-Za-z0-9_./-]+", path) and ".." not in Path(path).parts
|
|
2199
1984
|
and not (seed_dir == path.rstrip("/") or seed_dir.startswith(path.rstrip("/") + "/") or path.startswith(seed_dir + "/"))
|
|
2200
|
-
|
|
1985
|
+
]
|
|
1986
|
+
cpaths_sh = " ".join(f'"{p}"' for p in safe_paths)
|
|
2201
1987
|
if cpaths_sh:
|
|
2202
|
-
init_script = f"{
|
|
1988
|
+
init_script = f"{seed_dir}/opc-storage-init.sh"
|
|
1989
|
+
sync_py_path = f"{seed_dir}/sync_sqlite.py"
|
|
1990
|
+
sync_py_b64 = base64.b64encode(_ADDITIVE_SQLITE_SYNC_PY.encode("utf-8")).decode("ascii")
|
|
1991
|
+
init_sh_body = f"""#!/bin/sh
|
|
1992
|
+
set -e
|
|
1993
|
+
_opc_backup_mount() {{
|
|
1994
|
+
_d="$1"
|
|
1995
|
+
_bid="$2"
|
|
1996
|
+
_broot="$_d/.opc-backups"
|
|
1997
|
+
_marker="$_broot/.last-build-id"
|
|
1998
|
+
_prev=""
|
|
1999
|
+
if [ -f "$_marker" ]; then
|
|
2000
|
+
_prev="$(cat "$_marker" 2>/dev/null || true)"
|
|
2001
|
+
fi
|
|
2002
|
+
if [ "$_prev" != "$_bid" ] && [ -d "$_d" ]; then
|
|
2003
|
+
_has_data="$(find "$_d" -maxdepth 4 -path "$_broot" -prune -o -type f \\( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' -o -name '*.json' \\) -print 2>/dev/null | head -n 1 || true)"
|
|
2004
|
+
if [ -n "$_has_data" ]; then
|
|
2005
|
+
_snap="$_broot/pre-$_bid"
|
|
2006
|
+
mkdir -p "$_snap"
|
|
2007
|
+
(cd "$_d" && find . -maxdepth 4 -path './.opc-backups' -prune -o -type f \\( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' -o -name '*.db-wal' -o -name '*.db-shm' -o -name '*.sqlite-wal' -o -name '*.sqlite-shm' -o -name '*.sqlite3-wal' -o -name '*.sqlite3-shm' -o -name '*.json' \\) -print 2>/dev/null | while IFS= read -r _rel; do
|
|
2008
|
+
_rel="${{_rel#./}}"
|
|
2009
|
+
mkdir -p "$_snap/$(dirname "$_rel")"
|
|
2010
|
+
cp -a "$_d/$_rel" "$_snap/$_rel" 2>/dev/null || true
|
|
2011
|
+
done)
|
|
2012
|
+
_old_snaps="$(ls -1d "$_broot"/pre-* 2>/dev/null | sort || true)"
|
|
2013
|
+
_count="$(printf '%s\\n' "$_old_snaps" | grep -c '^' 2>/dev/null || echo 0)"
|
|
2014
|
+
if [ "$_count" -gt 3 ]; then
|
|
2015
|
+
printf '%s\\n' "$_old_snaps" | head -n $((_count - 3)) | while IFS= read -r _old; do
|
|
2016
|
+
rm -rf "$_old" 2>/dev/null || true
|
|
2017
|
+
done
|
|
2018
|
+
fi
|
|
2019
|
+
fi
|
|
2020
|
+
fi
|
|
2021
|
+
mkdir -p "$_broot"
|
|
2022
|
+
printf '%s' "$_bid" > "$_marker" 2>/dev/null || true
|
|
2023
|
+
}}
|
|
2024
|
+
_opc_copy_seed_dir() {{
|
|
2025
|
+
_s="$1"
|
|
2026
|
+
_d="$2"
|
|
2027
|
+
if [ ! -d "$_s" ]; then
|
|
2028
|
+
return 0
|
|
2029
|
+
fi
|
|
2030
|
+
mkdir -p "$_d"
|
|
2031
|
+
for _f in "$_s"/* "$_s"/.[!.]*; do
|
|
2032
|
+
[ -e "$_f" ] || continue
|
|
2033
|
+
_base="$(basename "$_f")"
|
|
2034
|
+
_target="$_d/$_base"
|
|
2035
|
+
if [ -d "$_f" ]; then
|
|
2036
|
+
_opc_copy_seed_dir "$_f" "$_target"
|
|
2037
|
+
elif [ ! -e "$_target" ]; then
|
|
2038
|
+
cp -a "$_f" "$_target" 2>/dev/null || true
|
|
2039
|
+
fi
|
|
2040
|
+
done
|
|
2041
|
+
}}
|
|
2042
|
+
_opc_build_id="1"
|
|
2043
|
+
if [ -f "{seed_dir}/.build-id" ]; then
|
|
2044
|
+
_opc_build_id="$(cat "{seed_dir}/.build-id" 2>/dev/null || echo 1)"
|
|
2045
|
+
fi
|
|
2046
|
+
i=0
|
|
2047
|
+
for d in {cpaths_sh}; do
|
|
2048
|
+
i=$((i+1))
|
|
2049
|
+
mkdir -p "$d"
|
|
2050
|
+
_opc_backup_mount "$d" "$_opc_build_id"
|
|
2051
|
+
_opc_copy_seed_dir "{seed_dir}/$i" "$d"
|
|
2052
|
+
if command -v python3 >/dev/null 2>&1 && [ -f "{sync_py_path}" ]; then
|
|
2053
|
+
python3 "{sync_py_path}" "{seed_dir}/$i" "$d" 2>/dev/null || true
|
|
2054
|
+
fi
|
|
2055
|
+
done
|
|
2056
|
+
exec "$@"
|
|
2057
|
+
"""
|
|
2058
|
+
init_sh_b64 = base64.b64encode(init_sh_body.encode("utf-8")).decode("ascii")
|
|
2203
2059
|
seed_snippet_lines = [
|
|
2204
2060
|
"# [OPC Persistent Storage] Backup initial seed data & hydrate external volume on startup",
|
|
2205
2061
|
f'RUN mkdir -p "{seed_dir}" {cpaths_sh} \\',
|
|
@@ -2209,16 +2065,25 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
|
|
|
2209
2065
|
f' mkdir -p "{seed_dir}/$i" && cp -a "$d/." "{seed_dir}/$i/" || exit 1; \\',
|
|
2210
2066
|
" fi; \\",
|
|
2211
2067
|
" done \\",
|
|
2212
|
-
f
|
|
2068
|
+
f' && (date +%s%N 2>/dev/null || date +%s) > "{seed_dir}/.build-id" \\',
|
|
2069
|
+
f' && printf \'%s\' \'{sync_py_b64}\' | (base64 -d 2>/dev/null || base64 --decode) > "{sync_py_path}" \\',
|
|
2070
|
+
f' && printf \'%s\' \'{init_sh_b64}\' | (base64 -d 2>/dev/null || base64 --decode) > "{init_script}" \\',
|
|
2213
2071
|
f' && chmod +x "{init_script}"',
|
|
2214
2072
|
"ENTRYPOINT " + json.dumps([init_script]),
|
|
2215
2073
|
]
|
|
2216
2074
|
seed_block = "\n".join(seed_snippet_lines)
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
)
|
|
2221
|
-
|
|
2075
|
+
base_lines = stripped_lines if already_has_opc_seed else lines
|
|
2076
|
+
base_stage_start = stripped_stage_start if already_has_opc_seed else final_stage_start
|
|
2077
|
+
last_copy_idx = base_stage_start
|
|
2078
|
+
for idx in range(base_stage_start + 1, len(base_lines)):
|
|
2079
|
+
t = base_lines[idx].strip().upper()
|
|
2080
|
+
if t.startswith(("COPY ", "COPY[", "ADD ", "ADD[")):
|
|
2081
|
+
last_copy_idx = idx
|
|
2082
|
+
insert_idx = len(base_lines)
|
|
2083
|
+
for idx in range(len(base_lines) - 1, last_copy_idx, -1):
|
|
2084
|
+
if re.match(r"\s*(?:CMD(?:\s+|\[)|USER\s+)", base_lines[idx], re.I):
|
|
2085
|
+
insert_idx = idx
|
|
2086
|
+
updated_df = "\n".join(base_lines[:insert_idx] + [seed_block] + base_lines[insert_idx:]) + "\n"
|
|
2222
2087
|
mount_summary = "、".join(m["container_path"] for m in mounts)
|
|
2223
2088
|
if "Dockerfile" in changes_by_path:
|
|
2224
2089
|
changes_by_path["Dockerfile"]["content"] = updated_df
|
|
@@ -2251,7 +2116,18 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
|
|
|
2251
2116
|
build_context = contract.get("build_context")
|
|
2252
2117
|
if not isinstance(build_context, list):
|
|
2253
2118
|
build_context = None
|
|
2254
|
-
|
|
2119
|
+
changes = [ch for ch in (proposal.get("changes") or []) if isinstance(ch, dict) and isinstance(ch.get("path"), str) and isinstance(ch.get("content"), str)]
|
|
2120
|
+
if changes:
|
|
2121
|
+
with tempfile.TemporaryDirectory(prefix="opc-scan-") as tmp_dir:
|
|
2122
|
+
scan_root = Path(tmp_dir) / "source"
|
|
2123
|
+
shutil.copytree(source, scan_root, symlinks=True)
|
|
2124
|
+
for ch in changes:
|
|
2125
|
+
target_file = delivery.relative(scan_root, ch["path"])
|
|
2126
|
+
target_file.parent.mkdir(parents=True, exist_ok=True)
|
|
2127
|
+
target_file.write_text(ch["content"], encoding="utf-8")
|
|
2128
|
+
scan = scan_app_routes_and_pages(scan_root, build_context=build_context)
|
|
2129
|
+
else:
|
|
2130
|
+
scan = scan_app_routes_and_pages(source, build_context=build_context)
|
|
2255
2131
|
sqlite_info = inspect_local_sqlite_files(source)
|
|
2256
2132
|
|
|
2257
2133
|
# 1. Merge any missing scanned APIs into contract['apis'] so 0 routes are omitted
|
|
@@ -2292,20 +2168,15 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
|
|
|
2292
2168
|
})
|
|
2293
2169
|
contract["pages"] = existing_pages
|
|
2294
2170
|
|
|
2295
|
-
# 2b.
|
|
2296
|
-
# Local file databases remain on volumes; historical migration receipts do not trigger code rewrites.
|
|
2297
|
-
|
|
2298
|
-
# 2b-2. Automatically detect & adapt persistent local file storage (JSON state, uploads, reports, backups)
|
|
2171
|
+
# 2b. Automatically detect & adapt persistent local file storage (SQLite, JSON state, uploads, reports, backups)
|
|
2299
2172
|
storage_mounts = _auto_adapt_persistent_file_storage(source, str(data.get("slug", "")), data, proposal)
|
|
2300
2173
|
|
|
2301
|
-
# 2c. Filter out internal tool/truncation/
|
|
2174
|
+
# 2c. Filter out internal tool/truncation/default-password/local-json noise from unresolved & decisions
|
|
2302
2175
|
internal_noise_re = re.compile(
|
|
2303
|
-
r"(截断|truncated|补齐.*扫描|完整接口详情|接口扫描显示|扫描结果不一致|自动路由扫描未识别|逐表差异|逐表核对|10,?264|14,?758|healthz
|
|
2176
|
+
r"(截断|truncated|补齐.*扫描|完整接口详情|接口扫描显示|扫描结果不一致|自动路由扫描未识别|逐表差异|逐表核对|10,?264|14,?758|healthz|健康检查接口|默认管理员密码|公开的默认.*密码|演示口令|仅存于容器本地文件|容器重建后.*丢失)",
|
|
2304
2177
|
re.I,
|
|
2305
2178
|
)
|
|
2306
|
-
has_migrated = bool(data.get("migrations"))
|
|
2307
2179
|
feedback_text = str(data.get("feedback") or "")
|
|
2308
|
-
user_confirmed_db = has_migrated or bool(re.search(r"(迁移|PostgreSQL|平台.*数据库|直接迁移|保留.*SQLite)", feedback_text, re.I))
|
|
2309
2180
|
user_confirmed_local_or_pwd = bool(re.search(r"(保留演示口令|保留文件存储|保留原|先保留)", feedback_text, re.I))
|
|
2310
2181
|
|
|
2311
2182
|
def _keep_question(q: str) -> bool:
|
|
@@ -2314,8 +2185,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
|
|
|
2314
2185
|
return False
|
|
2315
2186
|
if internal_noise_re.search(s):
|
|
2316
2187
|
return False
|
|
2317
|
-
if user_confirmed_db and re.search(r"(SQLite.*迁移|补迁|保留\s*SQLite|PostgreSQL\s*连接适配|数据库切换)", s, re.I):
|
|
2318
|
-
return False
|
|
2319
2188
|
if user_confirmed_local_or_pwd and re.search(r"(密码|口令|本地文件|文件存储|JSON)", s, re.I):
|
|
2320
2189
|
return False
|
|
2321
2190
|
return True
|
|
@@ -2368,12 +2237,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
|
|
|
2368
2237
|
|
|
2369
2238
|
sqlite_dbs = sqlite_info.get("sqlite_databases") or []
|
|
2370
2239
|
resources = select_and_filter_platform_resources(data.get("resources") or [], str(data.get("slug", "")))
|
|
2371
|
-
pg_resources = [
|
|
2372
|
-
r for r in resources
|
|
2373
|
-
if str(r.get("type", "")).lower() in {"postgres", "postgresql"}
|
|
2374
|
-
and not str(r.get("id", "")).lower().startswith("db-")
|
|
2375
|
-
]
|
|
2376
|
-
pg_resources.sort(key=lambda r: (0 if r.get("is_default") else 1, str(r.get("id", ""))))
|
|
2377
2240
|
storage_type = step1_ai.get("storage_type") or (
|
|
2378
2241
|
"sqlite" if sqlite_dbs else ("persistent_volume" if storage_mounts else "none")
|
|
2379
2242
|
)
|
|
@@ -2470,9 +2333,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
|
|
|
2470
2333
|
}
|
|
2471
2334
|
for db in sqlite_dbs
|
|
2472
2335
|
],
|
|
2473
|
-
"can_migrate_to_platform_pg": False,
|
|
2474
|
-
"platform_pg_resource": pg_resources[0]["name"] if pg_resources else "",
|
|
2475
|
-
"platform_pg_id": pg_resources[0]["id"] if pg_resources else "",
|
|
2476
2336
|
"storage_mounts": storage_mounts,
|
|
2477
2337
|
"storage_migration_plan": default_storage_plan if sqlite_dbs else (step2_ai.get("storage_migration") or default_storage_plan),
|
|
2478
2338
|
},
|