dsh-plugin-opc 0.1.19 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  """Step 1 & Step 2 helpers for OPC App Delivery:
2
2
  - Dockerfile template fetching from platform-opc (deploy-opc database)
3
- - Local SQLite / local DB (PostgreSQL, MySQL, SQLServer) / local Redis inspection and automatic migration to platform-opc server resources
3
+ - Local SQLite / local DB (PostgreSQL, MySQL, SQLServer) / local Redis inspection and persistent storage adaptation
4
4
  """
5
5
 
6
6
  from __future__ import annotations
@@ -8,9 +8,12 @@ from __future__ import annotations
8
8
  import ast
9
9
  import base64
10
10
  import json
11
+ import os
11
12
  from pathlib import Path
12
13
  import re
14
+ import shutil
13
15
  import sqlite3
16
+ import tempfile
14
17
  import urllib.error
15
18
  import urllib.parse
16
19
  import urllib.request
@@ -112,7 +115,6 @@ def inspect_local_file_storage(
112
115
  source: Path,
113
116
  slug: str = "",
114
117
  workdir: str = "/app",
115
- has_migrations: bool = False,
116
118
  ) -> dict:
117
119
  """Detect local JSON/file data directories, upload folders, report/backup folders, and seed files
118
120
  that require external persistent volume mounting (Docker Volume / Host Bind / S3-OSS).
@@ -143,7 +145,7 @@ def inspect_local_file_storage(
143
145
  ext = p.suffix.lower()
144
146
  if ext in _SOURCE_CODE_EXTENSIONS:
145
147
  dir_has_code.add(top)
146
- elif not (has_migrations and ext in {".sqlite", ".sqlite3", ".db"}):
148
+ else:
147
149
  dir_seed_files.setdefault(top, []).append(p.relative_to(source).as_posix())
148
150
 
149
151
  # 2. Check second-level persistent directories (e.g. reimburse-web/server/data)
@@ -159,7 +161,7 @@ def inspect_local_file_storage(
159
161
  ext = p.suffix.lower()
160
162
  if ext in _SOURCE_CODE_EXTENSIONS:
161
163
  dir_has_code.add(rel_dir)
162
- elif not (has_migrations and ext in {".sqlite", ".sqlite3", ".db"}):
164
+ else:
163
165
  dir_seed_files.setdefault(rel_dir, []).append(p.relative_to(source).as_posix())
164
166
 
165
167
  # 3. Scan source code for runtime mutable directories (e.g. join(root, "uploads"), join(root, "reports"))
@@ -352,75 +354,6 @@ def extract_sqlite_tables(sqlite_file: Path, max_rows_per_table: int | None = No
352
354
  return tables
353
355
 
354
356
 
355
- def migrate_local_data_to_platform(
356
- *,
357
- platform_url: str,
358
- slug: str,
359
- source_dir: Path,
360
- source_type: str = "sqlite",
361
- sqlite_path: str = "",
362
- target_type: str = "postgres",
363
- target_resource_id: str = "",
364
- tables_override: list | None = None,
365
- redis_keys: list | None = None,
366
- ) -> dict:
367
- """Extract schema & 100% of data from local SQLite (or provided tables/redis_keys) and upload to platform-opc POST /api/v1/apps/:slug/migrate-data."""
368
- tables = tables_override or []
369
- if not tables and source_type.lower() == "sqlite":
370
- if sqlite_path.strip():
371
- db_file = delivery.relative(source_dir, sqlite_path.strip())
372
- if db_file.is_file() and db_file.stat().st_size > 0:
373
- tables = extract_sqlite_tables(db_file, max_rows_per_table=None)
374
- if not tables:
375
- found = [
376
- p for p in sorted(source_dir.rglob("*"))
377
- if p.is_file() and p.suffix.lower() in {".sqlite", ".sqlite3", ".db"} and p.stat().st_size > 0
378
- ]
379
- if not found:
380
- raise ValueError("未在源码目录中找到非空 SQLite 数据库文件")
381
- # Sort non-empty SQLite files by size descending and merge all non-empty tables
382
- found.sort(key=lambda p: p.stat().st_size, reverse=True)
383
- seen_tables = set()
384
- for db_file in found:
385
- try:
386
- extracted = extract_sqlite_tables(db_file, max_rows_per_table=None)
387
- except Exception:
388
- continue
389
- for tbl in extracted:
390
- if tbl["name"] not in seen_tables:
391
- seen_tables.add(tbl["name"])
392
- tables.append(tbl)
393
-
394
- payload = {
395
- "source_type": source_type.lower().strip(),
396
- "target_type": target_type.lower().strip() or "postgres",
397
- "target_resource_id": target_resource_id.strip(),
398
- "tables": tables,
399
- "redis_keys": redis_keys or [],
400
- }
401
- base = (platform_url or "https://shtokendance.benlai.com").rstrip("/")
402
- url = f"{base}/api/v1/apps/{urllib.parse.quote(slug)}/migrate-data"
403
- body_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
404
- req = urllib.request.Request(
405
- url,
406
- data=body_bytes,
407
- headers={"Content-Type": "application/json", "Accept": "application/json"},
408
- method="POST",
409
- )
410
- opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
411
- try:
412
- with opener.open(req, timeout=600) as resp:
413
- resp_json = json.loads(resp.read().decode("utf-8"))
414
- if resp_json.get("code") == 0 and resp_json.get("data"):
415
- return resp_json["data"]
416
- return resp_json
417
- except urllib.error.HTTPError as exc:
418
- detail = exc.read().decode("utf-8", errors="replace")
419
- raise ValueError(f"平台数据迁移接口返回错误 ({exc.code}): {detail}") from exc
420
- except Exception as exc:
421
- raise ValueError(f"调用平台数据迁移接口失败: {exc}") from exc
422
-
423
-
424
357
  def _join_route_paths(*parts: str) -> str:
425
358
  cleaned = []
426
359
  for p in parts:
@@ -1784,6 +1717,8 @@ def _normalize_resource_type_group(raw_type: str) -> str:
1784
1717
  return "postgres"
1785
1718
  if t == "mssql":
1786
1719
  return "sqlserver"
1720
+ if t in {"local-directory", "aliyun-oss-volume"}:
1721
+ return "application-volume"
1787
1722
  return t
1788
1723
 
1789
1724
 
@@ -1856,258 +1791,97 @@ def select_and_filter_platform_resources(resources: list[dict] | None, slug: str
1856
1791
  return result
1857
1792
 
1858
1793
 
1859
- _PG_SQLITE3_COMPAT_SHIM = '''# --- OPC Platform PostgreSQL (DATABASE_URL) Compatibility Shim ---
1860
- import os as _opc_os, re as _opc_re
1861
- if _opc_os.getenv("DATABASE_URL", "").startswith(("postgres://", "postgresql://")):
1862
- import psycopg2 as _opc_pg
1863
- class _OpcPgRow(dict):
1864
- def __init__(self, cols, vals):
1865
- super().__init__(zip(cols, vals))
1866
- self._vals = tuple(vals)
1867
- def __getitem__(self, key):
1868
- return self._vals[key] if isinstance(key, (int, slice)) else super().__getitem__(key)
1869
- def _opc_translate_sql(sql: str) -> str:
1870
- s = sql.strip()
1871
- u = s.upper()
1872
- if u.startswith("BEGIN IMMEDIATE") or u.startswith("BEGIN EXCLUSIVE") or u == "BEGIN":
1873
- return "SELECT 1"
1874
- if _opc_re.match(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", s):
1875
- s = _opc_re.sub(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", "INSERT INTO", s, count=1).rstrip().rstrip(";") + " ON CONFLICT DO NOTHING"
1876
- out, in_sq, in_dq, i = [], False, False, 0
1877
- while i < len(s):
1878
- ch = s[i]
1879
- if ch == "'" and not in_dq:
1880
- in_sq = not in_sq
1881
- out.append(ch)
1882
- elif ch == '"' and not in_sq:
1883
- in_dq = not in_dq
1884
- out.append(ch)
1885
- elif ch == "?" and not in_sq and not in_dq:
1886
- out.append("%s")
1887
- elif ch == "%" and not in_sq and not in_dq and (i + 1 >= len(s) or s[i + 1] != "s"):
1888
- out.append("%%")
1889
- else:
1890
- out.append(ch)
1891
- i += 1
1892
- return "".join(out)
1893
- class _OpcPgCursor:
1894
- def __init__(self, rows=None, rowcount=0, lastrowid=None):
1895
- self._rows = list(rows or [])
1896
- self._idx = 0
1897
- self.rowcount = rowcount
1898
- self.lastrowid = lastrowid
1899
- def fetchone(self):
1900
- if self._idx >= len(self._rows):
1901
- return None
1902
- r = self._rows[self._idx]
1903
- self._idx += 1
1904
- return r
1905
- def fetchall(self):
1906
- r = self._rows[self._idx:]
1907
- self._idx = len(self._rows)
1908
- return r
1909
- def __iter__(self):
1910
- return iter(self._rows)
1911
- class _OpcPgConn:
1912
- def __init__(self, dsn: str):
1913
- self._conn = _opc_pg.connect(dsn)
1914
- self.row_factory = None
1915
- def execute(self, sql: str, params=()):
1916
- s = sql.strip()
1917
- u = s.upper()
1918
- if u.startswith("PRAGMA"):
1919
- m = _opc_re.match(r'(?is)^PRAGMA\\s+table_info\\(\\s*["\\\']?([A-Za-z0-9_]+)["\\\']?\\s*\\)', s)
1920
- if m and m.group(1).lower() != "invoice_registry":
1921
- tbl = m.group(1).lower()
1922
- with self._conn.cursor() as cur:
1923
- cur.execute("SELECT ordinal_position - 1 AS cid, column_name AS name, data_type AS type, CASE WHEN is_nullable = 'NO' THEN 1 ELSE 0 END AS notnull, column_default AS dflt_value, 0 AS pk FROM information_schema.columns WHERE table_schema = 'public' AND table_name = %s ORDER BY ordinal_position", (tbl,))
1924
- cols = [d[0] for d in cur.description]
1925
- return _OpcPgCursor([_OpcPgRow(cols, r) for r in cur.fetchall()])
1926
- return _OpcPgCursor([])
1927
- if "SQLITE_MASTER" in u:
1928
- return _OpcPgCursor([])
1929
- pg_sql = _opc_translate_sql(s)
1930
- with self._conn.cursor() as cur:
1931
- cur.execute(pg_sql, tuple(params) if params else None)
1932
- if cur.description:
1933
- cols = [d[0] for d in cur.description]
1934
- rows = [_OpcPgRow(cols, r) for r in cur.fetchall()]
1935
- return _OpcPgCursor(rows, cur.rowcount)
1936
- last_id = None
1937
- if u.startswith("INSERT"):
1938
- try:
1939
- cur.execute("SAVEPOINT _opc_lv; SELECT LASTVAL()")
1940
- r = cur.fetchone()
1941
- last_id = r[0] if r else None
1942
- cur.execute("RELEASE SAVEPOINT _opc_lv")
1943
- except Exception:
1944
- try:
1945
- cur.execute("ROLLBACK TO SAVEPOINT _opc_lv")
1946
- except Exception:
1947
- pass
1948
- return _OpcPgCursor([], cur.rowcount, last_id)
1949
- def executemany(self, sql: str, seq_of_params):
1950
- pg_sql = _opc_translate_sql(sql)
1951
- with self._conn.cursor() as cur:
1952
- for p in seq_of_params:
1953
- cur.execute(pg_sql, tuple(p))
1954
- return _OpcPgCursor([], cur.rowcount)
1955
- def executescript(self, script: str):
1956
- return _OpcPgCursor([])
1957
- def cursor(self):
1958
- return self
1959
- def commit(self):
1960
- self._conn.commit()
1961
- def rollback(self):
1962
- self._conn.rollback()
1963
- def close(self):
1964
- self._conn.close()
1965
- def __enter__(self):
1966
- return self
1967
- def __exit__(self, exc_type, exc, tb):
1968
- if exc_type is None:
1969
- self.commit()
1970
- else:
1971
- self.rollback()
1972
- sqlite3.connect = lambda *a, **kw: _OpcPgConn(_opc_os.environ["DATABASE_URL"])
1973
- # --- End OPC Platform PostgreSQL Compatibility Shim ---
1974
- '''
1975
-
1976
-
1977
- def _auto_adapt_sqlite_to_postgres_changes(source: Path, slug: str, data: dict, proposal: dict) -> None:
1978
- """When the user has migrated local SQLite to platform PostgreSQL (`data['migrations']`),
1979
- guarantee that:
1980
- 1. All SQLite database connection modules in Python (e.g. `db.py`) are adapted to connect via `DATABASE_URL` (`psycopg2`).
1981
- 2. `requirements.txt` includes `psycopg2-binary`.
1982
- 3. `contract.configs` declares `DATABASE_URL` and `contract.resources` includes the migrated database resource.
1983
- """
1984
- migrations = [m for m in (data.get("migrations") or []) if isinstance(m, dict)]
1985
- pg_migs = [m for m in migrations if str(m.get("target_type", "postgres")).lower() in {"postgres", "postgresql"}]
1986
- if not pg_migs:
1987
- return
1988
-
1989
- latest_mig = pg_migs[-1]
1990
- res_id = str(latest_mig.get("resource_id") or f"db-{slug}")
1991
- db_name = str(latest_mig.get("database_name") or f"app_{slug.replace('-', '_')}")
1992
-
1993
- contract = proposal.setdefault("contract", {})
1994
- changes: list[dict] = proposal.setdefault("changes", [])
1995
- changes_by_path: dict[str, dict] = {c.get("path", ""): c for c in changes if isinstance(c, dict) and c.get("path")}
1996
- used_ids: set[str] = {str(c.get("id", "")) for c in changes if isinstance(c, dict)}
1997
-
1998
- def _unique_change_id(base_id: str) -> str:
1999
- cid = base_id
2000
- idx = 2
2001
- while cid in used_ids:
2002
- cid = f"{base_id}-{idx}"
2003
- idx += 1
2004
- used_ids.add(cid)
2005
- return cid
2006
-
2007
- # 1. Find Python files that call sqlite3.connect(...)
2008
- sqlite_py_files: list[Path] = []
2009
- for p in sorted(source.rglob("*.py")):
2010
- if not p.is_file() or p.is_symlink():
1794
+ def _strip_opc_seed_block(df_text: str) -> str:
1795
+ lines = df_text.splitlines()
1796
+ out: list[str] = []
1797
+ in_cont = False
1798
+ for line in lines:
1799
+ trimmed = line.strip()
1800
+ if in_cont:
1801
+ in_cont = trimmed.endswith("\\")
2011
1802
  continue
2012
- rel_parts = p.relative_to(source).parts
2013
- if any(part in delivery.EXCLUDED or part in {"tests", "test"} for part in rel_parts):
1803
+ if trimmed.startswith("# [OPC Persistent Storage]") or trimmed.startswith("# OPC-PERSISTENT-STORAGE-"):
2014
1804
  continue
2015
- try:
2016
- txt = p.read_text(encoding="utf-8")
2017
- except Exception:
1805
+ if ".opc-seed" in trimmed or "opc-storage-init.sh" in trimmed:
1806
+ in_cont = trimmed.endswith("\\")
2018
1807
  continue
2019
- if "import sqlite3" in txt and "sqlite3.connect" in txt:
2020
- sqlite_py_files.append(p)
2021
-
2022
- # Prioritize files named db.py / database.py / storage.py
2023
- sqlite_py_files.sort(key=lambda p: (0 if p.stem.lower() in {"db", "database", "storage", "sqlite"} else 1, len(p.parts)))
2024
-
2025
- for py_file in sqlite_py_files:
2026
- rel_path = py_file.relative_to(source).as_posix()
2027
- existing_change = changes_by_path.get(rel_path)
2028
- current_content = existing_change["content"] if (existing_change and isinstance(existing_change.get("content"), str)) else py_file.read_text(encoding="utf-8")
2029
- if "DATABASE_URL" in current_content or "psycopg" in current_content:
1808
+ out.append(line)
1809
+ return "\n".join(out) + ("\n" if df_text.endswith("\n") else "")
1810
+
1811
+
1812
+ _ADDITIVE_SQLITE_SYNC_PY = """import os, sys, sqlite3
1813
+ seed_root, dst_root = sys.argv[1], sys.argv[2]
1814
+ if not os.path.isdir(seed_root) or not os.path.isdir(dst_root):
1815
+ sys.exit(0)
1816
+ def q(ident):
1817
+ return '"' + str(ident).replace('"', '""') + '"'
1818
+ for root, dirs, files in os.walk(seed_root):
1819
+ dirs[:] = [d for d in dirs if d != '.opc-backups']
1820
+ rel_dir = os.path.relpath(root, seed_root)
1821
+ for fn in files:
1822
+ if not fn.lower().endswith(('.db', '.sqlite', '.sqlite3')):
2030
1823
  continue
2031
- # Inject _PG_SQLITE3_COMPAT_SHIM right after `import sqlite3`
2032
- new_content, n_subs = re.subn(
2033
- r"(?m)^(\s*import\s+sqlite3\b[^\n]*\n)",
2034
- lambda m: m.group(1) + _PG_SQLITE3_COMPAT_SHIM,
2035
- current_content,
2036
- count=1,
2037
- )
2038
- if n_subs == 0:
1824
+ sp = os.path.join(root, fn)
1825
+ dp = os.path.join(dst_root, fn) if rel_dir == '.' else os.path.join(dst_root, rel_dir, fn)
1826
+ if not os.path.isfile(sp) or not os.path.isfile(dp):
2039
1827
  continue
2040
- if existing_change:
2041
- existing_change["content"] = new_content
2042
- if "PostgreSQL" not in existing_change.get("reason", ""):
2043
- existing_change["reason"] = existing_change.get("reason", "").rstrip("。") + ";并接入平台 PostgreSQL 数据库 (DATABASE_URL)"
2044
- else:
2045
- change_obj = {
2046
- "id": _unique_change_id("adapt-postgres-db"),
2047
- "path": rel_path,
2048
- "group": "迁移并接入平台 PostgreSQL 数据库",
2049
- "reason": f"将本地 SQLite 数据库连接切换至平台默认 PostgreSQL(专属库 {db_name},读取 DATABASE_URL 环境变量,保留原业务查询逻辑不变)",
2050
- "content": new_content,
2051
- "executable": False,
2052
- }
2053
- changes.append(change_obj)
2054
- changes_by_path[rel_path] = change_obj
2055
-
2056
- # 2. Ensure requirements.txt includes psycopg2-binary
2057
- req_files: list[Path] = []
2058
- for p in sorted(source.rglob("requirements*.txt")):
2059
- if p.is_file() and not p.is_symlink() and not any(part in delivery.EXCLUDED for part in p.relative_to(source).parts):
2060
- req_files.append(p)
2061
- req_files.sort(key=lambda p: (0 if p.name == "requirements.txt" else 1, len(p.parts)))
2062
- if req_files:
2063
- req_file = req_files[0]
2064
- req_rel = req_file.relative_to(source).as_posix()
2065
- existing_req = changes_by_path.get(req_rel)
2066
- req_content = existing_req["content"] if (existing_req and isinstance(existing_req.get("content"), str)) else req_file.read_text(encoding="utf-8")
2067
- if not re.search(r"(?i)\bpsycopg2(?:-binary)?\b", req_content):
2068
- updated_req = req_content.rstrip() + "\npsycopg2-binary>=2.9\n"
2069
- if existing_req:
2070
- existing_req["content"] = updated_req
2071
- else:
2072
- req_change = {
2073
- "id": _unique_change_id("add-postgres-driver"),
2074
- "path": req_rel,
2075
- "group": "迁移并接入平台 PostgreSQL 数据库",
2076
- "reason": "添加 PostgreSQL 数据库驱动依赖 psycopg2-binary,使应用在容器内可直连平台 PostgreSQL",
2077
- "content": updated_req,
2078
- "executable": False,
2079
- }
2080
- changes.append(req_change)
2081
- changes_by_path[req_rel] = req_change
2082
-
2083
- # 3. Ensure DATABASE_URL is declared in contract.configs
2084
- configs: list[dict] = contract.setdefault("configs", [])
2085
- if not any(isinstance(c, dict) and c.get("key") == "DATABASE_URL" for c in configs):
2086
- configs.append({
2087
- "key": "DATABASE_URL",
2088
- "default": "",
2089
- "secret": True,
2090
- "required": True,
2091
- "description": f"平台 PostgreSQL 数据库连接地址(自动绑定专属迁移库 {db_name})",
2092
- })
2093
-
2094
- # 4. Ensure migrated resource is listed in contract.resources
2095
- c_resources: list[dict] = contract.setdefault("resources", [])
2096
- if not any(isinstance(r, dict) and r.get("id") == res_id for r in c_resources):
2097
- c_resources.append({
2098
- "id": res_id,
2099
- "type": "postgres",
2100
- "purpose": f"平台自动迁移创建的专属 PostgreSQL 数据库 ({db_name})",
2101
- })
2102
-
2103
- # 5. Ensure build_context covers any newly added/modified files
2104
- build_ctx: list[str] = contract.setdefault("build_context", [])
2105
- if isinstance(build_ctx, list) and build_ctx and "." not in build_ctx:
2106
- for ch_path in changes_by_path:
2107
- covered = any(ch_path == entry or ch_path.startswith(entry.rstrip("/") + "/") for entry in build_ctx if isinstance(entry, str))
2108
- if not covered:
2109
- top_dir = ch_path.split("/")[0]
2110
- build_ctx.append(top_dir if "/" in ch_path else ch_path)
1828
+ try:
1829
+ with open(sp, 'rb') as f:
1830
+ if f.read(16) != b'SQLite format 3\\x00':
1831
+ continue
1832
+ sc = sqlite3.connect('file:' + sp + '?mode=ro', uri=True)
1833
+ dc = sqlite3.connect(dp, timeout=10.0)
1834
+ dc.execute('PRAGMA busy_timeout=10000')
1835
+ existing_tables = {r[0] for r in dc.execute("SELECT name FROM sqlite_master WHERE type='table'")}
1836
+ for tbl, ddl in sc.execute("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND sql IS NOT NULL").fetchall():
1837
+ try:
1838
+ if tbl not in existing_tables:
1839
+ dc.execute(ddl)
1840
+ rows = sc.execute('SELECT * FROM ' + q(tbl)).fetchall()
1841
+ if rows:
1842
+ ph = ','.join('?' for _ in rows[0])
1843
+ dc.executemany('INSERT OR IGNORE INTO ' + q(tbl) + ' VALUES (' + ph + ')', rows)
1844
+ else:
1845
+ dst_cols = {r[1] for r in dc.execute('PRAGMA table_info(' + q(tbl) + ')').fetchall()}
1846
+ for cid, cname, ctype, notnull, dflt, pk in sc.execute('PRAGMA table_info(' + q(tbl) + ')').fetchall():
1847
+ if cname in dst_cols or pk:
1848
+ continue
1849
+ t = (ctype or 'TEXT').strip() or 'TEXT'
1850
+ tu = t.upper()
1851
+ if 'INT' in tu or 'NUM' in tu or 'REAL' in tu or 'FLOA' in tu or 'DOUB' in tu:
1852
+ fallback_dflt = '0'
1853
+ elif 'BLOB' in tu:
1854
+ fallback_dflt = "X''"
1855
+ else:
1856
+ fallback_dflt = "''"
1857
+ clause = q(cname) + ' ' + t
1858
+ if dflt is not None:
1859
+ clause += ' DEFAULT ' + str(dflt)
1860
+ elif notnull:
1861
+ clause += ' NOT NULL DEFAULT ' + fallback_dflt
1862
+ try:
1863
+ dc.execute('ALTER TABLE ' + q(tbl) + ' ADD COLUMN ' + clause)
1864
+ except Exception:
1865
+ try:
1866
+ dc.execute('ALTER TABLE ' + q(tbl) + ' ADD COLUMN ' + q(cname) + ' ' + t + ' DEFAULT ' + fallback_dflt)
1867
+ except Exception:
1868
+ pass
1869
+ except Exception:
1870
+ pass
1871
+ for idx_name, idx_sql in sc.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND sql IS NOT NULL"):
1872
+ try:
1873
+ stmt = idx_sql
1874
+ if 'IF NOT EXISTS' not in stmt.upper():
1875
+ stmt = stmt.replace('CREATE UNIQUE INDEX ', 'CREATE UNIQUE INDEX IF NOT EXISTS ', 1).replace('CREATE INDEX ', 'CREATE INDEX IF NOT EXISTS ', 1)
1876
+ dc.execute(stmt)
1877
+ except Exception:
1878
+ pass
1879
+ dc.commit()
1880
+ sc.close()
1881
+ dc.close()
1882
+ except Exception:
1883
+ pass
1884
+ """
2111
1885
 
2112
1886
 
2113
1887
  def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, proposal: dict) -> list[dict]:
@@ -2142,7 +1916,7 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
2142
1916
  workdir = workdirs[-1].rstrip("/") or "/app"
2143
1917
 
2144
1918
  # 2. Deterministically inspect local file storage & merge with any AI-declared mounts
2145
- scanned_storage = inspect_local_file_storage(source, slug=slug, workdir=workdir, has_migrations=False)
1919
+ scanned_storage = inspect_local_file_storage(source, slug=slug, workdir=workdir)
2146
1920
  mounts_by_cpath: dict[str, dict] = {}
2147
1921
  for item in scanned_storage.get("storage_mounts") or []:
2148
1922
  if isinstance(item, dict) and item.get("container_path"):
@@ -2190,16 +1964,98 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
2190
1964
  build_ctx.append(top_dir)
2191
1965
 
2192
1966
  # 4. Seed the actual volume destinations without replacing a custom startup.
2193
- has_entrypoint = bool(re.search(r"(?im)^\s*ENTRYPOINT\s+", final_stage))
2194
- if df_text and not has_entrypoint and ".opc-seed" not in final_stage and "opc-storage-init.sh" not in final_stage:
1967
+ # If Dockerfile already contains an OPC seed block from v1, update it when mounts change.
1968
+ stripped_df = _strip_opc_seed_block(df_text)
1969
+ stripped_lines = stripped_df.splitlines()
1970
+ stripped_stage_start = next(
1971
+ (index for index in range(len(stripped_lines) - 1, -1, -1) if re.match(r"\s*FROM\s+", stripped_lines[index], re.I)),
1972
+ 0,
1973
+ )
1974
+ stripped_final_stage = "\n".join(stripped_lines[stripped_stage_start:])
1975
+ has_custom_entrypoint = bool(re.search(r"(?im)^\s*ENTRYPOINT\s+", stripped_final_stage))
1976
+ already_has_opc_seed = ".opc-seed" in final_stage or "opc-storage-init.sh" in final_stage
1977
+ all_mounts_covered = all(m["container_path"] in final_stage for m in mounts)
1978
+
1979
+ if df_text and not has_custom_entrypoint and (not already_has_opc_seed or not all_mounts_covered):
2195
1980
  seed_dir = f"{workdir}/.opc-seed"
2196
- cpaths_sh = " ".join(
2197
- f'"{path}"' for path in (m["container_path"] for m in mounts)
1981
+ safe_paths = [
1982
+ path for path in (m["container_path"] for m in mounts)
2198
1983
  if re.fullmatch(r"/[A-Za-z0-9_./-]+", path) and ".." not in Path(path).parts
2199
1984
  and not (seed_dir == path.rstrip("/") or seed_dir.startswith(path.rstrip("/") + "/") or path.startswith(seed_dir + "/"))
2200
- )
1985
+ ]
1986
+ cpaths_sh = " ".join(f'"{p}"' for p in safe_paths)
2201
1987
  if cpaths_sh:
2202
- init_script = f"{workdir}/.opc-seed/opc-storage-init.sh"
1988
+ init_script = f"{seed_dir}/opc-storage-init.sh"
1989
+ sync_py_path = f"{seed_dir}/sync_sqlite.py"
1990
+ sync_py_b64 = base64.b64encode(_ADDITIVE_SQLITE_SYNC_PY.encode("utf-8")).decode("ascii")
1991
+ init_sh_body = f"""#!/bin/sh
1992
+ set -e
1993
+ _opc_backup_mount() {{
1994
+ _d="$1"
1995
+ _bid="$2"
1996
+ _broot="$_d/.opc-backups"
1997
+ _marker="$_broot/.last-build-id"
1998
+ _prev=""
1999
+ if [ -f "$_marker" ]; then
2000
+ _prev="$(cat "$_marker" 2>/dev/null || true)"
2001
+ fi
2002
+ if [ "$_prev" != "$_bid" ] && [ -d "$_d" ]; then
2003
+ _has_data="$(find "$_d" -maxdepth 4 -path "$_broot" -prune -o -type f \\( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' -o -name '*.json' \\) -print 2>/dev/null | head -n 1 || true)"
2004
+ if [ -n "$_has_data" ]; then
2005
+ _snap="$_broot/pre-$_bid"
2006
+ mkdir -p "$_snap"
2007
+ (cd "$_d" && find . -maxdepth 4 -path './.opc-backups' -prune -o -type f \\( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' -o -name '*.db-wal' -o -name '*.db-shm' -o -name '*.sqlite-wal' -o -name '*.sqlite-shm' -o -name '*.sqlite3-wal' -o -name '*.sqlite3-shm' -o -name '*.json' \\) -print 2>/dev/null | while IFS= read -r _rel; do
2008
+ _rel="${{_rel#./}}"
2009
+ mkdir -p "$_snap/$(dirname "$_rel")"
2010
+ cp -a "$_d/$_rel" "$_snap/$_rel" 2>/dev/null || true
2011
+ done)
2012
+ _old_snaps="$(ls -1d "$_broot"/pre-* 2>/dev/null | sort || true)"
2013
+ _count="$(printf '%s\\n' "$_old_snaps" | grep -c '^' 2>/dev/null || echo 0)"
2014
+ if [ "$_count" -gt 3 ]; then
2015
+ printf '%s\\n' "$_old_snaps" | head -n $((_count - 3)) | while IFS= read -r _old; do
2016
+ rm -rf "$_old" 2>/dev/null || true
2017
+ done
2018
+ fi
2019
+ fi
2020
+ fi
2021
+ mkdir -p "$_broot"
2022
+ printf '%s' "$_bid" > "$_marker" 2>/dev/null || true
2023
+ }}
2024
+ _opc_copy_seed_dir() {{
2025
+ _s="$1"
2026
+ _d="$2"
2027
+ if [ ! -d "$_s" ]; then
2028
+ return 0
2029
+ fi
2030
+ mkdir -p "$_d"
2031
+ for _f in "$_s"/* "$_s"/.[!.]*; do
2032
+ [ -e "$_f" ] || continue
2033
+ _base="$(basename "$_f")"
2034
+ _target="$_d/$_base"
2035
+ if [ -d "$_f" ]; then
2036
+ _opc_copy_seed_dir "$_f" "$_target"
2037
+ elif [ ! -e "$_target" ]; then
2038
+ cp -a "$_f" "$_target" 2>/dev/null || true
2039
+ fi
2040
+ done
2041
+ }}
2042
+ _opc_build_id="1"
2043
+ if [ -f "{seed_dir}/.build-id" ]; then
2044
+ _opc_build_id="$(cat "{seed_dir}/.build-id" 2>/dev/null || echo 1)"
2045
+ fi
2046
+ i=0
2047
+ for d in {cpaths_sh}; do
2048
+ i=$((i+1))
2049
+ mkdir -p "$d"
2050
+ _opc_backup_mount "$d" "$_opc_build_id"
2051
+ _opc_copy_seed_dir "{seed_dir}/$i" "$d"
2052
+ if command -v python3 >/dev/null 2>&1 && [ -f "{sync_py_path}" ]; then
2053
+ python3 "{sync_py_path}" "{seed_dir}/$i" "$d" 2>/dev/null || true
2054
+ fi
2055
+ done
2056
+ exec "$@"
2057
+ """
2058
+ init_sh_b64 = base64.b64encode(init_sh_body.encode("utf-8")).decode("ascii")
2203
2059
  seed_snippet_lines = [
2204
2060
  "# [OPC Persistent Storage] Backup initial seed data & hydrate external volume on startup",
2205
2061
  f'RUN mkdir -p "{seed_dir}" {cpaths_sh} \\',
@@ -2209,16 +2065,25 @@ def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, pro
2209
2065
  f' mkdir -p "{seed_dir}/$i" && cp -a "$d/." "{seed_dir}/$i/" || exit 1; \\',
2210
2066
  " fi; \\",
2211
2067
  " done \\",
2212
- f" && printf '#!/bin/sh\\nset -e\\ni=0\\nfor d in {cpaths_sh}; do\\n i=$((i+1))\\n mkdir -p \"$d\"\\n if [ -d \"{seed_dir}/$i\" ]; then\\n for f in \"{seed_dir}/$i\"/* \"{seed_dir}/$i\"/.[!.]*; do\\n [ -e \"$f\" ] || continue\\n bn=\"$(basename \"$f\")\"\\n [ -e \"$d/$bn\" ] || cp -a \"$f\" \"$d/$bn\"\\n done\\n fi\\ndone\\nexec \"$@\"\\n' > \"{init_script}\" \\",
2068
+ f' && (date +%s%N 2>/dev/null || date +%s) > "{seed_dir}/.build-id" \\',
2069
+ f' && printf \'%s\' \'{sync_py_b64}\' | (base64 -d 2>/dev/null || base64 --decode) > "{sync_py_path}" \\',
2070
+ f' && printf \'%s\' \'{init_sh_b64}\' | (base64 -d 2>/dev/null || base64 --decode) > "{init_script}" \\',
2213
2071
  f' && chmod +x "{init_script}"',
2214
2072
  "ENTRYPOINT " + json.dumps([init_script]),
2215
2073
  ]
2216
2074
  seed_block = "\n".join(seed_snippet_lines)
2217
- insert_idx = next(
2218
- (index for index in range(len(lines) - 1, final_stage_start, -1) if re.match(r"\s*CMD(?:\s+|\[)", lines[index], re.I)),
2219
- len(lines),
2220
- )
2221
- updated_df = "\n".join(lines[:insert_idx] + [seed_block] + lines[insert_idx:]) + "\n"
2075
+ base_lines = stripped_lines if already_has_opc_seed else lines
2076
+ base_stage_start = stripped_stage_start if already_has_opc_seed else final_stage_start
2077
+ last_copy_idx = base_stage_start
2078
+ for idx in range(base_stage_start + 1, len(base_lines)):
2079
+ t = base_lines[idx].strip().upper()
2080
+ if t.startswith(("COPY ", "COPY[", "ADD ", "ADD[")):
2081
+ last_copy_idx = idx
2082
+ insert_idx = len(base_lines)
2083
+ for idx in range(len(base_lines) - 1, last_copy_idx, -1):
2084
+ if re.match(r"\s*(?:CMD(?:\s+|\[)|USER\s+)", base_lines[idx], re.I):
2085
+ insert_idx = idx
2086
+ updated_df = "\n".join(base_lines[:insert_idx] + [seed_block] + base_lines[insert_idx:]) + "\n"
2222
2087
  mount_summary = "、".join(m["container_path"] for m in mounts)
2223
2088
  if "Dockerfile" in changes_by_path:
2224
2089
  changes_by_path["Dockerfile"]["content"] = updated_df
@@ -2251,7 +2116,18 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
2251
2116
  build_context = contract.get("build_context")
2252
2117
  if not isinstance(build_context, list):
2253
2118
  build_context = None
2254
- scan = scan_app_routes_and_pages(source, build_context=build_context)
2119
+ changes = [ch for ch in (proposal.get("changes") or []) if isinstance(ch, dict) and isinstance(ch.get("path"), str) and isinstance(ch.get("content"), str)]
2120
+ if changes:
2121
+ with tempfile.TemporaryDirectory(prefix="opc-scan-") as tmp_dir:
2122
+ scan_root = Path(tmp_dir) / "source"
2123
+ shutil.copytree(source, scan_root, symlinks=True)
2124
+ for ch in changes:
2125
+ target_file = delivery.relative(scan_root, ch["path"])
2126
+ target_file.parent.mkdir(parents=True, exist_ok=True)
2127
+ target_file.write_text(ch["content"], encoding="utf-8")
2128
+ scan = scan_app_routes_and_pages(scan_root, build_context=build_context)
2129
+ else:
2130
+ scan = scan_app_routes_and_pages(source, build_context=build_context)
2255
2131
  sqlite_info = inspect_local_sqlite_files(source)
2256
2132
 
2257
2133
  # 1. Merge any missing scanned APIs into contract['apis'] so 0 routes are omitted
@@ -2292,20 +2168,15 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
2292
2168
  })
2293
2169
  contract["pages"] = existing_pages
2294
2170
 
2295
- # 2b. When user has migrated SQLite to platform PostgreSQL, auto-complete any missing DB code/driver/config adaptations
2296
- # Local file databases remain on volumes; historical migration receipts do not trigger code rewrites.
2297
-
2298
- # 2b-2. Automatically detect & adapt persistent local file storage (JSON state, uploads, reports, backups)
2171
+ # 2b. Automatically detect & adapt persistent local file storage (SQLite, JSON state, uploads, reports, backups)
2299
2172
  storage_mounts = _auto_adapt_persistent_file_storage(source, str(data.get("slug", "")), data, proposal)
2300
2173
 
2301
- # 2c. Filter out internal tool/truncation/row-diff/db-adaptation/default-password/local-json noise from unresolved & decisions
2174
+ # 2c. Filter out internal tool/truncation/default-password/local-json noise from unresolved & decisions
2302
2175
  internal_noise_re = re.compile(
2303
- r"(截断|truncated|补齐.*扫描|完整接口详情|接口扫描显示|扫描结果不一致|自动路由扫描未识别|逐表差异|逐表核对|10,?264|14,?758|healthz|健康检查接口|尚无.*PostgreSQL.*连接适配|数据库切换不能视为已完成|专属\s*PostgreSQL.*连接适配|默认管理员密码|公开的默认.*密码|演示口令|仅存于容器本地文件|容器重建后.*丢失)",
2176
+ r"(截断|truncated|补齐.*扫描|完整接口详情|接口扫描显示|扫描结果不一致|自动路由扫描未识别|逐表差异|逐表核对|10,?264|14,?758|healthz|健康检查接口|默认管理员密码|公开的默认.*密码|演示口令|仅存于容器本地文件|容器重建后.*丢失)",
2304
2177
  re.I,
2305
2178
  )
2306
- has_migrated = bool(data.get("migrations"))
2307
2179
  feedback_text = str(data.get("feedback") or "")
2308
- user_confirmed_db = has_migrated or bool(re.search(r"(迁移|PostgreSQL|平台.*数据库|直接迁移|保留.*SQLite)", feedback_text, re.I))
2309
2180
  user_confirmed_local_or_pwd = bool(re.search(r"(保留演示口令|保留文件存储|保留原|先保留)", feedback_text, re.I))
2310
2181
 
2311
2182
  def _keep_question(q: str) -> bool:
@@ -2314,8 +2185,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
2314
2185
  return False
2315
2186
  if internal_noise_re.search(s):
2316
2187
  return False
2317
- if user_confirmed_db and re.search(r"(SQLite.*迁移|补迁|保留\s*SQLite|PostgreSQL\s*连接适配|数据库切换)", s, re.I):
2318
- return False
2319
2188
  if user_confirmed_local_or_pwd and re.search(r"(密码|口令|本地文件|文件存储|JSON)", s, re.I):
2320
2189
  return False
2321
2190
  return True
@@ -2368,12 +2237,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
2368
2237
 
2369
2238
  sqlite_dbs = sqlite_info.get("sqlite_databases") or []
2370
2239
  resources = select_and_filter_platform_resources(data.get("resources") or [], str(data.get("slug", "")))
2371
- pg_resources = [
2372
- r for r in resources
2373
- if str(r.get("type", "")).lower() in {"postgres", "postgresql"}
2374
- and not str(r.get("id", "")).lower().startswith("db-")
2375
- ]
2376
- pg_resources.sort(key=lambda r: (0 if r.get("is_default") else 1, str(r.get("id", ""))))
2377
2240
  storage_type = step1_ai.get("storage_type") or (
2378
2241
  "sqlite" if sqlite_dbs else ("persistent_volume" if storage_mounts else "none")
2379
2242
  )
@@ -2470,9 +2333,6 @@ def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dic
2470
2333
  }
2471
2334
  for db in sqlite_dbs
2472
2335
  ],
2473
- "can_migrate_to_platform_pg": False,
2474
- "platform_pg_resource": pg_resources[0]["name"] if pg_resources else "",
2475
- "platform_pg_id": pg_resources[0]["id"] if pg_resources else "",
2476
2336
  "storage_mounts": storage_mounts,
2477
2337
  "storage_migration_plan": default_storage_plan if sqlite_dbs else (step2_ai.get("storage_migration") or default_storage_plan),
2478
2338
  },