dsh-plugin-opc 0.0.0-stage → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +257 -2
- package/config/overlay.example.yml +9 -0
- package/docs/acceptance-matrix.md +30 -0
- package/docs/images/01-plugins.png +0 -0
- package/docs/images/02-install-menu.png +0 -0
- package/docs/images/03-install-address.png +0 -0
- package/docs/images/04-installed.png +0 -0
- package/docs/images/05-open-sidebar.png +0 -0
- package/docs/images/06-opc-entries.png +0 -0
- package/docs/images/07-space-and-knowledge.png +0 -0
- package/docs/images/08-search-results.png +0 -0
- package/docs/images/09-document.png +0 -0
- package/docs/images/10-mention-menu.png +0 -0
- package/docs/images/11-mention-question.png +0 -0
- package/docs/images/12-answer-sources.png +0 -0
- package/docs/images/13-skills-install.png +0 -0
- package/docs/images/14-skill-enabled.png +0 -0
- package/docs/images/15-skill-slash.png +0 -0
- package/docs/images/16-mcp-list.png +0 -0
- package/docs/images/17-mcp-tool.png +0 -0
- package/docs/images/chat-mention.png +0 -0
- package/docs/images/git-source.png +0 -0
- package/docs/images/mcp-services.png +0 -0
- package/docs/implementation-status.md +40 -0
- package/lib/client.js +31892 -0
- package/lib/index.js +3477 -0
- package/package.json +90 -4
- package/runtime/app_delivery/LICENSE +18 -0
- package/runtime/app_delivery/PROVENANCE.md +1 -0
- package/runtime/app_delivery/SKILL.md +38 -0
- package/runtime/app_delivery/__init__.py +1 -0
- package/runtime/app_delivery/assets/invoke_api.py.txt +269 -0
- package/runtime/app_delivery/references/contract.md +90 -0
- package/runtime/app_delivery/references/framework-checks.md +59 -0
- package/runtime/app_delivery/references/integration.md +23 -0
- package/runtime/app_delivery/references/preparation-checks.md +26 -0
- package/runtime/app_delivery/scripts/__init__.py +1 -0
- package/runtime/app_delivery/scripts/delivery.py +650 -0
- package/runtime/app_delivery/scripts/invoke_api.py +269 -0
- package/runtime/app_delivery/scripts/sqlite_snapshot.py +139 -0
- package/runtime/app_delivery_auth.py +113 -0
- package/runtime/app_delivery_config.py +108 -0
- package/runtime/app_delivery_migration.py +2452 -0
- package/runtime/app_delivery_repair.py +48 -0
- package/runtime/app_delivery_sources.py +39 -0
- package/runtime/vendor/tomli/LICENSE +21 -0
- package/runtime/vendor/tomli/__init__.py +8 -0
- package/runtime/vendor/tomli/_parser.py +793 -0
- package/runtime/vendor/tomli/_re.py +119 -0
- package/runtime/vendor/tomli/_types.py +10 -0
- package/runtime/vendor/yaml/LICENSE +20 -0
- package/runtime/vendor/yaml/__init__.py +389 -0
- package/runtime/vendor/yaml/composer.py +138 -0
- package/runtime/vendor/yaml/constructor.py +748 -0
- package/runtime/vendor/yaml/cyaml.py +100 -0
- package/runtime/vendor/yaml/dumper.py +61 -0
- package/runtime/vendor/yaml/emitter.py +1137 -0
- package/runtime/vendor/yaml/error.py +74 -0
- package/runtime/vendor/yaml/events.py +85 -0
- package/runtime/vendor/yaml/loader.py +63 -0
- package/runtime/vendor/yaml/nodes.py +48 -0
- package/runtime/vendor/yaml/parser.py +588 -0
- package/runtime/vendor/yaml/reader.py +185 -0
- package/runtime/vendor/yaml/representer.py +388 -0
- package/runtime/vendor/yaml/resolver.py +226 -0
- package/runtime/vendor/yaml/scanner.py +1435 -0
- package/runtime/vendor/yaml/serializer.py +110 -0
- package/runtime/vendor/yaml/tokens.py +103 -0
- package/runtime/worker.py +574 -0
|
@@ -0,0 +1,2452 @@
|
|
|
1
|
+
"""Step 1 & Step 2 helpers for OPC App Delivery:
|
|
2
|
+
- Dockerfile template fetching from platform-opc (deploy-opc database)
|
|
3
|
+
- Local SQLite / local DB (PostgreSQL, MySQL, SQLServer) / local Redis inspection and automatic migration to platform-opc server resources
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import ast
|
|
9
|
+
import base64
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import re
|
|
13
|
+
import sqlite3
|
|
14
|
+
import urllib.error
|
|
15
|
+
import urllib.parse
|
|
16
|
+
import urllib.request
|
|
17
|
+
|
|
18
|
+
from app_delivery.scripts import delivery
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def fetch_platform_dockerfile_template(platform_url: str, language: str, version: str = "") -> dict:
|
|
22
|
+
"""Step 2 helper: fetch Dockerfile template for (language, version) from platform-opc deploy-opc DB."""
|
|
23
|
+
base = (platform_url or "https://shtokendance.benlai.com").rstrip("/")
|
|
24
|
+
query = urllib.parse.urlencode({"language": language.strip(), "version": version.strip()})
|
|
25
|
+
url = f"{base}/api/v1/apps/dockerfile-templates?{query}"
|
|
26
|
+
req = urllib.request.Request(url, headers={"Accept": "application/json"}, method="GET")
|
|
27
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
28
|
+
try:
|
|
29
|
+
with opener.open(req, timeout=10) as resp:
|
|
30
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
31
|
+
return {
|
|
32
|
+
"ok": True,
|
|
33
|
+
"source": "platform-opc",
|
|
34
|
+
"matched_template": data.get("matched_template"),
|
|
35
|
+
"templates_count": len(data.get("templates") or []),
|
|
36
|
+
}
|
|
37
|
+
except Exception as exc:
|
|
38
|
+
return {
|
|
39
|
+
"ok": False,
|
|
40
|
+
"source": "fallback",
|
|
41
|
+
"error": str(exc),
|
|
42
|
+
"matched_template": _fallback_template(language, version),
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _fallback_template(language: str, version: str) -> dict:
|
|
47
|
+
lang = language.lower().strip()
|
|
48
|
+
ver = version.strip() or ("3.11" if "py" in lang else "20" if "node" in lang else "1.22" if "go" in lang else "latest")
|
|
49
|
+
if "py" in lang:
|
|
50
|
+
return {
|
|
51
|
+
"language": "python",
|
|
52
|
+
"runtime_version": ver,
|
|
53
|
+
"base_image": f"python:{ver}-slim",
|
|
54
|
+
"template_content": (
|
|
55
|
+
f"FROM python:{ver}-slim\nWORKDIR /app\n"
|
|
56
|
+
"ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/\n"
|
|
57
|
+
"COPY requirements*.txt ./\nRUN pip install --no-cache-dir -r requirements.txt\n"
|
|
58
|
+
"COPY . .\nEXPOSE 8080\nCMD [\"python\", \"main.py\"]"
|
|
59
|
+
),
|
|
60
|
+
}
|
|
61
|
+
if "node" in lang or "js" in lang or "ts" in lang:
|
|
62
|
+
return {
|
|
63
|
+
"language": "nodejs",
|
|
64
|
+
"runtime_version": ver,
|
|
65
|
+
"base_image": f"node:{ver}-alpine",
|
|
66
|
+
"template_content": (
|
|
67
|
+
f"FROM node:{ver}-alpine\nWORKDIR /app\n"
|
|
68
|
+
"RUN npm config set registry https://registry.npmmirror.com\n"
|
|
69
|
+
"COPY package*.json ./\nRUN npm install\nCOPY . .\nEXPOSE 8080\nCMD [\"npm\", \"start\"]"
|
|
70
|
+
),
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
"language": lang or "generic",
|
|
74
|
+
"runtime_version": ver,
|
|
75
|
+
"base_image": f"{lang}:{ver}",
|
|
76
|
+
"template_content": f"FROM {lang}:{ver}\nWORKDIR /app\nCOPY . .\nEXPOSE 8080\n",
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_PERSISTENT_DIR_CATALOG: dict[str, dict[str, str]] = {
|
|
81
|
+
"data": {"purpose": "业务状态与 JSON/本地数据文件持久化目录", "kind": "data_dir"},
|
|
82
|
+
"uploads": {"purpose": "用户上传文件与现场证据附件持久化目录", "kind": "upload_dir"},
|
|
83
|
+
"upload": {"purpose": "用户上传文件持久化目录", "kind": "upload_dir"},
|
|
84
|
+
"reports": {"purpose": "评估报告与导出文件持久化目录", "kind": "report_dir"},
|
|
85
|
+
"backups": {"purpose": "业务数据快照与备份归档目录", "kind": "backup_dir"},
|
|
86
|
+
"backup": {"purpose": "业务数据快照与备份归档目录", "kind": "backup_dir"},
|
|
87
|
+
"storage": {"purpose": "应用运行时本地持久化存储目录", "kind": "data_dir"},
|
|
88
|
+
"attachments": {"purpose": "业务附件与凭证持久化目录", "kind": "upload_dir"},
|
|
89
|
+
"exports": {"purpose": "业务导出报表持久化目录", "kind": "report_dir"},
|
|
90
|
+
"db_data": {"purpose": "本地数据库与数据文件持久化目录", "kind": "data_dir"},
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
_EXCLUDED_PERSISTENT_PARENT_DIRS = {
|
|
94
|
+
"src", "lib", "app", "web", "public", "static", "assets", "views", "pages",
|
|
95
|
+
"components", "routes", "controllers", "models", "services", "utils", "helpers",
|
|
96
|
+
"middleware", "test", "tests", "__tests__", "spec", "docs", "doc", "miniprogram",
|
|
97
|
+
"node_modules", "vendor", "dist", "build", ".git", ".github", ".vscode", ".idea",
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
_SOURCE_CODE_EXTENSIONS = {
|
|
101
|
+
".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx",
|
|
102
|
+
".go", ".java", ".kt", ".cs", ".rb", ".php", ".vue", ".svelte",
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
_CODE_MUTABLE_DIR_RE = re.compile(
|
|
106
|
+
r"""(?i)(?:join|resolve)\s*\([^)]*?['"](data|uploads|upload|reports|backups|backup|storage|attachments|exports|db_data)['"]\s*\)|"""
|
|
107
|
+
r"""(?:DATA_DIR|UPLOAD_DIR|UPLOAD_FOLDER|REPORT_DIR|BACKUP_DIR|STORAGE_DIR)\s*(?:=|\|\||\?\?)\s*[^;\n]*?['"](/[^'"]+|data|uploads|upload|reports|backups|backup|storage|attachments|exports)['"]"""
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def inspect_local_file_storage(
|
|
112
|
+
source: Path,
|
|
113
|
+
slug: str = "",
|
|
114
|
+
workdir: str = "/app",
|
|
115
|
+
has_migrations: bool = False,
|
|
116
|
+
) -> dict:
|
|
117
|
+
"""Detect local JSON/file data directories, upload folders, report/backup folders, and seed files
|
|
118
|
+
that require external persistent volume mounting (Docker Volume / Host Bind / S3-OSS).
|
|
119
|
+
"""
|
|
120
|
+
clean_workdir = "/" + (workdir or "/app").strip().strip("/")
|
|
121
|
+
clean_slug = re.sub(r"[^a-z0-9-]+", "-", (slug or "app").lower()).strip("-") or "app"
|
|
122
|
+
|
|
123
|
+
dir_seed_files: dict[str, list[str]] = {}
|
|
124
|
+
dir_has_code: set[str] = set()
|
|
125
|
+
code_referenced_dirs: set[str] = set()
|
|
126
|
+
|
|
127
|
+
for p in sorted(source.rglob("*")):
|
|
128
|
+
if p.is_symlink():
|
|
129
|
+
continue
|
|
130
|
+
rel_parts = p.relative_to(source).parts
|
|
131
|
+
if not rel_parts or any(part in delivery.EXCLUDED for part in rel_parts):
|
|
132
|
+
continue
|
|
133
|
+
top = rel_parts[0]
|
|
134
|
+
if top in _EXCLUDED_PERSISTENT_PARENT_DIRS:
|
|
135
|
+
continue
|
|
136
|
+
|
|
137
|
+
# 1. Check top-level persistent directories (e.g. data/, uploads/, reports/, backups/)
|
|
138
|
+
if top in _PERSISTENT_DIR_CATALOG:
|
|
139
|
+
if p.is_dir() and len(rel_parts) == 1:
|
|
140
|
+
dir_seed_files.setdefault(top, [])
|
|
141
|
+
elif p.is_file():
|
|
142
|
+
ext = p.suffix.lower()
|
|
143
|
+
if ext in _SOURCE_CODE_EXTENSIONS:
|
|
144
|
+
dir_has_code.add(top)
|
|
145
|
+
elif not (has_migrations and ext in {".sqlite", ".sqlite3", ".db"}):
|
|
146
|
+
dir_seed_files.setdefault(top, []).append(p.relative_to(source).as_posix())
|
|
147
|
+
|
|
148
|
+
# 2. Check second-level persistent directories (e.g. reimburse-web/server/data)
|
|
149
|
+
if len(rel_parts) >= 2 and top not in _PERSISTENT_DIR_CATALOG and not any(part in _EXCLUDED_PERSISTENT_PARENT_DIRS for part in rel_parts[:-1]):
|
|
150
|
+
if p.is_dir() and len(rel_parts) <= 3 and rel_parts[-1] in _PERSISTENT_DIR_CATALOG:
|
|
151
|
+
rel_dir = p.relative_to(source).as_posix()
|
|
152
|
+
dir_seed_files.setdefault(rel_dir, [])
|
|
153
|
+
elif p.is_file():
|
|
154
|
+
for idx in range(2, min(4, len(rel_parts))):
|
|
155
|
+
cand_name = rel_parts[idx - 1]
|
|
156
|
+
if cand_name in _PERSISTENT_DIR_CATALOG:
|
|
157
|
+
rel_dir = "/".join(rel_parts[:idx])
|
|
158
|
+
ext = p.suffix.lower()
|
|
159
|
+
if ext in _SOURCE_CODE_EXTENSIONS:
|
|
160
|
+
dir_has_code.add(rel_dir)
|
|
161
|
+
elif not (has_migrations and ext in {".sqlite", ".sqlite3", ".db"}):
|
|
162
|
+
dir_seed_files.setdefault(rel_dir, []).append(p.relative_to(source).as_posix())
|
|
163
|
+
|
|
164
|
+
# 3. Scan source code for runtime mutable directories (e.g. join(root, "uploads"), join(root, "reports"))
|
|
165
|
+
if p.is_file() and p.suffix.lower() in {".js", ".mjs", ".cjs", ".ts", ".py", ".go", ".java", ".cs"} and p.stat().st_size <= 2_000_000:
|
|
166
|
+
try:
|
|
167
|
+
text = p.read_text(encoding="utf-8", errors="ignore")
|
|
168
|
+
except Exception:
|
|
169
|
+
continue
|
|
170
|
+
for m in _CODE_MUTABLE_DIR_RE.finditer(text):
|
|
171
|
+
raw_dir = (m.group(1) or m.group(2) or "").strip().strip("/")
|
|
172
|
+
if not raw_dir:
|
|
173
|
+
continue
|
|
174
|
+
base_dir = raw_dir.split("/")[-1].lower()
|
|
175
|
+
if base_dir in _PERSISTENT_DIR_CATALOG:
|
|
176
|
+
code_referenced_dirs.add(base_dir)
|
|
177
|
+
|
|
178
|
+
# SQLite may live outside conventional data/ directories. Mount its parent,
|
|
179
|
+
# including WAL/SHM companions, but never cover application code or the root.
|
|
180
|
+
for db_file in sorted(source.rglob("*")):
|
|
181
|
+
if db_file.is_symlink() or not db_file.is_file():
|
|
182
|
+
continue
|
|
183
|
+
if db_file.suffix.lower() not in {".sqlite", ".sqlite3", ".db"} and not delivery.is_sqlite(db_file):
|
|
184
|
+
continue
|
|
185
|
+
rel = db_file.relative_to(source)
|
|
186
|
+
if len(rel.parts) < 2 or any(part in delivery.EXCLUDED or part in _EXCLUDED_PERSISTENT_PARENT_DIRS for part in rel.parts[:-1]):
|
|
187
|
+
continue
|
|
188
|
+
parent = db_file.parent
|
|
189
|
+
if any(p.suffix.lower() in _SOURCE_CODE_EXTENSIONS for p in parent.rglob("*") if p.is_file()):
|
|
190
|
+
continue
|
|
191
|
+
rel_dir = parent.relative_to(source).as_posix()
|
|
192
|
+
dir_seed_files.setdefault(rel_dir, []).extend(p.relative_to(source).as_posix() for p in parent.rglob("*") if p.is_file() and not p.is_symlink())
|
|
193
|
+
|
|
194
|
+
for d in sorted(code_referenced_dirs):
|
|
195
|
+
if d not in dir_has_code:
|
|
196
|
+
dir_seed_files.setdefault(d, [])
|
|
197
|
+
|
|
198
|
+
candidate_dirs = sorted(d for d in dir_seed_files if d not in dir_has_code)
|
|
199
|
+
# Deduplicate nested directories (if both 'data' and 'data/uploads' matched, keep outer 'data')
|
|
200
|
+
filtered_dirs: list[str] = []
|
|
201
|
+
for d in candidate_dirs:
|
|
202
|
+
if any(parent != d and d.startswith(parent + "/") for parent in candidate_dirs):
|
|
203
|
+
continue
|
|
204
|
+
# Only keep a directory if it either has seed files OR was explicitly referenced in code OR is a top-level existing directory
|
|
205
|
+
if dir_seed_files.get(d) or d in code_referenced_dirs or ("/" not in d and (source / d).is_dir()):
|
|
206
|
+
filtered_dirs.append(d)
|
|
207
|
+
|
|
208
|
+
mounts: list[dict] = []
|
|
209
|
+
for rel_dir in filtered_dirs:
|
|
210
|
+
base_name = rel_dir.split("/")[-1].lower()
|
|
211
|
+
meta = _PERSISTENT_DIR_CATALOG.get(base_name, {"purpose": "应用持久化数据目录", "kind": "data_dir"})
|
|
212
|
+
seeds = sorted(set(dir_seed_files.get(rel_dir, [])))
|
|
213
|
+
container_path = f"{clean_workdir}/{rel_dir}".replace("//", "/")
|
|
214
|
+
vol_suffix = re.sub(r"[^a-z0-9-]+", "-", rel_dir.lower()).strip("-") or base_name
|
|
215
|
+
purpose = meta["purpose"]
|
|
216
|
+
if seeds:
|
|
217
|
+
sample_names = ", ".join(Path(s).name for s in seeds[:3])
|
|
218
|
+
if len(seeds) > 3:
|
|
219
|
+
sample_names += f" 等 {len(seeds)} 个文件"
|
|
220
|
+
purpose = f"{meta['purpose']}(含初始种子文件:{sample_names})"
|
|
221
|
+
mounts.append({
|
|
222
|
+
"name": base_name if "/" not in rel_dir else vol_suffix,
|
|
223
|
+
"relative_path": rel_dir,
|
|
224
|
+
"container_path": container_path,
|
|
225
|
+
"purpose": purpose,
|
|
226
|
+
"kind": meta["kind"],
|
|
227
|
+
"volume_name": f"opc-vol-{clean_slug}-{vol_suffix}",
|
|
228
|
+
"storage_backend": "docker_volume",
|
|
229
|
+
"seed_files_count": len(seeds),
|
|
230
|
+
"seed_files": seeds[:10],
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
return {
|
|
234
|
+
"has_persistent_file_storage": bool(mounts),
|
|
235
|
+
"storage_mounts": mounts,
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def inspect_local_sqlite_files(source: Path, sqlite_rel_path: str = "") -> dict:
|
|
240
|
+
"""Inspect SQLite database files AND local JSON/file storage mounts inside the application source directory."""
|
|
241
|
+
candidates = []
|
|
242
|
+
if sqlite_rel_path.strip():
|
|
243
|
+
p = delivery.relative(source, sqlite_rel_path.strip())
|
|
244
|
+
if p.is_file() and p.stat().st_size > 0:
|
|
245
|
+
candidates.append(p)
|
|
246
|
+
else:
|
|
247
|
+
for p in sorted(source.rglob("*")):
|
|
248
|
+
if p.is_file() and not p.is_symlink() and p.stat().st_size > 0:
|
|
249
|
+
if p.suffix.lower() in {".sqlite", ".sqlite3", ".db"} or delivery.is_sqlite(p):
|
|
250
|
+
candidates.append(p)
|
|
251
|
+
|
|
252
|
+
databases = []
|
|
253
|
+
for db_file in candidates:
|
|
254
|
+
rel = db_file.relative_to(source).as_posix()
|
|
255
|
+
try:
|
|
256
|
+
tables = extract_sqlite_tables(db_file, max_rows_per_table=5)
|
|
257
|
+
except Exception:
|
|
258
|
+
continue
|
|
259
|
+
if not tables and len(candidates) > 1:
|
|
260
|
+
continue
|
|
261
|
+
total_rows = sum(int(t.get("total_rows", len(t["rows"]))) for t in tables)
|
|
262
|
+
databases.append({
|
|
263
|
+
"path": rel,
|
|
264
|
+
"size_bytes": db_file.stat().st_size,
|
|
265
|
+
"table_count": len(tables),
|
|
266
|
+
"total_rows": total_rows,
|
|
267
|
+
"tables": [
|
|
268
|
+
{
|
|
269
|
+
"name": t["name"],
|
|
270
|
+
"columns": t["columns"],
|
|
271
|
+
"row_count": int(t.get("total_rows", len(t["rows"]))),
|
|
272
|
+
"row_count_preview": len(t["rows"]),
|
|
273
|
+
}
|
|
274
|
+
for t in tables
|
|
275
|
+
],
|
|
276
|
+
})
|
|
277
|
+
databases.sort(key=lambda d: (d.get("total_rows", 0), d.get("table_count", 0), d.get("size_bytes", 0)), reverse=True)
|
|
278
|
+
file_storage = inspect_local_file_storage(source)
|
|
279
|
+
return {
|
|
280
|
+
"sqlite_databases": databases,
|
|
281
|
+
"local_file_storage": file_storage,
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def extract_sqlite_tables(sqlite_file: Path, max_rows_per_table: int | None = None) -> list[dict]:
|
|
286
|
+
"""Read full schema (columns + types + PKs) and ALL rows from a local SQLite file for server migration."""
|
|
287
|
+
tables = []
|
|
288
|
+
preexisting_aux = {
|
|
289
|
+
sfx: Path(f"{sqlite_file}{sfx}").exists()
|
|
290
|
+
for sfx in ("-shm", "-wal")
|
|
291
|
+
}
|
|
292
|
+
conn = sqlite3.connect(f"file:{sqlite_file}?mode=ro", uri=True)
|
|
293
|
+
conn.row_factory = sqlite3.Row
|
|
294
|
+
try:
|
|
295
|
+
cur = conn.execute("SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
|
|
296
|
+
for table_name, ddl in cur.fetchall():
|
|
297
|
+
cols_info = conn.execute(f'PRAGMA table_info("{table_name}")').fetchall()
|
|
298
|
+
columns = []
|
|
299
|
+
for c in cols_info:
|
|
300
|
+
col_type = str(c["type"] or "TEXT").upper()
|
|
301
|
+
is_pk = bool(c["pk"])
|
|
302
|
+
auto_inc = is_pk and "INT" in col_type and "AUTOINCREMENT" in (ddl or "").upper()
|
|
303
|
+
columns.append({
|
|
304
|
+
"name": c["name"],
|
|
305
|
+
"type": col_type,
|
|
306
|
+
"primary_key": is_pk,
|
|
307
|
+
"auto_increment": auto_inc,
|
|
308
|
+
"not_null": bool(c["notnull"]),
|
|
309
|
+
"default_value": str(c["dflt_value"]) if c["dflt_value"] is not None else "",
|
|
310
|
+
})
|
|
311
|
+
total_count = 0
|
|
312
|
+
try:
|
|
313
|
+
cnt_row = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone()
|
|
314
|
+
if cnt_row:
|
|
315
|
+
total_count = int(cnt_row[0])
|
|
316
|
+
except Exception:
|
|
317
|
+
pass
|
|
318
|
+
if max_rows_per_table is not None and int(max_rows_per_table) > 0:
|
|
319
|
+
rows_raw = conn.execute(f'SELECT * FROM "{table_name}" LIMIT {int(max_rows_per_table)}').fetchall()
|
|
320
|
+
else:
|
|
321
|
+
rows_raw = conn.execute(f'SELECT * FROM "{table_name}"').fetchall()
|
|
322
|
+
rows = []
|
|
323
|
+
for r in rows_raw:
|
|
324
|
+
row_dict = {}
|
|
325
|
+
for k in r.keys():
|
|
326
|
+
val = r[k]
|
|
327
|
+
if isinstance(val, bytes):
|
|
328
|
+
val = base64.b64encode(val).decode("ascii")
|
|
329
|
+
row_dict[k] = val
|
|
330
|
+
rows.append(row_dict)
|
|
331
|
+
tables.append({
|
|
332
|
+
"name": table_name,
|
|
333
|
+
"ddl": ddl or "",
|
|
334
|
+
"columns": columns,
|
|
335
|
+
"total_rows": total_count or len(rows),
|
|
336
|
+
"rows": rows,
|
|
337
|
+
})
|
|
338
|
+
finally:
|
|
339
|
+
conn.close()
|
|
340
|
+
# Clean up only transient shm/wal files created during readonly inspection that did not exist beforehand
|
|
341
|
+
for sfx, existed in preexisting_aux.items():
|
|
342
|
+
if existed:
|
|
343
|
+
continue
|
|
344
|
+
aux = Path(f"{sqlite_file}{sfx}")
|
|
345
|
+
if aux.exists():
|
|
346
|
+
try:
|
|
347
|
+
aux.unlink()
|
|
348
|
+
except OSError:
|
|
349
|
+
pass
|
|
350
|
+
return tables
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def migrate_local_data_to_platform(
|
|
354
|
+
*,
|
|
355
|
+
platform_url: str,
|
|
356
|
+
slug: str,
|
|
357
|
+
source_dir: Path,
|
|
358
|
+
source_type: str = "sqlite",
|
|
359
|
+
sqlite_path: str = "",
|
|
360
|
+
target_type: str = "postgres",
|
|
361
|
+
target_resource_id: str = "",
|
|
362
|
+
tables_override: list | None = None,
|
|
363
|
+
redis_keys: list | None = None,
|
|
364
|
+
) -> dict:
|
|
365
|
+
"""Extract schema & 100% of data from local SQLite (or provided tables/redis_keys) and upload to platform-opc POST /api/v1/apps/:slug/migrate-data."""
|
|
366
|
+
tables = tables_override or []
|
|
367
|
+
if not tables and source_type.lower() == "sqlite":
|
|
368
|
+
if sqlite_path.strip():
|
|
369
|
+
db_file = delivery.relative(source_dir, sqlite_path.strip())
|
|
370
|
+
if db_file.is_file() and db_file.stat().st_size > 0:
|
|
371
|
+
tables = extract_sqlite_tables(db_file, max_rows_per_table=None)
|
|
372
|
+
if not tables:
|
|
373
|
+
found = [
|
|
374
|
+
p for p in sorted(source_dir.rglob("*"))
|
|
375
|
+
if p.is_file() and p.suffix.lower() in {".sqlite", ".sqlite3", ".db"} and p.stat().st_size > 0
|
|
376
|
+
]
|
|
377
|
+
if not found:
|
|
378
|
+
raise ValueError("未在源码目录中找到非空 SQLite 数据库文件")
|
|
379
|
+
# Sort non-empty SQLite files by size descending and merge all non-empty tables
|
|
380
|
+
found.sort(key=lambda p: p.stat().st_size, reverse=True)
|
|
381
|
+
seen_tables = set()
|
|
382
|
+
for db_file in found:
|
|
383
|
+
try:
|
|
384
|
+
extracted = extract_sqlite_tables(db_file, max_rows_per_table=None)
|
|
385
|
+
except Exception:
|
|
386
|
+
continue
|
|
387
|
+
for tbl in extracted:
|
|
388
|
+
if tbl["name"] not in seen_tables:
|
|
389
|
+
seen_tables.add(tbl["name"])
|
|
390
|
+
tables.append(tbl)
|
|
391
|
+
|
|
392
|
+
payload = {
|
|
393
|
+
"source_type": source_type.lower().strip(),
|
|
394
|
+
"target_type": target_type.lower().strip() or "postgres",
|
|
395
|
+
"target_resource_id": target_resource_id.strip(),
|
|
396
|
+
"tables": tables,
|
|
397
|
+
"redis_keys": redis_keys or [],
|
|
398
|
+
}
|
|
399
|
+
base = (platform_url or "https://shtokendance.benlai.com").rstrip("/")
|
|
400
|
+
url = f"{base}/api/v1/apps/{urllib.parse.quote(slug)}/migrate-data"
|
|
401
|
+
body_bytes = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
402
|
+
req = urllib.request.Request(
|
|
403
|
+
url,
|
|
404
|
+
data=body_bytes,
|
|
405
|
+
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
|
406
|
+
method="POST",
|
|
407
|
+
)
|
|
408
|
+
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
409
|
+
try:
|
|
410
|
+
with opener.open(req, timeout=600) as resp:
|
|
411
|
+
resp_json = json.loads(resp.read().decode("utf-8"))
|
|
412
|
+
if resp_json.get("code") == 0 and resp_json.get("data"):
|
|
413
|
+
return resp_json["data"]
|
|
414
|
+
return resp_json
|
|
415
|
+
except urllib.error.HTTPError as exc:
|
|
416
|
+
detail = exc.read().decode("utf-8", errors="replace")
|
|
417
|
+
raise ValueError(f"平台数据迁移接口返回错误 ({exc.code}): {detail}") from exc
|
|
418
|
+
except Exception as exc:
|
|
419
|
+
raise ValueError(f"调用平台数据迁移接口失败: {exc}") from exc
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _join_route_paths(*parts: str) -> str:
|
|
423
|
+
cleaned = []
|
|
424
|
+
for p in parts:
|
|
425
|
+
s = (p or "").strip()
|
|
426
|
+
if not s or s == "/":
|
|
427
|
+
continue
|
|
428
|
+
cleaned.append(s.strip("/"))
|
|
429
|
+
return "/" + "/".join(cleaned) if cleaned else "/"
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _humanize_identifier(name: str, method: str = "GET", path: str = "") -> str:
|
|
433
|
+
"""Generate a clear Chinese purpose description from a route handler name and path when docstring is absent."""
|
|
434
|
+
n = (name or "").strip().strip("_")
|
|
435
|
+
verb_map = {
|
|
436
|
+
"list": "查询列表",
|
|
437
|
+
"get": "获取详情",
|
|
438
|
+
"read": "读取",
|
|
439
|
+
"fetch": "获取",
|
|
440
|
+
"query": "查询",
|
|
441
|
+
"search": "搜索",
|
|
442
|
+
"export": "导出",
|
|
443
|
+
"download": "下载",
|
|
444
|
+
"preview": "预览",
|
|
445
|
+
"create": "新建",
|
|
446
|
+
"add": "新增",
|
|
447
|
+
"save": "保存",
|
|
448
|
+
"submit": "提交",
|
|
449
|
+
"upload": "上传",
|
|
450
|
+
"import": "导入",
|
|
451
|
+
"parse": "解析",
|
|
452
|
+
"extract": "提取",
|
|
453
|
+
"recognize": "识别",
|
|
454
|
+
"match": "匹配",
|
|
455
|
+
"validate": "校验",
|
|
456
|
+
"verify": "验证",
|
|
457
|
+
"check": "检查",
|
|
458
|
+
"update": "更新",
|
|
459
|
+
"edit": "编辑",
|
|
460
|
+
"patch": "修改",
|
|
461
|
+
"set": "设置",
|
|
462
|
+
"toggle": "切换状态",
|
|
463
|
+
"enable": "启用",
|
|
464
|
+
"disable": "禁用",
|
|
465
|
+
"approve": "审批通过",
|
|
466
|
+
"reject": "驳回",
|
|
467
|
+
"cancel": "取消",
|
|
468
|
+
"delete": "删除",
|
|
469
|
+
"remove": "移除",
|
|
470
|
+
"clear": "清空",
|
|
471
|
+
"batch": "批量处理",
|
|
472
|
+
"sync": "同步",
|
|
473
|
+
"login": "登录",
|
|
474
|
+
"logout": "退出登录",
|
|
475
|
+
"register": "注册",
|
|
476
|
+
"stats": "统计汇总",
|
|
477
|
+
"summary": "汇总信息",
|
|
478
|
+
"config": "配置",
|
|
479
|
+
"settings": "设置",
|
|
480
|
+
"template": "模板",
|
|
481
|
+
"health": "健康检查",
|
|
482
|
+
"healthz": "健康检查",
|
|
483
|
+
}
|
|
484
|
+
noun_map = {
|
|
485
|
+
"reimburse": "报销单",
|
|
486
|
+
"form": "单据",
|
|
487
|
+
"forms": "单据列表",
|
|
488
|
+
"invoice": "发票",
|
|
489
|
+
"invoices": "发票",
|
|
490
|
+
"attachment": "附件",
|
|
491
|
+
"attachments": "附件",
|
|
492
|
+
"payee": "收款人/领款人",
|
|
493
|
+
"payees": "收款人/领款人",
|
|
494
|
+
"sales": "销售",
|
|
495
|
+
"order": "订单",
|
|
496
|
+
"orders": "订单",
|
|
497
|
+
"travel": "差旅",
|
|
498
|
+
"itinerary": "行程",
|
|
499
|
+
"allowance": "补贴",
|
|
500
|
+
"city": "城市/地点",
|
|
501
|
+
"cities": "城市/地点",
|
|
502
|
+
"company": "公司主体",
|
|
503
|
+
"companies": "公司主体",
|
|
504
|
+
"department": "部门",
|
|
505
|
+
"departments": "部门",
|
|
506
|
+
"category": "费用大类/科目",
|
|
507
|
+
"categories": "费用大类/科目",
|
|
508
|
+
"type": "类型",
|
|
509
|
+
"types": "类型",
|
|
510
|
+
"rule": "规则",
|
|
511
|
+
"rules": "规则",
|
|
512
|
+
"prompt": "提示词",
|
|
513
|
+
"prompts": "提示词",
|
|
514
|
+
"model": "模型",
|
|
515
|
+
"models": "模型",
|
|
516
|
+
"llm": "大模型配置",
|
|
517
|
+
"ocr": "OCR识别",
|
|
518
|
+
"admin": "管理后台",
|
|
519
|
+
"audit": "审计日志",
|
|
520
|
+
"log": "日志",
|
|
521
|
+
"logs": "日志",
|
|
522
|
+
"user": "用户",
|
|
523
|
+
"users": "用户",
|
|
524
|
+
"item": "明细项",
|
|
525
|
+
"items": "明细项",
|
|
526
|
+
"file": "文件",
|
|
527
|
+
"files": "文件",
|
|
528
|
+
"voucher": "凭证",
|
|
529
|
+
"contract": "合同",
|
|
530
|
+
"dedup": "查重",
|
|
531
|
+
"duplicate": "重复项",
|
|
532
|
+
"supplier": "供应商",
|
|
533
|
+
"suppliers": "供应商",
|
|
534
|
+
"task": "任务",
|
|
535
|
+
"tasks": "任务",
|
|
536
|
+
"checklist": "检查表",
|
|
537
|
+
"corrective": "整改项",
|
|
538
|
+
"action": "处理项",
|
|
539
|
+
"actions": "处理项",
|
|
540
|
+
"evidence": "现场证据",
|
|
541
|
+
"finding": "发现项",
|
|
542
|
+
"findings": "发现项",
|
|
543
|
+
"signature": "签字确认",
|
|
544
|
+
"signatures": "签字确认",
|
|
545
|
+
"session": "登录会话",
|
|
546
|
+
"directory": "目录",
|
|
547
|
+
"activity": "活动记录",
|
|
548
|
+
"standard": "标准版本",
|
|
549
|
+
"reset": "重置",
|
|
550
|
+
"state": "状态数据",
|
|
551
|
+
"report": "评估报告",
|
|
552
|
+
"exports": "导出记录",
|
|
553
|
+
}
|
|
554
|
+
tokens = [t.lower() for t in re.split(r"[_\-/]+", n) if t]
|
|
555
|
+
if not tokens:
|
|
556
|
+
tokens = [t.lower() for t in re.split(r"[_\-/{}]+", path) if t and t != "api"]
|
|
557
|
+
zh_parts = []
|
|
558
|
+
for tok in tokens:
|
|
559
|
+
if tok in verb_map:
|
|
560
|
+
zh_parts.append(verb_map[tok])
|
|
561
|
+
elif tok in noun_map:
|
|
562
|
+
zh_parts.append(noun_map[tok])
|
|
563
|
+
if zh_parts:
|
|
564
|
+
dedup = []
|
|
565
|
+
for p in zh_parts:
|
|
566
|
+
if not dedup or dedup[-1] != p:
|
|
567
|
+
dedup.append(p)
|
|
568
|
+
return "".join(dedup) + f" ({method.upper()} {path})"
|
|
569
|
+
return f"{n or '接口'} ({method.upper()} {path})"
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def scan_app_routes_and_pages(source: Path) -> dict:
|
|
573
|
+
"""Deterministically scan the entire application source tree for:
|
|
574
|
+
- All backend API endpoints (FastAPI, Flask, Express/Node, Go, etc.) with method, full path, parameters, request_body, auth, effect, and source line evidence.
|
|
575
|
+
- All frontend pages AND SPA sub-views/panels with path, purpose, and source line evidence.
|
|
576
|
+
- Runtime language, minimum version, syntax requirements (e.g. Python >=3.11 for StrEnum), and existing Dockerfile info.
|
|
577
|
+
"""
|
|
578
|
+
source_files: list[Path] = []
|
|
579
|
+
for p in sorted(source.rglob("*")):
|
|
580
|
+
if not p.is_file() or p.is_symlink():
|
|
581
|
+
continue
|
|
582
|
+
rel_parts = p.relative_to(source).parts
|
|
583
|
+
if any(part in delivery.EXCLUDED for part in rel_parts):
|
|
584
|
+
continue
|
|
585
|
+
source_files.append(p)
|
|
586
|
+
|
|
587
|
+
# 1. Runtime & syntax inspection
|
|
588
|
+
runtime_info = _scan_runtime_info(source, source_files)
|
|
589
|
+
|
|
590
|
+
# 2. Scan Python Pydantic models for request/response schemas
|
|
591
|
+
pydantic_models = _scan_pydantic_models(source, source_files)
|
|
592
|
+
|
|
593
|
+
# 3. Scan backend API routes across specs & frameworks
|
|
594
|
+
apis: list[dict] = []
|
|
595
|
+
# 3.1 OpenAPI / Swagger specifications (highest fidelity)
|
|
596
|
+
apis.extend(_scan_openapi_specs(source, source_files))
|
|
597
|
+
# 3.2 Python routes (FastAPI, Flask, Django, DRF, Sanic, etc.)
|
|
598
|
+
apis.extend(_scan_python_routes(source, source_files, pydantic_models))
|
|
599
|
+
# 3.3 Node.js / TypeScript routes (Express, Koa, Fastify, NestJS, Next.js, native HTTP)
|
|
600
|
+
apis.extend(_scan_js_ts_routes(source, source_files))
|
|
601
|
+
# 3.4 Go routes (Gin, Echo, Chi, Fiber, standard net/http, table-driven)
|
|
602
|
+
apis.extend(_scan_go_routes(source, source_files))
|
|
603
|
+
# 3.5 Java / Kotlin routes (Spring Boot, JAX-RS, JDK HttpServer)
|
|
604
|
+
apis.extend(_scan_java_routes(source, source_files))
|
|
605
|
+
# 3.6 C# / .NET routes (ASP.NET Core Minimal API, WebAPI Controllers)
|
|
606
|
+
apis.extend(_scan_dotnet_routes(source, source_files))
|
|
607
|
+
|
|
608
|
+
# Deduplicate APIs by (method, path), keeping the richest entry
|
|
609
|
+
dedup_apis: dict[tuple[str, str], dict] = {}
|
|
610
|
+
for api in apis:
|
|
611
|
+
key = (api["method"].upper(), api["path"])
|
|
612
|
+
if key not in dedup_apis:
|
|
613
|
+
dedup_apis[key] = api
|
|
614
|
+
else:
|
|
615
|
+
existing = dedup_apis[key]
|
|
616
|
+
if len(api.get("parameters") or []) > len(existing.get("parameters") or []):
|
|
617
|
+
dedup_apis[key] = api
|
|
618
|
+
|
|
619
|
+
sorted_apis = sorted(dedup_apis.values(), key=lambda a: (a["path"], a["method"]))
|
|
620
|
+
|
|
621
|
+
# Group APIs by module prefix
|
|
622
|
+
api_modules: dict[str, int] = {}
|
|
623
|
+
for a in sorted_apis:
|
|
624
|
+
parts = [seg for seg in a["path"].split("/") if seg and not seg.startswith("{")]
|
|
625
|
+
mod = "/" + "/".join(parts[:2]) if len(parts) >= 2 else ("/" + parts[0] if parts else "/")
|
|
626
|
+
api_modules[mod] = api_modules.get(mod, 0) + 1
|
|
627
|
+
|
|
628
|
+
# 4. Scan frontend pages and SPA sub-panels
|
|
629
|
+
pages = _scan_frontend_pages(source, source_files)
|
|
630
|
+
|
|
631
|
+
# 5. Scan local file storage (JSON state files, uploads, reports, backups)
|
|
632
|
+
local_file_storage = inspect_local_file_storage(source)
|
|
633
|
+
|
|
634
|
+
return {
|
|
635
|
+
"runtime": runtime_info,
|
|
636
|
+
"api_count": len(sorted_apis),
|
|
637
|
+
"api_modules": api_modules,
|
|
638
|
+
"apis": sorted_apis,
|
|
639
|
+
"page_count": len(pages),
|
|
640
|
+
"pages": pages,
|
|
641
|
+
"local_file_storage": local_file_storage,
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _scan_runtime_info(source: Path, files: list[Path]) -> dict:
|
|
646
|
+
has_root_node = (source / "package.json").is_file()
|
|
647
|
+
has_root_go = (source / "go.mod").is_file()
|
|
648
|
+
has_root_py = (source / "requirements.txt").is_file() or (source / "pyproject.toml").is_file() or (source / "Pipfile").is_file() or (source / "setup.py").is_file()
|
|
649
|
+
has_root_java = (source / "pom.xml").is_file() or (source / "build.gradle").is_file() or any(p.name.endswith(".java") and p.parent == source for p in files)
|
|
650
|
+
has_root_dotnet = any(p.suffix == ".csproj" for p in files)
|
|
651
|
+
has_dockerfile = (source / "Dockerfile").is_file()
|
|
652
|
+
|
|
653
|
+
py_count = sum(1 for p in files if p.suffix == ".py")
|
|
654
|
+
js_ts_count = sum(1 for p in files if p.suffix in {".js", ".ts", ".mjs", ".cjs", ".vue", ".jsx", ".tsx"})
|
|
655
|
+
go_count = sum(1 for p in files if p.suffix == ".go")
|
|
656
|
+
java_count = sum(1 for p in files if p.suffix in {".java", ".kt"})
|
|
657
|
+
dotnet_count = sum(1 for p in files if p.suffix in {".cs", ".fs"})
|
|
658
|
+
|
|
659
|
+
dockerfile_base = ""
|
|
660
|
+
if has_dockerfile:
|
|
661
|
+
try:
|
|
662
|
+
df_text = (source / "Dockerfile").read_text(encoding="utf-8", errors="ignore")
|
|
663
|
+
m = re.search(r"(?im)^\s*FROM\s+([^\s#]+)", df_text)
|
|
664
|
+
if m:
|
|
665
|
+
dockerfile_base = m.group(1)
|
|
666
|
+
except Exception:
|
|
667
|
+
pass
|
|
668
|
+
|
|
669
|
+
py_min_version = "3.10"
|
|
670
|
+
py_syntax_notes: list[str] = []
|
|
671
|
+
frameworks: list[str] = []
|
|
672
|
+
for p in files:
|
|
673
|
+
if p.suffix == ".py" and p.stat().st_size < 500_000:
|
|
674
|
+
try:
|
|
675
|
+
txt = p.read_text(encoding="utf-8", errors="ignore")
|
|
676
|
+
except Exception:
|
|
677
|
+
continue
|
|
678
|
+
if re.search(r"\bStrEnum\b|\btomllib\b|\bSelf\b|\bExceptionGroup\b", txt):
|
|
679
|
+
py_min_version = "3.11"
|
|
680
|
+
if "StrEnum (Python 3.11+)" not in py_syntax_notes and "StrEnum" in txt:
|
|
681
|
+
py_syntax_notes.append("StrEnum (Python 3.11+)")
|
|
682
|
+
if "FastAPI" in txt or "APIRouter" in txt:
|
|
683
|
+
if "FastAPI" not in frameworks:
|
|
684
|
+
frameworks.append("FastAPI")
|
|
685
|
+
elif "Flask" in txt and "Flask" not in frameworks:
|
|
686
|
+
frameworks.append("Flask")
|
|
687
|
+
elif "django" in txt and "Django" not in frameworks:
|
|
688
|
+
frameworks.append("Django")
|
|
689
|
+
elif p.suffix in {".js", ".ts", ".vue", ".jsx", ".tsx"} and p.stat().st_size < 500_000:
|
|
690
|
+
try:
|
|
691
|
+
txt = p.read_text(encoding="utf-8", errors="ignore")
|
|
692
|
+
except Exception:
|
|
693
|
+
continue
|
|
694
|
+
if "vue.esm-browser" in txt or "createApp" in txt:
|
|
695
|
+
if "Vue 3" not in frameworks:
|
|
696
|
+
frameworks.append("Vue 3")
|
|
697
|
+
if re.search(r"""\bfrom\s+['"]react['"]|\bReactDOM\b""", txt) and "React" not in frameworks:
|
|
698
|
+
frameworks.append("React")
|
|
699
|
+
|
|
700
|
+
# Determine primary runtime language by weighted project artifacts rather than simple any()
|
|
701
|
+
if has_root_node and (not has_root_py or js_ts_count >= py_count):
|
|
702
|
+
language = "nodejs"
|
|
703
|
+
elif has_root_go or (go_count > 0 and go_count >= py_count and go_count >= js_ts_count):
|
|
704
|
+
language = "golang"
|
|
705
|
+
elif has_root_dotnet or dotnet_count > 5:
|
|
706
|
+
language = "dotnet"
|
|
707
|
+
elif has_root_java or java_count > 5:
|
|
708
|
+
language = "java"
|
|
709
|
+
elif has_root_py or (py_count > 0 and py_count >= js_ts_count):
|
|
710
|
+
language = "python"
|
|
711
|
+
elif js_ts_count > 0:
|
|
712
|
+
language = "nodejs"
|
|
713
|
+
elif py_count > 0:
|
|
714
|
+
language = "python"
|
|
715
|
+
elif go_count > 0:
|
|
716
|
+
language = "golang"
|
|
717
|
+
else:
|
|
718
|
+
language = "unknown"
|
|
719
|
+
|
|
720
|
+
runtime_version = py_min_version if language == "python" else ("20" if language == "nodejs" else ("1.22" if language == "golang" else ("21" if language == "java" else ("8.0" if language == "dotnet" else "unknown"))))
|
|
721
|
+
return {
|
|
722
|
+
"language": language,
|
|
723
|
+
"runtime_version": runtime_version,
|
|
724
|
+
"frameworks": frameworks,
|
|
725
|
+
"syntax_requirements": py_syntax_notes,
|
|
726
|
+
"has_dockerfile": has_dockerfile,
|
|
727
|
+
"dockerfile_base_image": dockerfile_base,
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _scan_openapi_specs(source: Path, files: list[Path]) -> list[dict]:
|
|
732
|
+
"""Parse OpenAPI/Swagger JSON and YAML specs if present for highest fidelity API contracts."""
|
|
733
|
+
routes: list[dict] = []
|
|
734
|
+
spec_files: list[Path] = []
|
|
735
|
+
for p in files:
|
|
736
|
+
if p.suffix.lower() in {".json", ".yaml", ".yml"} and p.stat().st_size < 1_500_000:
|
|
737
|
+
name_lower = p.name.lower()
|
|
738
|
+
if any(name_lower.startswith(k) for k in ("swagger", "openapi")) or any(k in name_lower for k in ("swagger.", "openapi.")):
|
|
739
|
+
spec_files.append(p)
|
|
740
|
+
|
|
741
|
+
try:
|
|
742
|
+
import yaml
|
|
743
|
+
except Exception:
|
|
744
|
+
yaml = None
|
|
745
|
+
|
|
746
|
+
for p in spec_files:
|
|
747
|
+
rel = p.relative_to(source).as_posix()
|
|
748
|
+
try:
|
|
749
|
+
txt = p.read_text(encoding="utf-8", errors="ignore")
|
|
750
|
+
if p.suffix.lower() == ".json":
|
|
751
|
+
doc = json.loads(txt)
|
|
752
|
+
elif yaml is not None:
|
|
753
|
+
doc = yaml.safe_load(txt)
|
|
754
|
+
else:
|
|
755
|
+
continue
|
|
756
|
+
except Exception:
|
|
757
|
+
continue
|
|
758
|
+
|
|
759
|
+
if not isinstance(doc, dict):
|
|
760
|
+
continue
|
|
761
|
+
base_path = str(doc.get("basePath") or "").strip()
|
|
762
|
+
paths = doc.get("paths")
|
|
763
|
+
if not isinstance(paths, dict):
|
|
764
|
+
continue
|
|
765
|
+
|
|
766
|
+
for path_pattern, path_item in paths.items():
|
|
767
|
+
if not isinstance(path_item, dict):
|
|
768
|
+
continue
|
|
769
|
+
full_path = _join_route_paths(base_path, path_pattern)
|
|
770
|
+
for mth in ("get", "post", "put", "patch", "delete"):
|
|
771
|
+
op = path_item.get(mth)
|
|
772
|
+
if not isinstance(op, dict):
|
|
773
|
+
continue
|
|
774
|
+
mth_upper = mth.upper()
|
|
775
|
+
summary = str(op.get("summary") or op.get("description") or "").strip()
|
|
776
|
+
op_id = op.get("operationId") or ""
|
|
777
|
+
purpose = summary if summary else _humanize_identifier(op_id, mth_upper, full_path)
|
|
778
|
+
effect = "read" if mth_upper == "GET" else ("destructive" if mth_upper == "DELETE" else "write")
|
|
779
|
+
raw_id = re.sub(r"[^a-z0-9]+", "-", f"{mth}-{full_path.strip('/') or 'root'}").strip("-")
|
|
780
|
+
api_id = op_id if op_id and op_id[0].isalpha() else (raw_id if raw_id and raw_id[0].isalpha() else f"op-{raw_id or 'root'}")
|
|
781
|
+
|
|
782
|
+
params: list[dict] = []
|
|
783
|
+
for param in op.get("parameters") or []:
|
|
784
|
+
if isinstance(param, dict) and param.get("name"):
|
|
785
|
+
p_in = param.get("in", "query")
|
|
786
|
+
params.append({
|
|
787
|
+
"name": param["name"],
|
|
788
|
+
"in": p_in,
|
|
789
|
+
"type": param.get("type", "string"),
|
|
790
|
+
"required": bool(param.get("required")),
|
|
791
|
+
"description": param.get("description") or f"参数 {param['name']}",
|
|
792
|
+
"schema": param.get("schema") or {"type": param.get("type", "string")},
|
|
793
|
+
})
|
|
794
|
+
|
|
795
|
+
# Route discovery is not authentication analysis.
|
|
796
|
+
auth_obj = {"type": "unknown"}
|
|
797
|
+
|
|
798
|
+
routes.append({
|
|
799
|
+
"id": api_id,
|
|
800
|
+
"method": mth_upper,
|
|
801
|
+
"path": full_path,
|
|
802
|
+
"handler": op_id or f"openapi_{p.stem}",
|
|
803
|
+
"purpose": purpose,
|
|
804
|
+
"evidence": [f"{rel}:1"],
|
|
805
|
+
"parameters": params,
|
|
806
|
+
"request_body": None,
|
|
807
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
808
|
+
"auth": auth_obj,
|
|
809
|
+
"effect": effect,
|
|
810
|
+
"confirmation_required": effect != "read",
|
|
811
|
+
})
|
|
812
|
+
return routes
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _scan_pydantic_models(source: Path, files: list[Path]) -> dict[str, dict]:
|
|
816
|
+
models: dict[str, dict] = {}
|
|
817
|
+
for p in files:
|
|
818
|
+
if p.suffix != ".py" or p.stat().st_size > 500_000:
|
|
819
|
+
continue
|
|
820
|
+
try:
|
|
821
|
+
tree = ast.parse(p.read_text(encoding="utf-8", errors="ignore"))
|
|
822
|
+
except Exception:
|
|
823
|
+
continue
|
|
824
|
+
for node in ast.walk(tree):
|
|
825
|
+
if isinstance(node, ast.ClassDef):
|
|
826
|
+
base_names = [ast.unparse(b) for b in node.bases] if hasattr(ast, "unparse") else []
|
|
827
|
+
if not any("BaseModel" in b or "Schema" in b or "Request" in node.name for b in base_names + [node.name]):
|
|
828
|
+
continue
|
|
829
|
+
props: dict[str, dict] = {}
|
|
830
|
+
required: list[str] = []
|
|
831
|
+
for stmt in node.body:
|
|
832
|
+
if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name):
|
|
833
|
+
fname = stmt.target.id
|
|
834
|
+
if fname.startswith("_"):
|
|
835
|
+
continue
|
|
836
|
+
ann_str = ast.unparse(stmt.annotation) if hasattr(ast, "unparse") else "string"
|
|
837
|
+
jtype = "string"
|
|
838
|
+
if any(k in ann_str.lower() for k in ("int", "float", "decimal")):
|
|
839
|
+
jtype = "number"
|
|
840
|
+
elif "bool" in ann_str.lower():
|
|
841
|
+
jtype = "boolean"
|
|
842
|
+
elif any(k in ann_str.lower() for k in ("list", "sequence", "set")):
|
|
843
|
+
jtype = "array"
|
|
844
|
+
elif any(k in ann_str.lower() for k in ("dict", "mapping", "any")):
|
|
845
|
+
jtype = "object"
|
|
846
|
+
props[fname] = {"type": jtype}
|
|
847
|
+
if stmt.value is None and "None" not in ann_str and "Optional" not in ann_str:
|
|
848
|
+
required.append(fname)
|
|
849
|
+
if props:
|
|
850
|
+
models[node.name] = {
|
|
851
|
+
"type": "object",
|
|
852
|
+
"required": required,
|
|
853
|
+
"properties": props,
|
|
854
|
+
}
|
|
855
|
+
return models
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _extract_ast_str(node: ast.AST | None) -> str:
|
|
859
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
860
|
+
return node.value
|
|
861
|
+
return ""
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def _scan_python_routes(source: Path, files: list[Path], pydantic_models: dict[str, dict]) -> list[dict]:
|
|
865
|
+
py_files = [p for p in files if p.suffix == ".py" and p.stat().st_size < 500_000]
|
|
866
|
+
# Map module relative path -> router prefixes & include_router mounts
|
|
867
|
+
module_routers: dict[str, dict[str, dict]] = {}
|
|
868
|
+
include_mounts: list[dict] = []
|
|
869
|
+
|
|
870
|
+
parsed_trees: dict[str, tuple[Path, ast.Module, str]] = {}
|
|
871
|
+
for p in py_files:
|
|
872
|
+
rel = p.relative_to(source).as_posix()
|
|
873
|
+
try:
|
|
874
|
+
text = p.read_text(encoding="utf-8", errors="ignore")
|
|
875
|
+
tree = ast.parse(text)
|
|
876
|
+
parsed_trees[rel] = (p, tree, text)
|
|
877
|
+
except Exception:
|
|
878
|
+
continue
|
|
879
|
+
|
|
880
|
+
# First pass: collect APIRouter / Blueprint definitions and include_router mounts
|
|
881
|
+
for rel, (_p, tree, _text) in parsed_trees.items():
|
|
882
|
+
routers_in_file: dict[str, dict] = {}
|
|
883
|
+
imports_in_file: dict[str, str] = {}
|
|
884
|
+
for node in ast.walk(tree):
|
|
885
|
+
if isinstance(node, ast.ImportFrom) and node.module:
|
|
886
|
+
mod_path = node.module.replace(".", "/")
|
|
887
|
+
for alias in node.names:
|
|
888
|
+
local_name = alias.asname or alias.name
|
|
889
|
+
imports_in_file[local_name] = f"{mod_path}/{alias.name}"
|
|
890
|
+
elif isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
|
|
891
|
+
var_name = node.targets[0].id
|
|
892
|
+
if isinstance(node.value, ast.Call):
|
|
893
|
+
func_str = ast.unparse(node.value.func) if hasattr(ast, "unparse") else ""
|
|
894
|
+
if "APIRouter" in func_str or "Blueprint" in func_str:
|
|
895
|
+
prefix = ""
|
|
896
|
+
has_admin_dep = False
|
|
897
|
+
for kw in node.value.keywords:
|
|
898
|
+
if kw.arg in {"prefix", "url_prefix"}:
|
|
899
|
+
prefix = _extract_ast_str(kw.value)
|
|
900
|
+
elif kw.arg == "dependencies":
|
|
901
|
+
dep_str = ast.unparse(kw.value) if hasattr(ast, "unparse") else ""
|
|
902
|
+
if re.search(r"(?i)admin|token|auth", dep_str):
|
|
903
|
+
has_admin_dep = True
|
|
904
|
+
routers_in_file[var_name] = {"prefix": prefix, "admin_auth": has_admin_dep}
|
|
905
|
+
elif isinstance(node, ast.Call):
|
|
906
|
+
func_str = ast.unparse(node.func) if hasattr(ast, "unparse") else ""
|
|
907
|
+
if func_str.endswith(".include_router") and node.args:
|
|
908
|
+
arg_str = ast.unparse(node.args[0]) if hasattr(ast, "unparse") else ""
|
|
909
|
+
mount_prefix = ""
|
|
910
|
+
mount_admin = False
|
|
911
|
+
for kw in node.keywords:
|
|
912
|
+
if kw.arg == "prefix":
|
|
913
|
+
mount_prefix = _extract_ast_str(kw.value)
|
|
914
|
+
elif kw.arg == "dependencies":
|
|
915
|
+
dep_str = ast.unparse(kw.value) if hasattr(ast, "unparse") else ""
|
|
916
|
+
if re.search(r"(?i)admin|token|auth", dep_str):
|
|
917
|
+
mount_admin = True
|
|
918
|
+
include_mounts.append({
|
|
919
|
+
"file": rel,
|
|
920
|
+
"arg": arg_str,
|
|
921
|
+
"imports": imports_in_file,
|
|
922
|
+
"prefix": mount_prefix,
|
|
923
|
+
"admin_auth": mount_admin,
|
|
924
|
+
})
|
|
925
|
+
module_routers[rel] = routers_in_file
|
|
926
|
+
|
|
927
|
+
# Resolve include_router prefixes per file
|
|
928
|
+
file_mount_prefix: dict[str, tuple[str, bool]] = {}
|
|
929
|
+
for m in include_mounts:
|
|
930
|
+
arg = m["arg"]
|
|
931
|
+
base_name = arg.split(".")[0]
|
|
932
|
+
for target_rel in parsed_trees:
|
|
933
|
+
stem = Path(target_rel).stem
|
|
934
|
+
if base_name == stem or arg.endswith(f"{stem}.router") or base_name in m["imports"].get(base_name, ""):
|
|
935
|
+
if stem == base_name or m["imports"].get(base_name, "").endswith(f"/{stem}") or m["imports"].get(base_name, "").endswith(f"/{stem}/router"):
|
|
936
|
+
file_mount_prefix[target_rel] = (m["prefix"], m["admin_auth"])
|
|
937
|
+
|
|
938
|
+
# Second pass: extract all route handlers
|
|
939
|
+
routes: list[dict] = []
|
|
940
|
+
http_methods = {"get", "post", "put", "patch", "delete"}
|
|
941
|
+
for rel, (_p, tree, _text) in parsed_trees.items():
|
|
942
|
+
routers_in_file = module_routers.get(rel, {})
|
|
943
|
+
mount_pref, mount_admin = file_mount_prefix.get(rel, ("", False))
|
|
944
|
+
for node in ast.walk(tree):
|
|
945
|
+
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
946
|
+
continue
|
|
947
|
+
for dec in node.decorator_list:
|
|
948
|
+
if not isinstance(dec, ast.Call) or not isinstance(dec.func, ast.Attribute):
|
|
949
|
+
continue
|
|
950
|
+
attr = dec.func.attr.lower()
|
|
951
|
+
owner = dec.func.value.id if isinstance(dec.func.value, ast.Name) else ""
|
|
952
|
+
methods_found: list[str] = []
|
|
953
|
+
sub_path = ""
|
|
954
|
+
if attr in http_methods:
|
|
955
|
+
methods_found = [attr.upper()]
|
|
956
|
+
if dec.args:
|
|
957
|
+
sub_path = _extract_ast_str(dec.args[0])
|
|
958
|
+
for kw in dec.keywords:
|
|
959
|
+
if kw.arg == "path":
|
|
960
|
+
sub_path = _extract_ast_str(kw.value)
|
|
961
|
+
elif attr == "route":
|
|
962
|
+
if dec.args:
|
|
963
|
+
sub_path = _extract_ast_str(dec.args[0])
|
|
964
|
+
methods_found = ["GET"]
|
|
965
|
+
for kw in dec.keywords:
|
|
966
|
+
if kw.arg == "methods" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
967
|
+
ms = [_extract_ast_str(elt).upper() for elt in kw.value.elts if _extract_ast_str(elt)]
|
|
968
|
+
if ms:
|
|
969
|
+
methods_found = ms
|
|
970
|
+
if not methods_found:
|
|
971
|
+
continue
|
|
972
|
+
|
|
973
|
+
router_info = routers_in_file.get(owner, {"prefix": "", "admin_auth": False})
|
|
974
|
+
full_path = _join_route_paths(mount_pref, router_info["prefix"], sub_path)
|
|
975
|
+
|
|
976
|
+
# Extract summary/docstring
|
|
977
|
+
doc = (ast.get_docstring(node) or "").strip().splitlines()[0] if ast.get_docstring(node) else ""
|
|
978
|
+
for kw in dec.keywords:
|
|
979
|
+
if kw.arg in {"summary", "description"} and _extract_ast_str(kw.value):
|
|
980
|
+
doc = _extract_ast_str(kw.value).strip()
|
|
981
|
+
break
|
|
982
|
+
|
|
983
|
+
auth_obj = {"type": "unknown"}
|
|
984
|
+
|
|
985
|
+
# Extract path & query parameters + request body
|
|
986
|
+
path_param_names = set(re.findall(r"\{([a-zA-Z0-9_]+)\}", full_path))
|
|
987
|
+
params: list[dict] = []
|
|
988
|
+
seen_param_names: set[str] = set()
|
|
989
|
+
for pname in re.findall(r"\{([a-zA-Z0-9_]+)\}", full_path):
|
|
990
|
+
if pname in seen_param_names:
|
|
991
|
+
continue
|
|
992
|
+
seen_param_names.add(pname)
|
|
993
|
+
params.append({
|
|
994
|
+
"name": pname,
|
|
995
|
+
"in": "path",
|
|
996
|
+
"type": "string",
|
|
997
|
+
"required": True,
|
|
998
|
+
"description": f"路径参数 {pname}",
|
|
999
|
+
"schema": {"type": "string"},
|
|
1000
|
+
})
|
|
1001
|
+
|
|
1002
|
+
req_body: dict | None = None
|
|
1003
|
+
all_fn_args = list(node.args.args) + list(node.args.kwonlyargs)
|
|
1004
|
+
num_defaults = len(node.args.defaults)
|
|
1005
|
+
num_regular = len(node.args.args)
|
|
1006
|
+
for idx, arg in enumerate(all_fn_args):
|
|
1007
|
+
aname = arg.arg
|
|
1008
|
+
if aname in {"self", "cls", "request", "response", "db", "session", "conn", "background_tasks", "_"} or aname.startswith("_"):
|
|
1009
|
+
continue
|
|
1010
|
+
ann_str = ast.unparse(arg.annotation) if (arg.annotation and hasattr(ast, "unparse")) else ""
|
|
1011
|
+
if any(skip in ann_str for skip in ("Request", "Response", "BackgroundTasks", "Session", "Connection")):
|
|
1012
|
+
continue
|
|
1013
|
+
if "Depends" in ann_str or "Header" in ann_str:
|
|
1014
|
+
continue
|
|
1015
|
+
if aname in path_param_names or aname in seen_param_names:
|
|
1016
|
+
continue
|
|
1017
|
+
if "UploadFile" in ann_str or "File" in ann_str:
|
|
1018
|
+
req_body = {
|
|
1019
|
+
"required": True,
|
|
1020
|
+
"content_type": "multipart/form-data",
|
|
1021
|
+
"schema": {
|
|
1022
|
+
"type": "object",
|
|
1023
|
+
"required": [aname],
|
|
1024
|
+
"properties": {aname: {"type": "string", "description": "上传文件"}},
|
|
1025
|
+
},
|
|
1026
|
+
}
|
|
1027
|
+
continue
|
|
1028
|
+
clean_ann = re.sub(r"Optional\[(.*)\]|None|\|", "", ann_str).strip()
|
|
1029
|
+
if clean_ann in pydantic_models:
|
|
1030
|
+
req_body = {
|
|
1031
|
+
"required": True,
|
|
1032
|
+
"content_type": "application/json",
|
|
1033
|
+
"schema": pydantic_models[clean_ann],
|
|
1034
|
+
}
|
|
1035
|
+
continue
|
|
1036
|
+
if clean_ann in {"dict", "Dict", " dict[str, Any]", "dict[str, Any]"} or aname in {"payload", "body", "data", "req"}:
|
|
1037
|
+
req_body = {
|
|
1038
|
+
"required": True,
|
|
1039
|
+
"content_type": "application/json",
|
|
1040
|
+
"schema": {
|
|
1041
|
+
"type": "object",
|
|
1042
|
+
"required": [],
|
|
1043
|
+
"properties": {"data": {"type": "object"}},
|
|
1044
|
+
},
|
|
1045
|
+
}
|
|
1046
|
+
continue
|
|
1047
|
+
# Otherwise treat primitive parameters on GET/DELETE (or with Query) as query parameters
|
|
1048
|
+
has_default = idx >= (num_regular - num_defaults) if idx < num_regular else True
|
|
1049
|
+
jtype = "integer" if "int" in ann_str.lower() else ("boolean" if "bool" in ann_str.lower() else ("number" if "float" in ann_str.lower() else "string"))
|
|
1050
|
+
seen_param_names.add(aname)
|
|
1051
|
+
params.append({
|
|
1052
|
+
"name": aname,
|
|
1053
|
+
"in": "query",
|
|
1054
|
+
"type": jtype,
|
|
1055
|
+
"required": not has_default,
|
|
1056
|
+
"description": f"查询参数 {aname}",
|
|
1057
|
+
"schema": {"type": jtype},
|
|
1058
|
+
})
|
|
1059
|
+
|
|
1060
|
+
for mth in methods_found:
|
|
1061
|
+
if mth in {"POST", "PUT", "PATCH"} and req_body is None and not params:
|
|
1062
|
+
req_body = {
|
|
1063
|
+
"required": False,
|
|
1064
|
+
"content_type": "application/json",
|
|
1065
|
+
"schema": {
|
|
1066
|
+
"type": "object",
|
|
1067
|
+
"required": [],
|
|
1068
|
+
"properties": {"payload": {"type": "object"}},
|
|
1069
|
+
},
|
|
1070
|
+
}
|
|
1071
|
+
effect = "read" if mth == "GET" else ("destructive" if mth == "DELETE" else "write")
|
|
1072
|
+
raw_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{full_path.strip('/') or 'root'}").strip("-")
|
|
1073
|
+
api_id = raw_id if (raw_id and raw_id[0].isalpha()) else f"op-{raw_id or 'root'}"
|
|
1074
|
+
purpose = doc if doc else _humanize_identifier(node.name, mth, full_path)
|
|
1075
|
+
routes.append({
|
|
1076
|
+
"id": api_id,
|
|
1077
|
+
"method": mth,
|
|
1078
|
+
"path": full_path,
|
|
1079
|
+
"handler": node.name,
|
|
1080
|
+
"purpose": purpose,
|
|
1081
|
+
"evidence": [f"{rel}:{node.lineno}"],
|
|
1082
|
+
"parameters": params,
|
|
1083
|
+
"request_body": req_body if mth in {"POST", "PUT", "PATCH"} else None,
|
|
1084
|
+
"responses": {
|
|
1085
|
+
"200": {
|
|
1086
|
+
"description": f"{purpose}响应",
|
|
1087
|
+
"type": "object",
|
|
1088
|
+
"required": [],
|
|
1089
|
+
"properties": {"status": {"type": "string"}},
|
|
1090
|
+
}
|
|
1091
|
+
},
|
|
1092
|
+
"auth": auth_obj,
|
|
1093
|
+
"effect": effect,
|
|
1094
|
+
"confirmation_required": effect in {"write", "destructive", "external"},
|
|
1095
|
+
})
|
|
1096
|
+
|
|
1097
|
+
# Third pass: Django & Django REST Framework (urls.py)
|
|
1098
|
+
django_url_re = re.compile(r"""(?:path|re_path)\(\s*['"]([a-zA-Z0-9_\-/{}\<\>: \^\$\*\+\.\?]+)['"]\s*,\s*([a-zA-Z0-9_.]+)""")
|
|
1099
|
+
for rel, (_p, tree, text) in parsed_trees.items():
|
|
1100
|
+
if not ("urls" in rel or "urlpatterns" in text):
|
|
1101
|
+
continue
|
|
1102
|
+
for idx, line in enumerate(text.splitlines(), start=1):
|
|
1103
|
+
for m in django_url_re.finditer(line):
|
|
1104
|
+
raw_path = m.group(1).lstrip("^").rstrip("$")
|
|
1105
|
+
handler = m.group(2)
|
|
1106
|
+
# Normalize django path params <int:id> or <slug> to {id} or {slug}
|
|
1107
|
+
clean_path = "/" + re.sub(r"<(?:\w+:)?(\w+)>", r"{\1}", raw_path.strip("/"))
|
|
1108
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"get-{clean_path.strip('/') or 'root'}").strip("-")
|
|
1109
|
+
routes.append({
|
|
1110
|
+
"id": api_id,
|
|
1111
|
+
"method": "GET",
|
|
1112
|
+
"path": clean_path,
|
|
1113
|
+
"handler": handler,
|
|
1114
|
+
"purpose": _humanize_identifier(handler.split(".")[-1], "GET", clean_path),
|
|
1115
|
+
"evidence": [f"{rel}:{idx}"],
|
|
1116
|
+
"parameters": [],
|
|
1117
|
+
"request_body": None,
|
|
1118
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1119
|
+
"auth": {"type": "unknown"},
|
|
1120
|
+
"effect": "read",
|
|
1121
|
+
"confirmation_required": False,
|
|
1122
|
+
})
|
|
1123
|
+
|
|
1124
|
+
# Fourth pass: Python BaseHTTPRequestHandler (do_GET, do_POST)
|
|
1125
|
+
for rel, (_p, tree, text) in parsed_trees.items():
|
|
1126
|
+
if "BaseHTTPRequestHandler" not in text:
|
|
1127
|
+
continue
|
|
1128
|
+
for m in re.finditer(r"""def\s+(do_GET|do_POST|do_PUT|do_DELETE)\s*\(""", text):
|
|
1129
|
+
mth = m.group(1).replace("do_", "").upper()
|
|
1130
|
+
start_pos = m.start()
|
|
1131
|
+
method_block = text[start_pos:start_pos+3000]
|
|
1132
|
+
for pm in re.finditer(r"""(?:self\.path|url\.path)\s*==\s*['"](/[^'"]*)['"]""", method_block):
|
|
1133
|
+
path = pm.group(1)
|
|
1134
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{path.strip('/') or 'root'}").strip("-")
|
|
1135
|
+
routes.append({
|
|
1136
|
+
"id": api_id,
|
|
1137
|
+
"method": mth,
|
|
1138
|
+
"path": path,
|
|
1139
|
+
"handler": m.group(1),
|
|
1140
|
+
"purpose": _humanize_identifier("", mth, path),
|
|
1141
|
+
"evidence": [f"{rel}:1"],
|
|
1142
|
+
"parameters": [],
|
|
1143
|
+
"request_body": None,
|
|
1144
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1145
|
+
"auth": {"type": "unknown"},
|
|
1146
|
+
"effect": "read" if mth == "GET" else "write",
|
|
1147
|
+
"confirmation_required": mth != "GET",
|
|
1148
|
+
})
|
|
1149
|
+
|
|
1150
|
+
return routes
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def _scan_js_ts_routes(source: Path, files: list[Path]) -> list[dict]:
|
|
1154
|
+
routes: list[dict] = []
|
|
1155
|
+
route_re = re.compile(r"""\b(?:app|router|api|server)\.(get|post|put|patch|delete)\(\s*['"`](/[^'"`]*)['"`]""", re.I)
|
|
1156
|
+
native_exact_re = re.compile(
|
|
1157
|
+
r"""(?:(?:request|req)\.method|method)\s*===\s*['"](GET|POST|PUT|PATCH|DELETE)['"]\s*&&\s*(?:url\.)?pathname\s*===\s*['"](/[^'"]+)['"]"""
|
|
1158
|
+
r"""|(?:url\.)?pathname\s*===\s*['"](/[^'"]+)['"]\s*&&\s*(?:(?:request|req)\.method|method)\s*===\s*['"](GET|POST|PUT|PATCH|DELETE)['"]""",
|
|
1159
|
+
re.I,
|
|
1160
|
+
)
|
|
1161
|
+
native_method_re = re.compile(r"""(?:(?:request|req)\.method|method)\s*===\s*['"](GET|POST|PUT|PATCH|DELETE)['"]""", re.I)
|
|
1162
|
+
native_starts_re = re.compile(r"""(?:url\.)?pathname\.startsWith\(\s*['"](/api/[^'"]+)['"]\s*\)""")
|
|
1163
|
+
native_ends_re = re.compile(r"""(?:url\.)?pathname\.endsWith\(\s*['"](/[^'"]+)['"]\s*\)""")
|
|
1164
|
+
native_includes_re = re.compile(r"""(?:url\.)?pathname\.includes\(\s*['"](/[^'"]+)['"]\s*\)""")
|
|
1165
|
+
|
|
1166
|
+
def _make_js_route(mth: str, path: str, rel: str, idx: int) -> dict:
|
|
1167
|
+
mth = mth.upper()
|
|
1168
|
+
effect = "read" if mth == "GET" else ("destructive" if mth == "DELETE" else "write")
|
|
1169
|
+
raw_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{path.strip('/') or 'root'}").strip("-")
|
|
1170
|
+
api_id = raw_id if (raw_id and raw_id[0].isalpha()) else f"op-{raw_id or 'root'}"
|
|
1171
|
+
path_params = []
|
|
1172
|
+
seen_p: set[str] = set()
|
|
1173
|
+
for pname in re.findall(r"\{([a-zA-Z0-9_]+)\}", path):
|
|
1174
|
+
if pname not in seen_p:
|
|
1175
|
+
seen_p.add(pname)
|
|
1176
|
+
path_params.append({
|
|
1177
|
+
"name": pname,
|
|
1178
|
+
"in": "path",
|
|
1179
|
+
"type": "string",
|
|
1180
|
+
"required": True,
|
|
1181
|
+
"description": f"路径参数 {pname}",
|
|
1182
|
+
"schema": {"type": "string"},
|
|
1183
|
+
})
|
|
1184
|
+
req_body = (
|
|
1185
|
+
{
|
|
1186
|
+
"required": False,
|
|
1187
|
+
"content_type": "application/json",
|
|
1188
|
+
"schema": {"type": "object", "required": [], "properties": {"payload": {"type": "object"}}},
|
|
1189
|
+
}
|
|
1190
|
+
if mth in {"POST", "PUT", "PATCH"}
|
|
1191
|
+
else None
|
|
1192
|
+
)
|
|
1193
|
+
return {
|
|
1194
|
+
"id": api_id,
|
|
1195
|
+
"method": mth,
|
|
1196
|
+
"path": path,
|
|
1197
|
+
"handler": f"line_{idx}",
|
|
1198
|
+
"purpose": _humanize_identifier("", mth, path),
|
|
1199
|
+
"evidence": [f"{rel}:{idx}"],
|
|
1200
|
+
"parameters": path_params,
|
|
1201
|
+
"request_body": req_body,
|
|
1202
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1203
|
+
"auth": {"type": "unknown"},
|
|
1204
|
+
"effect": effect,
|
|
1205
|
+
"confirmation_required": effect != "read",
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
# NestJS Controller decorators
|
|
1209
|
+
nest_controller_re = re.compile(r"""@Controller\(\s*['"]?([^'"]*?)['"]?\s*\)[\s\S]*?class\s+([A-Za-z0-9_]+)""")
|
|
1210
|
+
nest_method_re = re.compile(r"""@(Get|Post|Put|Patch|Delete)\(\s*['"]?([^'"]*?)['"]?\s*\)[\s\S]*?(?:async\s+)?([a-zA-Z0-9_]+)\s*\(""")
|
|
1211
|
+
|
|
1212
|
+
for p in files:
|
|
1213
|
+
if p.suffix not in {".js", ".ts", ".mjs", ".cjs"} or p.stat().st_size > 500_000:
|
|
1214
|
+
continue
|
|
1215
|
+
rel = p.relative_to(source).as_posix()
|
|
1216
|
+
if any(seg in rel for seg in ("static/", "public/", "dist/", "assets/", "components/", "miniprogram/")):
|
|
1217
|
+
continue
|
|
1218
|
+
try:
|
|
1219
|
+
content = p.read_text(encoding="utf-8", errors="ignore")
|
|
1220
|
+
lines = content.splitlines()
|
|
1221
|
+
except Exception:
|
|
1222
|
+
continue
|
|
1223
|
+
|
|
1224
|
+
for idx, line in enumerate(lines, start=1):
|
|
1225
|
+
for m in route_re.finditer(line):
|
|
1226
|
+
routes.append(_make_js_route(m.group(1), m.group(2), rel, idx))
|
|
1227
|
+
for m in native_exact_re.finditer(line):
|
|
1228
|
+
if "return false" in line:
|
|
1229
|
+
continue
|
|
1230
|
+
mth = m.group(1) or m.group(4)
|
|
1231
|
+
path = m.group(2) or m.group(3)
|
|
1232
|
+
routes.append(_make_js_route(mth, path, rel, idx))
|
|
1233
|
+
# Node native if (url.pathname === '/api/...') without explicit method => GET
|
|
1234
|
+
if "return false" not in line and not native_exact_re.search(line):
|
|
1235
|
+
m_exact_path = re.search(r"""(?:url\.)?pathname\s*===\s*['"](/api/[^'"]+)['"]""", line)
|
|
1236
|
+
if m_exact_path:
|
|
1237
|
+
routes.append(_make_js_route("GET", m_exact_path.group(1), rel, idx))
|
|
1238
|
+
if "return false" not in line and "pathname.startsWith" in line:
|
|
1239
|
+
m_mth = native_method_re.search(line)
|
|
1240
|
+
m_st = native_starts_re.search(line)
|
|
1241
|
+
if m_mth and m_st:
|
|
1242
|
+
mth = m_mth.group(1)
|
|
1243
|
+
prefix = m_st.group(1).rstrip("/")
|
|
1244
|
+
m_end = native_ends_re.search(line)
|
|
1245
|
+
m_inc = native_includes_re.search(line)
|
|
1246
|
+
if m_end:
|
|
1247
|
+
suffix = m_end.group(1)
|
|
1248
|
+
synth_path = f"{prefix}/{{id}}{suffix}"
|
|
1249
|
+
elif m_inc:
|
|
1250
|
+
mid = m_inc.group(1).strip("/")
|
|
1251
|
+
synth_path = f"{prefix}/{{id}}/{mid}/{{item_id}}"
|
|
1252
|
+
else:
|
|
1253
|
+
synth_path = f"{prefix}/{{id}}"
|
|
1254
|
+
routes.append(_make_js_route(mth, synth_path, rel, idx))
|
|
1255
|
+
|
|
1256
|
+
# Check NestJS
|
|
1257
|
+
if "@Controller" in content:
|
|
1258
|
+
m_ctrl = nest_controller_re.search(content)
|
|
1259
|
+
ctrl_prefix = m_ctrl.group(1).strip("/") if m_ctrl else ""
|
|
1260
|
+
for m in nest_method_re.finditer(content):
|
|
1261
|
+
mth, sub, handler = m.group(1).upper(), m.group(2).strip("/"), m.group(3)
|
|
1262
|
+
full = "/" + "/".join(p for p in (ctrl_prefix, sub) if p)
|
|
1263
|
+
routes.append(_make_js_route(mth, full, rel, 1))
|
|
1264
|
+
|
|
1265
|
+
# Next.js App Router API routes (app/api/**/route.ts)
|
|
1266
|
+
for p in files:
|
|
1267
|
+
rel = p.relative_to(source).as_posix()
|
|
1268
|
+
if ("app/api/" in rel or "src/app/api/" in rel) and p.stem == "route" and p.suffix in {".js", ".ts"}:
|
|
1269
|
+
dir_path = p.parent.relative_to(source).as_posix()
|
|
1270
|
+
api_path = "/" + dir_path.split("app/", 1)[-1]
|
|
1271
|
+
try:
|
|
1272
|
+
txt = p.read_text(encoding="utf-8", errors="ignore")
|
|
1273
|
+
for mth in ("GET", "POST", "PUT", "PATCH", "DELETE"):
|
|
1274
|
+
if re.search(rf"\bexport\s+(?:async\s+)?function\s+{mth}\b", txt):
|
|
1275
|
+
routes.append(_make_js_route(mth, api_path, rel, 1))
|
|
1276
|
+
except Exception:
|
|
1277
|
+
continue
|
|
1278
|
+
|
|
1279
|
+
return routes
|
|
1280
|
+
|
|
1281
|
+
|
|
1282
|
+
def _scan_go_routes(source: Path, files: list[Path]) -> list[dict]:
|
|
1283
|
+
routes: list[dict] = []
|
|
1284
|
+
# 1. Match direct router method calls (.GET, .POST, .Get, .Post, .HandleFunc)
|
|
1285
|
+
route_re = re.compile(r"""([a-zA-Z0-9_.]*)\s*\.\s*(GET|POST|PUT|PATCH|DELETE|HandleFunc|Get|Post|Put|Patch|Delete)\(\s*"([^"]+)"\s*,\s*([a-zA-Z0-9_.]+)""")
|
|
1286
|
+
# 2. Table-driven RouterEntry{Path: "...", Method: http.MethodPost, Handler: app.Run}
|
|
1287
|
+
table_re = re.compile(r"""(?:AppendRouters|RouterEntry)\s*\(\s*(?:([a-zA-Z0-9_]+)\s*,\s*)?sdk\.RouterEntry\s*\{\s*Path:\s*"([^"]+)"\s*,\s*Method:\s*(?:http\.Method)?([A-Za-z]+)\s*,\s*Handler:\s*([a-zA-Z0-9_.]+)""")
|
|
1288
|
+
# 3. Swaggo comments: // @Router /path [method]
|
|
1289
|
+
swaggo_router_re = re.compile(r"""//\s*@Router\s+([^\s\[]+)\s*\[(get|post|put|patch|delete)\]""", re.I)
|
|
1290
|
+
swaggo_summary_re = re.compile(r"""//\s*@Summary\s+(.+)""")
|
|
1291
|
+
|
|
1292
|
+
# Map Group variables to path prefixes: v1 := r.Group("/api/v1")
|
|
1293
|
+
group_var_prefixes: dict[str, str] = {}
|
|
1294
|
+
const_prefixes: dict[str, str] = {"V1": "/api/v1", "v1": "/api/v1"}
|
|
1295
|
+
|
|
1296
|
+
for p in files:
|
|
1297
|
+
if p.suffix != ".go" or p.name.endswith("_test.go") or p.stat().st_size > 500_000:
|
|
1298
|
+
continue
|
|
1299
|
+
rel = p.relative_to(source).as_posix()
|
|
1300
|
+
try:
|
|
1301
|
+
content = p.read_text(encoding="utf-8", errors="ignore")
|
|
1302
|
+
lines = content.splitlines()
|
|
1303
|
+
except Exception:
|
|
1304
|
+
continue
|
|
1305
|
+
|
|
1306
|
+
# Extract const string definitions like const V1 = "/api/v1"
|
|
1307
|
+
for cm in re.finditer(r"""const\s*\(\s*([\s\S]*?)\)""", content):
|
|
1308
|
+
for line in cm.group(1).splitlines():
|
|
1309
|
+
m_kv = re.search(r"""([a-zA-Z0-9_]+)\s*=\s*"([^"]+)""", line)
|
|
1310
|
+
if m_kv:
|
|
1311
|
+
const_prefixes[m_kv.group(1)] = m_kv.group(2)
|
|
1312
|
+
for m_kv in re.finditer(r"""const\s+([a-zA-Z0-9_]+)\s*=\s*"([^"]+)""", content):
|
|
1313
|
+
const_prefixes[m_kv.group(1)] = m_kv.group(2)
|
|
1314
|
+
|
|
1315
|
+
# Extract group definitions
|
|
1316
|
+
for gm in re.finditer(r"""([a-zA-Z0-9_]+)\s*:=\s*([a-zA-Z0-9_.]*)\s*\.\s*Group\(\s*"([^"]+)"\s*\)""", content):
|
|
1317
|
+
gvar, parent, gpath = gm.group(1), gm.group(2), gm.group(3)
|
|
1318
|
+
pref = group_var_prefixes.get(parent, "")
|
|
1319
|
+
group_var_prefixes[gvar] = _join_route_paths(pref, gpath)
|
|
1320
|
+
|
|
1321
|
+
for idx, line in enumerate(lines, start=1):
|
|
1322
|
+
for m in route_re.finditer(line):
|
|
1323
|
+
obj, raw_mth, path, handler = m.group(1), m.group(2), m.group(3), m.group(4)
|
|
1324
|
+
mth = "GET" if raw_mth == "HandleFunc" else raw_mth.upper()
|
|
1325
|
+
pref = group_var_prefixes.get(obj.split(".")[-1], "")
|
|
1326
|
+
full_path = _join_route_paths(pref, path)
|
|
1327
|
+
effect = "read" if mth == "GET" else ("destructive" if mth == "DELETE" else "write")
|
|
1328
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{full_path.strip('/') or 'root'}").strip("-")
|
|
1329
|
+
routes.append({
|
|
1330
|
+
"id": api_id,
|
|
1331
|
+
"method": mth,
|
|
1332
|
+
"path": full_path,
|
|
1333
|
+
"handler": handler,
|
|
1334
|
+
"purpose": _humanize_identifier(handler.split(".")[-1], mth, full_path),
|
|
1335
|
+
"evidence": [f"{rel}:{idx}"],
|
|
1336
|
+
"parameters": [],
|
|
1337
|
+
"request_body": None,
|
|
1338
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"code": {"type": "integer"}}}},
|
|
1339
|
+
"auth": {"type": "unknown"},
|
|
1340
|
+
"effect": effect,
|
|
1341
|
+
"confirmation_required": effect != "read",
|
|
1342
|
+
})
|
|
1343
|
+
|
|
1344
|
+
for m in table_re.finditer(line):
|
|
1345
|
+
gconst, path, raw_mth, handler = m.group(1), m.group(2), m.group(3), m.group(4)
|
|
1346
|
+
mth = raw_mth.upper()
|
|
1347
|
+
pref = const_prefixes.get(gconst, "") if gconst else ""
|
|
1348
|
+
full_path = _join_route_paths(pref, path)
|
|
1349
|
+
effect = "read" if mth == "GET" else ("destructive" if mth == "DELETE" else "write")
|
|
1350
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{full_path.strip('/') or 'root'}").strip("-")
|
|
1351
|
+
routes.append({
|
|
1352
|
+
"id": api_id,
|
|
1353
|
+
"method": mth,
|
|
1354
|
+
"path": full_path,
|
|
1355
|
+
"handler": handler,
|
|
1356
|
+
"purpose": _humanize_identifier(handler.split(".")[-1], mth, full_path),
|
|
1357
|
+
"evidence": [f"{rel}:{idx}"],
|
|
1358
|
+
"parameters": [],
|
|
1359
|
+
"request_body": None,
|
|
1360
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"code": {"type": "integer"}}}},
|
|
1361
|
+
"auth": {"type": "unknown"},
|
|
1362
|
+
"effect": effect,
|
|
1363
|
+
"confirmation_required": effect != "read",
|
|
1364
|
+
})
|
|
1365
|
+
|
|
1366
|
+
# Scan swaggo comments
|
|
1367
|
+
curr_summary = ""
|
|
1368
|
+
for idx, line in enumerate(lines, start=1):
|
|
1369
|
+
line_s = line.strip()
|
|
1370
|
+
if line_s.startswith("//"):
|
|
1371
|
+
sm = swaggo_summary_re.search(line_s)
|
|
1372
|
+
if sm:
|
|
1373
|
+
curr_summary = sm.group(1).strip()
|
|
1374
|
+
rm = swaggo_router_re.search(line_s)
|
|
1375
|
+
if rm:
|
|
1376
|
+
path, mth = rm.group(1), rm.group(2).upper()
|
|
1377
|
+
effect = "read" if mth == "GET" else ("destructive" if mth == "DELETE" else "write")
|
|
1378
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{path.strip('/') or 'root'}").strip("-")
|
|
1379
|
+
purpose = curr_summary if curr_summary else _humanize_identifier("", mth, path)
|
|
1380
|
+
routes.append({
|
|
1381
|
+
"id": api_id,
|
|
1382
|
+
"method": mth,
|
|
1383
|
+
"path": path,
|
|
1384
|
+
"handler": f"swaggo_{Path(rel).stem}",
|
|
1385
|
+
"purpose": purpose,
|
|
1386
|
+
"evidence": [f"{rel}:{idx}"],
|
|
1387
|
+
"parameters": [],
|
|
1388
|
+
"request_body": None,
|
|
1389
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"code": {"type": "integer"}}}},
|
|
1390
|
+
"auth": {"type": "unknown"},
|
|
1391
|
+
"effect": effect,
|
|
1392
|
+
"confirmation_required": effect != "read",
|
|
1393
|
+
})
|
|
1394
|
+
else:
|
|
1395
|
+
if not line_s.startswith("//") and line_s:
|
|
1396
|
+
curr_summary = ""
|
|
1397
|
+
|
|
1398
|
+
return routes
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def _scan_java_routes(source: Path, files: list[Path]) -> list[dict]:
|
|
1402
|
+
"""Scan Java / Kotlin Spring Boot, JAX-RS, and JDK HttpServer routes."""
|
|
1403
|
+
routes: list[dict] = []
|
|
1404
|
+
# Spring Boot @RequestMapping on class
|
|
1405
|
+
spring_class_re = re.compile(r"""@(?:Rest)?Controller[\s\S]*?@RequestMapping\(\s*["']([^"']+)["']\s*\)[\s\S]*?class\s+([A-Za-z0-9_]+)""")
|
|
1406
|
+
# Spring Boot methods @GetMapping, @PostMapping, etc.
|
|
1407
|
+
spring_method_re = re.compile(r"""@(GetMapping|PostMapping|PutMapping|DeleteMapping|PatchMapping)(?:\(\s*(?:(?:value|path)\s*=\s*)?["']?([^"'\)]*)["']?\s*\))?[\s\S]*?(?:public|protected|private)?\s+[A-Za-z0-9_<>, \t\n\r]+\s+([A-Za-z0-9_]+)\s*\(""")
|
|
1408
|
+
# JDK HttpServer createContext("/api/stock", ...)
|
|
1409
|
+
jdk_context_re = re.compile(r"""\.createContext\(\s*["']([^"']+)["']""")
|
|
1410
|
+
|
|
1411
|
+
for p in files:
|
|
1412
|
+
if p.suffix not in {".java", ".kt"} or p.stat().st_size > 500_000:
|
|
1413
|
+
continue
|
|
1414
|
+
rel = p.relative_to(source).as_posix()
|
|
1415
|
+
try:
|
|
1416
|
+
content = p.read_text(encoding="utf-8", errors="ignore")
|
|
1417
|
+
except Exception:
|
|
1418
|
+
continue
|
|
1419
|
+
|
|
1420
|
+
class_prefix = ""
|
|
1421
|
+
m_cls = spring_class_re.search(content)
|
|
1422
|
+
if m_cls:
|
|
1423
|
+
class_prefix = m_cls.group(1).strip()
|
|
1424
|
+
|
|
1425
|
+
for m in spring_method_re.finditer(content):
|
|
1426
|
+
ann, sub, handler = m.group(1), m.group(2) or "", m.group(3)
|
|
1427
|
+
mth = ann.replace("Mapping", "").upper()
|
|
1428
|
+
full_path = _join_route_paths(class_prefix, sub)
|
|
1429
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{full_path.strip('/') or 'root'}").strip("-")
|
|
1430
|
+
routes.append({
|
|
1431
|
+
"id": api_id,
|
|
1432
|
+
"method": mth,
|
|
1433
|
+
"path": full_path,
|
|
1434
|
+
"handler": handler,
|
|
1435
|
+
"purpose": _humanize_identifier(handler, mth, full_path),
|
|
1436
|
+
"evidence": [f"{rel}:1"],
|
|
1437
|
+
"parameters": [],
|
|
1438
|
+
"request_body": None,
|
|
1439
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1440
|
+
"auth": {"type": "unknown"},
|
|
1441
|
+
"effect": "read" if mth == "GET" else "write",
|
|
1442
|
+
"confirmation_required": mth != "GET",
|
|
1443
|
+
})
|
|
1444
|
+
|
|
1445
|
+
for m in jdk_context_re.finditer(content):
|
|
1446
|
+
path = m.group(1)
|
|
1447
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"get-{path.strip('/') or 'root'}").strip("-")
|
|
1448
|
+
routes.append({
|
|
1449
|
+
"id": api_id,
|
|
1450
|
+
"method": "GET",
|
|
1451
|
+
"path": path,
|
|
1452
|
+
"handler": "createContext",
|
|
1453
|
+
"purpose": _humanize_identifier("", "GET", path),
|
|
1454
|
+
"evidence": [f"{rel}:1"],
|
|
1455
|
+
"parameters": [],
|
|
1456
|
+
"request_body": None,
|
|
1457
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1458
|
+
"auth": {"type": "unknown"},
|
|
1459
|
+
"effect": "read",
|
|
1460
|
+
"confirmation_required": False,
|
|
1461
|
+
})
|
|
1462
|
+
|
|
1463
|
+
return routes
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def _scan_dotnet_routes(source: Path, files: list[Path]) -> list[dict]:
|
|
1467
|
+
"""Scan C# / .NET Minimal APIs and Controllers."""
|
|
1468
|
+
routes: list[dict] = []
|
|
1469
|
+
# Minimal API: app.MapGet("/api/products/{sku}", ...)
|
|
1470
|
+
min_api_re = re.compile(r"""\.\s*Map(Get|Post|Put|Delete|Patch)\(\s*["']([^"']+)["']""")
|
|
1471
|
+
# Controller: [HttpGet("...")], [HttpPost("...")]
|
|
1472
|
+
controller_route_re = re.compile(r"""\[Route\(\s*["']([^"']+)["']\s*\)\][\s\S]*?class\s+([A-Za-z0-9_]+)""")
|
|
1473
|
+
controller_method_re = re.compile(r"""\[Http(Get|Post|Put|Delete|Patch)(?:\(\s*["']?([^"'\)]*)["']?\s*\))?\][\s\S]*?(?:public|protected)?\s+[A-Za-z0-9_<>, \t\n\r]+\s+([A-Za-z0-9_]+)\s*\(""")
|
|
1474
|
+
|
|
1475
|
+
for p in files:
|
|
1476
|
+
if p.suffix not in {".cs", ".fs"} or p.stat().st_size > 500_000:
|
|
1477
|
+
continue
|
|
1478
|
+
rel = p.relative_to(source).as_posix()
|
|
1479
|
+
try:
|
|
1480
|
+
content = p.read_text(encoding="utf-8", errors="ignore")
|
|
1481
|
+
except Exception:
|
|
1482
|
+
continue
|
|
1483
|
+
|
|
1484
|
+
for m in min_api_re.finditer(content):
|
|
1485
|
+
mth = m.group(1).upper()
|
|
1486
|
+
path = m.group(2)
|
|
1487
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{path.strip('/') or 'root'}").strip("-")
|
|
1488
|
+
routes.append({
|
|
1489
|
+
"id": api_id,
|
|
1490
|
+
"method": mth,
|
|
1491
|
+
"path": path,
|
|
1492
|
+
"handler": "MinimalApi",
|
|
1493
|
+
"purpose": _humanize_identifier("", mth, path),
|
|
1494
|
+
"evidence": [f"{rel}:1"],
|
|
1495
|
+
"parameters": [],
|
|
1496
|
+
"request_body": None,
|
|
1497
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1498
|
+
"auth": {"type": "unknown"},
|
|
1499
|
+
"effect": "read" if mth == "GET" else "write",
|
|
1500
|
+
"confirmation_required": mth != "GET",
|
|
1501
|
+
})
|
|
1502
|
+
|
|
1503
|
+
ctrl_prefix = ""
|
|
1504
|
+
m_ctrl = controller_route_re.search(content)
|
|
1505
|
+
if m_ctrl:
|
|
1506
|
+
raw_pref = m_ctrl.group(1)
|
|
1507
|
+
cname = m_ctrl.group(2).replace("Controller", "")
|
|
1508
|
+
ctrl_prefix = raw_pref.replace("[controller]", cname).replace("[action]", "")
|
|
1509
|
+
|
|
1510
|
+
for m in controller_method_re.finditer(content):
|
|
1511
|
+
mth, sub, handler = m.group(1).upper(), m.group(2) or "", m.group(3)
|
|
1512
|
+
full_path = _join_route_paths(ctrl_prefix, sub)
|
|
1513
|
+
api_id = re.sub(r"[^a-z0-9]+", "-", f"{mth.lower()}-{full_path.strip('/') or 'root'}").strip("-")
|
|
1514
|
+
routes.append({
|
|
1515
|
+
"id": api_id,
|
|
1516
|
+
"method": mth,
|
|
1517
|
+
"path": full_path,
|
|
1518
|
+
"handler": handler,
|
|
1519
|
+
"purpose": _humanize_identifier(handler, mth, full_path),
|
|
1520
|
+
"evidence": [f"{rel}:1"],
|
|
1521
|
+
"parameters": [],
|
|
1522
|
+
"request_body": None,
|
|
1523
|
+
"responses": {"200": {"type": "object", "required": [], "properties": {"ok": {"type": "boolean"}}}},
|
|
1524
|
+
"auth": {"type": "unknown"},
|
|
1525
|
+
"effect": "read" if mth == "GET" else "write",
|
|
1526
|
+
"confirmation_required": mth != "GET",
|
|
1527
|
+
})
|
|
1528
|
+
|
|
1529
|
+
return routes
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def _scan_frontend_pages(source: Path, files: list[Path]) -> list[dict]:
|
|
1533
|
+
pages: list[dict] = []
|
|
1534
|
+
seen_ids: set[str] = set()
|
|
1535
|
+
|
|
1536
|
+
# 1. Check index.html or root entry page (deduplicate multiple index.html)
|
|
1537
|
+
for p in files:
|
|
1538
|
+
rel = p.relative_to(source).as_posix()
|
|
1539
|
+
if p.name == "index.html" and "index-page" not in seen_ids:
|
|
1540
|
+
title = "应用主界面"
|
|
1541
|
+
try:
|
|
1542
|
+
html = p.read_text(encoding="utf-8", errors="ignore")
|
|
1543
|
+
tm = re.search(r"<title>(.*?)</title>", html, re.I | re.S)
|
|
1544
|
+
if tm and tm.group(1).strip():
|
|
1545
|
+
title = f"{tm.group(1).strip()}(主页面入口)"
|
|
1546
|
+
except Exception:
|
|
1547
|
+
pass
|
|
1548
|
+
pages.append({
|
|
1549
|
+
"id": "index-page",
|
|
1550
|
+
"path": "/",
|
|
1551
|
+
"purpose": title,
|
|
1552
|
+
"actions": ["打开应用主页面并执行业务操作"],
|
|
1553
|
+
"auth": {"type": "unknown"},
|
|
1554
|
+
"evidence": [f"{rel}:1"],
|
|
1555
|
+
})
|
|
1556
|
+
seen_ids.add("index-page")
|
|
1557
|
+
break
|
|
1558
|
+
|
|
1559
|
+
# 2. Scan SPA Nav groups, view metadata, and view section containers
|
|
1560
|
+
# e.g. NAV_GROUPS = [{ items: [{ view: "dashboard", label: "审厂看板" }] }]
|
|
1561
|
+
# VIEW_META = { dashboard: ["年度审厂看板", "..."] }
|
|
1562
|
+
# <section id="dashboard" class="view">
|
|
1563
|
+
for p in files:
|
|
1564
|
+
if p.suffix not in {".js", ".ts", ".mjs", ".vue", ".jsx", ".tsx", ".html"} or p.stat().st_size > 500_000:
|
|
1565
|
+
continue
|
|
1566
|
+
rel = p.relative_to(source).as_posix()
|
|
1567
|
+
try:
|
|
1568
|
+
text = p.read_text(encoding="utf-8", errors="ignore")
|
|
1569
|
+
except Exception:
|
|
1570
|
+
continue
|
|
1571
|
+
|
|
1572
|
+
# Match navigation items: view: "xxx", label: "yyy"
|
|
1573
|
+
for m in re.finditer(r"""view:\s*['"]([a-zA-Z0-9_-]+)['"],\s*label:\s*['"]([^'"]+)['"]""", text):
|
|
1574
|
+
vid, label = m.group(1), m.group(2).strip()
|
|
1575
|
+
if vid not in seen_ids:
|
|
1576
|
+
seen_ids.add(vid)
|
|
1577
|
+
pages.append({
|
|
1578
|
+
"id": vid,
|
|
1579
|
+
"path": f"/#/{vid}",
|
|
1580
|
+
"purpose": f"{label}(业务视图)",
|
|
1581
|
+
"actions": [f"查看或操作 {label}"],
|
|
1582
|
+
"auth": {"type": "unknown"},
|
|
1583
|
+
"evidence": [f"{rel}:1"],
|
|
1584
|
+
})
|
|
1585
|
+
|
|
1586
|
+
# Match VIEW_META = { dashboard: ["title", "desc"] } (ensure it's in a view/meta context)
|
|
1587
|
+
if "view_meta" in text.lower() or "nav_groups" in text.lower() or "viewmeta" in text.lower():
|
|
1588
|
+
for m in re.finditer(r"""([a-zA-Z0-9_-]+)\s*:\s*\[\s*['"]([^'"]+)['"]\s*,\s*['"]([^'"]+)['"]\s*\]""", text):
|
|
1589
|
+
vid, title, desc = m.group(1), m.group(2).strip(), m.group(3).strip()
|
|
1590
|
+
# Must be a Chinese or capitalized title, not code tokens like 'admin' / 'camera'
|
|
1591
|
+
if vid not in seen_ids and any('\u4e00' <= ch <= '\u9fff' for ch in title) and len(title) <= 30:
|
|
1592
|
+
seen_ids.add(vid)
|
|
1593
|
+
pages.append({
|
|
1594
|
+
"id": vid,
|
|
1595
|
+
"path": f"/#/{vid}",
|
|
1596
|
+
"purpose": f"{title}({desc[:35]})" if desc else f"{title}(业务视图)",
|
|
1597
|
+
"actions": [f"查看或操作 {title}"],
|
|
1598
|
+
"auth": {"type": "unknown"},
|
|
1599
|
+
"evidence": [f"{rel}:1"],
|
|
1600
|
+
})
|
|
1601
|
+
|
|
1602
|
+
# Match Vue / React Router configs: path: "/users", name: "...", meta: { title: "..." }
|
|
1603
|
+
for m in re.finditer(r"""path:\s*['"](/[^'"]*)['"]""", text):
|
|
1604
|
+
r_path = m.group(1)
|
|
1605
|
+
if r_path == "/" and "index-page" in seen_ids:
|
|
1606
|
+
continue
|
|
1607
|
+
window = text[m.start():m.start() + 350]
|
|
1608
|
+
m_name = re.search(r"""name:\s*['"]([^'"]+)['"]""", window)
|
|
1609
|
+
m_title = re.search(r"""(?:title|label):\s*['"]([^'"]+)['"]""", window)
|
|
1610
|
+
r_name = m_name.group(1) if m_name else r_path.strip("/").replace("/", "-") or "root"
|
|
1611
|
+
r_title = m_title.group(1) if m_title else r_name
|
|
1612
|
+
pid = re.sub(r"[^a-z0-9]+", "-", f"route-{r_name.lower()}").strip("-")
|
|
1613
|
+
if pid not in seen_ids:
|
|
1614
|
+
seen_ids.add(pid)
|
|
1615
|
+
pages.append({
|
|
1616
|
+
"id": pid,
|
|
1617
|
+
"path": r_path,
|
|
1618
|
+
"purpose": f"{r_title}(前端路由视图)",
|
|
1619
|
+
"actions": [f"访问并操作 {r_title} 页面"],
|
|
1620
|
+
"auth": {"type": "unknown"},
|
|
1621
|
+
"evidence": [f"{rel}:1"],
|
|
1622
|
+
})
|
|
1623
|
+
|
|
1624
|
+
# Match HTML <section id="xxx" class="view">
|
|
1625
|
+
for m in re.finditer(r"""<section\s+id=['"]([a-zA-Z0-9_-]+)['"]\s+class=['"][^'"]*\bview\b[^'"]*['"]""", text):
|
|
1626
|
+
vid = m.group(1)
|
|
1627
|
+
if vid not in seen_ids:
|
|
1628
|
+
seen_ids.add(vid)
|
|
1629
|
+
pages.append({
|
|
1630
|
+
"id": vid,
|
|
1631
|
+
"path": f"/#/{vid}",
|
|
1632
|
+
"purpose": f"{vid} 业务视图",
|
|
1633
|
+
"actions": [f"查看或操作 {vid}"],
|
|
1634
|
+
"auth": {"type": "unknown"},
|
|
1635
|
+
"evidence": [f"{rel}:1"],
|
|
1636
|
+
})
|
|
1637
|
+
|
|
1638
|
+
# 3. Scan pages/, views/, and *-page.* / *-panel.* components
|
|
1639
|
+
for p in files:
|
|
1640
|
+
if p.suffix not in {".js", ".ts", ".vue", ".jsx", ".tsx", ".html"} or p.stat().st_size > 500_000:
|
|
1641
|
+
continue
|
|
1642
|
+
rel = p.relative_to(source).as_posix()
|
|
1643
|
+
stem = p.stem.lower()
|
|
1644
|
+
is_page_dir = any(part in {"pages", "views", "screens", "panels"} for part in p.relative_to(source).parts)
|
|
1645
|
+
is_panel_or_page = stem.endswith("-page") or stem.endswith("-panel") or stem.endswith("page") or stem.endswith("view")
|
|
1646
|
+
if not (is_page_dir or is_panel_or_page):
|
|
1647
|
+
continue
|
|
1648
|
+
if p.name == "index.html":
|
|
1649
|
+
continue
|
|
1650
|
+
|
|
1651
|
+
try:
|
|
1652
|
+
text = p.read_text(encoding="utf-8", errors="ignore")
|
|
1653
|
+
except Exception:
|
|
1654
|
+
continue
|
|
1655
|
+
|
|
1656
|
+
page_id = re.sub(r"[^a-z0-9]+", "-", stem).strip("-")
|
|
1657
|
+
if page_id in seen_ids:
|
|
1658
|
+
continue
|
|
1659
|
+
seen_ids.add(page_id)
|
|
1660
|
+
|
|
1661
|
+
# Extract human-readable heading or docstring/comment from component
|
|
1662
|
+
headings = re.findall(r"<h[1-4][^>]*>\s*([^<{]+?)\s*</h[1-4]>", text)
|
|
1663
|
+
clean_headings = [h.strip() for h in headings if h.strip() and len(h.strip()) <= 30]
|
|
1664
|
+
comment_match = re.search(r"(?m)^\s*(?://|/\*\*?|\*)\s*([^\n*]{4,60})", text)
|
|
1665
|
+
|
|
1666
|
+
if "admin" in stem:
|
|
1667
|
+
path = "/#/admin"
|
|
1668
|
+
elif "list" in stem:
|
|
1669
|
+
path = "/#/list"
|
|
1670
|
+
elif "create" in stem:
|
|
1671
|
+
path = "/#/create"
|
|
1672
|
+
elif is_page_dir:
|
|
1673
|
+
path = f"/#/{stem.replace('-page', '')}"
|
|
1674
|
+
else:
|
|
1675
|
+
path = f"/#/{stem}"
|
|
1676
|
+
|
|
1677
|
+
if clean_headings:
|
|
1678
|
+
purpose = f"{clean_headings[0]}({p.name} 视图/面板:" + "、".join(clean_headings[:3]) + ")"
|
|
1679
|
+
elif comment_match:
|
|
1680
|
+
purpose = f"{comment_match.group(1).strip()} ({p.name})"
|
|
1681
|
+
else:
|
|
1682
|
+
purpose = f"{p.stem} 页面/业务视图 ({rel})"
|
|
1683
|
+
|
|
1684
|
+
actions = clean_headings[:4] if clean_headings else [f"查看或操作 {p.stem} 视图"]
|
|
1685
|
+
auth_obj = {"type": "unknown"}
|
|
1686
|
+
pages.append({
|
|
1687
|
+
"id": page_id,
|
|
1688
|
+
"path": path,
|
|
1689
|
+
"purpose": purpose,
|
|
1690
|
+
"actions": actions,
|
|
1691
|
+
"auth": auth_obj,
|
|
1692
|
+
"evidence": [f"{rel}:1"],
|
|
1693
|
+
})
|
|
1694
|
+
|
|
1695
|
+
return pages
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def _normalize_resource_type_group(raw_type: str) -> str:
|
|
1699
|
+
t = str(raw_type or "").strip().lower()
|
|
1700
|
+
if t == "postgresql":
|
|
1701
|
+
return "postgres"
|
|
1702
|
+
if t == "mssql":
|
|
1703
|
+
return "sqlserver"
|
|
1704
|
+
return t
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def _is_preferred_default_resource_id(res_id: str) -> bool:
|
|
1708
|
+
rid = str(res_id or "").strip().lower()
|
|
1709
|
+
return rid.endswith("-main") or rid.endswith("-default") or rid.endswith("-server")
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
def select_and_filter_platform_resources(resources: list[dict] | None, slug: str = "", *, default_only: bool = False) -> list[dict]:
|
|
1713
|
+
"""Filter out other apps' db-<slug> resources, ensure each resource type has an `is_default=True` resource,
|
|
1714
|
+
and optionally return only the single default resource per type (plus db-<slug> if present) so the AI
|
|
1715
|
+
never gets confused by multiple resources of the same type.
|
|
1716
|
+
"""
|
|
1717
|
+
if not isinstance(resources, list):
|
|
1718
|
+
return []
|
|
1719
|
+
slug_lower = str(slug or "").strip().lower()
|
|
1720
|
+
own_db_id = f"db-{slug_lower}" if slug_lower else ""
|
|
1721
|
+
|
|
1722
|
+
filtered: list[dict] = []
|
|
1723
|
+
for item in resources:
|
|
1724
|
+
if not isinstance(item, dict):
|
|
1725
|
+
continue
|
|
1726
|
+
rid = str(item.get("id", "")).strip()
|
|
1727
|
+
rid_lower = rid.lower()
|
|
1728
|
+
if not rid:
|
|
1729
|
+
continue
|
|
1730
|
+
if rid_lower.startswith("db-") and rid_lower != own_db_id:
|
|
1731
|
+
continue
|
|
1732
|
+
copy_item = dict(item)
|
|
1733
|
+
if rid_lower.startswith("db-"):
|
|
1734
|
+
copy_item["is_default"] = False
|
|
1735
|
+
filtered.append(copy_item)
|
|
1736
|
+
|
|
1737
|
+
by_type: dict[str, list[dict]] = {}
|
|
1738
|
+
for item in filtered:
|
|
1739
|
+
rid_lower = str(item.get("id", "")).strip().lower()
|
|
1740
|
+
if rid_lower.startswith("db-"):
|
|
1741
|
+
continue
|
|
1742
|
+
t_group = _normalize_resource_type_group(item.get("type", ""))
|
|
1743
|
+
if not t_group:
|
|
1744
|
+
continue
|
|
1745
|
+
by_type.setdefault(t_group, []).append(item)
|
|
1746
|
+
|
|
1747
|
+
for group in by_type.values():
|
|
1748
|
+
defaults = [r for r in group if r.get("is_default") is True]
|
|
1749
|
+
if not defaults:
|
|
1750
|
+
chosen = next((r for r in group if _is_preferred_default_resource_id(r.get("id", ""))), group[0])
|
|
1751
|
+
chosen["is_default"] = True
|
|
1752
|
+
elif len(defaults) > 1:
|
|
1753
|
+
keep = next((r for r in defaults if _is_preferred_default_resource_id(r.get("id", ""))), defaults[0])
|
|
1754
|
+
for r in defaults:
|
|
1755
|
+
r["is_default"] = (r is keep)
|
|
1756
|
+
|
|
1757
|
+
filtered.sort(key=lambda r: (0 if r.get("is_default") else 1, str(r.get("type", "")), str(r.get("id", ""))))
|
|
1758
|
+
|
|
1759
|
+
if not default_only:
|
|
1760
|
+
return filtered
|
|
1761
|
+
|
|
1762
|
+
result: list[dict] = []
|
|
1763
|
+
seen_types: set[str] = set()
|
|
1764
|
+
for item in filtered:
|
|
1765
|
+
rid_lower = str(item.get("id", "")).strip().lower()
|
|
1766
|
+
if rid_lower == own_db_id and own_db_id:
|
|
1767
|
+
result.append(item)
|
|
1768
|
+
continue
|
|
1769
|
+
t_group = _normalize_resource_type_group(item.get("type", ""))
|
|
1770
|
+
if item.get("is_default") and t_group not in seen_types:
|
|
1771
|
+
seen_types.add(t_group)
|
|
1772
|
+
result.append(item)
|
|
1773
|
+
return result
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
_PG_SQLITE3_COMPAT_SHIM = '''# --- OPC Platform PostgreSQL (DATABASE_URL) Compatibility Shim ---
|
|
1777
|
+
import os as _opc_os, re as _opc_re
|
|
1778
|
+
if _opc_os.getenv("DATABASE_URL", "").startswith(("postgres://", "postgresql://")):
|
|
1779
|
+
import psycopg2 as _opc_pg
|
|
1780
|
+
class _OpcPgRow(dict):
|
|
1781
|
+
def __init__(self, cols, vals):
|
|
1782
|
+
super().__init__(zip(cols, vals))
|
|
1783
|
+
self._vals = tuple(vals)
|
|
1784
|
+
def __getitem__(self, key):
|
|
1785
|
+
return self._vals[key] if isinstance(key, (int, slice)) else super().__getitem__(key)
|
|
1786
|
+
def _opc_translate_sql(sql: str) -> str:
|
|
1787
|
+
s = sql.strip()
|
|
1788
|
+
u = s.upper()
|
|
1789
|
+
if u.startswith("BEGIN IMMEDIATE") or u.startswith("BEGIN EXCLUSIVE") or u == "BEGIN":
|
|
1790
|
+
return "SELECT 1"
|
|
1791
|
+
if _opc_re.match(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", s):
|
|
1792
|
+
s = _opc_re.sub(r"(?is)^INSERT\\s+OR\\s+IGNORE\\s+INTO\\b", "INSERT INTO", s, count=1).rstrip().rstrip(";") + " ON CONFLICT DO NOTHING"
|
|
1793
|
+
out, in_sq, in_dq, i = [], False, False, 0
|
|
1794
|
+
while i < len(s):
|
|
1795
|
+
ch = s[i]
|
|
1796
|
+
if ch == "'" and not in_dq:
|
|
1797
|
+
in_sq = not in_sq
|
|
1798
|
+
out.append(ch)
|
|
1799
|
+
elif ch == '"' and not in_sq:
|
|
1800
|
+
in_dq = not in_dq
|
|
1801
|
+
out.append(ch)
|
|
1802
|
+
elif ch == "?" and not in_sq and not in_dq:
|
|
1803
|
+
out.append("%s")
|
|
1804
|
+
elif ch == "%" and not in_sq and not in_dq and (i + 1 >= len(s) or s[i + 1] != "s"):
|
|
1805
|
+
out.append("%%")
|
|
1806
|
+
else:
|
|
1807
|
+
out.append(ch)
|
|
1808
|
+
i += 1
|
|
1809
|
+
return "".join(out)
|
|
1810
|
+
class _OpcPgCursor:
|
|
1811
|
+
def __init__(self, rows=None, rowcount=0, lastrowid=None):
|
|
1812
|
+
self._rows = list(rows or [])
|
|
1813
|
+
self._idx = 0
|
|
1814
|
+
self.rowcount = rowcount
|
|
1815
|
+
self.lastrowid = lastrowid
|
|
1816
|
+
def fetchone(self):
|
|
1817
|
+
if self._idx >= len(self._rows):
|
|
1818
|
+
return None
|
|
1819
|
+
r = self._rows[self._idx]
|
|
1820
|
+
self._idx += 1
|
|
1821
|
+
return r
|
|
1822
|
+
def fetchall(self):
|
|
1823
|
+
r = self._rows[self._idx:]
|
|
1824
|
+
self._idx = len(self._rows)
|
|
1825
|
+
return r
|
|
1826
|
+
def __iter__(self):
|
|
1827
|
+
return iter(self._rows)
|
|
1828
|
+
class _OpcPgConn:
|
|
1829
|
+
def __init__(self, dsn: str):
|
|
1830
|
+
self._conn = _opc_pg.connect(dsn)
|
|
1831
|
+
self.row_factory = None
|
|
1832
|
+
def execute(self, sql: str, params=()):
|
|
1833
|
+
s = sql.strip()
|
|
1834
|
+
u = s.upper()
|
|
1835
|
+
if u.startswith("PRAGMA"):
|
|
1836
|
+
m = _opc_re.match(r'(?is)^PRAGMA\\s+table_info\\(\\s*["\\\']?([A-Za-z0-9_]+)["\\\']?\\s*\\)', s)
|
|
1837
|
+
if m and m.group(1).lower() != "invoice_registry":
|
|
1838
|
+
tbl = m.group(1).lower()
|
|
1839
|
+
with self._conn.cursor() as cur:
|
|
1840
|
+
cur.execute("SELECT ordinal_position - 1 AS cid, column_name AS name, data_type AS type, CASE WHEN is_nullable = 'NO' THEN 1 ELSE 0 END AS notnull, column_default AS dflt_value, 0 AS pk FROM information_schema.columns WHERE table_schema = 'public' AND table_name = %s ORDER BY ordinal_position", (tbl,))
|
|
1841
|
+
cols = [d[0] for d in cur.description]
|
|
1842
|
+
return _OpcPgCursor([_OpcPgRow(cols, r) for r in cur.fetchall()])
|
|
1843
|
+
return _OpcPgCursor([])
|
|
1844
|
+
if "SQLITE_MASTER" in u:
|
|
1845
|
+
return _OpcPgCursor([])
|
|
1846
|
+
pg_sql = _opc_translate_sql(s)
|
|
1847
|
+
with self._conn.cursor() as cur:
|
|
1848
|
+
cur.execute(pg_sql, tuple(params) if params else None)
|
|
1849
|
+
if cur.description:
|
|
1850
|
+
cols = [d[0] for d in cur.description]
|
|
1851
|
+
rows = [_OpcPgRow(cols, r) for r in cur.fetchall()]
|
|
1852
|
+
return _OpcPgCursor(rows, cur.rowcount)
|
|
1853
|
+
last_id = None
|
|
1854
|
+
if u.startswith("INSERT"):
|
|
1855
|
+
try:
|
|
1856
|
+
cur.execute("SAVEPOINT _opc_lv; SELECT LASTVAL()")
|
|
1857
|
+
r = cur.fetchone()
|
|
1858
|
+
last_id = r[0] if r else None
|
|
1859
|
+
cur.execute("RELEASE SAVEPOINT _opc_lv")
|
|
1860
|
+
except Exception:
|
|
1861
|
+
try:
|
|
1862
|
+
cur.execute("ROLLBACK TO SAVEPOINT _opc_lv")
|
|
1863
|
+
except Exception:
|
|
1864
|
+
pass
|
|
1865
|
+
return _OpcPgCursor([], cur.rowcount, last_id)
|
|
1866
|
+
def executemany(self, sql: str, seq_of_params):
|
|
1867
|
+
pg_sql = _opc_translate_sql(sql)
|
|
1868
|
+
with self._conn.cursor() as cur:
|
|
1869
|
+
for p in seq_of_params:
|
|
1870
|
+
cur.execute(pg_sql, tuple(p))
|
|
1871
|
+
return _OpcPgCursor([], cur.rowcount)
|
|
1872
|
+
def executescript(self, script: str):
|
|
1873
|
+
return _OpcPgCursor([])
|
|
1874
|
+
def cursor(self):
|
|
1875
|
+
return self
|
|
1876
|
+
def commit(self):
|
|
1877
|
+
self._conn.commit()
|
|
1878
|
+
def rollback(self):
|
|
1879
|
+
self._conn.rollback()
|
|
1880
|
+
def close(self):
|
|
1881
|
+
self._conn.close()
|
|
1882
|
+
def __enter__(self):
|
|
1883
|
+
return self
|
|
1884
|
+
def __exit__(self, exc_type, exc, tb):
|
|
1885
|
+
if exc_type is None:
|
|
1886
|
+
self.commit()
|
|
1887
|
+
else:
|
|
1888
|
+
self.rollback()
|
|
1889
|
+
sqlite3.connect = lambda *a, **kw: _OpcPgConn(_opc_os.environ["DATABASE_URL"])
|
|
1890
|
+
# --- End OPC Platform PostgreSQL Compatibility Shim ---
|
|
1891
|
+
'''
|
|
1892
|
+
|
|
1893
|
+
|
|
1894
|
+
def _auto_adapt_sqlite_to_postgres_changes(source: Path, slug: str, data: dict, proposal: dict) -> None:
|
|
1895
|
+
"""When the user has migrated local SQLite to platform PostgreSQL (`data['migrations']`),
|
|
1896
|
+
guarantee that:
|
|
1897
|
+
1. All SQLite database connection modules in Python (e.g. `db.py`) are adapted to connect via `DATABASE_URL` (`psycopg2`).
|
|
1898
|
+
2. `requirements.txt` includes `psycopg2-binary`.
|
|
1899
|
+
3. `contract.configs` declares `DATABASE_URL` and `contract.resources` includes the migrated database resource.
|
|
1900
|
+
"""
|
|
1901
|
+
migrations = [m for m in (data.get("migrations") or []) if isinstance(m, dict)]
|
|
1902
|
+
pg_migs = [m for m in migrations if str(m.get("target_type", "postgres")).lower() in {"postgres", "postgresql"}]
|
|
1903
|
+
if not pg_migs:
|
|
1904
|
+
return
|
|
1905
|
+
|
|
1906
|
+
latest_mig = pg_migs[-1]
|
|
1907
|
+
res_id = str(latest_mig.get("resource_id") or f"db-{slug}")
|
|
1908
|
+
db_name = str(latest_mig.get("database_name") or f"app_{slug.replace('-', '_')}")
|
|
1909
|
+
|
|
1910
|
+
contract = proposal.setdefault("contract", {})
|
|
1911
|
+
changes: list[dict] = proposal.setdefault("changes", [])
|
|
1912
|
+
changes_by_path: dict[str, dict] = {c.get("path", ""): c for c in changes if isinstance(c, dict) and c.get("path")}
|
|
1913
|
+
used_ids: set[str] = {str(c.get("id", "")) for c in changes if isinstance(c, dict)}
|
|
1914
|
+
|
|
1915
|
+
def _unique_change_id(base_id: str) -> str:
|
|
1916
|
+
cid = base_id
|
|
1917
|
+
idx = 2
|
|
1918
|
+
while cid in used_ids:
|
|
1919
|
+
cid = f"{base_id}-{idx}"
|
|
1920
|
+
idx += 1
|
|
1921
|
+
used_ids.add(cid)
|
|
1922
|
+
return cid
|
|
1923
|
+
|
|
1924
|
+
# 1. Find Python files that call sqlite3.connect(...)
|
|
1925
|
+
sqlite_py_files: list[Path] = []
|
|
1926
|
+
for p in sorted(source.rglob("*.py")):
|
|
1927
|
+
if not p.is_file() or p.is_symlink():
|
|
1928
|
+
continue
|
|
1929
|
+
rel_parts = p.relative_to(source).parts
|
|
1930
|
+
if any(part in delivery.EXCLUDED or part in {"tests", "test"} for part in rel_parts):
|
|
1931
|
+
continue
|
|
1932
|
+
try:
|
|
1933
|
+
txt = p.read_text(encoding="utf-8")
|
|
1934
|
+
except Exception:
|
|
1935
|
+
continue
|
|
1936
|
+
if "import sqlite3" in txt and "sqlite3.connect" in txt:
|
|
1937
|
+
sqlite_py_files.append(p)
|
|
1938
|
+
|
|
1939
|
+
# Prioritize files named db.py / database.py / storage.py
|
|
1940
|
+
sqlite_py_files.sort(key=lambda p: (0 if p.stem.lower() in {"db", "database", "storage", "sqlite"} else 1, len(p.parts)))
|
|
1941
|
+
|
|
1942
|
+
for py_file in sqlite_py_files:
|
|
1943
|
+
rel_path = py_file.relative_to(source).as_posix()
|
|
1944
|
+
existing_change = changes_by_path.get(rel_path)
|
|
1945
|
+
current_content = existing_change["content"] if (existing_change and isinstance(existing_change.get("content"), str)) else py_file.read_text(encoding="utf-8")
|
|
1946
|
+
if "DATABASE_URL" in current_content or "psycopg" in current_content:
|
|
1947
|
+
continue
|
|
1948
|
+
# Inject _PG_SQLITE3_COMPAT_SHIM right after `import sqlite3`
|
|
1949
|
+
new_content, n_subs = re.subn(
|
|
1950
|
+
r"(?m)^(\s*import\s+sqlite3\b[^\n]*\n)",
|
|
1951
|
+
lambda m: m.group(1) + _PG_SQLITE3_COMPAT_SHIM,
|
|
1952
|
+
current_content,
|
|
1953
|
+
count=1,
|
|
1954
|
+
)
|
|
1955
|
+
if n_subs == 0:
|
|
1956
|
+
continue
|
|
1957
|
+
if existing_change:
|
|
1958
|
+
existing_change["content"] = new_content
|
|
1959
|
+
if "PostgreSQL" not in existing_change.get("reason", ""):
|
|
1960
|
+
existing_change["reason"] = existing_change.get("reason", "").rstrip("。") + ";并接入平台 PostgreSQL 数据库 (DATABASE_URL)"
|
|
1961
|
+
else:
|
|
1962
|
+
change_obj = {
|
|
1963
|
+
"id": _unique_change_id("adapt-postgres-db"),
|
|
1964
|
+
"path": rel_path,
|
|
1965
|
+
"group": "迁移并接入平台 PostgreSQL 数据库",
|
|
1966
|
+
"reason": f"将本地 SQLite 数据库连接切换至平台默认 PostgreSQL(专属库 {db_name},读取 DATABASE_URL 环境变量,保留原业务查询逻辑不变)",
|
|
1967
|
+
"content": new_content,
|
|
1968
|
+
"executable": False,
|
|
1969
|
+
}
|
|
1970
|
+
changes.append(change_obj)
|
|
1971
|
+
changes_by_path[rel_path] = change_obj
|
|
1972
|
+
|
|
1973
|
+
# 2. Ensure requirements.txt includes psycopg2-binary
|
|
1974
|
+
req_files: list[Path] = []
|
|
1975
|
+
for p in sorted(source.rglob("requirements*.txt")):
|
|
1976
|
+
if p.is_file() and not p.is_symlink() and not any(part in delivery.EXCLUDED for part in p.relative_to(source).parts):
|
|
1977
|
+
req_files.append(p)
|
|
1978
|
+
req_files.sort(key=lambda p: (0 if p.name == "requirements.txt" else 1, len(p.parts)))
|
|
1979
|
+
if req_files:
|
|
1980
|
+
req_file = req_files[0]
|
|
1981
|
+
req_rel = req_file.relative_to(source).as_posix()
|
|
1982
|
+
existing_req = changes_by_path.get(req_rel)
|
|
1983
|
+
req_content = existing_req["content"] if (existing_req and isinstance(existing_req.get("content"), str)) else req_file.read_text(encoding="utf-8")
|
|
1984
|
+
if not re.search(r"(?i)\bpsycopg2(?:-binary)?\b", req_content):
|
|
1985
|
+
updated_req = req_content.rstrip() + "\npsycopg2-binary>=2.9\n"
|
|
1986
|
+
if existing_req:
|
|
1987
|
+
existing_req["content"] = updated_req
|
|
1988
|
+
else:
|
|
1989
|
+
req_change = {
|
|
1990
|
+
"id": _unique_change_id("add-postgres-driver"),
|
|
1991
|
+
"path": req_rel,
|
|
1992
|
+
"group": "迁移并接入平台 PostgreSQL 数据库",
|
|
1993
|
+
"reason": "添加 PostgreSQL 数据库驱动依赖 psycopg2-binary,使应用在容器内可直连平台 PostgreSQL",
|
|
1994
|
+
"content": updated_req,
|
|
1995
|
+
"executable": False,
|
|
1996
|
+
}
|
|
1997
|
+
changes.append(req_change)
|
|
1998
|
+
changes_by_path[req_rel] = req_change
|
|
1999
|
+
|
|
2000
|
+
# 3. Ensure DATABASE_URL is declared in contract.configs
|
|
2001
|
+
configs: list[dict] = contract.setdefault("configs", [])
|
|
2002
|
+
if not any(isinstance(c, dict) and c.get("key") == "DATABASE_URL" for c in configs):
|
|
2003
|
+
configs.append({
|
|
2004
|
+
"key": "DATABASE_URL",
|
|
2005
|
+
"default": "",
|
|
2006
|
+
"secret": True,
|
|
2007
|
+
"required": True,
|
|
2008
|
+
"description": f"平台 PostgreSQL 数据库连接地址(自动绑定专属迁移库 {db_name})",
|
|
2009
|
+
})
|
|
2010
|
+
|
|
2011
|
+
# 4. Ensure migrated resource is listed in contract.resources
|
|
2012
|
+
c_resources: list[dict] = contract.setdefault("resources", [])
|
|
2013
|
+
if not any(isinstance(r, dict) and r.get("id") == res_id for r in c_resources):
|
|
2014
|
+
c_resources.append({
|
|
2015
|
+
"id": res_id,
|
|
2016
|
+
"type": "postgres",
|
|
2017
|
+
"purpose": f"平台自动迁移创建的专属 PostgreSQL 数据库 ({db_name})",
|
|
2018
|
+
})
|
|
2019
|
+
|
|
2020
|
+
# 5. Ensure build_context covers any newly added/modified files
|
|
2021
|
+
build_ctx: list[str] = contract.setdefault("build_context", [])
|
|
2022
|
+
if isinstance(build_ctx, list) and build_ctx and "." not in build_ctx:
|
|
2023
|
+
for ch_path in changes_by_path:
|
|
2024
|
+
covered = any(ch_path == entry or ch_path.startswith(entry.rstrip("/") + "/") for entry in build_ctx if isinstance(entry, str))
|
|
2025
|
+
if not covered:
|
|
2026
|
+
top_dir = ch_path.split("/")[0]
|
|
2027
|
+
build_ctx.append(top_dir if "/" in ch_path else ch_path)
|
|
2028
|
+
|
|
2029
|
+
|
|
2030
|
+
def _auto_adapt_persistent_file_storage(source: Path, slug: str, data: dict, proposal: dict) -> list[dict]:
|
|
2031
|
+
"""Automatically detect local JSON/file data directories and upload/report/backup folders,
|
|
2032
|
+
populate contract['storage_mounts'], include existing seed folders in contract['build_context'],
|
|
2033
|
+
and inject non-destructive .opc-seed backup & startup seed hydration into Dockerfile.
|
|
2034
|
+
"""
|
|
2035
|
+
contract = proposal.setdefault("contract", {})
|
|
2036
|
+
changes: list[dict] = proposal.setdefault("changes", [])
|
|
2037
|
+
changes_by_path: dict[str, dict] = {
|
|
2038
|
+
str(c.get("path", "")): c for c in changes if isinstance(c, dict) and c.get("path")
|
|
2039
|
+
}
|
|
2040
|
+
|
|
2041
|
+
# 1. Extract WORKDIR from proposed or existing Dockerfile
|
|
2042
|
+
df_text = ""
|
|
2043
|
+
if "Dockerfile" in changes_by_path and isinstance(changes_by_path["Dockerfile"].get("content"), str):
|
|
2044
|
+
df_text = changes_by_path["Dockerfile"]["content"]
|
|
2045
|
+
elif (source / "Dockerfile").is_file():
|
|
2046
|
+
try:
|
|
2047
|
+
df_text = (source / "Dockerfile").read_text(encoding="utf-8", errors="ignore")
|
|
2048
|
+
except Exception:
|
|
2049
|
+
pass
|
|
2050
|
+
workdir = "/app"
|
|
2051
|
+
m_wd = re.search(r"(?im)^\s*WORKDIR\s+([^\s#]+)", df_text)
|
|
2052
|
+
if m_wd and m_wd.group(1).startswith("/"):
|
|
2053
|
+
workdir = m_wd.group(1).rstrip("/") or "/app"
|
|
2054
|
+
|
|
2055
|
+
# 2. Deterministically inspect local file storage & merge with any AI-declared mounts
|
|
2056
|
+
scanned_storage = inspect_local_file_storage(source, slug=slug, workdir=workdir, has_migrations=False)
|
|
2057
|
+
mounts_by_cpath: dict[str, dict] = {}
|
|
2058
|
+
for item in scanned_storage.get("storage_mounts") or []:
|
|
2059
|
+
if isinstance(item, dict) and item.get("container_path"):
|
|
2060
|
+
mounts_by_cpath[item["container_path"]] = dict(item)
|
|
2061
|
+
|
|
2062
|
+
for raw in contract.get("storage_mounts") or []:
|
|
2063
|
+
if not isinstance(raw, dict):
|
|
2064
|
+
continue
|
|
2065
|
+
rel = str(raw.get("relative_path") or raw.get("name") or "").strip().strip("/")
|
|
2066
|
+
cpath = str(raw.get("container_path") or (f"{workdir}/{rel}" if rel else "")).strip()
|
|
2067
|
+
if not cpath.startswith("/") or cpath in {"/", "/app", workdir}:
|
|
2068
|
+
continue
|
|
2069
|
+
if cpath not in mounts_by_cpath:
|
|
2070
|
+
base_name = cpath.rstrip("/").split("/")[-1].lower()
|
|
2071
|
+
clean_slug = re.sub(r"[^a-z0-9-]+", "-", (slug or "app").lower()).strip("-") or "app"
|
|
2072
|
+
mounts_by_cpath[cpath] = {
|
|
2073
|
+
"name": str(raw.get("name") or base_name),
|
|
2074
|
+
"relative_path": rel or base_name,
|
|
2075
|
+
"container_path": cpath,
|
|
2076
|
+
"purpose": str(raw.get("purpose") or "应用持久化数据与文件目录"),
|
|
2077
|
+
"kind": str(raw.get("kind") or "data_dir"),
|
|
2078
|
+
"volume_name": str(raw.get("volume_name") or f"opc-vol-{clean_slug}-{base_name}"),
|
|
2079
|
+
"storage_backend": str(raw.get("storage_backend") or "docker_volume"),
|
|
2080
|
+
"seed_files_count": int(raw.get("seed_files_count") or 0),
|
|
2081
|
+
"seed_files": list(raw.get("seed_files") or []),
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
mounts = sorted(mounts_by_cpath.values(), key=lambda m: m["container_path"])
|
|
2085
|
+
if not mounts:
|
|
2086
|
+
return []
|
|
2087
|
+
|
|
2088
|
+
contract["storage_mounts"] = mounts
|
|
2089
|
+
|
|
2090
|
+
# 3. Ensure build_context includes existing seed directories so initial JSON/files enter the image
|
|
2091
|
+
build_ctx: list[str] = contract.setdefault("build_context", [])
|
|
2092
|
+
if isinstance(build_ctx, list) and build_ctx and "." not in build_ctx:
|
|
2093
|
+
for m in mounts:
|
|
2094
|
+
rel_dir = str(m.get("relative_path") or "").strip().strip("/")
|
|
2095
|
+
if not rel_dir or not (source / rel_dir).exists():
|
|
2096
|
+
continue
|
|
2097
|
+
covered = any(rel_dir == entry.rstrip("/") or rel_dir.startswith(entry.rstrip("/") + "/") for entry in build_ctx if isinstance(entry, str))
|
|
2098
|
+
if not covered:
|
|
2099
|
+
top_dir = rel_dir.split("/")[0]
|
|
2100
|
+
if (source / top_dir).exists() and top_dir not in build_ctx:
|
|
2101
|
+
build_ctx.append(top_dir)
|
|
2102
|
+
|
|
2103
|
+
# 4. Inject .opc-seed build-time snapshot & /opc-storage-init.sh non-destructive startup seed hydration into Dockerfile
|
|
2104
|
+
if df_text and ".opc-seed" not in df_text and "opc-storage-init.sh" not in df_text:
|
|
2105
|
+
rel_dirs_sh = " ".join(
|
|
2106
|
+
f'"{m["relative_path"]}"' for m in mounts if re.fullmatch(r"[A-Za-z0-9_./-]+", str(m.get("relative_path", "")))
|
|
2107
|
+
)
|
|
2108
|
+
cpaths_sh = " ".join(
|
|
2109
|
+
f'"{m["container_path"]}"' for m in mounts if re.fullmatch(r"[A-Za-z0-9_./-]+", str(m.get("container_path", "")))
|
|
2110
|
+
)
|
|
2111
|
+
if rel_dirs_sh and cpaths_sh:
|
|
2112
|
+
has_entrypoint = bool(re.search(r"(?im)^\s*ENTRYPOINT\s+", df_text))
|
|
2113
|
+
seed_snippet_lines = [
|
|
2114
|
+
"# [OPC Persistent Storage] Backup initial seed data & hydrate external volume on startup",
|
|
2115
|
+
f"RUN mkdir -p {workdir}/.opc-seed {cpaths_sh} \\",
|
|
2116
|
+
f" && for d in {rel_dirs_sh}; do \\",
|
|
2117
|
+
f' if [ -d "{workdir}/$d" ] && [ -n "$(ls -A "{workdir}/$d" 2>/dev/null)" ]; then \\',
|
|
2118
|
+
f' mkdir -p "{workdir}/.opc-seed/$d" && cp -a "{workdir}/$d/." "{workdir}/.opc-seed/$d/"; \\',
|
|
2119
|
+
" fi; \\",
|
|
2120
|
+
" done \\",
|
|
2121
|
+
f" && printf '#!/bin/sh\\nset -e\\nfor d in {rel_dirs_sh}; do\\n mkdir -p \"{workdir}/$d\"\\n if [ -d \"{workdir}/.opc-seed/$d\" ]; then\\n for f in \"{workdir}/.opc-seed/$d\"/* \"{workdir}/.opc-seed/$d\"/.[!.]*; do\\n [ -e \"$f\" ] || continue\\n bn=\"$(basename \"$f\")\"\\n [ -e \"{workdir}/$d/$bn\" ] || cp -a \"$f\" \"{workdir}/$d/$bn\"\\n done\\n fi\\ndone\\nexec \"$@\"\\n' > /opc-storage-init.sh \\",
|
|
2122
|
+
" && chmod +x /opc-storage-init.sh",
|
|
2123
|
+
]
|
|
2124
|
+
if not has_entrypoint:
|
|
2125
|
+
seed_snippet_lines.append('ENTRYPOINT ["/opc-storage-init.sh"]')
|
|
2126
|
+
seed_block = "\n".join(seed_snippet_lines)
|
|
2127
|
+
|
|
2128
|
+
lines = df_text.splitlines()
|
|
2129
|
+
insert_idx = len(lines)
|
|
2130
|
+
for idx in range(len(lines) - 1, -1, -1):
|
|
2131
|
+
stripped = lines[idx].strip().upper()
|
|
2132
|
+
if stripped.startswith("CMD ") or stripped.startswith("CMD[") or stripped.startswith("EXPOSE "):
|
|
2133
|
+
insert_idx = idx
|
|
2134
|
+
updated_df = "\n".join(lines[:insert_idx] + [seed_block] + lines[insert_idx:]) + "\n"
|
|
2135
|
+
mount_summary = "、".join(m["container_path"] for m in mounts)
|
|
2136
|
+
if "Dockerfile" in changes_by_path:
|
|
2137
|
+
changes_by_path["Dockerfile"]["content"] = updated_df
|
|
2138
|
+
old_reason = str(changes_by_path["Dockerfile"].get("reason") or "")
|
|
2139
|
+
if "持久化" not in old_reason and "外部卷" not in old_reason:
|
|
2140
|
+
changes_by_path["Dockerfile"]["reason"] = (
|
|
2141
|
+
f"{old_reason.rstrip('。')};自动配置 {mount_summary} 外部持久化卷种子保护,应用更新或重启不丢失本地数据。"
|
|
2142
|
+
)
|
|
2143
|
+
elif (source / "Dockerfile").is_file():
|
|
2144
|
+
df_change = {
|
|
2145
|
+
"id": "dockerfile-persistent-storage",
|
|
2146
|
+
"path": "Dockerfile",
|
|
2147
|
+
"group": "持久化数据与文件保护",
|
|
2148
|
+
"reason": f"配置 {mount_summary} 外部持久化存储卷与初始种子数据保护,确保版本升级不丢失本地文件",
|
|
2149
|
+
"content": updated_df,
|
|
2150
|
+
"executable": False,
|
|
2151
|
+
}
|
|
2152
|
+
changes.append(df_change)
|
|
2153
|
+
changes_by_path["Dockerfile"] = df_change
|
|
2154
|
+
|
|
2155
|
+
return mounts
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
def enrich_contract_and_discovery_report(source: Path, data: dict, proposal: dict) -> dict:
|
|
2159
|
+
"""Ensure 100% of statically scanned APIs & pages are present in proposal['contract'],
|
|
2160
|
+
and build a fixed-format Step 1 Discovery Report (proposal['discovery_report']) and
|
|
2161
|
+
Step 2 Transformation Report (proposal['transformation_report']).
|
|
2162
|
+
"""
|
|
2163
|
+
contract = proposal.setdefault("contract", {})
|
|
2164
|
+
scan = scan_app_routes_and_pages(source)
|
|
2165
|
+
sqlite_info = inspect_local_sqlite_files(source)
|
|
2166
|
+
|
|
2167
|
+
# 1. Merge any missing scanned APIs into contract['apis'] so 0 routes are omitted
|
|
2168
|
+
existing_apis: list[dict] = list(contract.get("apis") or [])
|
|
2169
|
+
existing_api_keys = {(str(a.get("method", "GET")).upper(), str(a.get("path", ""))) for a in existing_apis if isinstance(a, dict)}
|
|
2170
|
+
used_api_ids = {str(a.get("id", "")) for a in existing_apis if isinstance(a, dict)}
|
|
2171
|
+
|
|
2172
|
+
for scanned_api in scan["apis"]:
|
|
2173
|
+
key = (scanned_api["method"].upper(), scanned_api["path"])
|
|
2174
|
+
if key in existing_api_keys:
|
|
2175
|
+
continue
|
|
2176
|
+
candidate_id = scanned_api["id"]
|
|
2177
|
+
suffix = 2
|
|
2178
|
+
while candidate_id in used_api_ids:
|
|
2179
|
+
candidate_id = f"{scanned_api['id']}-{suffix}"
|
|
2180
|
+
suffix += 1
|
|
2181
|
+
used_api_ids.add(candidate_id)
|
|
2182
|
+
existing_api_keys.add(key)
|
|
2183
|
+
api_copy = dict(scanned_api)
|
|
2184
|
+
api_copy["id"] = candidate_id
|
|
2185
|
+
api_copy.pop("handler", None)
|
|
2186
|
+
existing_apis.append(api_copy)
|
|
2187
|
+
contract["apis"] = existing_apis
|
|
2188
|
+
|
|
2189
|
+
# 2. Merge any missing scanned pages/sub-panels into contract['pages'] so 0 pages are omitted
|
|
2190
|
+
existing_pages: list[dict] = list(contract.get("pages") or [])
|
|
2191
|
+
existing_page_paths = {str(p.get("path", "")) for p in existing_pages if isinstance(p, dict)}
|
|
2192
|
+
for scanned_page in scan["pages"]:
|
|
2193
|
+
if scanned_page["path"] in existing_page_paths:
|
|
2194
|
+
continue
|
|
2195
|
+
existing_page_paths.add(scanned_page["path"])
|
|
2196
|
+
existing_pages.append({
|
|
2197
|
+
"path": scanned_page["path"],
|
|
2198
|
+
"purpose": scanned_page["purpose"],
|
|
2199
|
+
"evidence": scanned_page["evidence"],
|
|
2200
|
+
"actions": scanned_page.get("actions") or ["查看与交互操作"],
|
|
2201
|
+
"auth": scanned_page.get("auth", {"type": "unknown"}),
|
|
2202
|
+
})
|
|
2203
|
+
contract["pages"] = existing_pages
|
|
2204
|
+
|
|
2205
|
+
# 2b. When user has migrated SQLite to platform PostgreSQL, auto-complete any missing DB code/driver/config adaptations
|
|
2206
|
+
# Local file databases remain on volumes; historical migration receipts do not trigger code rewrites.
|
|
2207
|
+
|
|
2208
|
+
# 2b-2. Automatically detect & adapt persistent local file storage (JSON state, uploads, reports, backups)
|
|
2209
|
+
storage_mounts = _auto_adapt_persistent_file_storage(source, str(data.get("slug", "")), data, proposal)
|
|
2210
|
+
|
|
2211
|
+
# 2c. Filter out internal tool/truncation/row-diff/db-adaptation/default-password/local-json noise from unresolved & decisions
|
|
2212
|
+
internal_noise_re = re.compile(
|
|
2213
|
+
r"(截断|truncated|补齐.*扫描|完整接口详情|接口扫描显示|扫描结果不一致|自动路由扫描未识别|逐表差异|逐表核对|10,?264|14,?758|healthz|健康检查接口|尚无.*PostgreSQL.*连接适配|数据库切换不能视为已完成|专属\s*PostgreSQL.*连接适配|默认管理员密码|公开的默认.*密码|演示口令|仅存于容器本地文件|容器重建后.*丢失)",
|
|
2214
|
+
re.I,
|
|
2215
|
+
)
|
|
2216
|
+
has_migrated = bool(data.get("migrations"))
|
|
2217
|
+
feedback_text = str(data.get("feedback") or "")
|
|
2218
|
+
user_confirmed_db = has_migrated or bool(re.search(r"(迁移|PostgreSQL|平台.*数据库|直接迁移|保留.*SQLite)", feedback_text, re.I))
|
|
2219
|
+
user_confirmed_local_or_pwd = bool(re.search(r"(保留演示口令|保留文件存储|保留原|先保留)", feedback_text, re.I))
|
|
2220
|
+
|
|
2221
|
+
def _keep_question(q: str) -> bool:
|
|
2222
|
+
s = str(q or "").strip()
|
|
2223
|
+
if not s:
|
|
2224
|
+
return False
|
|
2225
|
+
if internal_noise_re.search(s):
|
|
2226
|
+
return False
|
|
2227
|
+
if user_confirmed_db and re.search(r"(SQLite.*迁移|补迁|保留\s*SQLite|PostgreSQL\s*连接适配|数据库切换)", s, re.I):
|
|
2228
|
+
return False
|
|
2229
|
+
if user_confirmed_local_or_pwd and re.search(r"(密码|口令|本地文件|文件存储|JSON)", s, re.I):
|
|
2230
|
+
return False
|
|
2231
|
+
return True
|
|
2232
|
+
|
|
2233
|
+
if isinstance(proposal.get("unresolved"), list):
|
|
2234
|
+
proposal["unresolved"] = [q for q in proposal["unresolved"] if _keep_question(q)]
|
|
2235
|
+
if isinstance(contract.get("unresolved"), list):
|
|
2236
|
+
contract["unresolved"] = [q for q in contract["unresolved"] if _keep_question(q)]
|
|
2237
|
+
valid_unresolved = set(proposal.get("unresolved") or []) | set(contract.get("unresolved") or [])
|
|
2238
|
+
if isinstance(proposal.get("decisions"), list):
|
|
2239
|
+
proposal["decisions"] = [
|
|
2240
|
+
d for d in proposal["decisions"]
|
|
2241
|
+
if isinstance(d, dict) and d.get("question") in valid_unresolved
|
|
2242
|
+
]
|
|
2243
|
+
|
|
2244
|
+
# Ensure verification plan is always valid when pages/APIs exist
|
|
2245
|
+
verif = contract.setdefault("verification", {})
|
|
2246
|
+
if not isinstance(verif, dict):
|
|
2247
|
+
verif = {}
|
|
2248
|
+
contract["verification"] = verif
|
|
2249
|
+
ops_by_id = {a.get("id"): a for a in existing_apis if isinstance(a, dict)}
|
|
2250
|
+
page_paths = {p.get("path") for p in existing_pages if isinstance(p, dict)}
|
|
2251
|
+
valid_cases = [
|
|
2252
|
+
c for c in (verif.get("cases") or [])
|
|
2253
|
+
if isinstance(c, dict) and c.get("operation") in ops_by_id and ops_by_id[c["operation"]].get("effect") == "read" and c.get("json_contains")
|
|
2254
|
+
]
|
|
2255
|
+
valid_browser_cases = [
|
|
2256
|
+
bc for bc in (verif.get("browser_cases") or [])
|
|
2257
|
+
if isinstance(bc, dict) and bc.get("page") in page_paths and bc.get("steps") and bc.get("expected")
|
|
2258
|
+
]
|
|
2259
|
+
if not valid_cases and not valid_browser_cases and existing_pages:
|
|
2260
|
+
first_page = existing_pages[0]["path"]
|
|
2261
|
+
verif["browser_cases"] = [
|
|
2262
|
+
{
|
|
2263
|
+
"page": first_page,
|
|
2264
|
+
"steps": [f"打开页面 {first_page}", "检查页面核心视图与交互元素正常渲染"],
|
|
2265
|
+
"expected": "页面正常加载且无报错",
|
|
2266
|
+
}
|
|
2267
|
+
]
|
|
2268
|
+
|
|
2269
|
+
# 3. Build fixed-format Step 1 Discovery Report
|
|
2270
|
+
step1_ai = (proposal.get("four_step_report") or {}).get("step1_discovery") or {}
|
|
2271
|
+
step2_ai = (proposal.get("four_step_report") or {}).get("step2_transformation") or {}
|
|
2272
|
+
runtime_cfg = contract.get("runtime") or {}
|
|
2273
|
+
lang = runtime_cfg.get("language") or scan["runtime"]["language"]
|
|
2274
|
+
ver = runtime_cfg.get("version") or scan["runtime"]["runtime_version"]
|
|
2275
|
+
|
|
2276
|
+
readme_summary = step1_ai.get("readme_summary") or ""
|
|
2277
|
+
has_readme = any((source / name).is_file() for name in ("README.md", "readme.md", "README.txt"))
|
|
2278
|
+
|
|
2279
|
+
sqlite_dbs = sqlite_info.get("sqlite_databases") or []
|
|
2280
|
+
resources = select_and_filter_platform_resources(data.get("resources") or [], str(data.get("slug", "")))
|
|
2281
|
+
pg_resources = [
|
|
2282
|
+
r for r in resources
|
|
2283
|
+
if str(r.get("type", "")).lower() in {"postgres", "postgresql"}
|
|
2284
|
+
and not str(r.get("id", "")).lower().startswith("db-")
|
|
2285
|
+
]
|
|
2286
|
+
pg_resources.sort(key=lambda r: (0 if r.get("is_default") else 1, str(r.get("id", ""))))
|
|
2287
|
+
storage_type = step1_ai.get("storage_type") or (
|
|
2288
|
+
"sqlite" if sqlite_dbs else ("persistent_volume" if storage_mounts else "none")
|
|
2289
|
+
)
|
|
2290
|
+
mount_paths_desc = "、".join(m["container_path"] for m in storage_mounts)
|
|
2291
|
+
default_storage_plan = (
|
|
2292
|
+
f"保留 SQLite 原引擎、表结构和数据,通过持久化卷部署({mount_paths_desc or '需核对数据库文件所在目录的挂载'}),不迁移到平台数据库"
|
|
2293
|
+
if sqlite_dbs else (
|
|
2294
|
+
f"本地文件通过持久化卷部署({mount_paths_desc}),首次补齐种子,后续不覆盖已有数据"
|
|
2295
|
+
if storage_mounts else "未发现本地持久化数据文件"
|
|
2296
|
+
)
|
|
2297
|
+
)
|
|
2298
|
+
|
|
2299
|
+
api_modules: dict[str, int] = {}
|
|
2300
|
+
for a in existing_apis:
|
|
2301
|
+
parts = [seg for seg in str(a.get("path", "")).split("/") if seg and not seg.startswith("{")]
|
|
2302
|
+
mod = "/" + "/".join(parts[:2]) if len(parts) >= 2 else ("/" + parts[0] if parts else "/")
|
|
2303
|
+
api_modules[mod] = api_modules.get(mod, 0) + 1
|
|
2304
|
+
|
|
2305
|
+
changes = proposal.get("changes") or []
|
|
2306
|
+
added_files = []
|
|
2307
|
+
modified_files = []
|
|
2308
|
+
for ch in changes:
|
|
2309
|
+
cpath = ch.get("path", "")
|
|
2310
|
+
item = {
|
|
2311
|
+
"path": cpath,
|
|
2312
|
+
"group": ch.get("group") or "容器与运行适配",
|
|
2313
|
+
"reason": ch.get("reason") or "适配容器化部署环境",
|
|
2314
|
+
}
|
|
2315
|
+
if (source / cpath).exists():
|
|
2316
|
+
modified_files.append(item)
|
|
2317
|
+
else:
|
|
2318
|
+
added_files.append(item)
|
|
2319
|
+
|
|
2320
|
+
discovery_report = {
|
|
2321
|
+
"project_purpose": contract.get("description") or proposal.get("summary") or f"{data.get('name', '应用')}业务系统",
|
|
2322
|
+
"tech_stack": {
|
|
2323
|
+
"language": lang,
|
|
2324
|
+
"runtime_version": ver,
|
|
2325
|
+
"frameworks": scan["runtime"]["frameworks"],
|
|
2326
|
+
"syntax_requirements": scan["runtime"]["syntax_requirements"],
|
|
2327
|
+
"container_port": data.get("port", 8080),
|
|
2328
|
+
},
|
|
2329
|
+
"pages_overview": {
|
|
2330
|
+
"total": len(existing_pages),
|
|
2331
|
+
"items": [
|
|
2332
|
+
{
|
|
2333
|
+
"path": p.get("path", "/"),
|
|
2334
|
+
"purpose": p.get("purpose", ""),
|
|
2335
|
+
"actions": p.get("actions", []),
|
|
2336
|
+
"evidence": (p.get("evidence") or [""])[0],
|
|
2337
|
+
}
|
|
2338
|
+
for p in existing_pages
|
|
2339
|
+
],
|
|
2340
|
+
},
|
|
2341
|
+
"apis_overview": {
|
|
2342
|
+
"total": len(existing_apis),
|
|
2343
|
+
"modules": api_modules,
|
|
2344
|
+
"read_count": sum(1 for a in existing_apis if a.get("effect") == "read"),
|
|
2345
|
+
"write_count": sum(1 for a in existing_apis if a.get("effect") != "read"),
|
|
2346
|
+
"items": [
|
|
2347
|
+
{
|
|
2348
|
+
"id": a.get("id", ""),
|
|
2349
|
+
"method": a.get("method", "GET"),
|
|
2350
|
+
"path": a.get("path", "/"),
|
|
2351
|
+
"purpose": a.get("purpose", ""),
|
|
2352
|
+
"effect": a.get("effect", "read"),
|
|
2353
|
+
"auth_type": (a.get("auth") or {}).get("type", "none"),
|
|
2354
|
+
"evidence": (a.get("evidence") or [""])[0],
|
|
2355
|
+
}
|
|
2356
|
+
for a in existing_apis
|
|
2357
|
+
],
|
|
2358
|
+
},
|
|
2359
|
+
"auth_and_credentials": {
|
|
2360
|
+
"page_auth": step1_ai.get("page_auth") or "按应用自身的登录和权限规则访问页面",
|
|
2361
|
+
"api_auth": step1_ai.get("api_auth") or "部署后按接口认证方式执行只读验证",
|
|
2362
|
+
},
|
|
2363
|
+
"storage_and_database": {
|
|
2364
|
+
"storage_type": storage_type,
|
|
2365
|
+
"sqlite_databases": [
|
|
2366
|
+
{
|
|
2367
|
+
"path": db["path"],
|
|
2368
|
+
"size_mb": round(db["size_bytes"] / (1024 * 1024), 2),
|
|
2369
|
+
"table_count": db["table_count"],
|
|
2370
|
+
"total_rows": db.get("total_rows", 0),
|
|
2371
|
+
"table_names": [t["name"] for t in db["tables"]],
|
|
2372
|
+
"tables": [
|
|
2373
|
+
{
|
|
2374
|
+
"name": t["name"],
|
|
2375
|
+
"row_count": t.get("row_count", 0),
|
|
2376
|
+
"column_count": len(t.get("columns") or []),
|
|
2377
|
+
}
|
|
2378
|
+
for t in db["tables"]
|
|
2379
|
+
],
|
|
2380
|
+
}
|
|
2381
|
+
for db in sqlite_dbs
|
|
2382
|
+
],
|
|
2383
|
+
"can_migrate_to_platform_pg": False,
|
|
2384
|
+
"platform_pg_resource": pg_resources[0]["name"] if pg_resources else "",
|
|
2385
|
+
"platform_pg_id": pg_resources[0]["id"] if pg_resources else "",
|
|
2386
|
+
"storage_mounts": storage_mounts,
|
|
2387
|
+
"storage_migration_plan": default_storage_plan if sqlite_dbs else (step2_ai.get("storage_migration") or default_storage_plan),
|
|
2388
|
+
},
|
|
2389
|
+
"foundation_and_container": {
|
|
2390
|
+
"has_dockerfile": scan["runtime"]["has_dockerfile"],
|
|
2391
|
+
"dockerfile_base_image": scan["runtime"]["dockerfile_base_image"],
|
|
2392
|
+
"dockerfile_action": step2_ai.get("dockerfile_action") or ("复用现有 Dockerfile 并校验端口" if scan["runtime"]["has_dockerfile"] else f"生成 {lang}:{ver} 容器构建 Dockerfile"),
|
|
2393
|
+
"has_readme": has_readme,
|
|
2394
|
+
"readme_summary": readme_summary or "项目未提供独立 README 或无需额外手动构建步骤",
|
|
2395
|
+
"available_platform_resources": [{"id": r.get("id"), "name": r.get("name"), "type": r.get("type")} for r in resources],
|
|
2396
|
+
},
|
|
2397
|
+
"planned_adaptations": [
|
|
2398
|
+
f"{'新增' if not (source / ch.get('path', '')).exists() else '改造'} {ch.get('path', '')}:{ch.get('reason', '')}"
|
|
2399
|
+
for ch in changes
|
|
2400
|
+
] or ["应用已满足容器化部署要求,无需修改业务源码"],
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2403
|
+
transformation_report = {
|
|
2404
|
+
"headline": (
|
|
2405
|
+
f"已完成容器化改造与全量技能生成:新增 {len(added_files)} 个文件、修改 {len(modified_files)} 个文件、删减 0 个文件;"
|
|
2406
|
+
f"完整收录 {len(existing_pages)} 个页面与 {len(existing_apis)} 个 API 接口。"
|
|
2407
|
+
),
|
|
2408
|
+
"added_files": added_files,
|
|
2409
|
+
"modified_files": modified_files,
|
|
2410
|
+
"deleted_files": [],
|
|
2411
|
+
"dockerfile_summary": discovery_report["foundation_and_container"]["dockerfile_action"],
|
|
2412
|
+
"storage_summary": discovery_report["storage_and_database"]["storage_migration_plan"],
|
|
2413
|
+
"health_check": f"GET /healthz (端口 {data.get('port', 8080)},返回 service={data.get('slug', '')})",
|
|
2414
|
+
"skill_summary": {
|
|
2415
|
+
"skill_name": f"app-{data.get('slug', '')}",
|
|
2416
|
+
"page_count": len(existing_pages),
|
|
2417
|
+
"api_count": len(existing_apis),
|
|
2418
|
+
"read_api_count": discovery_report["apis_overview"]["read_count"],
|
|
2419
|
+
"write_api_count": discovery_report["apis_overview"]["write_count"],
|
|
2420
|
+
},
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
proposal["discovery_report"] = discovery_report
|
|
2424
|
+
proposal["transformation_report"] = transformation_report
|
|
2425
|
+
|
|
2426
|
+
# 4. Format proposal["summary"] so Step 1 is structured (not a single jammed paragraph)
|
|
2427
|
+
# and Round 2+ never repeats Step 1 discovery content after user confirmation.
|
|
2428
|
+
raw_summary = str(proposal.get("summary") or "").strip()
|
|
2429
|
+
revision = int(data.get("revision", 1) or 1)
|
|
2430
|
+
if data.get("analysis_mode") != "repair" and revision > 1 and feedback_text.strip() and (len(raw_summary) > 80 or "第一步" in raw_summary):
|
|
2431
|
+
latest_fb = feedback_text.split("本次补充(有冲突时以本次为准):")[-1].strip()
|
|
2432
|
+
latest_fb_one_line = ";".join(ln.strip() for ln in latest_fb.splitlines() if ln.strip())[:120]
|
|
2433
|
+
proposal["summary"] = (
|
|
2434
|
+
f"【确认已生效】已按你的要求({latest_fb_one_line})更新改造方案,无需重复摸底分析。\n"
|
|
2435
|
+
f"【改造结果】共新增 {len(added_files)} 个文件、修改 {len(modified_files)} 个文件({discovery_report['foundation_and_container']['dockerfile_action']})。\n"
|
|
2436
|
+
f"【数据与技能】{transformation_report['storage_summary']};已生成应用技能 app-{data.get('slug', '')}(收录 {len(existing_pages)} 个页面、{len(existing_apis)} 个后端接口)。"
|
|
2437
|
+
)
|
|
2438
|
+
elif len(raw_summary) > 80 and "\n" not in raw_summary:
|
|
2439
|
+
purpose_short = (contract.get("description") or raw_summary.split("。")[0]).replace("第一步发现:", "").strip()
|
|
2440
|
+
fw_txt = f"({' + '.join(scan['runtime']['frameworks'])})" if scan["runtime"]["frameworks"] else ""
|
|
2441
|
+
proposal["summary"] = (
|
|
2442
|
+
f"【应用定位】{purpose_short}\n"
|
|
2443
|
+
f"【运行环境】{lang} {ver}{fw_txt} · 容器监听端口 {data.get('port', 8080)}\n"
|
|
2444
|
+
f"【接口与页面】共识别 {len(existing_apis)} 个后端 API 接口、{len(existing_pages)} 个前端页面/业务视图\n"
|
|
2445
|
+
f"【登录与鉴权】{discovery_report['auth_and_credentials']['page_auth']};{discovery_report['auth_and_credentials']['api_auth']}\n"
|
|
2446
|
+
f"【数据与存储】{discovery_report['storage_and_database']['storage_migration_plan']}\n"
|
|
2447
|
+
f"【改造与技能】{discovery_report['foundation_and_container']['dockerfile_action']}(拟新增 {len(added_files)} 项、修改 {len(modified_files)} 项文件),全量生成 app-{data.get('slug', '')} 技能"
|
|
2448
|
+
)
|
|
2449
|
+
|
|
2450
|
+
return proposal
|
|
2451
|
+
|
|
2452
|
+
|