@hupan56/wlkj 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +289 -12
- package/package.json +1 -1
- package/templates/qoder/agents/insight-planning.md +67 -0
- package/templates/qoder/agents/insight-research.md +61 -0
- package/templates/qoder/agents/prd-quick.md +1 -0
- package/templates/qoder/agents/prd-reference.md +10 -2
- package/templates/qoder/commands/optional/wl-insight.md +275 -0
- package/templates/qoder/commands/{wl-report.md → optional/wl-report.md} +13 -5
- package/templates/qoder/commands/{wl-spec.md → optional/wl-spec.md} +1 -1
- package/templates/qoder/commands/{wl-status.md → optional/wl-status.md} +28 -2
- package/templates/qoder/commands/wl-code.md +10 -2
- package/templates/qoder/commands/wl-commit.md +1 -1
- package/templates/qoder/commands/wl-design-draw.md +78 -0
- package/templates/qoder/commands/wl-design-scan.md +108 -0
- package/templates/qoder/commands/wl-design-spec.md +154 -0
- package/templates/qoder/commands/wl-design.md +32 -0
- package/templates/qoder/commands/wl-init.md +24 -3
- package/templates/qoder/commands/wl-prd-full.md +226 -0
- package/templates/qoder/commands/wl-prd-quick.md +134 -0
- package/templates/qoder/commands/wl-prd-review.md +104 -0
- package/templates/qoder/commands/wl-prd.md +17 -288
- package/templates/qoder/commands/wl-search.md +66 -30
- package/templates/qoder/commands/wl-task.md +290 -59
- package/templates/qoder/commands/wl-test.md +92 -24
- package/templates/qoder/config.yaml +59 -15
- package/templates/qoder/hooks/inject-workflow-state.py +35 -9
- package/templates/qoder/hooks/session-start.py +144 -62
- package/templates/qoder/rules/wl-pipeline.md +216 -105
- package/templates/qoder/scripts/__pycache__/search_index.cpython-39.pyc +0 -0
- package/templates/qoder/scripts/archive_prd.py +377 -0
- package/templates/qoder/scripts/autotest.py +1715 -0
- package/templates/qoder/scripts/autotest_batch.py +224 -0
- package/templates/qoder/scripts/autotest_run.py +297 -0
- package/templates/qoder/scripts/benchmark.py +210 -209
- package/templates/qoder/scripts/build_style_index.py +444 -4
- package/templates/qoder/scripts/check_carriers.py +238 -0
- package/templates/qoder/scripts/check_mcp.py +298 -0
- package/templates/qoder/scripts/check_qoderwork_consistency.py +166 -0
- package/templates/qoder/scripts/common/developer.py +26 -19
- package/templates/qoder/scripts/common/events.py +46 -0
- package/templates/qoder/scripts/common/extract.py +419 -0
- package/templates/qoder/scripts/common/graph_traverse.py +533 -0
- package/templates/qoder/scripts/common/identity.py +6 -1
- package/templates/qoder/scripts/common/paths.py +89 -0
- package/templates/qoder/scripts/common/pip_install.py +144 -0
- package/templates/qoder/scripts/common/platform_guard.py +61 -0
- package/templates/qoder/scripts/common/search_engine.py +205 -205
- package/templates/qoder/scripts/common/terms.py +57 -0
- package/templates/qoder/scripts/common/ts_extract.py +536 -0
- package/templates/qoder/scripts/context_pack.py +73 -13
- package/templates/qoder/scripts/enrich_prompt.py +226 -0
- package/templates/qoder/scripts/eval_prd.py +318 -225
- package/templates/qoder/scripts/export.py +487 -487
- package/templates/qoder/scripts/extract_api_params.py +246 -0
- package/templates/qoder/scripts/extract_routes.py +54 -0
- package/templates/qoder/scripts/extract_routes_tree.py +78 -0
- package/templates/qoder/scripts/fill_prototype.py +707 -0
- package/templates/qoder/scripts/gen_design_doc.py +394 -0
- package/templates/qoder/scripts/git_sync.py +27 -15
- package/templates/qoder/scripts/init_doctor.py +306 -41
- package/templates/qoder/scripts/install_qoderwork.py +366 -9
- package/templates/qoder/scripts/kg.py +708 -0
- package/templates/qoder/scripts/kg_auto_login.py +196 -0
- package/templates/qoder/scripts/kg_build.py +612 -0
- package/templates/qoder/scripts/kg_build_db.py +327 -0
- package/templates/qoder/scripts/kg_duckdb.py +549 -0
- package/templates/qoder/scripts/kg_incremental.py +393 -0
- package/templates/qoder/scripts/kg_link_db.py +224 -0
- package/templates/qoder/scripts/kg_mcp_server.py +801 -0
- package/templates/qoder/scripts/kg_semantic.py +150 -0
- package/templates/qoder/scripts/kg_test_runner.py +241 -0
- package/templates/qoder/scripts/lanhu_stdio_wrapper.py +119 -0
- package/templates/qoder/scripts/learn.py +118 -39
- package/templates/qoder/scripts/learn_aggregate.py +201 -0
- package/templates/qoder/scripts/mcp_launcher.py +359 -0
- package/templates/qoder/scripts/mysql_mcp_server.py +396 -0
- package/templates/qoder/scripts/repo_root.py +106 -0
- package/templates/qoder/scripts/role.py +12 -0
- package/templates/qoder/scripts/run_weekly_update.bat +5 -0
- package/templates/qoder/scripts/run_weekly_update.sh +5 -0
- package/templates/qoder/scripts/search_index.py +307 -60
- package/templates/qoder/scripts/secure-ls.js +5640 -0
- package/templates/qoder/scripts/setup.py +706 -453
- package/templates/qoder/scripts/setup_lanhu.py +963 -0
- package/templates/qoder/scripts/status.py +250 -11
- package/templates/qoder/scripts/sync_carriers.py +259 -0
- package/templates/qoder/scripts/syncgate.py +5 -4
- package/templates/qoder/scripts/task.py +75 -0
- package/templates/qoder/scripts/team_sync.py +60 -4
- package/templates/qoder/scripts/workspace_init.py +1 -1
- package/templates/qoder/skills/design-import/SKILL.md +226 -0
- package/templates/qoder/skills/design-import/figma-workflow.md +81 -0
- package/templates/qoder/skills/design-review/SKILL.md +82 -25
- package/templates/qoder/skills/prd-generator/SKILL.md +185 -58
- package/templates/qoder/skills/prd-review/SKILL.md +18 -1
- package/templates/qoder/skills/prompt-enrich/SKILL.md +90 -0
- package/templates/qoder/skills/prototype-generator/SKILL.md +256 -141
- package/templates/qoder/skills/prototype-generator/SKILL.md.zcode-79180-2af4721f-f9a6-412c-88db-c0af680d211b.tmp +0 -0
- package/templates/qoder/skills/spec-coder/SKILL.md +18 -1
- package/templates/qoder/skills/spec-generator/SKILL.md +18 -1
- package/templates/qoder/skills/test-generator/SKILL.md +15 -2
- package/templates/qoder/skills/wl-code/SKILL.md +55 -36
- package/templates/qoder/skills/wl-commit/SKILL.md +89 -76
- package/templates/qoder/skills/wl-design/SKILL.md +55 -0
- package/templates/qoder/skills/wl-init/SKILL.md +76 -67
- package/templates/qoder/skills/wl-insight/SKILL.md +201 -81
- package/templates/qoder/skills/wl-prd-full/SKILL.md +69 -0
- package/templates/qoder/skills/wl-prd-quick/SKILL.md +49 -0
- package/templates/qoder/skills/wl-prd-review/SKILL.md +34 -0
- package/templates/qoder/skills/wl-report/SKILL.md +131 -107
- package/templates/qoder/skills/wl-search/SKILL.md +141 -75
- package/templates/qoder/skills/wl-spec/SKILL.md +49 -39
- package/templates/qoder/skills/wl-status/SKILL.md +83 -61
- package/templates/qoder/skills/wl-task/SKILL.md +132 -58
- package/templates/qoder/skills/wl-test/SKILL.md +406 -40
- package/templates/qoder/templates/prd-full-template.md +2 -0
- package/templates/qoder/templates/prd-quick-template.md +1 -0
- package/templates/qoder/templates/prototype-app.html +13 -8
- package/templates/qoder/templates/prototype-web.html +376 -93
- package/templates/root/AGENTS.md +89 -34
- package/templates/root/requirements.txt +21 -0
- package/templates/root//344/275/277/347/224/250/350/257/264/346/230/216.md +259 -259
- package/templates/root//346/226/260/346/211/213/346/214/207/345/215/227.md +186 -186
- package/templates/qoder/agents/prd-planning.md +0 -56
- package/templates/qoder/agents/prd-research.md +0 -33
- package/templates/qoder/commands/wl-insight.md +0 -51
- package/templates/qoder/skills/wl-prd/SKILL.md +0 -89
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Incremental UPSERT builder for the DuckDB knowledge graph.
|
|
4
|
+
|
|
5
|
+
Instead of full rebuild (build_style_index.py → kg_duckdb.py bulk_import),
|
|
6
|
+
this takes a list of changed files (from git diff) and only updates the
|
|
7
|
+
entities + edges that belong to those files.
|
|
8
|
+
|
|
9
|
+
Performance: ~0.2s per file vs 60s full rebuild.
|
|
10
|
+
For 500 changed files: ~2 minutes incremental vs 5+ minutes full rebuild.
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
python kg_incremental.py # auto-detect changes via git diff
|
|
14
|
+
python kg_incremental.py file1.vue file2.ts # explicit file list
|
|
15
|
+
"""
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
SCRIPTS = os.path.join(os.path.dirname(os.path.abspath(__file__)))
|
|
22
|
+
sys.path.insert(0, SCRIPTS)
|
|
23
|
+
|
|
24
|
+
from common.paths import DATA_INDEX_DIR, PROJECT_ROOT
|
|
25
|
+
from kg_duckdb import get_db, init_schema, upsert_entity, upsert_alias, upsert_edge, delete_edges_from
|
|
26
|
+
from common.ts_extract import ts_build_api_fn_table, ts_trace_clicks
|
|
27
|
+
from common.extract import build_api_fn_table
|
|
28
|
+
from common.test_extract import extract_test_assertions
|
|
29
|
+
|
|
30
|
+
CODE_DIR = str(PROJECT_ROOT / 'data' / 'code')
|
|
31
|
+
INDEX_DIR = str(DATA_INDEX_DIR)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_changed_files():
|
|
35
|
+
"""Auto-detect ALL changed files since last kg_build via stored commit hash.
|
|
36
|
+
|
|
37
|
+
Records the last-built commit per project in build_meta table.
|
|
38
|
+
Next run diffs from that commit to HEAD, capturing ALL changes (not just last 2 commits).
|
|
39
|
+
"""
|
|
40
|
+
changed = []
|
|
41
|
+
if not os.path.isdir(CODE_DIR):
|
|
42
|
+
return changed
|
|
43
|
+
import subprocess
|
|
44
|
+
|
|
45
|
+
# Read last-built commits from DuckDB build_meta
|
|
46
|
+
last_commits = {}
|
|
47
|
+
try:
|
|
48
|
+
con = get_db()
|
|
49
|
+
rows = con.execute("SELECT key, value FROM build_meta WHERE key LIKE 'last_commit_%'").fetchall()
|
|
50
|
+
con.close()
|
|
51
|
+
for k, v in rows:
|
|
52
|
+
last_commits[k.replace('last_commit_', '')] = v
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
current_commits = {}
|
|
57
|
+
for proj in os.listdir(CODE_DIR):
|
|
58
|
+
proj_path = os.path.join(CODE_DIR, proj)
|
|
59
|
+
if not os.path.isdir(os.path.join(proj_path, '.git')):
|
|
60
|
+
continue
|
|
61
|
+
try:
|
|
62
|
+
# Get current HEAD
|
|
63
|
+
r = subprocess.run(['git', 'rev-parse', 'HEAD'],
|
|
64
|
+
cwd=proj_path, capture_output=True, text=True, encoding='utf-8')
|
|
65
|
+
if r.returncode != 0:
|
|
66
|
+
continue
|
|
67
|
+
current_head = r.stdout.strip()
|
|
68
|
+
current_commits[proj] = current_head
|
|
69
|
+
|
|
70
|
+
# Determine diff range: from last-built commit to HEAD
|
|
71
|
+
last_built = last_commits.get(proj)
|
|
72
|
+
if last_built and last_built != current_head:
|
|
73
|
+
# Diff from last build to now — captures ALL changes since last build
|
|
74
|
+
diff_args = ['git', 'diff', '--name-only', last_built, current_head]
|
|
75
|
+
elif not last_built:
|
|
76
|
+
# First run — get last 7 days of changes (or all if repo is new)
|
|
77
|
+
r3 = subprocess.run(['git', 'log', '--since=7 days ago', '--oneline'],
|
|
78
|
+
cwd=proj_path, capture_output=True, text=True, encoding='utf-8')
|
|
79
|
+
log_lines = [l for l in r3.stdout.strip().split('\n') if l.strip()]
|
|
80
|
+
if len(log_lines) >= 2:
|
|
81
|
+
old_commit = log_lines[-1].split()[0]
|
|
82
|
+
diff_args = ['git', 'diff', '--name-only', old_commit, current_head]
|
|
83
|
+
else:
|
|
84
|
+
continue # No changes or single commit
|
|
85
|
+
else:
|
|
86
|
+
continue # Already up to date, no diff needed
|
|
87
|
+
|
|
88
|
+
r2 = subprocess.run(diff_args,
|
|
89
|
+
cwd=proj_path, capture_output=True, text=True, encoding='utf-8')
|
|
90
|
+
for f in r2.stdout.strip().split('\n'):
|
|
91
|
+
f = f.strip()
|
|
92
|
+
if f:
|
|
93
|
+
changed.append(os.path.join(proj_path, f.replace('/', os.sep)))
|
|
94
|
+
except Exception:
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
# Save current commits as last-built
|
|
98
|
+
try:
|
|
99
|
+
con = get_db()
|
|
100
|
+
for proj, commit in current_commits.items():
|
|
101
|
+
con.execute(
|
|
102
|
+
"INSERT OR REPLACE INTO build_meta VALUES (?, ?)",
|
|
103
|
+
['last_commit_%s' % proj, commit]
|
|
104
|
+
)
|
|
105
|
+
con.close()
|
|
106
|
+
except Exception:
|
|
107
|
+
pass
|
|
108
|
+
|
|
109
|
+
return changed
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def incremental_update(changed_files):
|
|
113
|
+
"""Update DuckDB incrementally for a list of changed files.
|
|
114
|
+
|
|
115
|
+
For each file:
|
|
116
|
+
- .vue: re-extract trace_clicks, UPSERT button entities + click edges
|
|
117
|
+
- .ts (api): re-extract api functions, UPSERT endpoint entities
|
|
118
|
+
- .java (test): re-extract test assertions, UPSERT test entities
|
|
119
|
+
"""
|
|
120
|
+
con = get_db()
|
|
121
|
+
init_schema(con)
|
|
122
|
+
|
|
123
|
+
stats = {'vue': 0, 'ts': 0, 'java_test': 0, 'edges_updated': 0, 'entities_upserted': 0}
|
|
124
|
+
|
|
125
|
+
# 判断是否有 api 文件变更 — 决定是否需要重建 api_table
|
|
126
|
+
has_api_changes = any(
|
|
127
|
+
f.replace('\\', '/').endswith('.ts') and '/api/' in f.replace('\\', '/')
|
|
128
|
+
for f in changed_files if os.path.isfile(f)
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
api_dir = _find_api_dir()
|
|
132
|
+
|
|
133
|
+
if has_api_changes:
|
|
134
|
+
# api 文件变了 — 必须重建完整 api_table
|
|
135
|
+
api_table = ts_build_api_fn_table(api_dir) if api_dir else {}
|
|
136
|
+
# 缓存到文件, 供下次增量复用
|
|
137
|
+
if api_table and api_dir:
|
|
138
|
+
_save_api_cache(api_table, api_dir)
|
|
139
|
+
else:
|
|
140
|
+
# api 文件没变 — 从缓存加载 (毫秒级, 不扫文件系统)
|
|
141
|
+
api_table = _load_api_cache(api_dir)
|
|
142
|
+
if not api_table:
|
|
143
|
+
# 缓存不存在 (首次运行) — 重建并缓存
|
|
144
|
+
api_table = ts_build_api_fn_table(api_dir) if api_dir else {}
|
|
145
|
+
if api_table and api_dir:
|
|
146
|
+
_save_api_cache(api_table, api_dir)
|
|
147
|
+
|
|
148
|
+
for filepath in changed_files:
|
|
149
|
+
if not os.path.isfile(filepath):
|
|
150
|
+
# File was deleted — clean up its entities
|
|
151
|
+
_cleanup_deleted_file(con, filepath)
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
rel_path = filepath.replace('\\', '/')
|
|
155
|
+
|
|
156
|
+
if filepath.endswith('.vue'):
|
|
157
|
+
stats['vue'] += 1
|
|
158
|
+
_update_vue_file(con, filepath, api_table, stats)
|
|
159
|
+
|
|
160
|
+
elif filepath.endswith('.ts') and '/api/' in rel_path:
|
|
161
|
+
stats['ts'] += 1
|
|
162
|
+
_update_api_file(con, filepath, stats)
|
|
163
|
+
|
|
164
|
+
elif filepath.endswith('Test.java'):
|
|
165
|
+
stats['java_test'] += 1
|
|
166
|
+
_update_test_file(con, filepath, stats)
|
|
167
|
+
|
|
168
|
+
# After file updates, refresh PRD + Wiki from git_sync's JSON output
|
|
169
|
+
_refresh_prds_and_wiki(con, stats)
|
|
170
|
+
|
|
171
|
+
con.close()
|
|
172
|
+
return stats
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _refresh_prds_and_wiki(con, stats):
|
|
176
|
+
"""Import PRD index + Wiki from git_sync's JSON output into DuckDB.
|
|
177
|
+
|
|
178
|
+
git_sync.py writes these as JSON; kg_incremental reads them and UPSERTs
|
|
179
|
+
into DuckDB so MCP tools can query them.
|
|
180
|
+
"""
|
|
181
|
+
INDEX = str(DATA_INDEX_DIR)
|
|
182
|
+
import json as _json
|
|
183
|
+
|
|
184
|
+
# PRD
|
|
185
|
+
prd_path = os.path.join(INDEX, '_build_cache', 'prd-index.json')
|
|
186
|
+
if not os.path.isfile(prd_path):
|
|
187
|
+
prd_path = os.path.join(INDEX, 'prd-index.json')
|
|
188
|
+
if os.path.isfile(prd_path):
|
|
189
|
+
try:
|
|
190
|
+
prds = _json.load(open(prd_path, encoding='utf-8'))
|
|
191
|
+
con.execute("DELETE FROM prds")
|
|
192
|
+
rows = [(f, info.get('title', ''),
|
|
193
|
+
_json.dumps(info.get('keywords', []), ensure_ascii=False),
|
|
194
|
+
_json.dumps(info.get('features', []), ensure_ascii=False),
|
|
195
|
+
_json.dumps(info.get('cn_terms', []), ensure_ascii=False))
|
|
196
|
+
for f, info in prds.items()]
|
|
197
|
+
if rows:
|
|
198
|
+
_batch_insert(con, 'prds', rows, ignore=True)
|
|
199
|
+
stats['prds_refreshed'] = len(rows)
|
|
200
|
+
except Exception:
|
|
201
|
+
pass
|
|
202
|
+
|
|
203
|
+
# Wiki
|
|
204
|
+
wiki_path = os.path.join(INDEX, 'wiki-index.json')
|
|
205
|
+
if os.path.isfile(wiki_path):
|
|
206
|
+
try:
|
|
207
|
+
wiki = _json.load(open(wiki_path, encoding='utf-8'))
|
|
208
|
+
con.execute("DELETE FROM wiki")
|
|
209
|
+
wiki_rows = []
|
|
210
|
+
for kw, entries in (wiki.get('index', {}) if isinstance(wiki, dict) else {}).items():
|
|
211
|
+
for e in (entries if isinstance(entries, list) else []):
|
|
212
|
+
wiki_rows.append((kw, e.get('project', ''), e.get('title', ''),
|
|
213
|
+
e.get('md_path', ''), e.get('description', '')))
|
|
214
|
+
if wiki_rows:
|
|
215
|
+
_batch_insert(con, 'wiki', wiki_rows)
|
|
216
|
+
stats['wiki_refreshed'] = len(wiki_rows)
|
|
217
|
+
except Exception:
|
|
218
|
+
pass
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _find_api_dir():
|
|
222
|
+
"""Find the fywl-ui api directory."""
|
|
223
|
+
apps = os.path.join(CODE_DIR, 'fywl-ui', 'apps')
|
|
224
|
+
if os.path.isdir(apps):
|
|
225
|
+
for app in os.listdir(apps):
|
|
226
|
+
candidate = os.path.join(apps, app, 'src', 'api')
|
|
227
|
+
if os.path.isdir(candidate):
|
|
228
|
+
return candidate
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _load_api_table_from_duckdb(con):
|
|
233
|
+
"""从 DuckDB 加载 api_table (api 函数名→端点), 避免重建。"""
|
|
234
|
+
table = {}
|
|
235
|
+
try:
|
|
236
|
+
rows = con.execute("SELECT id, props FROM entities WHERE type = 'ENDPOINT'").fetchall()
|
|
237
|
+
for eid, props_json in rows:
|
|
238
|
+
url = eid.replace('E:', '')
|
|
239
|
+
props = json.loads(props_json) if props_json else {}
|
|
240
|
+
fn_name = props.get('api_fn', '')
|
|
241
|
+
if fn_name:
|
|
242
|
+
table[fn_name] = {'url': url, 'verb': props.get('verb', '?'),
|
|
243
|
+
'file': props.get('source_file', '')}
|
|
244
|
+
except Exception:
|
|
245
|
+
pass
|
|
246
|
+
return table
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
_API_CACHE_PATH = os.path.join(INDEX_DIR, '.api-table-cache.json')
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _save_api_cache(api_table, api_dir):
|
|
253
|
+
"""保存 api_table 到缓存文件 (含 api_dir 的 mtime 作为失效依据)。"""
|
|
254
|
+
import time
|
|
255
|
+
cache = {
|
|
256
|
+
'api_dir_mtime': os.path.getmtime(api_dir) if os.path.isdir(api_dir) else 0,
|
|
257
|
+
'saved_at': time.time(),
|
|
258
|
+
'table': api_table,
|
|
259
|
+
}
|
|
260
|
+
with open(_API_CACHE_PATH, 'w', encoding='utf-8') as f:
|
|
261
|
+
json.dump(cache, f, ensure_ascii=False)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _load_api_cache(api_dir):
|
|
265
|
+
"""从缓存文件加载 api_table。如果缓存不存在或过期返回 {}。"""
|
|
266
|
+
if not os.path.isfile(_API_CACHE_PATH):
|
|
267
|
+
return {}
|
|
268
|
+
try:
|
|
269
|
+
cache = json.load(open(_API_CACHE_PATH, encoding='utf-8'))
|
|
270
|
+
# 检查 api_dir 的 mtime 是否变了
|
|
271
|
+
current_mtime = os.path.getmtime(api_dir) if api_dir and os.path.isdir(api_dir) else 0
|
|
272
|
+
if cache.get('api_dir_mtime') == current_mtime:
|
|
273
|
+
return cache.get('table', {})
|
|
274
|
+
except Exception:
|
|
275
|
+
pass
|
|
276
|
+
return {}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _update_vue_file(con, filepath, api_table, stats):
|
|
280
|
+
"""Re-extract traces from one Vue file and UPSERT its button entities + edges."""
|
|
281
|
+
try:
|
|
282
|
+
content = open(filepath, 'rb').read().decode('utf-8', errors='replace')
|
|
283
|
+
except Exception:
|
|
284
|
+
return
|
|
285
|
+
if '@click' not in content:
|
|
286
|
+
return
|
|
287
|
+
|
|
288
|
+
traces = ts_trace_clicks(filepath, content, api_table)
|
|
289
|
+
vue_norm = filepath.replace('\\', '/')
|
|
290
|
+
# Normalize vue_short — MUST match kg_build.py's format exactly
|
|
291
|
+
if '/views/' in vue_norm:
|
|
292
|
+
vue_short = vue_norm.split('/views/')[-1]
|
|
293
|
+
elif '/pages/' in vue_norm:
|
|
294
|
+
vue_short = vue_norm.split('/pages/')[-1]
|
|
295
|
+
else:
|
|
296
|
+
vue_short = os.path.basename(filepath)
|
|
297
|
+
|
|
298
|
+
for t in traces:
|
|
299
|
+
handler = t.get('handler', '')
|
|
300
|
+
eid = 'B:%s:%s' % (vue_short, handler)
|
|
301
|
+
# UPSERT entity (INSERT OR REPLACE, no duplicates)
|
|
302
|
+
upsert_entity(con, eid, 'BUTTON', handler, '',
|
|
303
|
+
{'button_text': t.get('button', ''),
|
|
304
|
+
'vue_file': vue_norm,
|
|
305
|
+
'confidence': t.get('confidence', '')})
|
|
306
|
+
upsert_alias(con, handler.lower(), eid)
|
|
307
|
+
stats['entities_upserted'] += 1
|
|
308
|
+
|
|
309
|
+
# Delete ALL old edges from this button first, then insert fresh
|
|
310
|
+
delete_edges_from(con, eid)
|
|
311
|
+
ep = t.get('endpoint')
|
|
312
|
+
if ep and ep != 'None':
|
|
313
|
+
to_eid = 'E:%s' % ep
|
|
314
|
+
upsert_edge(con, eid, to_eid, 'clicks', props={
|
|
315
|
+
'button': t.get('button', ''),
|
|
316
|
+
'confidence': t.get('confidence', ''),
|
|
317
|
+
})
|
|
318
|
+
stats['edges_updated'] += 1
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _update_api_file(con, filepath, stats):
|
|
322
|
+
"""Re-extract API functions from one .ts file and UPSERT endpoint entities."""
|
|
323
|
+
# Re-use the bulk api_table extraction on this single file's directory
|
|
324
|
+
api_dir = os.path.dirname(filepath)
|
|
325
|
+
file_table = build_api_fn_table(api_dir)
|
|
326
|
+
for fn_name, info in file_table.items():
|
|
327
|
+
# Only update functions defined in THIS file
|
|
328
|
+
if info.get('file', '').replace('\\', '/') != filepath.replace('\\', '/'):
|
|
329
|
+
continue
|
|
330
|
+
eid = 'E:%s' % info['url']
|
|
331
|
+
upsert_entity(con, eid, 'ENDPOINT', info['url'], '',
|
|
332
|
+
{'verb': info.get('verb', '?'),
|
|
333
|
+
'api_fn': fn_name,
|
|
334
|
+
'source_file': filepath.replace('\\', '/')})
|
|
335
|
+
upsert_alias(con, fn_name.lower(), eid)
|
|
336
|
+
stats['entities_upserted'] += 1
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _update_test_file(con, filepath, stats):
|
|
340
|
+
"""Re-extract test assertions from one Test.java and UPSERT test entities."""
|
|
341
|
+
tests = extract_test_assertions(os.path.dirname(filepath))
|
|
342
|
+
# Filter to only this file
|
|
343
|
+
file_norm = filepath.replace('\\', '/')
|
|
344
|
+
for t in tests:
|
|
345
|
+
if t.get('file', '').replace('\\', '/') != file_norm:
|
|
346
|
+
continue
|
|
347
|
+
name = '%s.%s' % (t['class'], t['method'])
|
|
348
|
+
eid = 'T:%s' % name
|
|
349
|
+
upsert_entity(con, eid, 'TEST', name, '',
|
|
350
|
+
{'display': t.get('display_name', ''),
|
|
351
|
+
'deps': t.get('injected_deps', []),
|
|
352
|
+
'assertions': t.get('assertions', []),
|
|
353
|
+
'package': t.get('package', '')})
|
|
354
|
+
upsert_alias(con, name.lower(), eid)
|
|
355
|
+
stats['entities_upserted'] += 1
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _cleanup_deleted_file(con, filepath):
|
|
359
|
+
"""Remove entities belonging to a deleted file."""
|
|
360
|
+
rel = filepath.replace('\\', '/')
|
|
361
|
+
# Delete buttons from this vue file
|
|
362
|
+
if '.vue' in rel:
|
|
363
|
+
vue_short = rel.split('/views/')[-1] if '/views/' in rel else os.path.basename(rel)
|
|
364
|
+
con.execute("DELETE FROM entities WHERE id LIKE 'B:%s:%%'" % vue_short.replace("'", "''"))
|
|
365
|
+
con.execute("DELETE FROM edges WHERE from_id LIKE 'B:%s:%%'" % vue_short.replace("'", "''"))
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
if __name__ == '__main__':
|
|
369
|
+
if len(sys.argv) > 1:
|
|
370
|
+
changed = sys.argv[1:]
|
|
371
|
+
else:
|
|
372
|
+
print('Auto-detecting changes via git diff...')
|
|
373
|
+
changed = get_changed_files()
|
|
374
|
+
|
|
375
|
+
if not changed:
|
|
376
|
+
print('No changes detected.')
|
|
377
|
+
sys.exit(0)
|
|
378
|
+
|
|
379
|
+
# Filter to relevant file types
|
|
380
|
+
relevant = [f for f in changed if f.endswith(('.vue', '.ts', '.java'))]
|
|
381
|
+
print('Changed files: %d total, %d relevant' % (len(changed), len(relevant)))
|
|
382
|
+
|
|
383
|
+
if relevant:
|
|
384
|
+
import time
|
|
385
|
+
t0 = time.time()
|
|
386
|
+
stats = incremental_update(relevant)
|
|
387
|
+
elapsed = time.time() - t0
|
|
388
|
+
print('Incremental update done in %.1fs' % elapsed)
|
|
389
|
+
print(' Vue: %d, TS: %d, Java: %d' % (stats['vue'], stats['ts'], stats['java_test']))
|
|
390
|
+
print(' Entities UPSERTed: %d' % stats['entities_upserted'])
|
|
391
|
+
print(' Edges updated: %d' % stats['edges_updated'])
|
|
392
|
+
else:
|
|
393
|
+
print('No relevant files to update.')
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""kg_link_db.py - 扫代码 @TableName 注解, 建 Entity 类 ↔ 数据库表 的语义桥。
|
|
4
|
+
|
|
5
|
+
解决"数据层孤岛"问题: kg_db.duckdb 有表结构, 但不知道对应代码里的哪个 Entity。
|
|
6
|
+
本脚本扫 Java 源码里的 @TableName("xxx") 注解, 提取:
|
|
7
|
+
Entity 类名 → 数据库表名 → (关联到) db_columns 的真实列
|
|
8
|
+
|
|
9
|
+
调研依据 (FalkorDB "结构遍历 > 语义匹配"):
|
|
10
|
+
- 不靠表名子串猜 endpoint (验证过 0 命中, 不准)
|
|
11
|
+
- 靠代码显式声明的关系 (@TableName 注解) 建边, 这才准
|
|
12
|
+
|
|
13
|
+
产出表 (写进 kg_db.duckdb, 不动 kg.duckdb 写锁):
|
|
14
|
+
entity_table_map:
|
|
15
|
+
entity_class TEXT -- QualityCaseBo (代码层类名)
|
|
16
|
+
table_name TEXT -- quality_case (数据库表名, 关联 db_tables)
|
|
17
|
+
source_file TEXT -- 哪个 .java 读到的
|
|
18
|
+
annotation TEXT -- @TableName("quality_case") 原文
|
|
19
|
+
|
|
20
|
+
建好后可遍历:
|
|
21
|
+
数据库表 → entity_table_map → Entity 类 → (kg.duckdb) Controller/接口
|
|
22
|
+
|
|
23
|
+
用法:
|
|
24
|
+
python kg_link_db.py # 扫 data/code/ 全量
|
|
25
|
+
python kg_link_db.py --rebuild # 清旧重建
|
|
26
|
+
python kg_link_db.py --stats # 看统计
|
|
27
|
+
"""
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
import sys
|
|
31
|
+
import time
|
|
32
|
+
|
|
33
|
+
SCRIPTS = os.path.dirname(os.path.abspath(__file__))
|
|
34
|
+
sys.path.insert(0, SCRIPTS)
|
|
35
|
+
|
|
36
|
+
import duckdb
|
|
37
|
+
from common.paths import DATA_INDEX_DIR, PROJECT_ROOT
|
|
38
|
+
|
|
39
|
+
CODE_DIR = str(PROJECT_ROOT / 'data' / 'code')
|
|
40
|
+
DB_PATH = str(DATA_INDEX_DIR / 'kg_db.duckdb')
|
|
41
|
+
|
|
42
|
+
# @TableName 的两种格式:
|
|
43
|
+
# @TableName("quality_case")
|
|
44
|
+
# @TableName(value = "quality_case")
|
|
45
|
+
RE_TABLENAME = re.compile(
|
|
46
|
+
r'@TableName\s*\(\s*(?:value\s*=\s*)?"([^"]+)"',
|
|
47
|
+
re.IGNORECASE
|
|
48
|
+
)
|
|
49
|
+
# 紧跟注解的 class 声明: class QualityCaseBo
|
|
50
|
+
RE_CLASS = re.compile(r'class\s+(\w+)')
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _normalize_table_name(raw):
|
|
54
|
+
"""归一化表名, 提升和数据库真实表的匹配率。
|
|
55
|
+
处理注解写法不规范的情况:
|
|
56
|
+
fywl_ics_cloud.sys_dept → sys_dept (去库名前缀)
|
|
57
|
+
empContractImport → emp_contract_import (camelCase→snake_case)
|
|
58
|
+
quality_case → quality_case (已是规范, 不变)
|
|
59
|
+
"""
|
|
60
|
+
t = raw.strip().lower()
|
|
61
|
+
# 去库名前缀 (db.table → table)
|
|
62
|
+
if '.' in t:
|
|
63
|
+
t = t.rsplit('.', 1)[-1]
|
|
64
|
+
# 去常见前缀 t_/tb_/t (数据库表常加这些, 注解有时不加)
|
|
65
|
+
# 注意: 不在这里去前缀, 因为 db_tables 里有的带 t_ 有的不带, 留原样让 JOIN 双向兜
|
|
66
|
+
# camelCase → snake_case (Java 注解里有时写驼峰表名)
|
|
67
|
+
if re.search(r'[a-z][A-Z]', t):
|
|
68
|
+
t = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', t).lower()
|
|
69
|
+
return t
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _scan_java_files():
|
|
73
|
+
"""扫 data/code/ 下所有 .java, 返回 [(entity_class, table_name, normalized_table, source_file, annotation)]。"""
|
|
74
|
+
mappings = []
|
|
75
|
+
scanned = 0
|
|
76
|
+
for root, dirs, files in os.walk(CODE_DIR):
|
|
77
|
+
# 跳过非业务目录
|
|
78
|
+
dirs[:] = [d for d in dirs if d not in (
|
|
79
|
+
'.git', 'node_modules', '__pycache__', 'target', 'test')]
|
|
80
|
+
for fn in files:
|
|
81
|
+
if not fn.endswith('.java'):
|
|
82
|
+
continue
|
|
83
|
+
scanned += 1
|
|
84
|
+
fpath = os.path.join(root, fn)
|
|
85
|
+
try:
|
|
86
|
+
for enc in ('utf-8', 'gbk'):
|
|
87
|
+
try:
|
|
88
|
+
with open(fpath, encoding=enc) as f:
|
|
89
|
+
content = f.read()
|
|
90
|
+
break
|
|
91
|
+
except (UnicodeDecodeError, OSError):
|
|
92
|
+
content = ''
|
|
93
|
+
continue
|
|
94
|
+
except OSError:
|
|
95
|
+
continue
|
|
96
|
+
if '@TableName' not in content:
|
|
97
|
+
continue
|
|
98
|
+
# 找所有 @TableName 注解
|
|
99
|
+
for m in RE_TABLENAME.finditer(content):
|
|
100
|
+
table_name = m.group(1)
|
|
101
|
+
normalized = _normalize_table_name(table_name)
|
|
102
|
+
annotation = content[m.start():m.end() + 1]
|
|
103
|
+
# 找注解之后最近的 class 声明 (Entity 类名)
|
|
104
|
+
after = content[m.end():m.end() + 500]
|
|
105
|
+
cm = RE_CLASS.search(after)
|
|
106
|
+
entity_class = cm.group(1) if cm else os.path.splitext(fn)[0]
|
|
107
|
+
rel = os.path.relpath(fpath, PROJECT_ROOT).replace('\\', '/')
|
|
108
|
+
mappings.append((entity_class, table_name, normalized, rel, annotation))
|
|
109
|
+
return mappings, scanned
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build(rebuild=False):
|
|
113
|
+
"""扫代码 @TableName → 写 entity_table_map。"""
|
|
114
|
+
t0 = time.time()
|
|
115
|
+
print('扫描 data/code/ 下的 @TableName 注解...')
|
|
116
|
+
mappings, scanned = _scan_java_files()
|
|
117
|
+
print(' 扫描 %d 个 .java 文件, 找到 %d 个 @TableName 映射 (%.1fs)'
|
|
118
|
+
% (scanned, len(mappings), time.time() - t0))
|
|
119
|
+
|
|
120
|
+
if not mappings:
|
|
121
|
+
print('未找到任何 @TableName 注解。检查 data/code/ 是否有 Java 源码。')
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
con = duckdb.connect(DB_PATH)
|
|
125
|
+
try:
|
|
126
|
+
con.execute("""CREATE TABLE IF NOT EXISTS entity_table_map (
|
|
127
|
+
entity_class TEXT,
|
|
128
|
+
table_name TEXT,
|
|
129
|
+
normalized_table TEXT,
|
|
130
|
+
source_file TEXT,
|
|
131
|
+
annotation TEXT
|
|
132
|
+
)""")
|
|
133
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_etm_table ON entity_table_map(table_name)")
|
|
134
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_etm_norm ON entity_table_map(normalized_table)")
|
|
135
|
+
con.execute("CREATE INDEX IF NOT EXISTS idx_etm_class ON entity_table_map(entity_class)")
|
|
136
|
+
|
|
137
|
+
if rebuild:
|
|
138
|
+
con.execute("DELETE FROM entity_table_map")
|
|
139
|
+
|
|
140
|
+
# 用 JSONL COPY 批量插入 (DuckDB 1.4 executemany 慢)
|
|
141
|
+
import json as _json
|
|
142
|
+
import tempfile
|
|
143
|
+
tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False,
|
|
144
|
+
mode='w', encoding='utf-8')
|
|
145
|
+
cols = ['entity_class', 'table_name', 'normalized_table', 'source_file', 'annotation']
|
|
146
|
+
for row in mappings:
|
|
147
|
+
obj = {cols[i]: (row[i] if row[i] is not None else '')
|
|
148
|
+
for i in range(len(cols))}
|
|
149
|
+
tmp.write(_json.dumps(obj, ensure_ascii=False) + '\n')
|
|
150
|
+
tmp.close()
|
|
151
|
+
try:
|
|
152
|
+
con.execute("COPY entity_table_map FROM '%s' (FORMAT JSON)"
|
|
153
|
+
% tmp.name.replace('\\', '/'))
|
|
154
|
+
finally:
|
|
155
|
+
os.unlink(tmp.name)
|
|
156
|
+
|
|
157
|
+
con.execute("INSERT OR REPLACE INTO build_meta(key, value) VALUES (?,?)",
|
|
158
|
+
['entity_table_map_built', str(time.time())])
|
|
159
|
+
print('写入 %d 条 Entity→表 映射到 entity_table_map' % len(mappings))
|
|
160
|
+
return len(mappings)
|
|
161
|
+
finally:
|
|
162
|
+
con.close()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def query_impact(table_name):
|
|
166
|
+
"""查某张表关联的 Entity 类 (给"改表影响分析"用)。"""
|
|
167
|
+
con = duckdb.connect(DB_PATH, read_only=True)
|
|
168
|
+
try:
|
|
169
|
+
rows = con.execute(
|
|
170
|
+
"SELECT DISTINCT entity_class, source_file FROM entity_table_map "
|
|
171
|
+
"WHERE table_name = ?", [table_name]
|
|
172
|
+
).fetchall()
|
|
173
|
+
return rows
|
|
174
|
+
finally:
|
|
175
|
+
con.close()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def stats():
|
|
179
|
+
"""映射统计 + 和 db_tables/db_columns 的关联率。"""
|
|
180
|
+
con = duckdb.connect(DB_PATH, read_only=True)
|
|
181
|
+
try:
|
|
182
|
+
total = con.execute("SELECT COUNT(*) FROM entity_table_map").fetchone()[0]
|
|
183
|
+
classes = con.execute("SELECT COUNT(DISTINCT entity_class) FROM entity_table_map").fetchone()[0]
|
|
184
|
+
tables_mapped = con.execute("SELECT COUNT(DISTINCT table_name) FROM entity_table_map").fetchone()[0]
|
|
185
|
+
# 关联率: 映射的表名 vs 数据库真实表名 (原始 + 归一化)
|
|
186
|
+
try:
|
|
187
|
+
raw_matched = con.execute("""
|
|
188
|
+
SELECT COUNT(DISTINCT m.table_name) FROM entity_table_map m
|
|
189
|
+
JOIN db_tables t ON m.table_name = t.table_name
|
|
190
|
+
""").fetchone()[0]
|
|
191
|
+
norm_matched = con.execute("""
|
|
192
|
+
SELECT COUNT(DISTINCT m.normalized_table) FROM entity_table_map m
|
|
193
|
+
JOIN db_tables t ON m.normalized_table = t.table_name
|
|
194
|
+
""").fetchone()[0]
|
|
195
|
+
db_total = con.execute("SELECT COUNT(DISTINCT table_name) FROM db_tables").fetchone()[0]
|
|
196
|
+
except Exception:
|
|
197
|
+
raw_matched, norm_matched, db_total = 0, 0, 0
|
|
198
|
+
print('entity_table_map 统计:')
|
|
199
|
+
print(' 总映射: %d' % total)
|
|
200
|
+
print(' Entity 类数: %d' % classes)
|
|
201
|
+
print(' 映射的表名数: %d' % tables_mapped)
|
|
202
|
+
if db_total:
|
|
203
|
+
print(' 原始表名匹配: %d/%d (%.0f%%)'
|
|
204
|
+
% (raw_matched, db_total, raw_matched / db_total * 100))
|
|
205
|
+
print(' 归一化后匹配: %d/%d (%.0f%%)'
|
|
206
|
+
% (norm_matched, db_total, norm_matched / db_total * 100))
|
|
207
|
+
# 抽样
|
|
208
|
+
sample = con.execute(
|
|
209
|
+
"SELECT entity_class, table_name FROM entity_table_map LIMIT 8"
|
|
210
|
+
).fetchall()
|
|
211
|
+
print(' 抽样:')
|
|
212
|
+
for ec, tn in sample:
|
|
213
|
+
print(' %s → %s' % (ec, tn))
|
|
214
|
+
finally:
|
|
215
|
+
con.close()
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
if __name__ == '__main__':
|
|
219
|
+
args = sys.argv[1:]
|
|
220
|
+
if '--stats' in args:
|
|
221
|
+
stats()
|
|
222
|
+
else:
|
|
223
|
+
build(rebuild='--rebuild' in args)
|
|
224
|
+
stats()
|