@hupan56/wlkj 2.7.12 → 3.1.2

package/templates/qoder/scripts/domain/kg/server/perf_bench.py

@@ -0,0 +1,197 @@
+# -*- coding: utf-8 -*-
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+from foundation.core.paths import get_repo_root
+
+"""
+perf_bench.py — 知识图谱关键路径性能基准
+
+诊断"用了工作流反而更慢"的根因, 并验证常驻化(daemon)加速层的效果。
+
+两种测量模式:
+  1. 冷启动 (subprocess 跑 kg.py) — 模拟真实用户体验: 起进程→import→连库→跑
+  2. daemon 热路径 (kgd_client)   — daemon 常驻时的 IPC 调用, 模型/连接已就绪
+
+用法:
+  python perf_bench.py                # 默认: 测冷启动 (--no-daemon)
+  python perf_bench.py --with-daemon  # 同时测 daemon 热路径 (需先 kgd.py start)
+  python perf_bench.py --quick        # 跳过 semantic (26s), 只测轻量路径
+"""
+import io
+import os
+import subprocess
+import sys
+import time
+
+# UTF-8 输出 (Windows 终端默认 GBK 会乱码)
+try:
+    sys.stdout.reconfigure(encoding='utf-8', errors='replace')
+except Exception:
+    pass
+
+KG_PY = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'kg.py')
+REPO_ROOT = get_repo_root()
+
+# ── 测试用例 (覆盖四层) ──────────────────────────────────────────────
+# (名称, kg子命令, 参数列表, 预期层)
+CASES = [
+    ('search-考勤(web)',    'search',    ['考勤', '--platform', 'web'],  '结构层'),
+    ('search-车辆(web)',    'search',    ['车辆', '--platform', 'web'],  '结构层'),
+    ('search-报销(app)',    'search',    ['报销', '--platform', 'app'],  '结构层'),
+    ('api-asset',           'api',       ['asset'],                      '结构层'),
+    ('context_pack-车辆',   'context',   ['车辆', '--platform', 'web'],  '①③+设计'),
+    ('impact-/asset',       'impact',    ['/asset'],                     '图谱层'),
+    ('feature-资产管理',    'feature',   ['资产管理'],                    '图谱层'),
+    ('semantic-报销',       'semantic',  ['报销'],                       '向量层'),  # 26s 慢王
+]
+
+
+def _run_kg(args, use_daemon=True, timeout=120):
+    """跑一次 kg.py 子命令, 返回 (耗时秒, 返回码)。
+
+    use_daemon=True  : 走默认 (kg.py 内部会优先 daemon)
+    use_daemon=False : 加 --no-daemon 强制冷路径
+    """
+    cmd = [sys.executable, KG_PY] + args
+    if not use_daemon:
+        cmd.append('--no-daemon')
+    env = dict(os.environ)
+    env['PYTHONIOENCODING'] = 'utf-8'
+    t0 = time.time()
+    try:
+        # stdout 重定向到 DEVNULL, 只测耗时, 不看内容
+        r = subprocess.run(cmd, stdout=subprocess.DEVNULL,
+                           stderr=subprocess.DEVNULL, cwd=REPO_ROOT,
+                           env=env, timeout=timeout)
+        return time.time() - t0, r.returncode
+    except subprocess.TimeoutExpired:
+        return float(timeout), -1
+
+
+def _median(samples):
+    """取中位数 (抗抖动)。"""
+    s = sorted(samples)
+    n = len(s)
+    return s[n // 2] if n % 2 else (s[n // 2 - 1] + s[n // 2]) / 2.0
+
+
+def bench_case(name, cmd, args, cold=True, daemon=False, repeats=3):
+    """跑单个用例, 返回 (中位数耗时, 状态)。
+
+    cold=True  : 测冷启动 (--no-daemon)
+    daemon=True: 测 daemon 热路径 (不加 --no-daemon, 但需 daemon 在线)
+    """
+    if cold and daemon:
+        raise ValueError('cold 和 daemon 互斥')
+    # 重量级命令 (semantic 冷启动 ~27s) 只跑 1 次, 避免基准脚本本身过慢
+    if cmd == 'semantic' and cold:
+        repeats = 1
+    use_daemon = daemon
+    samples = []
+    status = 'ok'
+    for _ in range(repeats):
+        elapsed, rc = _run_kg([cmd] + args, use_daemon=use_daemon)
+        samples.append(elapsed)
+        if rc != 0:
+            status = 'rc=%d' % rc
+    return _median(samples), status
+
+
+def daemon_online():
+    """探测 daemon 是否在线 (通过 kg.py status, 1s 超时)。"""
+    try:
+        from kgd import kgd_client
+        return kgd_client.ping(timeout=1.0)
+    except Exception:
+        return False
+
+
+def fmt_speedup(cold_s, hot_s):
+    """格式化加速比。"""
+    if hot_s <= 0.001:
+        return '-'
+    return '%.0fx' % (cold_s / hot_s)
+
+
+def main():
+    args = sys.argv[1:]
+    quick = '--quick' in args
+    want_daemon = '--with-daemon' in args
+
+    print('=' * 72)
+    print('知识图谱性能基准  (repo: %s)' % os.path.basename(REPO_ROOT))
+    print('=' * 72)
+
+    if quick:
+        print('[quick 模式] 跳过 semantic (向量层 26s), 只测轻量路径\n')
+        cases = [c for c in CASES if c[1] != 'semantic']
+    else:
+        cases = CASES
+
+    cold_results = []  # [(name, secs, status, layer)]
+    hot_results = {}   # name -> (secs, status)
+
+    # ── 1. 冷启动基线 (真实用户体验: 每次起一个进程) ──
+    print('[1/2] 冷启动基线 (--no-daemon, 模拟现状) ...')
+    for name, cmd, cargs, layer in cases:
+        secs, status = bench_case(name, cmd, cargs, cold=True)
+        cold_results.append((name, secs, status, layer))
+        marker = '  ' if status == 'ok' else '! '
+        print('  %s%-22s %6.2fs  [%s] %s' % (marker, name, secs, layer, status))
+    print()
+
+    # ── 2. daemon 热路径 (常驻加速层) ──
+    if want_daemon:
+        online = daemon_online()
+        if not online:
+            print('[2/2] daemon 未在线 — 跳过热路径测试。')
+            print('      (先启动: python .qoder/scripts/domain/kg/server/kgd.py start)')
+        else:
+            print('[2/2] daemon 在线, 测热路径 ...')
+            for name, cmd, cargs, layer in cases:
+                secs, status = bench_case(name, cmd, cargs, cold=False, daemon=True)
+                hot_results[name] = (secs, status)
+                marker = '  ' if status == 'ok' else '! '
+                print('  %s%-22s %6.2fs  [%s] %s' % (marker, name, secs, layer, status))
+        print()
+
+    # ── 汇总对比表 ──
+    print('-' * 72)
+    print('%-22s %10s %10s %10s' % ('用例', '冷启动', 'daemon', '加速'))
+    print('%-22s %10s %10s %10s' % ('─' * 18, '─' * 8, '─' * 8, '─' * 6))
+    for name, cold_s, status, layer in cold_results:
+        hot_s = hot_results.get(name, (None, None))[0]
+        hot_str = '%.2fs' % hot_s if hot_s else '-'
+        speed = fmt_speedup(cold_s, hot_s) if hot_s else '-'
+        print('%-22s %8.2fs %10s %10s' % (name, cold_s, hot_str, speed))
+    print('-' * 72)
+
+    # ── 诊断结论 ──
+    print('\n诊断:')
+    semantic_s = next((s for n, s, _, _ in cold_results if 'semantic' in n), None)
+    if semantic_s and semantic_s > 10:
+        print('  🔴 semantic %.1fs — 模型冷加载占 99%%, 是头号慢源' % semantic_s)
+        print('     解法: kgd.py start 常驻模型 → <0.5s')
+    elif semantic_s:
+        print('  🟢 semantic %.1fs — 在可接受范围' % semantic_s)
+
+    total_cold = sum(s for _, s, _, _ in cold_results)
+    print('  冷启动累积: %.1fs (一次 PRD 流程的隐性等待)' % total_cold)
+    if hot_results:
+        total_hot = sum(hot_results[n][0] for n in hot_results)
+        print('  daemon 累积: %.1fs  → 整体提速 %.0fx' %
+              (total_hot, total_cold / max(total_hot, 0.01)))
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/templates/qoder/scripts/{kg_duckdb.py → domain/kg/storage/kg_duckdb.py}

@@ -1,4 +1,15 @@
 # -*- coding: utf-8 -*-
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+
 """
 DuckDB knowledge graph store — replaces the JSON flat files with a single
 embedded database for fast incremental updates.
@@ -21,17 +32,42 @@ import sys
 
 import duckdb
 
-SCRIPTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.qoder', 'scripts')
-sys.path.insert(0, SCRIPTS)
-from common.paths import DATA_INDEX_DIR
-
-DB_PATH = str(DATA_INDEX_DIR / 'kg.duckdb')
+# 深度 4: scripts/ → domain/ → kg/ → storage/ → kg_duckdb.py
+SCRIPTS = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))  # storage→kg→domain→scripts/根
+if SCRIPTS not in sys.path:
+    sys.path.insert(0, SCRIPTS)
+from foundation.core.paths import DATA_INDEX_DIR
+from foundation.protocol.duckdb_conn import get_db, get_db_readonly, DB_PATH  # L0 基础设施
 
 
-def get_db():
-    """Open DuckDB connection (creates file if not exists)."""
-    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
-    return duckdb.connect(DB_PATH)
+def _load_index_json(name):
+    """统一加载索引 JSON。build_style_index 把产物写到 _build_cache/,
+    旧布局/手动放的可能在 INDEX 直接目录。两个位置都查, 优先 _build_cache。
+    所有 import_*_layer 共用这个, 避免各写各的 load() 导致路径不一致。"""
+    for d in (str(DATA_INDEX_DIR / '_build_cache'), str(DATA_INDEX_DIR)):
+        p = os.path.join(d, name)
+        if os.path.isfile(p):
+            try:
+                return json.load(open(p, encoding='utf-8'))
+            except (json.JSONDecodeError, OSError):
+                pass
+    return {}
+
+
+def get_db_readonly():
+    """Open read-only DuckDB connection. Returns None if file missing.
+
+    语义检索 (semantic_search / semantic_search_files) 只读向量数据,
+    但 search_index.py 已用 read_only=True 打开了同一个 kg.duckdb。
+    DuckDB 不允许对同一文件混用 read_only 和 read_write 连接, 所以
+    只读路径必须全程用 read_only, 避免连接冲突。文件不存在则返回 None。
+    """
+    if not os.path.isfile(DB_PATH):
+        return None
+    try:
+        return duckdb.connect(DB_PATH, read_only=True)
+    except Exception:
+        return None
 
 
 def init_schema(con):
@@ -115,7 +151,7 @@ def init_schema(con):
     # 自动补列: events.py 注册的新统计列, 表里没有就加 (ALTER ... ADD COLUMN IF NOT EXISTS)
     # 这样 events.py 加埋点不用手动改表结构。
     try:
-        from common.events import event_to_sql_filter
+        from foundation.data.events import event_to_sql_filter
         existing = {r[0] for r in con.execute(
             "SELECT column_name FROM information_schema.columns "
             "WHERE table_name = 'learning_stats'"
@@ -128,6 +164,19 @@ def init_schema(con):
                 )
     except Exception:
         pass  # events.py 不可用 (循环导入等) = 退化为固定 5 列, 不影响
+
+    # 语义检索向量表 (v3.1: bge-small-zh embedding)
+    # entity_id 关联 entities.id; text 是被 embed 的文本 (canonical + cn);
+    # vec 存 FLOAT 数组 (DuckDB 用 list 类型, 运行时 cast)。
+    con.execute("""
+        CREATE TABLE IF NOT EXISTS embeddings (
+            entity_id TEXT NOT NULL,
+            text TEXT NOT NULL,
+            vec FLOAT[],
+            model TEXT DEFAULT 'bge-small-zh-v1.5'
+        )
+    """)
+
     # Indexes
     con.execute("CREATE INDEX IF NOT EXISTS idx_ent_type ON entities(type)")
     con.execute("CREATE INDEX IF NOT EXISTS idx_alias ON aliases(alias)")
@@ -245,20 +294,12 @@ def multi_hop(con, start_id, edge_types=None, max_depth=5):
 
 def bulk_import_from_json(con):
     """One-time full import from existing JSON indexes into DuckDB."""
-    INDEX = str(DATA_INDEX_DIR / '_build_cache')
-
-    def load(name):
-        p = os.path.join(INDEX, name)
-        if os.path.isfile(p):
-            return json.load(open(p, encoding='utf-8'))
-        return {}
-
     con.execute("DELETE FROM entities")
     con.execute("DELETE FROM aliases")
     con.execute("DELETE FROM edges")
 
-    er = load('entity-registry.json')
-    rel = load('relations.json')
+    er = _load_index_json('entity-registry.json')
+    rel = _load_index_json('relations.json')
     edges_data = rel.get('edges', {})
 
     entity_rows = []
@@ -330,27 +371,22 @@ def bulk_import_from_json(con):
 
 def import_search_layer(con):
     """导入搜索层: keywords/endpoints/prds/wiki → DuckDB 表。"""
-    INDEX = str(DATA_INDEX_DIR)
-    def load(name):
-        p = os.path.join(INDEX, name)
-        return json.load(open(p, encoding='utf-8')) if os.path.isfile(p) else {}
-
     # keywords (code-keyword.json: {keyword: [files]})
-    kw = load('code-keyword.json')
+    kw = _load_index_json('code-keyword.json')
     kw_rows = [(k, f) for k, files in kw.items() for f in files]
     con.execute("DELETE FROM keywords")
     if kw_rows:
         _batch_insert(con, 'keywords', kw_rows)
 
     # endpoints (code-api.json: {endpoint: controller_file})
-    api = load('code-api.json')
+    api = _load_index_json('code-api.json')
     con.execute("DELETE FROM endpoints")
     ep_rows = [(ep, f) for ep, f in api.items()]
     if ep_rows:
         _batch_insert(con, 'endpoints', ep_rows, ignore=True)
 
     # prds (prd-index.json)
-    prds = load('prd-index.json')
+    prds = _load_index_json('prd-index.json')
     con.execute("DELETE FROM prds")
     prd_rows = [(f, info.get('title', ''),
                  json.dumps(info.get('keywords', []), ensure_ascii=False),
@@ -361,7 +397,7 @@ def import_search_layer(con):
         _batch_insert(con, 'prds', prd_rows, ignore=True)
 
     # wiki (wiki-index.json)
-    wiki = load('wiki-index.json')
+    wiki = _load_index_json('wiki-index.json')
     con.execute("DELETE FROM wiki")
     wiki_rows = []
     for kw, entries in (wiki.get('index', {}) if isinstance(wiki, dict) else {}).items():
@@ -376,12 +412,7 @@ def import_search_layer(con):
 
 def import_ui_layer(con):
     """导入UI层: pages/fields/modules → DuckDB 表。"""
-    INDEX = str(DATA_INDEX_DIR)
-    def load(name):
-        p = os.path.join(INDEX, name)
-        return json.load(open(p, encoding='utf-8')) if os.path.isfile(p) else {}
-
-    si = load('ui-style.json')
+    si = _load_index_json('ui-style.json')
 
     # pages (from ui-style projects.page_examples + page_types)
     con.execute("DELETE FROM pages")
@@ -408,7 +439,7 @@ def import_ui_layer(con):
         _batch_insert(con, 'fields', field_rows)
 
     # modules (from ui-modules.json)
-    mi = load('ui-modules.json')
+    mi = _load_index_json('ui-modules.json')
     con.execute("DELETE FROM modules")
     mod_rows = [(m['dir'], m['cn'], proj, m['path'])
                 for proj, mods in mi.get('projects', {}).items() for m in mods]
@@ -434,9 +465,7 @@ def import_ui_layer(con):
 
 def import_trace_layer(con):
     """导入trace层: traces/api_functions → DuckDB 表。"""
-    INDEX = str(DATA_INDEX_DIR)
-    tc = json.load(open(os.path.join(INDEX, 'trace-chain.json'), encoding='utf-8')) \
-        if os.path.isfile(os.path.join(INDEX, 'trace-chain.json')) else {}
+    tc = _load_index_json('trace-chain.json')
 
     con.execute("DELETE FROM traces")
     trace_rows = [(
@@ -458,9 +487,7 @@ def import_trace_layer(con):
 
 def import_test_layer(con):
     """导入test层: tests → DuckDB 表。"""
-    INDEX = str(DATA_INDEX_DIR)
-    ta = json.load(open(os.path.join(INDEX, 'test-assertions.json'), encoding='utf-8')) \
-        if os.path.isfile(os.path.join(INDEX, 'test-assertions.json')) else {}
+    ta = _load_index_json('test-assertions.json')
 
     con.execute("DELETE FROM tests")
     test_rows = [(
@@ -478,9 +505,7 @@ def import_test_layer(con):
 
 def import_workflow_layer(con):
     """导入workflow层 → DuckDB 表。"""
-    INDEX = str(DATA_INDEX_DIR)
-    wf = json.load(open(os.path.join(INDEX, 'workflow-index.json'), encoding='utf-8')) \
-        if os.path.isfile(os.path.join(INDEX, 'workflow-index.json')) else {}
+    wf = _load_index_json('workflow-index.json')
 
     con.execute("DELETE FROM workflows")
     wf_rows = []