@hupan56/wlkj 3.3.13 → 3.3.14

package/templates/qoder/scripts/domain/kg/graph/kg_link_db.py

@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+
+"""kg_link_db.py - 扫代码 @TableName 注解, 建 Entity 类 ↔ 数据库表 的语义桥。
+
+解决"数据层孤岛"问题: kg_db.duckdb 有表结构, 但不知道对应代码里的哪个 Entity。
+本脚本扫 Java 源码里的 @TableName("xxx") 注解, 提取:
+  Entity 类名 → 数据库表名 → (关联到) db_columns 的真实列
+
+调研依据 (FalkorDB "结构遍历 > 语义匹配"):
+  - 不靠表名子串猜 endpoint (验证过 0 命中, 不准)
+  - 靠代码显式声明的关系 (@TableName 注解) 建边, 这才准
+
+产出表 (写进 kg_db.duckdb, 不动 kg.duckdb 写锁):
+  entity_table_map:
+    entity_class  TEXT   -- QualityCaseBo (代码层类名)
+    table_name    TEXT   -- quality_case (数据库表名, 关联 db_tables)
+    source_file   TEXT   -- 哪个 .java 读到的
+    annotation    TEXT   -- @TableName("quality_case") 原文
+
+建好后可遍历:
+  数据库表 → entity_table_map → Entity 类 → (kg.duckdb) Controller/接口
+
+用法:
+  python kg_link_db.py              # 扫 data/code/ 全量
+  python kg_link_db.py --rebuild    # 清旧重建
+  python kg_link_db.py --stats      # 看统计
+"""
+import os
+import re
+import sys
+import time
+
+SCRIPTS = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))  # 子包→scripts/根
+sys.path.insert(0, SCRIPTS)
+
+import duckdb
+from foundation.core.paths import DATA_INDEX_DIR, PROJECT_ROOT
+
+CODE_DIR = str(PROJECT_ROOT / 'data' / 'code')
+DB_PATH = str(DATA_INDEX_DIR / 'kg_db.duckdb')
+
+# @TableName 的两种格式:
+#   @TableName("quality_case")
+#   @TableName(value = "quality_case")
+RE_TABLENAME = re.compile(
+    r'@TableName\s*\(\s*(?:value\s*=\s*)?"([^"]+)"',
+    re.IGNORECASE
+)
+# 紧跟注解的 class 声明: class QualityCaseBo
+RE_CLASS = re.compile(r'class\s+(\w+)')
+
+
+def _normalize_table_name(raw):
+    """归一化表名, 提升和数据库真实表的匹配率。
+    处理注解写法不规范的情况:
+      fywl_ics_cloud.sys_dept → sys_dept (去库名前缀)
+      empContractImport → emp_contract_import (camelCase→snake_case)
+      quality_case → quality_case (已是规范, 不变)
+    """
+    t = raw.strip().lower()
+    # 去库名前缀 (db.table → table)
+    if '.' in t:
+        t = t.rsplit('.', 1)[-1]
+    # 去常见前缀 t_/tb_/t (数据库表常加这些, 注解有时不加)
+    # 注意: 不在这里去前缀, 因为 db_tables 里有的带 t_ 有的不带, 留原样让 JOIN 双向兜
+    # camelCase → snake_case (Java 注解里有时写驼峰表名)
+    if re.search(r'[a-z][A-Z]', t):
+        t = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', t).lower()
+    return t
+
+
+def _scan_java_files():
+    """扫 data/code/ 下所有 .java, 返回 [(entity_class, table_name, normalized_table, source_file, annotation)]。"""
+    mappings = []
+    scanned = 0
+    for root, dirs, files in os.walk(CODE_DIR):
+        # 跳过非业务目录
+        dirs[:] = [d for d in dirs if d not in (
+            '.git', 'node_modules', '__pycache__', 'target', 'test')]
+        for fn in files:
+            if not fn.endswith('.java'):
+                continue
+            scanned += 1
+            fpath = os.path.join(root, fn)
+            try:
+                for enc in ('utf-8', 'gbk'):
+                    try:
+                        with open(fpath, encoding=enc) as f:
+                            content = f.read()
+                        break
+                    except (UnicodeDecodeError, OSError):
+                        content = ''
+                        continue
+            except OSError:
+                continue
+            if '@TableName' not in content:
+                continue
+            # 找所有 @TableName 注解
+            for m in RE_TABLENAME.finditer(content):
+                table_name = m.group(1)
+                normalized = _normalize_table_name(table_name)
+                annotation = content[m.start():m.end() + 1]
+                # 找注解之后最近的 class 声明 (Entity 类名)
+                after = content[m.end():m.end() + 500]
+                cm = RE_CLASS.search(after)
+                entity_class = cm.group(1) if cm else os.path.splitext(fn)[0]
+                rel = os.path.relpath(fpath, PROJECT_ROOT).replace('\\', '/')
+                mappings.append((entity_class, table_name, normalized, rel, annotation))
+    return mappings, scanned
+
+
+def build(rebuild=False):
+    """扫代码 @TableName → 写 entity_table_map。"""
+    t0 = time.time()
+    print('扫描 data/code/ 下的 @TableName 注解...')
+    mappings, scanned = _scan_java_files()
+    print('  扫描 %d 个 .java 文件, 找到 %d 个 @TableName 映射 (%.1fs)'
+          % (scanned, len(mappings), time.time() - t0))
+
+    if not mappings:
+        print('未找到任何 @TableName 注解。检查 data/code/ 是否有 Java 源码。')
+        return 0
+
+    con = duckdb.connect(DB_PATH)
+    try:
+        con.execute("""CREATE TABLE IF NOT EXISTS entity_table_map (
+            entity_class TEXT,
+            table_name TEXT,
+            normalized_table TEXT,
+            source_file TEXT,
+            annotation TEXT
+        )""")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_table ON entity_table_map(table_name)")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_norm ON entity_table_map(normalized_table)")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_class ON entity_table_map(entity_class)")
+
+        if rebuild:
+            con.execute("DELETE FROM entity_table_map")
+
+        # 用 JSONL COPY 批量插入 (DuckDB 1.4 executemany 慢)
+        import json as _json
+        import tempfile
+        tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False,
+                                          mode='w', encoding='utf-8')
+        cols = ['entity_class', 'table_name', 'normalized_table', 'source_file', 'annotation']
+        for row in mappings:
+            obj = {cols[i]: (row[i] if row[i] is not None else '')
+                   for i in range(len(cols))}
+            tmp.write(_json.dumps(obj, ensure_ascii=False) + '\n')
+        tmp.close()
+        try:
+            con.execute("COPY entity_table_map FROM '%s' (FORMAT JSON)"
+                        % tmp.name.replace('\\', '/'))
+        finally:
+            os.unlink(tmp.name)
+
+        con.execute("INSERT OR REPLACE INTO build_meta(key, value) VALUES (?,?)",
+                    ['entity_table_map_built', str(time.time())])
+        print('写入 %d 条 Entity→表 映射到 entity_table_map' % len(mappings))
+        return len(mappings)
+    finally:
+        con.close()
+
+
+def query_impact(table_name):
+    """查某张表关联的 Entity 类 (给"改表影响分析"用)。"""
+    con = duckdb.connect(DB_PATH, read_only=True)
+    try:
+        rows = con.execute(
+            "SELECT DISTINCT entity_class, source_file FROM entity_table_map "
+            "WHERE table_name = ?", [table_name]
+        ).fetchall()
+        return rows
+    finally:
+        con.close()
+
+
+def stats():
+    """映射统计 + 和 db_tables/db_columns 的关联率。"""
+    con = duckdb.connect(DB_PATH, read_only=True)
+    try:
+        total = con.execute("SELECT COUNT(*) FROM entity_table_map").fetchone()[0]
+        classes = con.execute("SELECT COUNT(DISTINCT entity_class) FROM entity_table_map").fetchone()[0]
+        tables_mapped = con.execute("SELECT COUNT(DISTINCT table_name) FROM entity_table_map").fetchone()[0]
+        # 关联率: 映射的表名 vs 数据库真实表名 (原始 + 归一化)
+        try:
+            raw_matched = con.execute("""
+                SELECT COUNT(DISTINCT m.table_name) FROM entity_table_map m
+                JOIN db_tables t ON m.table_name = t.table_name
+            """).fetchone()[0]
+            norm_matched = con.execute("""
+                SELECT COUNT(DISTINCT m.normalized_table) FROM entity_table_map m
+                JOIN db_tables t ON m.normalized_table = t.table_name
+            """).fetchone()[0]
+            db_total = con.execute("SELECT COUNT(DISTINCT table_name) FROM db_tables").fetchone()[0]
+        except Exception:
+            raw_matched, norm_matched, db_total = 0, 0, 0
+        print('entity_table_map 统计:')
+        print('  总映射: %d' % total)
+        print('  Entity 类数: %d' % classes)
+        print('  映射的表名数: %d' % tables_mapped)
+        if db_total:
+            print('  原始表名匹配: %d/%d (%.0f%%)'
+                  % (raw_matched, db_total, raw_matched / db_total * 100))
+            print('  归一化后匹配: %d/%d (%.0f%%)'
+                  % (norm_matched, db_total, norm_matched / db_total * 100))
+        # 抽样
+        sample = con.execute(
+            "SELECT entity_class, table_name FROM entity_table_map LIMIT 8"
+        ).fetchall()
+        print('  抽样:')
+        for ec, tn in sample:
+            print('    %s → %s' % (ec, tn))
+    finally:
+        con.close()
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    if '--stats' in args:
+        stats()
+    else:
+        build(rebuild='--rebuild' in args)
+        stats()

package/templates/qoder/scripts/domain/kg/graph/kg_semantic.py

@@ -278,31 +278,6 @@ def build_embeddings(batch_size=64):
         con.close()
 
 
-def _cloud_rag_search(query, top_k=10):
-    """云优先语义搜索：调工作台 MCP rag_search（本地无 KG/embedding，全迁云）。
-
-    Returns:
-        [(entity_id, text, score), ...] 或 None（云不可用时返回 None，让调用方退本地）。
-    """
-    try:
-        from domain.kg.search._remote import call_remote
-        data = call_remote("rag_search", {"query": query, "top_k": top_k})
-    except Exception:
-        return None
-    items = data.get("items") or []
-    out = []
-    for it in items[:top_k]:
-        eid = it.get("entity_id") or it.get("id") or ""
-        text = it.get("text") or it.get("name") or ""
-        score = it.get("score")
-        try:
-            score = float(score) if score is not None else 0.0
-        except Exception:
-            score = 0.0
-        out.append((eid, text, score))
-    return out
-
-
 def semantic_search(query, top_k=10):
     """语义搜索: query→向量→余弦相似度→top_k 实体。
 
@@ -681,24 +656,13 @@ if __name__ == '__main__':
         n = build_embeddings()
         print('Built %d embeddings' % n)
 
-    elif (len(sys.argv) > 2 and sys.argv[1] == 'search') or \
-         (len(sys.argv) > 1 and sys.argv[1] not in ('--check', '--label', 'build-embeddings', '--status', '--backfill-cn')):
-        # 兼容两种姿势: `search <词>` 和裸词 `<词>`（wlkj.py semantic 保险异常 透传过来）。
-        # ★ 云优先：本地无 KG/embedding（全迁云），先调云 rag_search；云不可用才退本地 semantic_search。
-        query = sys.argv[2] if sys.argv[1] == 'search' else sys.argv[1]
-        results = _cloud_rag_search(query)
-        if results is None:
-            # 云不可用 → 退本地（开发期有本地 embedding 时仍可用）
-            results = semantic_search(query)
+    elif len(sys.argv) > 2 and sys.argv[1] == 'search':
+        results = semantic_search(sys.argv[2])
         if results:
-            for row in results:
-                if len(row) == 3:
-                    eid, text, score = row
-                    print('  %.3f  %s' % (score, text[:70]))
-                else:
-                    print('  %s' % row)
+            for eid, text, score in results:
+                print('  %.3f  %s' % (score, text[:70]))
         else:
-            print('(无结果或云 rag_search 不可用 + 本地 embedding 未构建)')
+            print('(无结果或 embedding 未构建)')
 
     elif len(sys.argv) > 1 and sys.argv[1] == '--status':
         st = embedding_status()

package/templates/qoder/scripts/domain/kg/kg.py

@@ -872,20 +872,6 @@ def main(argv=None):
         return 0
     cmd = argv[0]
     rest = argv[1:]
-    # ★ MCP 名兼容：AI 常把 MCP 工具名(search_code)当 kg.py 子命令调 → "未知子命令"。
-    # 在此统一归一化，AI 猜哪个名（kg.py 子命令 / MCP 工具名）都能命中（治工作流混名）。
-    _MCP_ALIASES = {
-        'search_code': 'search', 'search_api': 'api', 'search_prds': 'prd',
-        'search_field': 'search', 'search_style': 'search', 'search_wiki': 'wiki',
-        'rag_search': 'semantic', 'ask_corpus': 'semantic',
-        'context_pack': 'context', 'context_360': 'context360',
-        'get_impact': 'impact', 'multi_hop': 'hop',
-        'coverage_matrix': 'coverage', 'feature_overview': 'feature',
-        'get_workflow': 'workflow', 'get_design_system': 'design-system',
-        'fill_prototype': 'fill-prototype',
-    }
-    if cmd in _MCP_ALIASES:
-        cmd = _MCP_ALIASES[cmd]
     # 查找子命令
     handler = None
     for name, fn, _ in COMMANDS:

package/templates/qoder/scripts/domain/kg/search/enrich_prompt.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+from foundation.core.paths import get_repo_root
+
+"""enrich_prompt.py - 极速提示词润色 (用户提问后第 0 步)
+
+目标: 在用户提问后, <10ms 内判断这句话命中哪些业务关键词,
+输出一个「润色后的提示词」给用户确认, 再走原命令。
+
+设计:
+  - 纯内存查表 (不跑重检索 context_pack, 那个留给确认后)
+  - 中文分词复用 common/terms.py 的 expand_chinese_query
+  - 命中 keyword-index / api-index / prd-index 的键
+  - 命中 0 个 = 非业务消息 (谢谢/继续), 直接放行不润色
+
+用法:
+  python .qoder/scripts/enrich_prompt.py "保险批改那个异常统计页加个审批按钮"
+  python .qoder/scripts/enrich_prompt.py "保险批改..." --json     # 机器可读
+
+输出 (默认人类可读):
+  ── 极速润色 ──
+  业务词: 保险→insurance, 批改→amend, 异常→abnormal...
+  相关代码符号: 108 处 (abnormalovertime, approval-cell...)
+  相关API: 5 个
+  历史 PRD: 2 篇
+  ──────────────
+  将以上下文继续。回复"改:xxx"调整, 任意其它内容=接受。
+"""
+import os, sys, json, time
+
+if sys.platform == 'win32':
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+BASE = get_repo_root()
+sys.path.insert(0, os.path.join(BASE, '.qoder', 'scripts'))
+
+IDX_DIR = os.path.join(BASE, 'data', 'index')
+# 缓存: 同一进程内只加载一次索引
+_CACHE = {}
+
+
+def _load(name):
+    """加载索引 (优先 DuckDB, 回退 JSON)。复用 search_index.load_index 的 fallback 逻辑。"""
+    if name in _CACHE:
+        return _CACHE[name]
+    try:
+        from domain.kg.search.search_index import load_index
+        d = load_index(name) or {}
+    except ImportError:
+        # search_index 不可用时, 直接读 JSON (老逻辑)
+        d = {}
+        path = os.path.join(IDX_DIR, name)
+        if os.path.isfile(path):
+            try:
+                for enc in ('utf-8', 'gbk'):
+                    try:
+                        with open(path, encoding=enc) as f:
+                            d = json.load(f)
+                        break
+                    except (OSError, UnicodeDecodeError):
+                        continue
+            except Exception:
+                pass
+    _CACHE[name] = d
+    return d
+
+
+# 短而无意义的英文 token 黑名单 (避免子串误命中 'cs'/'fl'/'in' 这类噪声)
+# 提到模块级: _match_keys 和 code_total 共用一份
+NOISE_KEYS = {
+    'app', 'api', 'all', 'add', 'age', 'and', 'are', 'ars', 'ase', 'ask',
+    'ate', 'aud', 'bar', 'big', 'btn', 'can', 'car', 'cas', 'cat', 'code',
+    'con', 'cor', 'cs', 'cur', 'dae', 'dat', 'day', 'del', 'dep', 'des',
+    'det', 'dim', 'div', 'doc', 'dos', 'ear', 'ecs', 'ema', 'end', 'ent',
+    'err', 'eva', 'exe', 'exp', 'ext', 'fil', 'fix', 'fl', 'for', 'ge',
+    'gen', 'get', 'has', 'her', 'his', 'ics', 'idx', 'imp', 'int', 'inv',
+    'io', 'is', 'iso', 'ite', 'its', 'key', 'lab', 'lan', 'lat', 'let',
+    'lin', 'lis', 'log', 'ma', 'map', 'max', 'min', 'mod', 'msg', 'new',
+    'nil', 'not', 'now', 'num', 'obj', 'off', 'one', 'opt', 'ord', 'out',
+    'ove', 'own', 'pag', 'par', 'pas', 'pay', 'pen', 'per', 'pos', 'pre',
+    'pro', 'put', 'que', 'rec', 'red', 'ref', 'reg', 'rep', 'res', 'ret',
+    'rev', 'row', 'run', 'sa', 'sel', 'set', 'sev', 'sie', 'sit', 'sor',
+    'src', 'sta', 'sub', 'sum', 'sys', 'tab', 'tar', 'tes', 'tex', 'tim',
+    'tmp', 'tok', 'tom', 'top', 'tot', 'tru', 'try', 'typ', 'udp', 'uni',
+    'upd', 'url', 'use', 'usr', 'val', 'ver', 'vid', 'vio', 'vis', 'voi',
+    'was', 'way', 'who', 'why', 'win', 'wri', 'xml', 'you', 'your', 'zone',
+}
+
+
+def _match_keys(query_lower, keys, max_hits=15, min_len=3):
+    """在 keys 里找被 query 包含的 (子串匹配), 返回命中列表 + 总数。
+    min_len: 短于这个长度的键忽略 (避免 'cs'/'fl'/'in' 这类噪声子串)。"""
+    hits = []
+    for k in keys:
+        kl = k.lower()
+        if len(kl) < min_len:
+            continue
+        if kl in NOISE_KEYS:
+            continue
+        if kl in query_lower:
+            hits.append(k)
+            if len(hits) >= max_hits:
+                break
+    return hits
+    for k in keys:
+        kl = k.lower()
+        if len(kl) < min_len:
+            continue
+        if kl in NOISE:
+            continue
+        if kl in query_lower:
+            hits.append(k)
+            if len(hits) >= max_hits:
+                break
+    return hits
+
+
+def enrich(query):
+    """润色主函数。返回 dict:
+       {
+         'is_business': bool,        # 是否业务消息 (有命中才算)
+         'cn_words': [...],          # 中文分词出的英文搜索词
+         'code_hits': [...],         # 命中的代码符号 (样例)
+         'code_total': int,          # 代码命中总数
+         'api_hits': [...],          # 命中的 API
+         'prd_hits': [...],          # 命中的历史 PRD
+         'elapsed_ms': float,        # 耗时
+         'polished': str,            # 润色后的提示词 (人类可读)
+       }
+    """
+    t0 = time.perf_counter()
+    from foundation.integrations.terms import expand_chinese_query
+
+    # 1. 中文分词 → 英文搜索词
+    cn_words = list(dict.fromkeys(expand_chinese_query(query)))  # 去重保序
+
+    # 2. 构造匹配串: 原句 + 扩展出的英文词, 一起拿去命中索引
+    match_str = query.lower()
+    for w in cn_words:
+        match_str += ' ' + w.lower()
+
+    # 3. 命中三类索引
+    code_idx = _load('code-keyword.json')
+    code_keys = list(code_idx.keys()) if isinstance(code_idx, dict) else []
+    code_hits = _match_keys(match_str, code_keys)
+
+    api_idx = _load('code-api.json')
+    api_keys = []
+    if isinstance(api_idx, dict):
+        api_keys = list(api_idx.keys())
+    api_hits = _match_keys(match_str, api_keys)
+
+    prd_idx = _load('prd-index.json')
+    prd_keys = []
+    if isinstance(prd_idx, dict):
+        prd_keys = list(prd_idx.keys())
+    prd_hits = _match_keys(match_str, prd_keys, max_hits=5)
+
+    # 4. 代码命中总数 (不限 max, 用于展示规模; 用与 _match_keys 一致的过滤)
+    code_total = 0
+    if code_keys:
+        code_total = sum(1 for k in code_keys
+                         if len(k) >= 3 and k.lower() not in NOISE_KEYS
+                         and k.lower() in match_str)
+
+    elapsed_ms = (time.perf_counter() - t0) * 1000
+    is_business = bool(code_hits or api_hits or prd_hits or cn_words)
+
+    # 5. 组装润色后提示词 (人类可读)
+    lines = []
+    if not is_business:
+        lines.append('（非业务消息，无需润色，直接放行）')
+    else:
+        if cn_words:
+            shown = ', '.join(cn_words[:8])
+            lines.append('业务词: ' + shown)
+        if code_total:
+            sample = ', '.join(code_hits[:8])
+            more = '' if code_total <= len(code_hits) else ' ...'
+            lines.append('相关代码符号: %d 处 (%s%s)' % (code_total, sample, more))
+        if api_hits:
+            lines.append('相关 API: %d 个 (%s)' % (len(api_hits), ', '.join(api_hits[:5])))
+        if prd_hits:
+            lines.append('历史 PRD: %s' % ', '.join(prd_hits[:5]))
+
+    polished = '\n'.join(lines)
+
+    return {
+        'is_business': is_business,
+        'cn_words': cn_words,
+        'code_hits': code_hits,
+        'code_total': code_total,
+        'api_hits': api_hits,
+        'prd_hits': prd_hits,
+        'elapsed_ms': round(elapsed_ms, 2),
+        'polished': polished,
+    }
+
+
+def main():
+    args = sys.argv[1:]
+    if not args:
+        print(__doc__)
+        print('错误: 缺少查询参数。用法: enrich_prompt.py "你的问题"')
+        sys.exit(2)
+    query = ' '.join(args[:1]) if args[0] in ('--json',) else ' '.join(args)
+    # 处理 --json 标志
+    as_json = '--json' in args
+    query = query.replace('--json', '').strip()
+
+    result = enrich(query)
+    if as_json:
+        print(json.dumps(result, ensure_ascii=False))
+        return
+
+    # 人类可读输出
+    print('── 极速润色 (%.1fms) ──' % result['elapsed_ms'])
+    print(result['polished'])
+    if result['is_business']:
+        print('──────────────')
+        print('将以上下文继续。回复"改:xxx"调整，任意其它内容=接受。')
+
+
+if __name__ == '__main__':
+    main()