@hupan56/wlkj 2.7.12 → 3.0.0

package/templates/qoder/scripts/{kg_link_db.py → kg/kg_link_db.py}

@@ -1,224 +1,230 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""kg_link_db.py - 扫代码 @TableName 注解, 建 Entity 类 ↔ 数据库表 的语义桥。
-
-解决"数据层孤岛"问题: kg_db.duckdb 有表结构, 但不知道对应代码里的哪个 Entity。
-本脚本扫 Java 源码里的 @TableName("xxx") 注解, 提取:
-  Entity 类名 → 数据库表名 → (关联到) db_columns 的真实列
-
-调研依据 (FalkorDB "结构遍历 > 语义匹配"):
-  - 不靠表名子串猜 endpoint (验证过 0 命中, 不准)
-  - 靠代码显式声明的关系 (@TableName 注解) 建边, 这才准
-
-产出表 (写进 kg_db.duckdb, 不动 kg.duckdb 写锁):
-  entity_table_map:
-    entity_class  TEXT   -- QualityCaseBo (代码层类名)
-    table_name    TEXT   -- quality_case (数据库表名, 关联 db_tables)
-    source_file   TEXT   -- 哪个 .java 读到的
-    annotation    TEXT   -- @TableName("quality_case") 原文
-
-建好后可遍历:
-  数据库表 → entity_table_map → Entity 类 → (kg.duckdb) Controller/接口
-
-用法:
-  python kg_link_db.py              # 扫 data/code/ 全量
-  python kg_link_db.py --rebuild    # 清旧重建
-  python kg_link_db.py --stats      # 看统计
-"""
-import os
-import re
-import sys
-import time
-
-SCRIPTS = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, SCRIPTS)
-
-import duckdb
-from common.paths import DATA_INDEX_DIR, PROJECT_ROOT
-
-CODE_DIR = str(PROJECT_ROOT / 'data' / 'code')
-DB_PATH = str(DATA_INDEX_DIR / 'kg_db.duckdb')
-
-# @TableName 的两种格式:
-#   @TableName("quality_case")
-#   @TableName(value = "quality_case")
-RE_TABLENAME = re.compile(
-    r'@TableName\s*\(\s*(?:value\s*=\s*)?"([^"]+)"',
-    re.IGNORECASE
-)
-# 紧跟注解的 class 声明: class QualityCaseBo
-RE_CLASS = re.compile(r'class\s+(\w+)')
-
-
-def _normalize_table_name(raw):
-    """归一化表名, 提升和数据库真实表的匹配率。
-    处理注解写法不规范的情况:
-      fywl_ics_cloud.sys_dept → sys_dept (去库名前缀)
-      empContractImport → emp_contract_import (camelCase→snake_case)
-      quality_case → quality_case (已是规范, 不变)
-    """
-    t = raw.strip().lower()
-    # 去库名前缀 (db.table → table)
-    if '.' in t:
-        t = t.rsplit('.', 1)[-1]
-    # 去常见前缀 t_/tb_/t (数据库表常加这些, 注解有时不加)
-    # 注意: 不在这里去前缀, 因为 db_tables 里有的带 t_ 有的不带, 留原样让 JOIN 双向兜
-    # camelCase → snake_case (Java 注解里有时写驼峰表名)
-    if re.search(r'[a-z][A-Z]', t):
-        t = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', t).lower()
-    return t
-
-
-def _scan_java_files():
-    """扫 data/code/ 下所有 .java, 返回 [(entity_class, table_name, normalized_table, source_file, annotation)]。"""
-    mappings = []
-    scanned = 0
-    for root, dirs, files in os.walk(CODE_DIR):
-        # 跳过非业务目录
-        dirs[:] = [d for d in dirs if d not in (
-            '.git', 'node_modules', '__pycache__', 'target', 'test')]
-        for fn in files:
-            if not fn.endswith('.java'):
-                continue
-            scanned += 1
-            fpath = os.path.join(root, fn)
-            try:
-                for enc in ('utf-8', 'gbk'):
-                    try:
-                        with open(fpath, encoding=enc) as f:
-                            content = f.read()
-                        break
-                    except (UnicodeDecodeError, OSError):
-                        content = ''
-                        continue
-            except OSError:
-                continue
-            if '@TableName' not in content:
-                continue
-            # 找所有 @TableName 注解
-            for m in RE_TABLENAME.finditer(content):
-                table_name = m.group(1)
-                normalized = _normalize_table_name(table_name)
-                annotation = content[m.start():m.end() + 1]
-                # 找注解之后最近的 class 声明 (Entity 类名)
-                after = content[m.end():m.end() + 500]
-                cm = RE_CLASS.search(after)
-                entity_class = cm.group(1) if cm else os.path.splitext(fn)[0]
-                rel = os.path.relpath(fpath, PROJECT_ROOT).replace('\\', '/')
-                mappings.append((entity_class, table_name, normalized, rel, annotation))
-    return mappings, scanned
-
-
-def build(rebuild=False):
-    """扫代码 @TableName → 写 entity_table_map。"""
-    t0 = time.time()
-    print('扫描 data/code/ 下的 @TableName 注解...')
-    mappings, scanned = _scan_java_files()
-    print('  扫描 %d 个 .java 文件, 找到 %d 个 @TableName 映射 (%.1fs)'
-          % (scanned, len(mappings), time.time() - t0))
-
-    if not mappings:
-        print('未找到任何 @TableName 注解。检查 data/code/ 是否有 Java 源码。')
-        return 0
-
-    con = duckdb.connect(DB_PATH)
-    try:
-        con.execute("""CREATE TABLE IF NOT EXISTS entity_table_map (
-            entity_class TEXT,
-            table_name TEXT,
-            normalized_table TEXT,
-            source_file TEXT,
-            annotation TEXT
-        )""")
-        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_table ON entity_table_map(table_name)")
-        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_norm ON entity_table_map(normalized_table)")
-        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_class ON entity_table_map(entity_class)")
-
-        if rebuild:
-            con.execute("DELETE FROM entity_table_map")
-
-        # 用 JSONL COPY 批量插入 (DuckDB 1.4 executemany 慢)
-        import json as _json
-        import tempfile
-        tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False,
-                                          mode='w', encoding='utf-8')
-        cols = ['entity_class', 'table_name', 'normalized_table', 'source_file', 'annotation']
-        for row in mappings:
-            obj = {cols[i]: (row[i] if row[i] is not None else '')
-                   for i in range(len(cols))}
-            tmp.write(_json.dumps(obj, ensure_ascii=False) + '\n')
-        tmp.close()
-        try:
-            con.execute("COPY entity_table_map FROM '%s' (FORMAT JSON)"
-                        % tmp.name.replace('\\', '/'))
-        finally:
-            os.unlink(tmp.name)
-
-        con.execute("INSERT OR REPLACE INTO build_meta(key, value) VALUES (?,?)",
-                    ['entity_table_map_built', str(time.time())])
-        print('写入 %d 条 Entity→表 映射到 entity_table_map' % len(mappings))
-        return len(mappings)
-    finally:
-        con.close()
-
-
-def query_impact(table_name):
-    """查某张表关联的 Entity 类 (给"改表影响分析"用)。"""
-    con = duckdb.connect(DB_PATH, read_only=True)
-    try:
-        rows = con.execute(
-            "SELECT DISTINCT entity_class, source_file FROM entity_table_map "
-            "WHERE table_name = ?", [table_name]
-        ).fetchall()
-        return rows
-    finally:
-        con.close()
-
-
-def stats():
-    """映射统计 + 和 db_tables/db_columns 的关联率。"""
-    con = duckdb.connect(DB_PATH, read_only=True)
-    try:
-        total = con.execute("SELECT COUNT(*) FROM entity_table_map").fetchone()[0]
-        classes = con.execute("SELECT COUNT(DISTINCT entity_class) FROM entity_table_map").fetchone()[0]
-        tables_mapped = con.execute("SELECT COUNT(DISTINCT table_name) FROM entity_table_map").fetchone()[0]
-        # 关联率: 映射的表名 vs 数据库真实表名 (原始 + 归一化)
-        try:
-            raw_matched = con.execute("""
-                SELECT COUNT(DISTINCT m.table_name) FROM entity_table_map m
-                JOIN db_tables t ON m.table_name = t.table_name
-            """).fetchone()[0]
-            norm_matched = con.execute("""
-                SELECT COUNT(DISTINCT m.normalized_table) FROM entity_table_map m
-                JOIN db_tables t ON m.normalized_table = t.table_name
-            """).fetchone()[0]
-            db_total = con.execute("SELECT COUNT(DISTINCT table_name) FROM db_tables").fetchone()[0]
-        except Exception:
-            raw_matched, norm_matched, db_total = 0, 0, 0
-        print('entity_table_map 统计:')
-        print('  总映射: %d' % total)
-        print('  Entity 类数: %d' % classes)
-        print('  映射的表名数: %d' % tables_mapped)
-        if db_total:
-            print('  原始表名匹配: %d/%d (%.0f%%)'
-                  % (raw_matched, db_total, raw_matched / db_total * 100))
-            print('  归一化后匹配: %d/%d (%.0f%%)'
-                  % (norm_matched, db_total, norm_matched / db_total * 100))
-        # 抽样
-        sample = con.execute(
-            "SELECT entity_class, table_name FROM entity_table_map LIMIT 8"
-        ).fetchall()
-        print('  抽样:')
-        for ec, tn in sample:
-            print('    %s → %s' % (ec, tn))
-    finally:
-        con.close()
-
-
-if __name__ == '__main__':
-    args = sys.argv[1:]
-    if '--stats' in args:
-        stats()
-    else:
-        build(rebuild='--rebuild' in args)
-        stats()
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_cp = _o.path.join(_o.path.dirname(_o.path.dirname(_o.path.abspath(__file__))), 'common')
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+
+"""kg_link_db.py - 扫代码 @TableName 注解, 建 Entity 类 ↔ 数据库表 的语义桥。
+
+解决"数据层孤岛"问题: kg_db.duckdb 有表结构, 但不知道对应代码里的哪个 Entity。
+本脚本扫 Java 源码里的 @TableName("xxx") 注解, 提取:
+  Entity 类名 → 数据库表名 → (关联到) db_columns 的真实列
+
+调研依据 (FalkorDB "结构遍历 > 语义匹配"):
+  - 不靠表名子串猜 endpoint (验证过 0 命中, 不准)
+  - 靠代码显式声明的关系 (@TableName 注解) 建边, 这才准
+
+产出表 (写进 kg_db.duckdb, 不动 kg.duckdb 写锁):
+  entity_table_map:
+    entity_class  TEXT   -- QualityCaseBo (代码层类名)
+    table_name    TEXT   -- quality_case (数据库表名, 关联 db_tables)
+    source_file   TEXT   -- 哪个 .java 读到的
+    annotation    TEXT   -- @TableName("quality_case") 原文
+
+建好后可遍历:
+  数据库表 → entity_table_map → Entity 类 → (kg.duckdb) Controller/接口
+
+用法:
+  python kg_link_db.py              # 扫 data/code/ 全量
+  python kg_link_db.py --rebuild    # 清旧重建
+  python kg_link_db.py --stats      # 看统计
+"""
+import os
+import re
+import sys
+import time
+
+SCRIPTS = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))  # 子包→scripts/根
+sys.path.insert(0, SCRIPTS)
+
+import duckdb
+from common.paths import DATA_INDEX_DIR, PROJECT_ROOT
+
+CODE_DIR = str(PROJECT_ROOT / 'data' / 'code')
+DB_PATH = str(DATA_INDEX_DIR / 'kg_db.duckdb')
+
+# @TableName 的两种格式:
+#   @TableName("quality_case")
+#   @TableName(value = "quality_case")
+RE_TABLENAME = re.compile(
+    r'@TableName\s*\(\s*(?:value\s*=\s*)?"([^"]+)"',
+    re.IGNORECASE
+)
+# 紧跟注解的 class 声明: class QualityCaseBo
+RE_CLASS = re.compile(r'class\s+(\w+)')
+
+
+def _normalize_table_name(raw):
+    """归一化表名, 提升和数据库真实表的匹配率。
+    处理注解写法不规范的情况:
+      fywl_ics_cloud.sys_dept → sys_dept (去库名前缀)
+      empContractImport → emp_contract_import (camelCase→snake_case)
+      quality_case → quality_case (已是规范, 不变)
+    """
+    t = raw.strip().lower()
+    # 去库名前缀 (db.table → table)
+    if '.' in t:
+        t = t.rsplit('.', 1)[-1]
+    # 去常见前缀 t_/tb_/t (数据库表常加这些, 注解有时不加)
+    # 注意: 不在这里去前缀, 因为 db_tables 里有的带 t_ 有的不带, 留原样让 JOIN 双向兜
+    # camelCase → snake_case (Java 注解里有时写驼峰表名)
+    if re.search(r'[a-z][A-Z]', t):
+        t = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', t).lower()
+    return t
+
+
+def _scan_java_files():
+    """扫 data/code/ 下所有 .java, 返回 [(entity_class, table_name, normalized_table, source_file, annotation)]。"""
+    mappings = []
+    scanned = 0
+    for root, dirs, files in os.walk(CODE_DIR):
+        # 跳过非业务目录
+        dirs[:] = [d for d in dirs if d not in (
+            '.git', 'node_modules', '__pycache__', 'target', 'test')]
+        for fn in files:
+            if not fn.endswith('.java'):
+                continue
+            scanned += 1
+            fpath = os.path.join(root, fn)
+            try:
+                for enc in ('utf-8', 'gbk'):
+                    try:
+                        with open(fpath, encoding=enc) as f:
+                            content = f.read()
+                        break
+                    except (UnicodeDecodeError, OSError):
+                        content = ''
+                        continue
+            except OSError:
+                continue
+            if '@TableName' not in content:
+                continue
+            # 找所有 @TableName 注解
+            for m in RE_TABLENAME.finditer(content):
+                table_name = m.group(1)
+                normalized = _normalize_table_name(table_name)
+                annotation = content[m.start():m.end() + 1]
+                # 找注解之后最近的 class 声明 (Entity 类名)
+                after = content[m.end():m.end() + 500]
+                cm = RE_CLASS.search(after)
+                entity_class = cm.group(1) if cm else os.path.splitext(fn)[0]
+                rel = os.path.relpath(fpath, PROJECT_ROOT).replace('\\', '/')
+                mappings.append((entity_class, table_name, normalized, rel, annotation))
+    return mappings, scanned
+
+
+def build(rebuild=False):
+    """扫代码 @TableName → 写 entity_table_map。"""
+    t0 = time.time()
+    print('扫描 data/code/ 下的 @TableName 注解...')
+    mappings, scanned = _scan_java_files()
+    print('  扫描 %d 个 .java 文件, 找到 %d 个 @TableName 映射 (%.1fs)'
+          % (scanned, len(mappings), time.time() - t0))
+
+    if not mappings:
+        print('未找到任何 @TableName 注解。检查 data/code/ 是否有 Java 源码。')
+        return 0
+
+    con = duckdb.connect(DB_PATH)
+    try:
+        con.execute("""CREATE TABLE IF NOT EXISTS entity_table_map (
+            entity_class TEXT,
+            table_name TEXT,
+            normalized_table TEXT,
+            source_file TEXT,
+            annotation TEXT
+        )""")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_table ON entity_table_map(table_name)")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_norm ON entity_table_map(normalized_table)")
+        con.execute("CREATE INDEX IF NOT EXISTS idx_etm_class ON entity_table_map(entity_class)")
+
+        if rebuild:
+            con.execute("DELETE FROM entity_table_map")
+
+        # 用 JSONL COPY 批量插入 (DuckDB 1.4 executemany 慢)
+        import json as _json
+        import tempfile
+        tmp = tempfile.NamedTemporaryFile(suffix='.jsonl', delete=False,
+                                          mode='w', encoding='utf-8')
+        cols = ['entity_class', 'table_name', 'normalized_table', 'source_file', 'annotation']
+        for row in mappings:
+            obj = {cols[i]: (row[i] if row[i] is not None else '')
+                   for i in range(len(cols))}
+            tmp.write(_json.dumps(obj, ensure_ascii=False) + '\n')
+        tmp.close()
+        try:
+            con.execute("COPY entity_table_map FROM '%s' (FORMAT JSON)"
+                        % tmp.name.replace('\\', '/'))
+        finally:
+            os.unlink(tmp.name)
+
+        con.execute("INSERT OR REPLACE INTO build_meta(key, value) VALUES (?,?)",
+                    ['entity_table_map_built', str(time.time())])
+        print('写入 %d 条 Entity→表 映射到 entity_table_map' % len(mappings))
+        return len(mappings)
+    finally:
+        con.close()
+
+
+def query_impact(table_name):
+    """查某张表关联的 Entity 类 (给"改表影响分析"用)。"""
+    con = duckdb.connect(DB_PATH, read_only=True)
+    try:
+        rows = con.execute(
+            "SELECT DISTINCT entity_class, source_file FROM entity_table_map "
+            "WHERE table_name = ?", [table_name]
+        ).fetchall()
+        return rows
+    finally:
+        con.close()
+
+
+def stats():
+    """映射统计 + 和 db_tables/db_columns 的关联率。"""
+    con = duckdb.connect(DB_PATH, read_only=True)
+    try:
+        total = con.execute("SELECT COUNT(*) FROM entity_table_map").fetchone()[0]
+        classes = con.execute("SELECT COUNT(DISTINCT entity_class) FROM entity_table_map").fetchone()[0]
+        tables_mapped = con.execute("SELECT COUNT(DISTINCT table_name) FROM entity_table_map").fetchone()[0]
+        # 关联率: 映射的表名 vs 数据库真实表名 (原始 + 归一化)
+        try:
+            raw_matched = con.execute("""
+                SELECT COUNT(DISTINCT m.table_name) FROM entity_table_map m
+                JOIN db_tables t ON m.table_name = t.table_name
+            """).fetchone()[0]
+            norm_matched = con.execute("""
+                SELECT COUNT(DISTINCT m.normalized_table) FROM entity_table_map m
+                JOIN db_tables t ON m.normalized_table = t.table_name
+            """).fetchone()[0]
+            db_total = con.execute("SELECT COUNT(DISTINCT table_name) FROM db_tables").fetchone()[0]
+        except Exception:
+            raw_matched, norm_matched, db_total = 0, 0, 0
+        print('entity_table_map 统计:')
+        print('  总映射: %d' % total)
+        print('  Entity 类数: %d' % classes)
+        print('  映射的表名数: %d' % tables_mapped)
+        if db_total:
+            print('  原始表名匹配: %d/%d (%.0f%%)'
+                  % (raw_matched, db_total, raw_matched / db_total * 100))
+            print('  归一化后匹配: %d/%d (%.0f%%)'
+                  % (norm_matched, db_total, norm_matched / db_total * 100))
+        # 抽样
+        sample = con.execute(
+            "SELECT entity_class, table_name FROM entity_table_map LIMIT 8"
+        ).fetchall()
+        print('  抽样:')
+        for ec, tn in sample:
+            print('    %s → %s' % (ec, tn))
+    finally:
+        con.close()
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    if '--stats' in args:
+        stats()
+    else:
+        build(rebuild='--rebuild' in args)
+        stats()