@hupan56/wlkj 3.1.32 → 3.3.0

package/templates/qoder/scripts/domain/kg/extract/build_goal4.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""GOAL 4 统一入口: PRD LLM + 资产中心 5 表 + REQ-ID 锚定。
+
+被 kg_build.py 调用, 或独立 CLI:
+    python -m domain.kg.extract.build_goal4 [--project-id PID] [--embed]
+"""
+from __future__ import annotations
+import os, sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def build(project_id: str = None, do_embed: bool = False, logger=print):
+    """GOAL 4 主入口: 资产中心 5 表吸收。"""
+    from domain.kg.extract.asset import (prd_importer, prototype_importer,
+                                          discussion_importer, asset_tree, returns_importer)
+
+    logger('  [goal4] === PRD LLM 抽取 + REQ-ID 锚定 ===')
+    prd_r = prd_importer.import_all_prds(project_id, do_embed=do_embed, logger=logger)
+    logger('  [goal4] PRD: %s' % {k: v for k, v in prd_r.items() if k != 'upsert'})
+
+    logger('  [goal4] === 原型 + 标注 ===')
+    proto_r = prototype_importer.import_prototypes(project_id, logger=logger)
+
+    logger('  [goal4] === 讨论 ===')
+    disc_r = discussion_importer.import_discussions(project_id, logger=logger)
+
+    logger('  [goal4] === 资产树 ===')
+    tree_r = asset_tree.import_asset_tree(project_id, logger=logger)
+
+    logger('  [goal4] === 回流 ===')
+    ret_r = returns_importer.import_returns(project_id, logger=logger)
+
+    return {'prd': prd_r, 'prototype': proto_r, 'discussion': disc_r,
+            'asset_tree': tree_r, 'returns': ret_r}
+
+
+if __name__ == '__main__':
+    import argparse, json
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--project-id', default='')
+    ap.add_argument('--embed', action='store_true')
+    args = ap.parse_args()
+    try:
+        from dotenv import load_dotenv
+        for p in ['wlinkj-workspace/backend/.env', '.env']:
+            if os.path.isfile(p): load_dotenv(p); break
+    except: pass
+    st = build(project_id=args.project_id or None, do_embed=args.embed)
+    print(json.dumps(st, ensure_ascii=False, indent=2, default=str))

package/templates/qoder/scripts/domain/kg/extract/build_goal5.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""GOAL 5 统一入口: L3 弱关联推理 + C 方案重算。
+
+CLI:
+    # 限量推理弱关联 (前500个无强关联实体)
+    python -m domain.kg.extract.build_goal5 --infer --limit 500
+    # C方案重算 (过期弱边续期/删除)
+    python -m domain.kg.extract.build_goal5 --recompute
+    # 升格 (删冗余弱边)
+    python -m domain.kg.extract.build_goal5 --promote
+"""
+from __future__ import annotations
+import os, sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def run_infer(limit: int = 500, logger=print):
+    """推理弱关联 (限量)。"""
+    from domain.kg.extract.inference import weak_link
+    from domain.kg.extract.java import pg_upsert
+    from sqlalchemy import text
+
+    engine, _ = pg_upsert._get_pg_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+
+    with engine.connect() as conn:
+        # 取无强关联的代码实体
+        entities = conn.execute(text(f"""
+            SELECT id, repo_id, type, canonical, cn, props FROM entities
+            WHERE type IN ('FUNCTION','BUTTON','ENDPOINT','CONTROLLER','SERVICE',
+                           'MAPPER','ENTITY','TABLE','COLUMN','FEATURE')
+              AND id NOT IN (
+                SELECT from_id FROM edges
+                WHERE source IN ('explicit','commit')
+                  AND edge_type IN ('implements_req','anchors')
+              )
+            LIMIT {int(limit)}
+        """)).all()
+        # 取全部 anchors
+        anchors = conn.execute(text("SELECT id, title, kind FROM anchors")).all()
+
+    ent_list = [{'id': r[0], 'repo_id': r[1], 'type': r[2],
+                 'canonical': r[3], 'cn': r[4], 'props': r[5] or {}} for r in entities]
+    anchor_list = [{'id': r[0], 'title': r[1], 'kind': r[2]} for r in anchors]
+    logger('  [goal5] %d 实体待推理, %d 锚点' % (len(ent_list), len(anchor_list)))
+
+    edges, st = weak_link.infer_batch(ent_list, anchor_list, logger=logger)
+
+    # 写 PG (用 pg_upsert, 它会处理 expires_at/fingerprint 字段)
+    upsert_r = pg_upsert.upsert_batch([], edges) if edges else {'ok': False, 'edges_ok': 0}
+    st['upsert'] = upsert_r
+    return st
+
+
+def run_recompute(logger=print):
+    """C 方案重算。"""
+    from domain.kg.extract.inference import recompute
+    return recompute.recompute_expired(logger=logger)
+
+
+def run_promote(logger=print):
+    """弱边升格。"""
+    from domain.kg.extract.inference import promote
+    return promote.promote_strong_over_weak(logger=logger)
+
+
+if __name__ == '__main__':
+    import argparse, json
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--infer', action='store_true')
+    ap.add_argument('--recompute', action='store_true')
+    ap.add_argument('--promote', action='store_true')
+    ap.add_argument('--limit', type=int, default=500)
+    args = ap.parse_args()
+    try:
+        from dotenv import load_dotenv
+        for p in ['wlinkj-workspace/backend/.env', '.env']:
+            if os.path.isfile(p): load_dotenv(p); break
+    except: pass
+
+    if args.infer:
+        print(json.dumps(run_infer(args.limit), ensure_ascii=False, indent=2, default=str))
+    elif args.recompute:
+        print(json.dumps(run_recompute(), ensure_ascii=False, indent=2, default=str))
+    elif args.promote:
+        print(json.dumps(run_promote(), ensure_ascii=False, indent=2, default=str))
+    else:
+        ap.print_help()

package/templates/qoder/scripts/domain/kg/extract/db/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""DB schema 提取器 (GOAL 3)。
+
+子模块:
+  schema_extractor  连 information_schema → 灌 db_tables/db_columns
+  fk_extractor      外键关系 → references 边
+  data_profile      数据特征 (null率/枚举分布, 选做)
+"""

package/templates/qoder/scripts/domain/kg/extract/db/data_profile.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""数据特征 profile (选做): 表行数、列 null 率、枚举分布。
+
+价值: 让 KG 知道"这张表有 50 万行""这个字段 null 率 80%"——数据质量知识。
+本轮 best-effort, 连不上 DB 跳过。
+"""
+from __future__ import annotations
+from . import schema_extractor as _se
+
+
+def profile_tables(repo_id: str = 'fywl-ics', sample_size: int = 100, logger=print):
+    """采样表的数据特征。Returns: list[dict] 连不上返回 []。"""
+    if not _se._can_connect():
+        return []
+    try:
+        import pymysql
+    except ImportError:
+        return []
+    # 本轮简化: 只返回行数 (table_rows 已在 schema_extractor 取过)
+    # 完整版可加: SELECT COUNT(*), 列 null 率, DISTINCT 值分布
+    logger('  [db-profile] 本轮跳过详细 profile (行数已在 schema_extractor 取)')
+    return []

package/templates/qoder/scripts/domain/kg/extract/db/fk_extractor.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""外键关系提取器: KEY_COLUMN_USAGE → references 边。
+
+让 table_impact 真正能做"改表影响谁"的跨表追溯。
+"""
+from __future__ import annotations
+from . import schema_extractor as _se
+
+
+def extract_foreign_keys(repo_id: str = 'fywl-ics', logger=print):
+    """连 MySQL → 提取外键关系 → references 边。
+
+    Returns: [{from_id, to_id, edge_type:'references', ...}] 连不上返回 []
+    """
+    if not _se._can_connect():
+        return []
+    try:
+        import pymysql
+    except ImportError:
+        return []
+
+    conn = pymysql.connect(host=_se.MYSQL_HOST, port=_se.MYSQL_PORT, user=_se.MYSQL_USER,
+                           password=_se.MYSQL_PASSWORD, charset='utf8mb4',
+                           cursorclass=pymysql.cursors.DictCursor,
+                           connect_timeout=5, read_timeout=30)
+    edges = []
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT schema_name FROM information_schema.schemata "
+                    "WHERE schema_name LIKE %s", (_se.DB_NAME_PREFIX + '%',))
+        dbs = [r['schema_name'] for r in cur.fetchall()]
+        for db in dbs:
+            # 外键关系
+            cur.execute("""
+                SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
+                FROM information_schema.KEY_COLUMN_USAGE
+                WHERE TABLE_SCHEMA=%s AND REFERENCED_TABLE_NAME IS NOT NULL
+            """, (db,))
+            for r in cur.fetchall():
+                from_tbl = r['TABLE_NAME'].lower()
+                to_tbl = r['REFERENCED_TABLE_NAME'].lower()
+                edges.append({
+                    'from_id': 'TBL:%s:%s' % (repo_id, from_tbl),
+                    'to_id': 'TBL:%s:%s' % (repo_id, to_tbl),
+                    'edge_type': 'references',
+                    'from_repo': repo_id, 'to_repo': repo_id,
+                    'confidence': 1.0, 'source': 'db_fk',
+                    'props': {'column': r['COLUMN_NAME'],
+                              'ref_column': r['REFERENCED_COLUMN_NAME'],
+                              'schema': db},
+                })
+        logger('  [db-fk] %d 条外键关系' % len(edges))
+    finally:
+        conn.close()
+    return edges

package/templates/qoder/scripts/domain/kg/extract/db/schema_extractor.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+"""连 MySQL information_schema → 灌 db_tables / db_columns。
+
+复用 kg_build_db.py 的 MySQL 连接配置 (内网 10.54.6.9, fywl_ics_* 库)。
+连不上内网时 best-effort 返回空, 不阻塞 (GOAL 3 主链路是 mapper XML)。
+"""
+from __future__ import annotations
+import os
+import sys
+import socket
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+# 复用 kg_build_db 的配置
+MYSQL_HOST = os.environ.get('MYSQL_HOST', '10.54.6.9')
+MYSQL_PORT = int(os.environ.get('MYSQL_PORT', '3306'))
+MYSQL_USER = os.environ.get('MYSQL_USER', 'fywl_ics_test')
+MYSQL_PASSWORD = os.environ.get('MYSQL_PASSWORD', 'Fywl_ics_test56476')
+DB_NAME_PREFIX = os.environ.get('DB_NAME_PREFIX', 'fywl_ics_')
+
+
+def _can_connect(timeout=3):
+    """探测内网 MySQL 是否可达。"""
+    try:
+        with socket.create_connection((MYSQL_HOST, MYSQL_PORT), timeout=timeout):
+            return True
+    except Exception:
+        return False
+
+
+def extract_all_schema(repo_id: str = 'fywl-ics', logger=print):
+    """连 MySQL → 灌 db_tables + db_columns。
+
+    Returns: (tables: list[dict], columns: list[dict]) 连不上返回 ([], [])
+    """
+    if not _can_connect():
+        logger('  [db] ⚠️ 内网 MySQL 不可达 (%s:%s), 跳过 schema 提取' % (MYSQL_HOST, MYSQL_PORT))
+        return [], []
+    try:
+        import pymysql
+    except ImportError:
+        logger('  [db] pymysql 未装, 跳过')
+        return [], []
+
+    logger('  [db] 连接 MySQL %s:%s ...' % (MYSQL_HOST, MYSQL_PORT))
+    conn = pymysql.connect(host=MYSQL_HOST, port=MYSQL_PORT, user=MYSQL_USER,
+                           password=MYSQL_PASSWORD, charset='utf8mb4',
+                           cursorclass=pymysql.cursors.DictCursor,
+                           connect_timeout=5, read_timeout=30)
+    tables, columns = [], []
+    try:
+        cur = conn.cursor()
+        cur.execute("SELECT schema_name FROM information_schema.schemata "
+                    "WHERE schema_name LIKE %s ORDER BY schema_name",
+                    (DB_NAME_PREFIX + '%'))
+        dbs = [r['schema_name'] for r in cur.fetchall()]
+        logger('  [db] 发现 %d 个业务库: %s' % (len(dbs), dbs[:3]))
+
+        for db in dbs:
+            # 表
+            cur.execute("SELECT table_name, table_comment, table_rows "
+                        "FROM information_schema.tables WHERE table_schema=%s", (db,))
+            for r in cur.fetchall():
+                tables.append({
+                    'repo_id': repo_id, 'schema_name': db,
+                    'table_name': r['table_name'], 'table_comment': r.get('table_comment') or '',
+                    'rows_count': r.get('table_rows'),
+                })
+            # 列
+            cur.execute("SELECT table_name, column_name, data_type, is_nullable, "
+                        "column_comment, ordinal_position "
+                        "FROM information_schema.columns WHERE table_schema=%s "
+                        "ORDER BY table_name, ordinal_position", (db,))
+            for r in cur.fetchall():
+                columns.append({
+                    'repo_id': repo_id, 'schema_name': db,
+                    'table_name': r['table_name'], 'column_name': r['column_name'],
+                    'data_type': r['data_type'], 'is_nullable': r['is_nullable'],
+                    'column_comment': r.get('column_comment') or '',
+                    'ordinal_position': r.get('ordinal_position'),
+                })
+        logger('  [db] 完成: %d 表 / %d 列' % (len(tables), len(columns)))
+    finally:
+        conn.close()
+    return tables, columns

package/templates/qoder/scripts/domain/kg/extract/extract.py

@@ -428,3 +428,87 @@ def trace_clicks(vue_path, vue_content, api_table):
         traces.append(trace)
 
     return traces
+
+
+# ============================================================
+# GOAL 3 泛化: Carmg-H5 (Vant/H5) 风格 API + 调用提取
+# ============================================================
+# Carmg-H5 的 API 定义方式和 fywl-ui 不同:
+#   fywl-ui:    enum Api { X='/url' } + requestClient.post(Api.X)
+#   Carmg-H5:   export function fn(data){ const params={url:'/x',method:'post'}; return fetch(params) }
+# 以及 van-button 的 @click/@tap 事件。
+# 下面两个函数让 extract.py 能同时处理两种风格。
+
+# Carmg-H5 API: export function fnName(...) { ... url: '/xxx' ... method: 'post' ... }
+RE_H5_API_FN = re.compile(
+    r"export\s+function\s+(\w+)\s*\([^)]*\)\s*\{(?:(?!\}).)*?"
+    r"url:\s*['\"]([^'\"]+)['\"](?:(?!\}).)*?"
+    r"(?:method:\s*['\"](\w+)['\"])?",
+    re.DOTALL
+)
+
+
+def build_api_fn_table_h5(api_dir):
+    """扫 src/api/**/*.js (Carmg-H5 风格) → {fnName: {url, verb, file}}。
+
+    识别 export function + url: 字面量模式。
+    """
+    table = {}
+    if not os.path.isdir(api_dir):
+        return table
+    for root, dirs, files in os.walk(api_dir):
+        dirs[:] = [d for d in dirs if d not in ('dist', 'node_modules', '__pycache__')]
+        for f in files:
+            if not (f.endswith('.js') or f.endswith('.ts')):
+                continue
+            fp = os.path.join(root, f)
+            try:
+                content = open(fp, encoding='utf-8', errors='ignore').read()
+            except Exception:
+                continue
+            for m in RE_H5_API_FN.finditer(content):
+                fn_name, url, verb = m.group(1), m.group(2), m.group(3) or 'get'
+                table[fn_name] = {'url': url, 'verb': verb.lower(), 'file': fp}
+    return table
+
+
+def trace_clicks_h5(vue_path, vue_content, api_table):
+    """Vant/H5 风格的 click→api 追踪。
+
+    和 trace_clicks 区别:
+      - 同时认 @click 和 @tap (Vant 用 @tap)
+      - API 调用是 import { fn } from '@/api/xx' + fn(args) (不是 requestClient.xx)
+    """
+    if not api_table:
+        return []
+    traces = []
+    # 收集 @click / @tap
+    clicks = list(re.finditer(r"@(?:click|tap)(?:\.\w+)?\s*=\s*['\"]([^'\"]+)['\"]", vue_content))
+    if not clicks:
+        return []
+    # 收集 import { fn1, fn2 } from '@/api/...'
+    imported = set()
+    for m in re.finditer(r"import\s*\{([^}]+)\}\s*from\s*['\"][^'\"]*api[^'\"]*['\"]", vue_content):
+        for n in re.split(r'[,\s]+', m.group(1)):
+            n = n.strip()
+            if n:
+                imported.add(n)
+    # script 里直接调用的 api 函数 (imported 集合内的)
+    for click in clicks:
+        handler = click.group(1).split('(')[0].strip()
+        if not handler:
+            continue
+        # 在 script 里找 handler 函数体, 看它调了哪些 imported api
+        handler_body = _get_fn_body(vue_content, handler)
+        if not handler_body:
+            continue
+        for api_fn in imported:
+            if api_fn in api_table and re.search(r'\b' + re.escape(api_fn) + r'\s*\(', handler_body):
+                info = api_table[api_fn]
+                traces.append({
+                    'vue_file': vue_path, 'button_text': handler,
+                    'handler': handler, 'api_fn': api_fn,
+                    'endpoint': info['url'], 'verb': info['verb'],
+                    'api_file': info['file'], 'confidence': 'high',
+                })
+    return traces