@hupan56/wlkj 3.1.32 → 3.3.0

package/templates/qoder/scripts/domain/kg/extract/java/_parser.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+"""tree-sitter Java 引擎 + AST 通用遍历工具。
+
+所有 java/* extractor 共用这个模块，避免重复 parse。
+一次解析一个文件 → root node → 各 extractor 各自遍历提取所需信息。
+
+tree-sitter-java 0.23.5 API: tsj.language() (不是 language_java)
+"""
+from __future__ import annotations
+import os
+import hashlib
+from typing import Optional
+
+try:
+    from tree_sitter import Language, Parser
+    import tree_sitter_java as tsj
+    _LANG = Language(tsj.language())
+    _PARSER = Parser(_LANG)
+    AVAILABLE = True
+except Exception as _e:  # pragma: no cover
+    AVAILABLE = False
+    _PARSER = None
+    _LAST_ERR = str(_e)
+
+
+def parse(content: str):
+    """解析 Java 源码 → tree-sitter root node（不可用返回 None）。
+
+    注意: _PARSER.parse 返回 Tree, 这里取 .root_node 返回 Node。
+    """
+    if not AVAILABLE or not content:
+        return None
+    try:
+        tree = _PARSER.parse(content.encode('utf-8', errors='replace'))
+        return tree.root_node if tree is not None else None
+    except Exception:
+        return None
+
+
+# ============ AST 遍历工具 ============
+
+def _text(node) -> str:
+    """取节点文本（tree-sitter 0.23 node.text 可用）。"""
+    try:
+        return node.text.decode('utf-8', errors='replace')
+    except Exception:
+        return ''
+
+
+def find_all(node, node_type: str):
+    """递归找所有指定类型的子节点。"""
+    out = []
+    if node is None:
+        return out
+    stack = [node]
+    while stack:
+        n = stack.pop()
+        if n.type == node_type:
+            out.append(n)
+        for c in n.children:
+            stack.append(c)
+    return out
+
+
+def find_first(node, node_type: str):
+    """找第一个匹配类型的子节点。"""
+    rs = find_all(node, node_type)
+    return rs[0] if rs else None
+
+
+def get_class_decls(root):
+    """所有 class_declaration / interface_declaration / enum_declaration。"""
+    return (find_all(root, 'class_declaration')
+            + find_all(root, 'interface_declaration')
+            + find_all(root, 'enum_declaration'))
+
+
+def get_class_name(cls_node) -> Optional[str]:
+    """类名（identifier 子节点）。"""
+    for c in cls_node.children:
+        if c.type == 'identifier':
+            return _text(c)
+    return None
+
+
+def get_superclass(cls_node) -> Optional[str]:
+    """父类（extends）。"""
+    sup = find_first(cls_node, 'superclass')
+    if sup:
+        ti = find_first(sup, 'type_identifier') or find_first(sup, 'generic_type')
+        if ti:
+            return _text(ti).split('<')[0]
+    return None
+
+
+def get_super_interfaces(cls_node) -> list:
+    """implements 的接口列表。"""
+    out = []
+    for tl in find_all(cls_node, 'super_interfaces'):
+        for ti in find_all(tl, 'type_identifier'):
+            out.append(_text(ti))
+    return out
+
+
+def get_annotations(node) -> dict:
+    """提取节点的所有注解 → {注解名(不含@): 参数文本 or ''}。
+
+    marker_annotation: @Service (无参数)
+    annotation:        @GetMapping("/x") (有参数)
+    """
+    anns = {}
+    # 注解只在 modifiers / 直接子节点里
+    mods = None
+    for c in node.children:
+        if c.type in ('modifiers', 'annotation', 'marker_annotation'):
+            if c.type == 'modifiers':
+                mods = c
+            else:
+                _collect_one_annotation(c, anns)
+    if mods:
+        for c in mods.children:
+            if c.type in ('annotation', 'marker_annotation'):
+                _collect_one_annotation(c, anns)
+    return anns
+
+
+def _collect_one_annotation(ann_node, out: dict):
+    """收集单个注解。结构:
+       marker_annotation: @ + identifier
+       annotation:        @ + identifier + (可选) annotation_argument_list
+    """
+    name = None
+    arg_text = ''
+    for c in ann_node.children:
+        if c.type == 'identifier' and name is None:
+            name = _text(c)
+        elif c.type == 'annotation_argument_list':
+            arg_text = _text(c).strip('()').strip()
+    if name:
+        out[name] = arg_text
+
+
+def get_fields(cls_node) -> list:
+    """类里所有 field_declaration → [{name,type,annotations,is_final,is_static}]。"""
+    out = []
+    for fd in find_all(cls_node, 'field_declaration'):
+        mods = find_first(fd, 'modifiers')
+        anns = {}
+        is_final = is_static = False
+        if mods:
+            for c in mods.children:
+                if c.type in ('annotation', 'marker_annotation'):
+                    _collect_one_annotation(c, anns)
+                elif c.type == 'final_modifier':
+                    is_final = True
+                elif c.type == 'static' or _text(c) == 'static':
+                    is_static = True
+        # 类型
+        ftype = ''
+        ti = find_first(fd, 'type_identifier') or find_first(fd, 'generic_type') \
+             or find_first(fd, 'array_type') or find_first(fd, 'basic_type')
+        if ti:
+            ftype = _text(ti).split('<')[0].strip('[]')
+        # 字段名（variable_declarator 内 identifier）
+        names = [_text(n) for n in find_all(fd, 'identifier')
+                 if n.parent and n.parent.type == 'variable_declarator']
+        # javadoc 注释（字段中文说明）
+        comment = _get_leading_comment(fd)
+        for nm in names:
+            out.append({
+                'name': nm, 'type': ftype, 'annotations': anns,
+                'is_final': is_final, 'is_static': is_static,
+                'comment': comment,
+            })
+    return out
+
+
+def get_methods(cls_node) -> list:
+    """类里所有 method_declaration → [{name, annotations, return_type, body_node}]。"""
+    out = []
+    for md in find_all(cls_node, 'method_declaration'):
+        name = None
+        for c in md.children:
+            if c.type == 'identifier' and name is None:
+                name = _text(c)
+                break
+        anns = get_annotations(md)
+        # 返回类型
+        rtype = ''
+        for c in md.children:
+            if c.type in ('type_identifier', 'generic_type', 'void_type', 'basic_type', 'array_type'):
+                rtype = _text(c).split('<')[0].strip('[]')
+                break
+        body = find_first(md, 'block')
+        out.append({
+            'name': name, 'annotations': anns,
+            'return_type': rtype, 'body_node': body,
+        })
+    return out
+
+
+def get_method_calls(body_node) -> list:
+    """方法体内所有方法调用 → [{object, method}]。
+
+    remoteClientService.getById(x) → {object:'remoteClientService', method:'getById'}
+    method_invocation 直接子节点顺序: identifier(object) . identifier(method) argument_list
+    """
+    out = []
+    if body_node is None:
+        return out
+    for mi in find_all(body_node, 'method_invocation'):
+        # 取 method_invocation 的直接子 identifier（按出现顺序）
+        ids = []
+        for c in mi.children:
+            if c.type == 'identifier':
+                ids.append(_text(c))
+        if len(ids) >= 2:
+            out.append({'object': ids[0], 'method': ids[1]})
+        elif len(ids) == 1:
+            out.append({'object': '', 'method': ids[0]})
+    return out
+
+
+def _get_leading_comment(node) -> str:
+    """取节点前的 javadoc/块注释（字段中文含义来源）。
+
+    tree-sitter 的 comment 节点通常是兄弟节点。这里用 prev_sibling 兜底。
+    """
+    try:
+        prev = node.prev_sibling
+        if prev and prev.type in ('comment', 'block_comment'):
+            txt = _text(prev).strip()
+            # 清理 /** ... */ 和 *
+            txt = txt.strip('/*').strip()
+            lines = [l.strip().lstrip('*').strip() for l in txt.split('\n')]
+            return ' '.join(l for l in lines if l)
+    except Exception:
+        pass
+    return ''
+
+
+# ============ 辅助: entity_id / repo_id ============
+
+def repo_id_from_path(filepath: str) -> str:
+    """从文件路径解析 repo_id（顶层项目目录名）。
+
+    data/code/fywl-ics/.../Foo.java → 'fywl-ics'
+    data/code/Carmg-H5/.../bar.vue → 'Carmg-H5'
+    """
+    parts = filepath.replace('\\', '/').split('/')
+    for i, p in enumerate(parts):
+        if p == 'code' and i + 1 < len(parts):
+            return parts[i + 1]
+    # 兜底: 取 data/ 之后的第一段
+    for i, p in enumerate(parts):
+        if p == 'data' and i + 1 < len(parts):
+            return parts[i + 1]
+    return 'unknown'
+
+
+def entity_id(repo_id: str, etype: str, canonical: str) -> str:
+    """构造实体 id（对齐现有规则: C:EsCrudController / F:fywl-ui:system）。
+
+    Java 层用: SVC:fywl-ics:FooServiceImpl / MAP:fywl-ics:AssetMapper
+    """
+    return '%s:%s:%s' % (etype, repo_id, canonical)
+
+
+def file_fingerprint(content: str) -> str:
+    """文件内容 hash（C 方案弱关联重算依赖）。"""
+    return hashlib.md5(content.encode('utf-8', errors='replace')).hexdigest()[:16]

package/templates/qoder/scripts/domain/kg/extract/java/all.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+"""Java 全栈提取协调器。
+
+单文件一次 tree-sitter 解析 → 多 extractor 各自提取 → 累积 → 小批量 PG upsert。
+
+使用:
+    from domain.kg.extract.java.all import extract_repo
+    extract_repo(repo_dir='/data/code/fywl-ics', repo_id='fywl-ics')
+"""
+from __future__ import annotations
+import os
+import sys
+import glob
+import time
+import json
+
+# 路径自举 (兼容被 kg_build / WlkjBridge / 独立脚本调用)
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p)
+        break
+    _THIS = _p
+
+from domain.kg.extract.java import _parser as P
+from domain.kg.extract.java import class_extractor as ce
+from domain.kg.extract.java import member_extractor as me
+from domain.kg.extract.java import domain_extractor as de
+
+
+def extract_file(filepath: str, repo_id: str):
+    """单文件提取 → (entities, edges)。
+
+    一次 parse, 多 extractor 复用 root node。
+    """
+    try:
+        with open(filepath, encoding='utf-8', errors='replace') as f:
+            content = f.read()
+    except Exception:
+        return [], []
+    if not content.strip():
+        return [], []
+
+    root = P.parse(content)
+    if root is None:
+        return [], []
+
+    # ① 类级提取 (实体 + 注入)
+    cls_entities, cls_edges = ce.extract(root, repo_id, filepath)
+    # 构造 owner 信息给 member_extractor
+    owner_classes = []
+    for e in cls_entities:
+        owner_classes.append({
+            'name': e['canonical'],
+            'etype': e['type'],
+            'id': e['id'],
+            'anns_dict': e['props'].get('annotations', []),
+        })
+    # 类级 RequestMapping 在 props.base_path, member 要用
+    for oc in owner_classes:
+        for e in cls_entities:
+            if e['canonical'] == oc['name']:
+                # 把类级注解 dict 重建 (从 props.annotations list)
+                oc['anns_dict'] = {'RequestMapping': e['props'].get('base_path', '')} if e['props'].get('base_path') else {}
+
+    # ② 方法级提取 (端点 + 权限 + 调用链)
+    mem_entities, mem_edges = me.extract(root, repo_id, filepath, owner_classes)
+
+    # ③ 领域对象提取 (Entity/DTO + 字段映射 + 校验)
+    dom_entities, dom_edges = de.extract(root, repo_id, filepath)
+
+    all_entities = cls_entities + mem_entities + dom_entities
+    all_edges = cls_edges + mem_edges + dom_edges
+
+    # 给每个实体附 file (member/domain extractor 已加, class_extractor 也在 props.file)
+    for e in all_entities:
+        if 'props' not in e:
+            e['props'] = {}
+        if 'file' not in e['props']:
+            e['props']['file'] = filepath
+
+    return all_entities, all_edges
+
+
+def extract_repo(repo_dir: str, repo_id: str, logger=print):
+    """扫整个仓库的 .java → 提取 → 返回 (entities, edges, stats)。
+
+    不直接写 PG (写库由调用方决定, 便于测试和复用)。
+    """
+    t0 = time.time()
+    java_files = glob.glob(os.path.join(repo_dir, '**', '*.java'), recursive=True)
+    # 跳过 test 目录
+    java_files = [f for f in java_files if '/test/' not in f.replace('\\', '/').lower()
+                  and '\\test\\' not in f.lower()
+                  and not f.endswith('Test.java')
+                  and not f.endswith('Tests.java')]
+    total_files = len(java_files)
+    logger('  [java] 扫描 %d 个 .java 文件 (repo=%s)' % (total_files, repo_id))
+
+    all_entities, all_edges = [], []
+    err_count = 0
+    for i, jf in enumerate(java_files):
+        try:
+            es, eds = extract_file(jf, repo_id)
+            all_entities += es
+            all_edges += eds
+        except Exception as e:
+            err_count += 1
+            if err_count <= 3:
+                logger('  [java] 提取失败 %s: %s' % (jf, str(e)[:60]))
+        if (i + 1) % 2000 == 0:
+            logger('  [java] 进度 %d/%d (实体%d 边%d, 错%d)' % (
+                i + 1, total_files, len(all_entities), len(all_edges), err_count))
+
+    dt = time.time() - t0
+    type_dist = {}
+    for e in all_entities:
+        type_dist[e['type']] = type_dist.get(e['type'], 0) + 1
+    edge_dist = {}
+    for e in all_edges:
+        edge_dist[e['edge_type']] = edge_dist.get(e['edge_type'], 0) + 1
+
+    stats = {
+        'files': total_files, 'entities': len(all_entities),
+        'edges': len(all_edges), 'errors': err_count, 'seconds': round(dt, 1),
+        'type_dist': type_dist, 'edge_dist': edge_dist,
+    }
+    logger('  [java] 完成: %d 实体 / %d 边 / %d 错 / %.1fs' % (
+        len(all_entities), len(all_edges), err_count, dt))
+    logger('  [java] 实体分布: %s' % type_dist)
+    logger('  [java] 边分布: %s' % edge_dist)
+    return all_entities, all_edges, stats
+
+
+if __name__ == '__main__':
+    # CLI: 直接对某 repo 跑一次, 不写库, 只看统计
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('repo_dir')
+    ap.add_argument('--repo-id', default='')
+    args = ap.parse_args()
+    rid = args.repo_id or os.path.basename(args.repo_dir.rstrip('/\\'))
+    es, eds, st = extract_repo(args.repo_dir, rid)
+    print(json.dumps(st, ensure_ascii=False, indent=2))

package/templates/qoder/scripts/domain/kg/extract/java/build_java_to_pg.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""Java 全栈提取 + PG 写入 一体化入口。
+
+被 kg_build.py 或 WlkjBridge.rebuild_kg 调用, 也可独立 CLI 运行:
+    python -m domain.kg.extract.java.build_java_to_pg --repo-dir data/code/fywl-ics --repo-id fywl-ics
+
+作用: 扫描 repo 下所有 .java → 提取后端全栈实体/边 → upsert 到 PG。
+这是 GOAL 2 的集成交付物, 让 11905 个 Java 文件从 0 吸收到全吸收。
+"""
+from __future__ import annotations
+import os
+import sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p)
+        break
+    _THIS = _p
+
+from domain.kg.extract.java.all import extract_repo
+from domain.kg.extract.java import pg_upsert
+
+
+def build(repo_dir: str, repo_id: str, write_pg: bool = True, logger=print):
+    """提取一个 repo 的 Java → 写 PG。
+
+    Args:
+        repo_dir: 代码目录 (data/code/fywl-ics)
+        repo_id: 仓库 id (fywl-ics)
+        write_pg: True 写库; False 只提取不写 (测试用)
+    Returns: dict {stats, upsert_result}
+    """
+    logger('  [java-build] 开始提取 %s (%s)' % (repo_id, repo_dir))
+    entities, edges, stats = extract_repo(repo_dir, repo_id, logger=logger)
+
+    if write_pg:
+        logger('  [java-build] 写入 PG (%d 实体 / %d 边)...' % (len(entities), len(edges)))
+        result = pg_upsert.upsert_batch(entities, edges)
+        logger('  [java-build] PG 写入: %s' % result)
+        # 新鲜度追踪
+        try:
+            pg_upsert.upsert_freshness(entities, {})
+        except Exception as e:
+            logger('  [java-build] freshness 写入跳过: %s' % str(e)[:50])
+        stats['upsert'] = result
+    return stats
+
+
+def build_all_repos(code_root: str = None, write_pg: bool = True, logger=print):
+    """扫 code_root 下所有 repo 目录, 逐个提取。
+
+    code_root: 默认 data/code (引擎根)
+    """
+    if code_root is None:
+        from foundation.core.paths import PROJECT_ROOT
+        code_root = os.path.join(str(PROJECT_ROOT), 'data', 'code')
+    if not os.path.isdir(code_root):
+        logger('  [java-build] code_root 不存在: %s' % code_root)
+        return
+
+    repos = [d for d in os.listdir(code_root)
+             if os.path.isdir(os.path.join(code_root, d)) and not d.startswith('.')]
+    logger('  [java-build] 发现 %d 个 repo: %s' % (len(repos), repos))
+    total_stats = []
+    for rid in repos:
+        rdir = os.path.join(code_root, rid)
+        # 只处理含 .java 的 repo
+        has_java = any(f.endswith('.java') for f in os.listdir(rdir)) or \
+                   any(f.endswith('.java') for _, _, fs in os.walk(rdir) for f in fs)
+        if not has_java:
+            logger('  [java-build] %s 无 .java, 跳过' % rid)
+            continue
+        try:
+            st = build(rdir, rid, write_pg=write_pg, logger=logger)
+            total_stats.append({'repo': rid, **st})
+        except Exception as e:
+            logger('  [java-build] %s 失败: %s' % (rid, str(e)[:80]))
+    return total_stats
+
+
+if __name__ == '__main__':
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--repo-dir', help='单个 repo 目录')
+    ap.add_argument('--repo-id', help='repo id')
+    ap.add_argument('--all', action='store_true', help='扫所有 repo')
+    ap.add_argument('--dry-run', action='store_true', help='只提取不写库')
+    args = ap.parse_args()
+
+    if args.all:
+        rs = build_all_repos(write_pg=not args.dry_run)
+        import json
+        print(json.dumps(rs, ensure_ascii=False, indent=2))
+    elif args.repo_dir:
+        rid = args.repo_id or os.path.basename(args.repo_dir.rstrip('/\\'))
+        st = build(args.repo_dir, rid, write_pg=not args.dry_run)
+        import json
+        print(json.dumps(st, ensure_ascii=False, indent=2))
+    else:
+        ap.print_help()

package/templates/qoder/scripts/domain/kg/extract/java/call_chain.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""调用链提取器（方法体内方法调用 → calls 边）。
+
+职责: 把 Service.method() 内部对 Mapper/其他 Service 的调用提取为 calls 边。
+这是 get_impact 5 层穿透的关键链路 (Controller→Service→Mapper→Table)。
+共享 member_extractor 的方法遍历逻辑。
+"""
+from __future__ import annotations
+from . import _parser as P
+from . import member_extractor as _me
+
+
+def extract_calls_in_method(body_node, field_types: dict, owner_id: str, repo_id: str) -> list:
+    """从方法体提取调用边。
+
+    Args:
+        body_node: tree-sitter block 节点
+        field_types: {字段名: 字段类型} (从类的 field_declaration 解析)
+        owner_id: 调用方实体 id
+        repo_id: 仓库 id
+    Returns:
+        [{from_id, to_id, edge_type:'calls', props:{method, callee_field}}]
+    """
+    edges = []
+    if body_node is None:
+        return edges
+    for call in P.get_method_calls(body_node):
+        obj_name = call['object']
+        method = call['method']
+        if not obj_name:
+            continue
+        dep_type = field_types.get(obj_name) or _me._guess_type_by_name(obj_name)
+        if not dep_type:
+            continue
+        dep_prefix = _me._guess_dep_prefix(dep_type)
+        edges.append({
+            'from_id': owner_id,
+            'to_id': '%s:%s:%s' % (dep_prefix, repo_id, dep_type),
+            'edge_type': 'calls',
+            'from_repo': repo_id, 'to_repo': repo_id,
+            'confidence': 1.0, 'source': 'explicit',
+            'props': {'method': method, 'callee_field': obj_name},
+        })
+    return edges
+
+
+def extract(root, repo_id: str, filepath: str, owner_classes: list = None):
+    """调用链提取（转发到 member_extractor, calls 边在那里产出）。"""
+    return _me.extract(root, repo_id, filepath, owner_classes or [])