@hupan56/wlkj 3.1.32 → 3.3.0

package/templates/qoder/scripts/domain/kg/extract/inference/weak_link.py

@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+"""embedding 弱关联推理 (L3 核心)。
+
+对没有 REQ-ID 标注的老代码:
+  1. 算实体 embedding (DashScope text-embedding-v3)
+  2. 向量召回最近 anchor (pgvector HNSW; 不可用降级命名匹配)
+  3. 按相似度建弱边:
+     > 0.85 → confidence=0.9, expires_at=now+30天
+     0.6-0.85 → confidence=0.7, expires_at=now+14天
+  4. 绑 fingerprint (C方案重算依赖)
+
+embedding 成本高 (37515 实体), 用 llm_limit 限量 + 增量。
+"""
+from __future__ import annotations
+import os
+import sys
+import time
+import json
+import urllib.request
+from datetime import datetime, timedelta
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+from domain.kg.extract.inference import naming_matcher
+
+
+def embed_one(text: str) -> list:
+    """调 DashScope text-embedding-v3 算单个向量 (1024维)。失败返回 []。"""
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    if not key or not text:
+        return []
+    payload = json.dumps({
+        'model': 'text-embedding-v3',
+        'input': [text[:500]],
+        'dimensions': 1024,
+    }).encode('utf-8')
+    req = urllib.request.Request(
+        'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
+        data=payload,
+        headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            r = json.loads(resp.read())
+        return r['data'][0]['embedding']
+    except Exception:
+        return []
+
+
+def cosine_sim(a: list, b: list) -> float:
+    """余弦相似度 (纯 Python, 不依赖 numpy)。"""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = sum(x * x for x in a) ** 0.5
+    nb = sum(y * y for y in b) ** 0.5
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+def infer_one(entity: dict, anchors: list, anchor_vecs: dict = None,
+              vec_matrix=None, vec_ids=None, preset_vec=None):
+    """对单个实体推理弱关联 → edges 列表。
+
+    Args:
+        entity: {id, canonical, cn, type, props}
+        anchors: [{id, title, kind}] 全部锚点
+        anchor_vecs: {anchor_id: vec} 锚点向量缓存 (可选, 没有则用命名匹配)
+        vec_matrix: numpy 矩阵 (N, 1024) 持久化向量 (GOAL A 矩阵召回)
+        vec_ids: 矩阵对应的 entity_id 列表 (长度 N)
+        preset_vec: 预计算的实体向量 (GOAL A 批量 embed, 避免逐个调 API)
+    Returns: [{from_id, to_id, edge_type, confidence, source, expires_at, fingerprint, props}]
+    """
+    eid = entity['id']
+    desc = (entity.get('canonical', '') + ' ' + entity.get('cn', '')).strip() or entity['id']
+
+    # 路1: numpy 矩阵召回 (GOAL A, 最快) — 一次矩阵乘算 top-K
+    if vec_matrix is not None and vec_ids is not None:
+        # ★ 优先用预计算向量 (GOAL A 批量 embed), 没有才逐个 embed
+        ent_vec = preset_vec or embed_one(desc)
+        if ent_vec:
+            scored = _numpy_recall(ent_vec, vec_matrix, vec_ids, top_k=5, min_score=0.6)
+            if scored:
+                return _build_edges(eid, scored, entity, anchors, via='embedding')
+
+    # 路2: 逐个 cosine_sim 召回 (有 anchor_vecs 时, 旧逻辑)
+    if anchor_vecs:
+        ent_vec = preset_vec or embed_one(desc)
+        if ent_vec:
+            scored = []
+            for aid, avec in anchor_vecs.items():
+                s = cosine_sim(ent_vec, avec)
+                if s >= 0.6:
+                    scored.append((aid, s))
+            scored.sort(key=lambda x: x[1], reverse=True)
+            if scored:
+                return _build_edges(eid, scored, entity, anchors, via='embedding')
+    # 路3: 命名匹配降级 (embedding 不可用或没 anchor_vecs 或 embedding 0命中)
+    matches = naming_matcher.match_to_anchors(entity.get('canonical', ''), anchors)
+    return _build_edges(eid, matches, entity, anchors, via='naming')
+
+
+def _numpy_recall(query_vec: list, matrix, ids: list, top_k: int = 5,
+                  min_score: float = 0.6) -> list:
+    """numpy 矩阵召回 top-K (GOAL A 核心, 替代逐个 cosine_sim)。
+
+    query_vec × matrix 一次矩阵乘 → 全量相似度 → top-K。
+    比逐个 Python cosine_sim 快 100x。
+    Returns: [(id, score)] 按 score 降序
+    """
+    try:
+        import numpy as np
+        q = np.array(query_vec, dtype=np.float32)
+        # 归一化 (余弦相似度 = 点积/模长)
+        q_norm = q / (np.linalg.norm(q) + 1e-8)
+        m = matrix  # 已归一化的矩阵 (N, dim)
+        # 一次矩阵乘: (1, dim) × (dim, N) → (1, N)
+        scores = m @ q_norm  # m 已归一化, q 已归一化 → 点积=余弦
+        # top-K
+        top_idx = np.argsort(scores)[::-1][:top_k]
+        out = []
+        for idx in top_idx:
+            s = float(scores[idx])
+            if s >= min_score:
+                out.append((ids[idx], round(s, 4)))
+        return out
+    except Exception:
+        return []
+
+
+def _load_vec_matrix(engine, kinds=None) -> tuple:
+    """从持久化向量加载到 numpy 矩阵 (GOAL A)。
+
+    优先读 embeddings_ali (rag.py 的 19093 条, 已有), 兜底读 embeddings (GOAL A 新建)。
+    Returns: (matrix_normalized, id_list)  matrix 是 (N, 1024) 已归一化矩阵
+    """
+    import numpy as np
+    from sqlalchemy import text
+    import json
+    # 路1: embeddings_ali (rag.py 的向量源, 数据最多)
+    try:
+        with engine.connect() as conn:
+            rows = conn.execute(text("""
+                SELECT entity_id, vec FROM embeddings_ali
+                WHERE vec IS NOT NULL AND array_length(vec, 1) > 0
+                LIMIT 5000
+            """)).all()
+        if rows:
+            ids, vecs = [], []
+            for eid, v in rows:
+                if isinstance(v, list) and len(v) > 0:
+                    ids.append(eid)
+                    vecs.append(v)
+            if vecs:
+                matrix = np.array(vecs, dtype=np.float32)
+                norms = np.linalg.norm(matrix, axis=1, keepdims=True) + 1e-8
+                return matrix / norms, ids
+    except Exception:
+        pass
+    # 路2: embeddings (GOAL A 新建, vec_json JSONB)
+    try:
+        with engine.connect() as conn:
+            rows = conn.execute(text("""
+                SELECT entity_id, vec_json FROM embeddings
+                WHERE vec_json IS NOT NULL
+            """)).all()
+    except Exception:
+        return None, None
+    if not rows:
+        return None, None
+    ids, vecs = [], []
+    for eid, vj in rows:
+        try:
+            v = vj if isinstance(vj, list) else json.loads(vj)
+            if isinstance(v, list) and len(v) > 0:
+                ids.append(eid)
+                vecs.append(v)
+        except Exception:
+            continue
+    if not vecs:
+        return None, None
+    matrix = np.array(vecs, dtype=np.float32)
+    norms = np.linalg.norm(matrix, axis=1, keepdims=True) + 1e-8
+    matrix_norm = matrix / norms
+    return matrix_norm, ids
+
+
+def _build_edges(entity_id: str, scored: list, entity: dict, anchors: list, via: str = 'naming'):
+    """按相似度分档建弱边。
+
+    via: 'embedding' 或 'naming' (影响 props 标注)
+    fingerprint: 从 file_signatures 表读 content_hash (C方案重算依赖真实文件指纹)
+    """
+    edges = []
+    repo_id = entity.get('repo_id', '')
+    # fingerprint: 优先从 file_signatures 表读真实 content_hash
+    fp = _get_file_fingerprint(entity)
+    now = datetime.utcnow()
+
+    for anchor_id, score in scored[:3]:  # 最多建3条弱边
+        if score > 0.85:
+            conf, days = 0.9, 30
+        elif score >= 0.6:
+            conf, days = 0.7, 14
+        else:
+            continue
+        edges.append({
+            'from_id': entity_id, 'to_id': anchor_id,
+            'edge_type': 'relates_to', 'from_repo': repo_id, 'to_repo': repo_id,
+            'confidence': conf, 'source': 'inferred',
+            'expires_at': now + timedelta(days=days),
+            'fingerprint': fp,
+            'props': {'score': round(score, 3), 'via': via},
+        })
+    return edges
+
+
+def _get_file_fingerprint(entity: dict) -> str:
+    """从 file_signatures 表读实体文件的 content_hash。
+
+    entity.props.file 是相对路径 (data/code/fywl-ics\...),
+    file_signatures.file_path 是绝对路径, 用路径段末尾匹配。
+    读不到时降级用 canonical hash (best-effort)。
+    """
+    import hashlib
+    props = entity.get('props') or {}
+    if isinstance(props, str):
+        try:
+            import json
+            props = json.loads(props)
+        except Exception:
+            props = {}
+    file_path = props.get('file', '') if isinstance(props, dict) else ''
+    if not file_path:
+        # 无文件信息 → 用 canonical hash 兜底
+        return hashlib.md5((entity.get('canonical', '') + entity.get('cn', '')).encode()).hexdigest()[:16]
+    # 从 file_signatures 查 content_hash (路径末尾段匹配)
+    try:
+        from domain.kg.extract.java import pg_upsert
+        engine, _ = pg_upsert._get_pg_engine()
+        if engine is None:
+            return hashlib.md5(file_path.encode()).hexdigest()[:16]
+        from sqlalchemy import text
+        # 标准化路径: 取最后两段做 LIKE 匹配 (跨绝对/相对路径)
+        norm = file_path.replace('\\', '/').split('/')
+        tail = '/'.join(norm[-3:]) if len(norm) >= 3 else file_path.replace('\\', '/')
+        with engine.connect() as conn:
+            row = conn.execute(text("""
+                SELECT content_hash FROM file_signatures
+                WHERE file_path ILIKE :tail LIMIT 1
+            """), {'tail': '%' + tail}).first()
+            if row and row[0]:
+                return row[0][:16]
+    except Exception:
+        pass
+    return hashlib.md5(file_path.encode()).hexdigest()[:16]
+
+
+def build_anchor_vectors(anchors: list, logger=print) -> dict:
+    """批量 embed anchors 算向量缓存 → {anchor_id: vec}。
+
+    让 weak_link 的向量召回主路径真正跑通 (不依赖 pgvector, 用 Python cosine_sim)。
+    anchors 多时只 embed 前 N 个 (限量, 避免全量 154 个 anchors 调 154 次 API)。
+    """
+    out = {}
+    if not anchors:
+        return out
+    # 只 embed 前 30 个 anchors (够召回, 控成本)
+    for a in anchors[:30]:
+        desc = (a.get('title') or a.get('id', '')).strip()
+        if not desc:
+            continue
+        vec = embed_one(desc)
+        if vec:
+            out[a['id']] = vec
+    logger('  [infer] anchor 向量缓存: %d/%d' % (len(out), len(anchors)))
+    return out
+
+
+def infer_batch(entities: list, anchors: list, use_embedding: bool = True, logger=print):
+    """批量推理弱关联 (GOAL A 升级: numpy 矩阵召回)。
+
+    Args:
+        entities: 待推理实体 (无强关联的)
+        anchors: 锚点列表
+        use_embedding: True 走向量主路径; False 只命名匹配
+    Returns: (edges, stats)
+
+    GOAL A 升级: 优先从 embeddings 表加载持久化向量到 numpy 矩阵,
+    每个实体只 embed 一次自己, 用矩阵乘召回 top-K (不再每个 anchor 都 embed)。
+    """
+    t0 = time.time()
+    all_edges = []
+    matched = 0
+
+    # ★ GOAL A: 优先用持久化向量矩阵 (最快)
+    vec_matrix, vec_ids = None, None
+    if use_embedding:
+        try:
+            from domain.kg.extract.java import pg_upsert
+            engine, _ = pg_upsert._get_pg_engine()
+            if engine:
+                vec_matrix, vec_ids = _load_vec_matrix(engine)
+                if vec_matrix is not None:
+                    logger('  [infer] 加载持久化向量矩阵: %d × %d' % vec_matrix.shape)
+        except Exception as e:
+            logger('  [infer] 矩阵加载失败, 降级: %s' % str(e)[:50])
+
+    # 矩阵不可用 → 降级到 build_anchor_vectors (旧路径)
+    anchor_vecs = None
+    if vec_matrix is None and use_embedding:
+        anchor_vecs = build_anchor_vectors(anchors, logger=logger)
+
+    # ★ GOAL A: 优先复用持久化向量, 缺的才批量 embed
+    entity_vecs = {}
+    if vec_matrix is not None and use_embedding:
+        from domain.kg.extract.inference.embed_builder import embed_batch as _embed_batch
+        from domain.kg.extract.java import pg_upsert as _pg
+        _engine, _ = _pg._get_pg_engine()
+        # ① 先从 embeddings_ali 批量读已有向量 (免 API 调用, 分chunk避免参数过多)
+        all_ids = [e['id'] for e in entities]
+        if _engine and all_ids:
+            try:
+                # 分 chunk 查 (每 chunk 100 个 id, 避免 PG 参数上限)
+                import json as _json
+                for ci in range(0, len(all_ids), 100):
+                    chunk_ids = all_ids[ci:ci+100]
+                    # 用 ANY(ARRAY[...]) 避免构造大量 :params
+                    arr = "ARRAY[" + ','.join("'" + eid.replace("'", "''") + "'" for eid in chunk_ids) + "]"
+                    from sqlalchemy import text as _text_fn
+                    with _engine.connect() as conn:
+                        rows = conn.execute(_text_fn(
+                            f"SELECT entity_id, vec FROM embeddings_ali WHERE entity_id = ANY({arr}) AND vec IS NOT NULL"
+                        )).all()
+                    for eid, v in rows:
+                        if isinstance(v, list) and len(v) > 0:
+                            entity_vecs[eid] = v
+                logger('  [infer] 复用 embeddings_ali 已有向量: %d/%d' % (len(entity_vecs), len(all_ids)))
+            except Exception as _e:
+                logger('  [infer] 复用查询异常: %s' % str(_e)[:50])
+        # ② 缺的才并发批量 embed (4线程并行, 破10秒关键)
+        missing = [e for e in entities if e['id'] not in entity_vecs]
+        if missing:
+            from domain.kg.extract.inference.embed_builder import embed_batch as _embed2
+            from concurrent.futures import ThreadPoolExecutor
+            descs = [((e.get('canonical', '') + ' ' + e.get('cn', '')).strip() or e['id'])[:200] for e in missing]
+            logger('  [infer] 并发 embed %d 缺失实体 (4线程)...' % len(missing))
+            try:
+                # 分 4 块并发, 每块调 embed_batch (内含 10条/批)
+                chunk = max(50, len(descs) // 4)
+                chunks = [descs[i:i+chunk] for i in range(0, len(descs), chunk)]
+                with ThreadPoolExecutor(max_workers=4) as ex:
+                    results = list(ex.map(_embed2, chunks))
+                vecs = [v for cv in results for v in cv]
+                for ent, v in zip(missing, vecs):
+                    if v:
+                        entity_vecs[ent['id']] = v
+                logger('  [infer] 实体向量就绪: %d/%d' % (len(entity_vecs), len(entities)))
+            except Exception as e:
+                logger('  [infer] 并发 embed 失败: %s' % str(e)[:50])
+
+    for i, ent in enumerate(entities):
+        try:
+            eds = infer_one(ent, anchors, anchor_vecs=anchor_vecs,
+                            vec_matrix=vec_matrix, vec_ids=vec_ids,
+                            preset_vec=entity_vecs.get(ent['id']))
+            all_edges += eds
+            if eds:
+                matched += 1
+        except Exception:
+            pass
+        if (i + 1) % 100 == 0:
+            logger('  [infer] %d/%d (匹配%d, 边%d)' % (i + 1, len(entities), matched, len(all_edges)))
+    dt = time.time() - t0
+    logger('  [infer] 完成: %d 实体 → %d 匹配 → %d 弱边 / %.1fs' % (
+        len(entities), matched, len(all_edges), dt))
+
+    # ★ GOAL A: 把新 embed 的实体向量持久化 (下次跑免 API 调用)
+    if entity_vecs and vec_matrix is not None:
+        try:
+            _persist_entity_vecs(entity_vecs, entities)
+        except Exception:
+            pass
+
+    return all_edges, {'entities': len(entities), 'matched': matched,
+                       'edges': len(all_edges), 'seconds': round(dt, 1)}
+
+
+def _persist_entity_vecs(vecs: dict, entities: list, logger=print):
+    """把推理时计算的实体向量存回 embeddings_ali (下次复用, 免 API)。"""
+    from domain.kg.extract.java import pg_upsert
+    from sqlalchemy import text
+    import json
+    engine, _ = pg_upsert._get_pg_engine()
+    if engine is None:
+        return
+    ent_map = {e['id']: e for e in entities}
+    ok = 0
+    with engine.begin() as conn:
+        for eid, vec in vecs.items():
+            ent = ent_map.get(eid, {})
+            try:
+                # embeddings_ali 表 vec 是 ARRAY 类型
+                arr_str = '{' + ','.join(str(float(v)) for v in vec[:1024]) + '}'
+                conn.execute(text("""
+                    INSERT INTO embeddings_ali (entity_id, kind, text, vec, model, embedded_at)
+                    VALUES (:id, :kind, :txt, CAST(:vec AS FLOAT[]), 'text-embedding-v3', now())
+                    ON CONFLICT (entity_id) DO NOTHING
+                """), {'id': eid, 'kind': ent.get('type', 'entity'),
+                       'txt': (ent.get('canonical', '') or eid)[:200], 'vec': arr_str})
+                ok += 1
+            except Exception:
+                pass
+    if ok:
+        logger('  [infer] 持久化 %d 条实体向量 (下次复用)' % ok)

package/templates/qoder/scripts/domain/kg/extract/java/__init__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+"""Java 后端全栈提取器 (GOAL 2)。
+
+子模块:
+  _parser      tree-sitter Java 引擎 + AST 通用遍历
+  spring_extractor      Spring 注解 → SERVICE/CONTROLLER/CONFIG + injects
+  dubbo_extractor       Dubbo 注解 → RPC injects/exposes 边
+  satoken_extractor     Sa-Token → PERMISSION 实体 + guards 边
+  mybatisplus_extractor @TableName/@TableField → ENTITY/TABLE/COLUMN + maps_to
+  validation_extractor  @NotNull/@Size → VALIDATION 实体 + validates 边
+  call_chain            方法体内调用 → calls 边 (Service→Mapper 关键链)
+  endpoint_extractor    @GetMapping → ENDPOINT + handles 边
+  domain_extractor      Entity/BO/VO 字段定义 → DTO 实体 + 字段语义
+  all                   协调器: 单文件一次解析 → 多 extractor → 小批量 PG upsert
+"""