@hupan56/wlkj 3.3.15 → 3.3.16

package/templates/qoder/scripts/domain/kg/extract/inference/embed_builder.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""批量灌入实体向量 → embeddings 表 (GOAL A 核心)。
+
+对关键实体 (anchors + SERVICE + TABLE + PRD) 算 DashScope embedding,
+存进 embeddings.vec_json (JSONB 数组, 不依赖 pgvector)。
+
+被 build_goal_a.py 调用, 也可独立 CLI。
+"""
+from __future__ import annotations
+import os
+import sys
+import json
+import time
+import urllib.request
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def embed_batch(texts: list, batch_size: int = 10) -> list:
+    """批量调 DashScope text-embedding-v3 → [[float]]。失败返回空。"""
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    if not key or not texts:
+        return []
+    all_vecs = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        payload = json.dumps({
+            'model': 'text-embedding-v3', 'input': batch, 'dimensions': 1024,
+        }).encode('utf-8')
+        req = urllib.request.Request(
+            'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
+            data=payload,
+            headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                r = json.loads(resp.read())
+            for d in sorted(r['data'], key=lambda x: x['index']):
+                all_vecs.append(d['embedding'])
+        except Exception as e:
+            # 这批失败 → 填 None 占位 (保持顺序)
+            all_vecs.extend([None] * len(batch))
+    return all_vecs
+
+
+def build_embeddings_for_entities(entity_filter: str = None, limit: int = 500,
+                                   kinds: list = None, logger=print):
+    """对实体算 embedding → 灌 embeddings.vec_json。
+
+    Args:
+        entity_filter: SQL WHERE 条件 (如 "type='ANCHOR'")
+        limit: 上限 (控成本)
+        kinds: 实体类型白名单 (默认 anchors + SERVICE + TABLE + PRD)
+    """
+    from domain.kg.extract.java import pg_upsert
+    from sqlalchemy import text
+
+    engine, _ = pg_upsert._get_pg_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+
+    if kinds is None:
+        kinds = ['ANCHOR', 'SERVICE', 'TABLE', 'PRD', 'FEATURE']
+
+    # 取实体 (跳过已 embed 过的)
+    where = "WHERE e.type IN :kinds AND e.id NOT IN (SELECT entity_id FROM embeddings)"
+    if entity_filter:
+        where += " AND " + entity_filter
+    sql = f"""
+        SELECT e.id, e.repo_id, e.type, e.canonical, e.cn
+        FROM entities e
+        {where.replace(':kinds', ':kinds')}
+        LIMIT :lim
+    """
+    # 兼容 IN 参数
+    with engine.connect() as conn:
+        rows = conn.execute(text(sql.replace(':kinds', "('" + "','".join(kinds) + "')")),
+                            {'lim': limit}).all()
+
+    logger('  [embed] %d 实体待向量化 (类型=%s)' % (len(rows), kinds))
+    if not rows:
+        return {'embedded': 0}
+
+    # 批量 embed
+    descs = [(r[3] or r[4] or r[0])[:200] for r in rows]  # canonical 优先
+    t0 = time.time()
+    vecs = embed_batch(descs)
+    embed_time = time.time() - t0
+    valid = sum(1 for v in vecs if v)
+    logger('  [embed] embedding 完成: %d/%d 有效 / %.1fs' % (valid, len(vecs), embed_time))
+
+    # 灌 PG
+    ok = 0
+    with engine.begin() as conn:
+        for (eid, repo_id, etype, canonical, cn), vec in zip(rows, vecs):
+            if not vec:
+                continue
+            try:
+                conn.execute(text("""
+                    INSERT INTO embeddings (entity_id, repo_id, kind, content, vec_json, model, embedded_at)
+                    VALUES (:id, :repo, :kind, :content, CAST(:vec AS JSONB), 'text-embedding-v3', now())
+                    ON CONFLICT (entity_id) DO UPDATE SET
+                        vec_json=EXCLUDED.vec_json, content=EXCLUDED.content, embedded_at=now()
+                """), {
+                    'id': eid, 'repo': repo_id, 'kind': etype,
+                    'content': (canonical or cn or eid)[:200],
+                    'vec': json.dumps(vec),
+                })
+                ok += 1
+            except Exception:
+                pass
+    logger('  [embed] 灌入 %d 条向量' % ok)
+    return {'embedded': ok, 'time_sec': round(embed_time, 1)}
+
+
+if __name__ == '__main__':
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--limit', type=int, default=500)
+    args = ap.parse_args()
+    try:
+        from dotenv import load_dotenv
+        for p in ['wlinkj-workspace/backend/.env', '.env']:
+            if os.path.isfile(p): load_dotenv(p); break
+    except: pass
+    r = build_embeddings_for_entities(limit=args.limit)
+    print(json.dumps(r, ensure_ascii=False, indent=2))

package/templates/qoder/scripts/domain/kg/extract/inference/naming_matcher.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""命名同源匹配: 不依赖 embedding, 用命名规则建弱关联。
+
+AssetController ↔ asset 表 ↔ 资产管理 PRD → 词干相同 → 弱边。
+这是 embedding 不可用时的主路径, 也是 embedding 的补充 (硬规则置信度高)。
+"""
+from __future__ import annotations
+import re
+
+
+def extract_stem(name: str) -> str:
+    """提取名称的词干 (去掉类型后缀 + 小写 + 驼峰拆词)。
+
+    AssetController → 'asset'
+    EmployeeOnline → 'employee online' → 'employee'
+    IQrCodeLoginService → 'qr code login'
+    test_demo → 'test demo'
+    资产管理 → '资产管理'
+    """
+    if not name:
+        return ''
+    s = name.strip()
+    # 去掉常见类型后缀
+    for suffix in ['Controller', 'ServiceImpl', 'Service', 'Mapper',
+                   'Repository', 'Entity', 'Vo', 'BO', 'Bo', 'Dto', 'DTO',
+                   'Form', 'Body', 'Request', 'Response', 'Result']:
+        if s.endswith(suffix) and len(s) > len(suffix):
+            s = s[:-len(suffix)]
+            break
+    # 去掉接口前缀 I
+    if s.startswith('I') and len(s) > 1 and s[1].isupper():
+        s = s[1:]
+    # 驼峰拆词
+    words = re.findall(r'[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|[\u4e00-\u9fff]+', s)
+    if not words:
+        return s.lower()
+    # 中文直接返回, 英文取第一个词 (主词)
+    if any('\u4e00' <= c <= '\u9fff' for c in s):
+        return ''.join(w for w in words if '\u4e00' <= w[0] <= '\u9fff')[:6]
+    return words[0].lower() if words else s.lower()
+
+
+def naming_similarity(name_a: str, name_b: str) -> float:
+    """两个名称的命名同源相似度 → [0, 1]。
+
+    词干相同 → 0.8
+    词干包含关系 → 0.6
+    无关 → 0
+    """
+    stem_a = extract_stem(name_a)
+    stem_b = extract_stem(name_b)
+    if not stem_a or not stem_b:
+        return 0.0
+    if stem_a == stem_b:
+        return 0.8
+    if stem_a in stem_b or stem_b in stem_a:
+        return 0.6
+    # 词重叠 (多词情况)
+    words_a = set(stem_a.split())
+    words_b = set(stem_b.split())
+    if words_a and words_b:
+        overlap = len(words_a & words_b)
+        if overlap > 0:
+            return 0.5 * overlap / max(len(words_a), len(words_b))
+    return 0.0
+
+
+def match_to_anchors(entity_canonical: str, anchors: list) -> list:
+    """把实体和 anchors 做命名匹配 → [(anchor_id, score)]。
+
+    anchors: [{'id':..., 'title':..., 'kind':...}]
+    Returns: 按相似度降序, score >= 0.6 的
+    """
+    out = []
+    for a in anchors:
+        score = naming_similarity(entity_canonical, a.get('title', '') or a.get('id', ''))
+        if score >= 0.6:
+            out.append((a['id'], score))
+    out.sort(key=lambda x: x[1], reverse=True)
+    return out

package/templates/qoder/scripts/domain/kg/extract/inference/promote.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""弱边升格: 出现强关联时, 对应弱边自动删除。
+
+避免同一对 (entity, anchor) 既有强边又有弱边, 强边优先。
+"""
+from __future__ import annotations
+import os
+import sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def _get_engine():
+    from domain.kg.extract.java import pg_upsert
+    return pg_upsert._get_pg_engine()
+
+
+def promote_strong_over_weak(logger=print):
+    """删除有强关联的实体的弱边。
+
+    逻辑: entity 如果有 source='explicit'/'commit' 的 implements_req 边,
+    则该 entity 的所有 source='inferred' 的 relates_to 边删除 (被强边替代)。
+
+    Returns: {deleted}
+    """
+    engine, _ = _get_engine()
+    if engine is None:
+        return {'deleted': 0}
+    from sqlalchemy import text
+
+    deleted = 0
+    with engine.begin() as conn:
+        # 找有强关联的实体
+        strong_entities = conn.execute(text("""
+            SELECT DISTINCT from_id FROM edges
+            WHERE source IN ('explicit', 'commit')
+              AND edge_type IN ('implements_req', 'anchors')
+        """)).all()
+        strong_ids = [r[0] for r in strong_entities]
+        # 也包括 prd: 锚点本身 (它们的实体不该有弱边)
+        if strong_ids:
+            # 删除这些实体的弱边
+            result = conn.execute(text("""
+                DELETE FROM edges
+                WHERE source='inferred' AND from_id = ANY(:ids)
+            """), {'ids': strong_ids})
+            deleted = result.rowcount or 0
+
+    logger('  [promote] 删除 %d 条冗余弱边 (升格到强关联)' % deleted)
+    return {'deleted': deleted}
+
+
+if __name__ == '__main__':
+    print(promote_strong_over_weak())

package/templates/qoder/scripts/domain/kg/extract/inference/recompute.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""C 方案指纹重算: 过期弱边对比指纹 → 续期/重推理/删除。
+
+每天 celery 定时跑:
+  1. 找 expires_at < now 的弱边
+  2. 对比 fingerprint: 没变 → 续期; 变了 → 删除重推理
+  3. 升格: 弱边的 from 出现强关联 → 弱边删除 (在 promote.py)
+"""
+from __future__ import annotations
+import os
+import sys
+from datetime import datetime
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def _get_engine():
+    from domain.kg.extract.java import pg_upsert
+    return pg_upsert._get_pg_engine()
+
+
+def recompute_expired(logger=print):
+    """重算所有过期弱边 (C 方案主入口)。
+
+    Returns: {expired, renewed, deleted, errors}
+    """
+    engine, _ = _get_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+    from sqlalchemy import text
+
+    now = datetime.utcnow().isoformat()
+    stats = {'expired': 0, 'renewed': 0, 'deleted': 0, 'errors': 0}
+
+    with engine.begin() as conn:
+        # 找过期弱边
+        rows = conn.execute(text("""
+            SELECT from_id, to_id, fingerprint FROM edges
+            WHERE source='inferred' AND expires_at IS NOT NULL
+              AND expires_at < CAST(:now AS TIMESTAMPTZ)
+            LIMIT 200
+        """), {'now': now}).all()
+        stats['expired'] = len(rows)
+        logger('  [recompute] %d 条过期弱边' % len(rows))
+
+        for from_id, to_id, fingerprint in rows:
+            try:
+                # 对比当前指纹: 用和建边时相同的 _get_file_fingerprint 函数算
+                # (保证一致性: file_signatures 真实 hash 优先, 无 file 时 canonical hash 兜底)
+                ent_row = conn.execute(text("""
+                    SELECT id, repo_id, canonical, cn, props FROM entities WHERE id=:id
+                """), {'id': from_id}).first()
+                if not ent_row:
+                    # 实体已删 → 删弱边
+                    conn.execute(text("DELETE FROM edges WHERE from_id=:f AND to_id=:t AND source='inferred'"),
+                                 {'f': from_id, 't': to_id})
+                    stats['deleted'] += 1
+                    continue
+                ent = {'id': ent_row[0], 'repo_id': ent_row[1], 'canonical': ent_row[2],
+                       'cn': ent_row[3], 'props': ent_row[4] or {}}
+                # 用 weak_link 的统一 fingerprint 函数 (和建边时一致)
+                from domain.kg.extract.inference.weak_link import _get_file_fingerprint
+                cur_fp = _get_file_fingerprint(ent)
+
+                if cur_fp and fingerprint and str(cur_fp)[:16] == str(fingerprint)[:16]:
+                    # ★ 指纹没变 → 续期 30 天
+                    conn.execute(text("""
+                        UPDATE edges SET expires_at = now() + interval '30 days'
+                        WHERE from_id=:f AND to_id=:t AND source='inferred'
+                    """), {'f': from_id, 't': to_id})
+                    stats['renewed'] += 1
+                else:
+                    # ★ 指纹变了 → 删除旧弱边 (下次全量推理重建)
+                    conn.execute(text("""
+                        DELETE FROM edges WHERE from_id=:f AND to_id=:t AND source='inferred'
+                    """), {'f': from_id, 't': to_id})
+                    stats['deleted'] += 1
+            except Exception:
+                stats['errors'] += 1
+
+    logger('  [recompute] 续期%d 删除%d 错%d' % (
+        stats['renewed'], stats['deleted'], stats['errors']))
+    return stats
+
+
+if __name__ == '__main__':
+    r = recompute_expired()
+    print(r)