@hupan56/wlkj 3.2.0 → 3.3.0

package/templates/qoder/scripts/domain/kg/extract/inference/community_summarizer.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+"""D2: 补社区摘要 + 摘要向量。
+
+对 graph_communities 表里 summary 为空的社区:
+  ① 读 member_ids → 取成员的 canonical+cn → LLM 生成摘要
+  ② 对摘要算 embedding → summary_vec
+
+独立于现有 summarize_communities (不改动它), 直接补 PG。
+"""
+from __future__ import annotations
+import os
+import sys
+import json
+import time
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+_SUMMARY_PROMPT = """你是软件架构师。下面是一个代码社区(聚类在一起的相关功能)的成员列表。
+请用1-2句话概括这个社区负责什么业务功能。
+
+成员列表(类名/端点/按钮, 括号内是中文含义):
+%s
+
+只返回摘要文本, 不要解释。"""
+
+
+def _call_qwen(prompt: str, timeout: int = 20) -> str:
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    base = os.environ.get('ALI_CHAT_BASE', '').rstrip('/')
+    model = os.environ.get('ALI_CHAT_MODEL', 'qwen-plus')
+    if not key or not base:
+        return ''
+    req = urllib.request.Request(
+        base + '/chat/completions',
+        data=json.dumps({'model': model, 'messages': [{'role': 'user', 'content': prompt}],
+                         'max_tokens': 200, 'temperature': 0.2}).encode('utf-8'),
+        headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read())['choices'][0]['message']['content'].strip()
+    except Exception:
+        return ''
+
+
+def _embed(text: str) -> list:
+    """调 DashScope embedding, 返回 1024 维向量。"""
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    if not key or not text:
+        return []
+    req = urllib.request.Request(
+        'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
+        data=json.dumps({'model': 'text-embedding-v3', 'input': [text[:500]], 'dimensions': 1024}).encode('utf-8'),
+        headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            return json.loads(resp.read())['data'][0]['embedding']
+    except Exception:
+        return []
+
+
+def summarize_missing(limit: int = 2500, do_embed: bool = True, logger=print):
+    """补全缺摘要的社区。
+
+    Args:
+        limit: 上限 (默认 2500, 覆盖全部 2206 个缺的)
+        do_embed: True 同时补 summary_vec
+    """
+    from sqlalchemy import text
+    from domain.kg.extract.java import pg_upsert
+
+    engine, _ = pg_upsert._get_pg_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+
+    # 取缺摘要的社区
+    with engine.connect() as conn:
+        rows = conn.execute(text("""
+            SELECT id, level, size, member_ids FROM graph_communities
+            WHERE summary IS NULL OR summary = ''
+            LIMIT :lim
+        """), {'lim': limit}).all()
+    logger('  [d2] %d 个社区待补摘要' % len(rows))
+    if not rows:
+        return {'summarized': 0}
+
+    # 批量取成员的 canonical+cn (一次查)
+    all_member_ids = set()
+    for r in rows:
+        try:
+            mids = json.loads(r[3]) if isinstance(r[3], str) else (r[3] or [])
+            all_member_ids.update(mids[:20])  # 每社区最多取20个成员
+        except Exception:
+            pass
+
+    member_info = {}  # {entity_id: "canonical(cn)"}
+    if all_member_ids:
+        member_list = list(all_member_ids)[:5000]  # 限 5000 避免查询太大
+        # 分 chunk 查
+        for ci in range(0, len(member_list), 200):
+            chunk = member_list[ci:ci+200]
+            arr = "ARRAY[" + ','.join("'" + eid.replace("'", "''") + "'" for eid in chunk) + "]"
+            with engine.connect() as conn:
+                mrows = conn.execute(text(
+                    f"SELECT id, canonical, cn FROM entities WHERE id = ANY({arr})"
+                )).all()
+            for eid, canonical, cn in mrows:
+                desc = canonical or eid
+                if cn:
+                    desc += '(' + cn[:20] + ')'
+                member_info[eid] = desc[:50]
+
+    logger('  [d2] 成员信息: %d 条' % len(member_info))
+
+    # 批量生成摘要 (4线程并发)
+    def make_summary(row):
+        cid, level, size, member_ids_raw = row
+        try:
+            mids = json.loads(member_ids_raw) if isinstance(member_ids_raw, str) else (member_ids_raw or [])
+        except Exception:
+            mids = []
+        members_desc = [member_info.get(mid, mid) for mid in mids[:15]]
+        if not members_desc:
+            members_desc = [mid for mid in mids[:15]]
+        if not members_desc:
+            return (cid, '空社区')
+        prompt = _SUMMARY_PROMPT % '\n'.join(members_desc)
+        summary = _call_qwen(prompt)
+        if not summary:
+            # 兜底: 实体名拼接
+            summary = '、'.join(members_desc[:5]) + '等相关功能'
+        return (cid, summary[:200])
+
+    # 并发跑
+    t0 = time.time()
+    batches = [rows[i:i+10] for i in range(0, len(rows), 10)]
+    all_summaries = []
+    for bi in range(0, len(batches), 5):
+        chunk = batches[bi:bi+5]
+        with ThreadPoolExecutor(max_workers=4) as ex:
+            results = list(ex.map(make_summary, [r for b in chunk for r in b]))
+        all_summaries += results
+        if (bi//5+1) % 10 == 0:
+            logger('  [d2] %d/%d 社区摘要生成 (%d)' % (len(all_summaries), len(rows), len(all_summaries)))
+
+    logger('  [d2] 摘要生成完成: %d / %.1fs' % (len(all_summaries), time.time()-t0))
+
+    # 写 PG (分块提交)
+    ok = 0
+    vec_ok = 0
+    for i in range(0, len(all_summaries), 100):
+        chunk = all_summaries[i:i+100]
+        with engine.begin() as conn:
+            for cid, summary in chunk:
+                try:
+                    # 算摘要向量 (可选)
+                    vec_str = ''
+                    if do_embed:
+                        vec = _embed(summary)
+                        if vec:
+                            # summary_vec 存成 JSONB (兼容无 pgvector)
+                            vec_str = json.dumps(vec)
+                    if vec_str:
+                        # summary_vec 是 ARRAY 类型, 用 FLOAT[] 写入
+                        arr_str = '{' + ','.join(str(float(v)) for v in vec[:1024]) + '}'
+                        conn.execute(text("""
+                            UPDATE graph_communities SET summary=:s, summary_vec=CAST(:arr AS FLOAT[])
+                            WHERE id=:id
+                        """), {'s': summary, 'arr': arr_str, 'id': cid})
+                        vec_ok += 1
+                    else:
+                        conn.execute(text("""
+                            UPDATE graph_communities SET summary=:s WHERE id=:id
+                        """), {'s': summary, 'id': cid})
+                    ok += 1
+                except Exception:
+                    pass
+        if (i//100+1) % 5 == 0:
+            logger('  [d2] 写入 %d/%d (vec=%d)' % (ok, len(all_summaries), vec_ok))
+
+    dt = time.time() - t0
+    logger('  [d2] 完成: %d 摘要 / %d 向量 / %.1fs' % (ok, vec_ok, dt))
+    return {'summarized': ok, 'vectorized': vec_ok, 'seconds': round(dt, 1)}
+
+
+if __name__ == '__main__':
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--limit', type=int, default=2500)
+    ap.add_argument('--no-embed', action='store_true')
+    args = ap.parse_args()
+    try:
+        from dotenv import load_dotenv
+        for p in ['wlinkj-workspace/backend/.env', '.env']:
+            if os.path.isfile(p): load_dotenv(p); break
+    except: pass
+    r = summarize_missing(limit=args.limit, do_embed=not args.no_embed)
+    print(json.dumps(r, ensure_ascii=False, indent=2))

package/templates/qoder/scripts/domain/kg/extract/inference/embed_builder.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+"""批量灌入实体向量 → embeddings 表 (GOAL A 核心)。
+
+对关键实体 (anchors + SERVICE + TABLE + PRD) 算 DashScope embedding,
+存进 embeddings.vec_json (JSONB 数组, 不依赖 pgvector)。
+
+被 build_goal_a.py 调用, 也可独立 CLI。
+"""
+from __future__ import annotations
+import os
+import sys
+import json
+import time
+import urllib.request
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def embed_batch(texts: list, batch_size: int = 10) -> list:
+    """批量调 DashScope text-embedding-v3 → [[float]]。失败返回空。"""
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    if not key or not texts:
+        return []
+    all_vecs = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+        payload = json.dumps({
+            'model': 'text-embedding-v3', 'input': batch, 'dimensions': 1024,
+        }).encode('utf-8')
+        req = urllib.request.Request(
+            'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings',
+            data=payload,
+            headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                r = json.loads(resp.read())
+            for d in sorted(r['data'], key=lambda x: x['index']):
+                all_vecs.append(d['embedding'])
+        except Exception as e:
+            # 这批失败 → 填 None 占位 (保持顺序)
+            all_vecs.extend([None] * len(batch))
+    return all_vecs
+
+
+def build_embeddings_for_entities(entity_filter: str = None, limit: int = 500,
+                                   kinds: list = None, logger=print):
+    """对实体算 embedding → 灌 embeddings.vec_json。
+
+    Args:
+        entity_filter: SQL WHERE 条件 (如 "type='ANCHOR'")
+        limit: 上限 (控成本)
+        kinds: 实体类型白名单 (默认 anchors + SERVICE + TABLE + PRD)
+    """
+    from domain.kg.extract.java import pg_upsert
+    from sqlalchemy import text
+
+    engine, _ = pg_upsert._get_pg_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+
+    if kinds is None:
+        kinds = ['ANCHOR', 'SERVICE', 'TABLE', 'PRD', 'FEATURE']
+
+    # 取实体 (跳过已 embed 过的)
+    where = "WHERE e.type IN :kinds AND e.id NOT IN (SELECT entity_id FROM embeddings)"
+    if entity_filter:
+        where += " AND " + entity_filter
+    sql = f"""
+        SELECT e.id, e.repo_id, e.type, e.canonical, e.cn
+        FROM entities e
+        {where.replace(':kinds', ':kinds')}
+        LIMIT :lim
+    """
+    # 兼容 IN 参数
+    with engine.connect() as conn:
+        rows = conn.execute(text(sql.replace(':kinds', "('" + "','".join(kinds) + "')")),
+                            {'lim': limit}).all()
+
+    logger('  [embed] %d 实体待向量化 (类型=%s)' % (len(rows), kinds))
+    if not rows:
+        return {'embedded': 0}
+
+    # 批量 embed
+    descs = [(r[3] or r[4] or r[0])[:200] for r in rows]  # canonical 优先
+    t0 = time.time()
+    vecs = embed_batch(descs)
+    embed_time = time.time() - t0
+    valid = sum(1 for v in vecs if v)
+    logger('  [embed] embedding 完成: %d/%d 有效 / %.1fs' % (valid, len(vecs), embed_time))
+
+    # 灌 PG
+    ok = 0
+    with engine.begin() as conn:
+        for (eid, repo_id, etype, canonical, cn), vec in zip(rows, vecs):
+            if not vec:
+                continue
+            try:
+                conn.execute(text("""
+                    INSERT INTO embeddings (entity_id, repo_id, kind, content, vec_json, model, embedded_at)
+                    VALUES (:id, :repo, :kind, :content, CAST(:vec AS JSONB), 'text-embedding-v3', now())
+                    ON CONFLICT (entity_id) DO UPDATE SET
+                        vec_json=EXCLUDED.vec_json, content=EXCLUDED.content, embedded_at=now()
+                """), {
+                    'id': eid, 'repo': repo_id, 'kind': etype,
+                    'content': (canonical or cn or eid)[:200],
+                    'vec': json.dumps(vec),
+                })
+                ok += 1
+            except Exception:
+                pass
+    logger('  [embed] 灌入 %d 条向量' % ok)
+    return {'embedded': ok, 'time_sec': round(embed_time, 1)}
+
+
+if __name__ == '__main__':
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--limit', type=int, default=500)
+    args = ap.parse_args()
+    try:
+        from dotenv import load_dotenv
+        for p in ['wlinkj-workspace/backend/.env', '.env']:
+            if os.path.isfile(p): load_dotenv(p); break
+    except: pass
+    r = build_embeddings_for_entities(limit=args.limit)
+    print(json.dumps(r, ensure_ascii=False, indent=2))

package/templates/qoder/scripts/domain/kg/extract/inference/naming_matcher.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""命名同源匹配: 不依赖 embedding, 用命名规则建弱关联。
+
+AssetController ↔ asset 表 ↔ 资产管理 PRD → 词干相同 → 弱边。
+这是 embedding 不可用时的主路径, 也是 embedding 的补充 (硬规则置信度高)。
+"""
+from __future__ import annotations
+import re
+
+
+def extract_stem(name: str) -> str:
+    """提取名称的词干 (去掉类型后缀 + 小写 + 驼峰拆词)。
+
+    AssetController → 'asset'
+    EmployeeOnline → 'employee online' → 'employee'
+    IQrCodeLoginService → 'qr code login'
+    test_demo → 'test demo'
+    资产管理 → '资产管理'
+    """
+    if not name:
+        return ''
+    s = name.strip()
+    # 去掉常见类型后缀
+    for suffix in ['Controller', 'ServiceImpl', 'Service', 'Mapper',
+                   'Repository', 'Entity', 'Vo', 'BO', 'Bo', 'Dto', 'DTO',
+                   'Form', 'Body', 'Request', 'Response', 'Result']:
+        if s.endswith(suffix) and len(s) > len(suffix):
+            s = s[:-len(suffix)]
+            break
+    # 去掉接口前缀 I
+    if s.startswith('I') and len(s) > 1 and s[1].isupper():
+        s = s[1:]
+    # 驼峰拆词
+    words = re.findall(r'[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|[\u4e00-\u9fff]+', s)
+    if not words:
+        return s.lower()
+    # 中文直接返回, 英文取第一个词 (主词)
+    if any('\u4e00' <= c <= '\u9fff' for c in s):
+        return ''.join(w for w in words if '\u4e00' <= w[0] <= '\u9fff')[:6]
+    return words[0].lower() if words else s.lower()
+
+
+def naming_similarity(name_a: str, name_b: str) -> float:
+    """两个名称的命名同源相似度 → [0, 1]。
+
+    词干相同 → 0.8
+    词干包含关系 → 0.6
+    无关 → 0
+    """
+    stem_a = extract_stem(name_a)
+    stem_b = extract_stem(name_b)
+    if not stem_a or not stem_b:
+        return 0.0
+    if stem_a == stem_b:
+        return 0.8
+    if stem_a in stem_b or stem_b in stem_a:
+        return 0.6
+    # 词重叠 (多词情况)
+    words_a = set(stem_a.split())
+    words_b = set(stem_b.split())
+    if words_a and words_b:
+        overlap = len(words_a & words_b)
+        if overlap > 0:
+            return 0.5 * overlap / max(len(words_a), len(words_b))
+    return 0.0
+
+
+def match_to_anchors(entity_canonical: str, anchors: list) -> list:
+    """把实体和 anchors 做命名匹配 → [(anchor_id, score)]。
+
+    anchors: [{'id':..., 'title':..., 'kind':...}]
+    Returns: 按相似度降序, score >= 0.6 的
+    """
+    out = []
+    for a in anchors:
+        score = naming_similarity(entity_canonical, a.get('title', '') or a.get('id', ''))
+        if score >= 0.6:
+            out.append((a['id'], score))
+    out.sort(key=lambda x: x[1], reverse=True)
+    return out

package/templates/qoder/scripts/domain/kg/extract/inference/promote.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""弱边升格: 出现强关联时, 对应弱边自动删除。
+
+避免同一对 (entity, anchor) 既有强边又有弱边, 强边优先。
+"""
+from __future__ import annotations
+import os
+import sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def _get_engine():
+    from domain.kg.extract.java import pg_upsert
+    return pg_upsert._get_pg_engine()
+
+
+def promote_strong_over_weak(logger=print):
+    """删除有强关联的实体的弱边。
+
+    逻辑: entity 如果有 source='explicit'/'commit' 的 implements_req 边,
+    则该 entity 的所有 source='inferred' 的 relates_to 边删除 (被强边替代)。
+
+    Returns: {deleted}
+    """
+    engine, _ = _get_engine()
+    if engine is None:
+        return {'deleted': 0}
+    from sqlalchemy import text
+
+    deleted = 0
+    with engine.begin() as conn:
+        # 找有强关联的实体
+        strong_entities = conn.execute(text("""
+            SELECT DISTINCT from_id FROM edges
+            WHERE source IN ('explicit', 'commit')
+              AND edge_type IN ('implements_req', 'anchors')
+        """)).all()
+        strong_ids = [r[0] for r in strong_entities]
+        # 也包括 prd: 锚点本身 (它们的实体不该有弱边)
+        if strong_ids:
+            # 删除这些实体的弱边
+            result = conn.execute(text("""
+                DELETE FROM edges
+                WHERE source='inferred' AND from_id = ANY(:ids)
+            """), {'ids': strong_ids})
+            deleted = result.rowcount or 0
+
+    logger('  [promote] 删除 %d 条冗余弱边 (升格到强关联)' % deleted)
+    return {'deleted': deleted}
+
+
+if __name__ == '__main__':
+    print(promote_strong_over_weak())

package/templates/qoder/scripts/domain/kg/extract/inference/recompute.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""C 方案指纹重算: 过期弱边对比指纹 → 续期/重推理/删除。
+
+每天 celery 定时跑:
+  1. 找 expires_at < now 的弱边
+  2. 对比 fingerprint: 没变 → 续期; 变了 → 删除重推理
+  3. 升格: 弱边的 from 出现强关联 → 弱边删除 (在 promote.py)
+"""
+from __future__ import annotations
+import os
+import sys
+from datetime import datetime
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def _get_engine():
+    from domain.kg.extract.java import pg_upsert
+    return pg_upsert._get_pg_engine()
+
+
+def recompute_expired(logger=print):
+    """重算所有过期弱边 (C 方案主入口)。
+
+    Returns: {expired, renewed, deleted, errors}
+    """
+    engine, _ = _get_engine()
+    if engine is None:
+        return {'error': 'PG 不可用'}
+    from sqlalchemy import text
+
+    now = datetime.utcnow().isoformat()
+    stats = {'expired': 0, 'renewed': 0, 'deleted': 0, 'errors': 0}
+
+    with engine.begin() as conn:
+        # 找过期弱边
+        rows = conn.execute(text("""
+            SELECT from_id, to_id, fingerprint FROM edges
+            WHERE source='inferred' AND expires_at IS NOT NULL
+              AND expires_at < CAST(:now AS TIMESTAMPTZ)
+            LIMIT 200
+        """), {'now': now}).all()
+        stats['expired'] = len(rows)
+        logger('  [recompute] %d 条过期弱边' % len(rows))
+
+        for from_id, to_id, fingerprint in rows:
+            try:
+                # 对比当前指纹: 用和建边时相同的 _get_file_fingerprint 函数算
+                # (保证一致性: file_signatures 真实 hash 优先, 无 file 时 canonical hash 兜底)
+                ent_row = conn.execute(text("""
+                    SELECT id, repo_id, canonical, cn, props FROM entities WHERE id=:id
+                """), {'id': from_id}).first()
+                if not ent_row:
+                    # 实体已删 → 删弱边
+                    conn.execute(text("DELETE FROM edges WHERE from_id=:f AND to_id=:t AND source='inferred'"),
+                                 {'f': from_id, 't': to_id})
+                    stats['deleted'] += 1
+                    continue
+                ent = {'id': ent_row[0], 'repo_id': ent_row[1], 'canonical': ent_row[2],
+                       'cn': ent_row[3], 'props': ent_row[4] or {}}
+                # 用 weak_link 的统一 fingerprint 函数 (和建边时一致)
+                from domain.kg.extract.inference.weak_link import _get_file_fingerprint
+                cur_fp = _get_file_fingerprint(ent)
+
+                if cur_fp and fingerprint and str(cur_fp)[:16] == str(fingerprint)[:16]:
+                    # ★ 指纹没变 → 续期 30 天
+                    conn.execute(text("""
+                        UPDATE edges SET expires_at = now() + interval '30 days'
+                        WHERE from_id=:f AND to_id=:t AND source='inferred'
+                    """), {'f': from_id, 't': to_id})
+                    stats['renewed'] += 1
+                else:
+                    # ★ 指纹变了 → 删除旧弱边 (下次全量推理重建)
+                    conn.execute(text("""
+                        DELETE FROM edges WHERE from_id=:f AND to_id=:t AND source='inferred'
+                    """), {'f': from_id, 't': to_id})
+                    stats['deleted'] += 1
+            except Exception:
+                stats['errors'] += 1
+
+    logger('  [recompute] 续期%d 删除%d 错%d' % (
+        stats['renewed'], stats['deleted'], stats['errors']))
+    return stats
+
+
+if __name__ == '__main__':
+    r = recompute_expired()
+    print(r)