@hupan56/wlkj 3.3.12 → 3.3.14

package/templates/qoder/scripts/domain/kg/extract/mybatis/sql_extractor.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""SQL 文本提取器: 从 SQL 提取表名 + 列引用 + 操作类型。
+
+正则匹配 MyBatis 动态 SQL 里残留的 ${ew.customSqlSegment} 等,
+但 FROM/INTO/UPDATE/JOIN 后的表名通常稳定可提。
+"""
+from __future__ import annotations
+import re
+
+# 表名提取 (FROM/JOIN/INTO/UPDATE 后跟标识符)
+_TABLE_PATTERNS = [
+    re.compile(r'\bFROM\s+`?([a-z_][a-z0-9_]*)`?', re.IGNORECASE),
+    re.compile(r'\bJOIN\s+`?([a-z_][a-z0-9_]*)`?', re.IGNORECASE),
+    re.compile(r'\bINTO\s+`?([a-z_][a-z0-9_]*)`?', re.IGNORECASE),
+    re.compile(r'\bUPDATE\s+`?([a-z_][a-z0-9_]*)`?\s+SET', re.IGNORECASE),
+]
+
+# 排除: 显然不是表名的 (变量/占位符/SQL关键字)
+_NOT_TABLES = {'dual', 'select', 'where', 'set', 'values', 'from', 'join',
+               'and', 'or', 'not', 'in', 'as', 'on', 'left', 'right', 'inner',
+               'outer', 'group', 'order', 'limit', 'distinct', 'case', 'when',
+               'then', 'else', 'end', 'null', 'true', 'false'}
+
+
+def extract_table_names(sql: str) -> list:
+    """从 SQL 文本提取所有被访问的表名。
+
+    Returns: ['test_demo', 'sys_user'] (去重保序)
+    """
+    if not sql:
+        return []
+    found = []
+    seen = set()
+    for pat in _TABLE_PATTERNS:
+        for m in pat.finditer(sql):
+            tbl = m.group(1).lower()
+            if tbl in _NOT_TABLES or tbl.startswith('$') or tbl.startswith('#'):
+                continue
+            # 排除太短或含特殊字符的 (变量插值残留)
+            if len(tbl) < 2 or '$' in tbl or '{' in tbl:
+                continue
+            if tbl not in seen:
+                seen.add(tbl)
+                found.append(tbl)
+    return found
+
+
+def extract_column_refs(sql: str) -> list:
+    """提取 SQL 里引用的列名 (粗粒度, 主要看 SELECT 字段列表)。
+
+    Returns: ['id', 'user_id', 'name'] (去重)
+    """
+    if not sql:
+        return []
+    cols = set()
+    # SELECT field1, field2 FROM ... (字段列表)
+    m = re.search(r'SELECT\s+(DISTINCT\s+)?(.*?)\s+FROM', sql, re.IGNORECASE | re.DOTALL)
+    if m:
+        field_part = m.group(2)
+        if field_part.strip() != '*':
+            for token in re.split(r'[,\s]+', field_part):
+                token = token.strip().strip('`')
+                # 别名 as xx → 取前半
+                token = token.split(' as ')[0].split(' AS ')[0].strip()
+                if re.match(r'^[a-z_][a-z0-9_]*$', token, re.I) and len(token) >= 2:
+                    cols.add(token.lower())
+    # xx.column 形式 (表别名.列)
+    for m in re.finditer(r'\b[a-z]+\s*\.\s*([a-z_][a-z0-9_]*)', sql, re.IGNORECASE):
+        cols.add(m.group(1).lower())
+    return sorted(cols)
+
+
+def classify_operation(stmt_type: str) -> str:
+    """MyBatis statement 类型 → 操作分类。"""
+    return {
+        'select': 'READ', 'insert': 'WRITE',
+        'update': 'WRITE', 'delete': 'WRITE',
+    }.get(stmt_type.lower(), 'UNKNOWN')

package/templates/qoder/scripts/domain/kg/extract/prd/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""PRD 业务知识提取 (GOAL 4)。
+
+子模块:
+  prd_llm_extract  PRD 正文 → LLM 抽功能点/字段/规则/端点/权限
+  prd_chunk_embed  PRD 分块 + embedding (LightRAG vector 路)
+  req_anchor       REQ-ID 强关联 (PRD/commit/代码注释 → anchors 表)
+"""

package/templates/qoder/scripts/domain/kg/extract/prd/prd_chunk_embed.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""PRD 分块 + embedding → LightRAG vector 路。
+
+让 rag_search 能搜到 PRD 正文 (语义召回), 不再只搜标题。
+复用工作台 rag.py 的 embed_texts (DashScope text-embedding-v3, 1024维)。
+"""
+from __future__ import annotations
+import os
+import sys
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+
+def chunk_text(text: str, size: int = 500, overlap: int = 50) -> list:
+    """滑窗分块。Returns: [chunk1, chunk2, ...]"""
+    if not text:
+        return []
+    chunks = []
+    i = 0
+    while i < len(text):
+        chunk = text[i:i + size]
+        chunks.append(chunk)
+        i += size - overlap
+    return chunks
+
+
+def embed_prd(prd_id: str, content_md: str, repo_id: str = ''):
+    """PRD 分块 + embedding → 写 PG embeddings 表。
+
+    Returns: {chunks, embedded} 失败返回 {'chunks':0,'embedded':0}
+    """
+    chunks = chunk_text(content_md)
+    if not chunks:
+        return {'chunks': 0, 'embedded': 0}
+
+    # 复用工作台 rag.py 的 embed_texts
+    try:
+        ws_backend = os.path.join(os.getcwd(), 'wlinkj-workspace', 'backend')
+        sys.path.insert(0, ws_backend)
+        from app.rag import embed_texts
+    except Exception:
+        # 兜底: 直接调 DashScope embedding API
+        return _embed_fallback(prd_id, chunks, repo_id)
+
+    vecs = embed_texts(chunks)
+    if not vecs:
+        return {'chunks': len(chunks), 'embedded': 0}
+
+    # 写 PG embeddings 表
+    _upsert_embeddings(prd_id, chunks, vecs, repo_id)
+    return {'chunks': len(chunks), 'embedded': len(vecs)}
+
+
+def _embed_fallback(prd_id, chunks, repo_id):
+    """直接调 DashScope text-embedding-v3 (rag.py 不可用时)。"""
+    import json, urllib.request
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    if not key:
+        return {'chunks': len(chunks), 'embedded': 0}
+    # DashScope embedding API
+    url = 'https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings'
+    try:
+        req = urllib.request.Request(
+            url,
+            data=json.dumps({
+                'model': 'text-embedding-v3',
+                'input': chunks[:10],  # 单次上限10条
+                'dimensions': 1024,
+            }).encode('utf-8'),
+            headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+        )
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            r = json.loads(resp.read())
+        vecs = [d['embedding'] for d in r['data']]
+        _upsert_embeddings(prd_id, chunks[:len(vecs)], vecs, repo_id)
+        return {'chunks': len(chunks), 'embedded': len(vecs)}
+    except Exception:
+        return {'chunks': len(chunks), 'embedded': 0}
+
+
+def _upsert_embeddings(prd_id, chunks, vecs, repo_id):
+    """写 PG embeddings 表 (pgvector)。"""
+    try:
+        from domain.kg.extract.java import pg_upsert
+        engine, _ = pg_upsert._get_pg_engine()
+        if engine is None:
+            return
+        from sqlalchemy import text
+        import json
+        with engine.begin() as conn:
+            for chunk, vec in zip(chunks, vecs):
+                vec_str = '[' + ','.join(str(v) for v in vec) + ']'
+                eid = 'PRDEMB:%s:%d' % (prd_id[:12], hash(chunk) & 0xffff)
+                conn.execute(text("""
+                    INSERT INTO embeddings (entity_id, repo_id, kind, content, vec, embedded_at)
+                    VALUES (:id, :repo, 'prd', :content, CAST(:vec AS VECTOR), now())
+                    ON CONFLICT (entity_id) DO UPDATE SET content=EXCLUDED.content, vec=EXCLUDED.vec, embedded_at=now()
+                """), {'id': eid, 'repo': repo_id, 'content': chunk[:500], 'vec': vec_str})
+    except Exception as e:
+        print('  [prd-embed] 写入跳过:', str(e)[:60])

package/templates/qoder/scripts/domain/kg/extract/prd/prd_llm_extract.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""PRD 正文 → LLM 抽取功能点/字段/规则/端点/权限。
+
+输入: PRD markdown 正文
+输出: 结构化知识 (FEATURE/CONSTRAINT/PERMISSION 实体)
+用 DashScope qwen-plus (已接入) 做结构化抽取。
+
+被 asset/prd_importer.py 调用 (灌资产中心 prds 表时, 每个PRD正文过一遍LLM)。
+"""
+from __future__ import annotations
+import os
+import sys
+import json
+import re
+import urllib.request
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+PRD_EXTRACT_PROMPT = """从以下PRD正文提取结构化知识, 只返回严格JSON(不要markdown代码块, 不要解释), 格式:
+{"features":[{"name":"资产列表分页","desc":"支持按页查看"}],"fields":[{"name":"owner_id","label":"资产归属人","rule":"必填"}],"rules":[{"rule":"金额>1万需二次审批","type":"business"}],"endpoints":[{"path":"/asset/list","method":"GET","desc":"资产列表"}],"permissions":[{"code":"asset:list","desc":"资产查看权限"}]}
+若某类提取不到就返回空数组。PRD正文:
+---
+%s
+---"""
+
+
+def _call_qwen(prompt: str, timeout: int = 60) -> str:
+    """调 DashScope qwen-plus, 返回文本。失败返回 ''。"""
+    key = os.environ.get('DASHSCOPE_API_KEY', '')
+    base = os.environ.get('ALI_CHAT_BASE', '').rstrip('/')
+    model = os.environ.get('ALI_CHAT_MODEL', 'qwen-plus')
+    if not key or not base:
+        return ''
+    req = urllib.request.Request(
+        base + '/chat/completions',
+        data=json.dumps({
+            'model': model,
+            'messages': [{'role': 'user', 'content': prompt}],
+            'max_tokens': 2000, 'temperature': 0.1,
+        }).encode('utf-8'),
+        headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            r = json.loads(resp.read())
+            return r['choices'][0]['message']['content']
+    except Exception:
+        return ''
+
+
+def _parse_json_lenient(text: str) -> dict:
+    """宽松解析 LLM 返回的 JSON (可能有 markdown 包裹或前后缀)。"""
+    if not text:
+        return {}
+    # 剥 markdown 代码块
+    text = re.sub(r'^```(?:json)?\s*', '', text.strip())
+    text = re.sub(r'\s*```$', '', text.strip())
+    # 找第一个 { 到最后一个 }
+    s = text.find('{')
+    e = text.rfind('}')
+    if s >= 0 and e > s:
+        text = text[s:e + 1]
+    try:
+        return json.loads(text)
+    except Exception:
+        return {}
+
+
+def extract_prd_knowledge(content_md: str) -> dict:
+    """PRD 正文 → LLM 抽取 → 结构化 dict。
+
+    Returns: {features, fields, rules, endpoints, permissions}
+    """
+    if not content_md or len(content_md) < 50:
+        return {'features': [], 'fields': [], 'rules': [], 'endpoints': [], 'permissions': []}
+    # 截断过长正文 (避免超 token)
+    content = content_md[:6000]
+    raw = _call_qwen(PRD_EXTRACT_PROMPT % content)
+    parsed = _parse_json_lenient(raw)
+    # 兜底确保字段齐全
+    return {
+        'features': parsed.get('features', []),
+        'fields': parsed.get('fields', []),
+        'rules': parsed.get('rules', []),
+        'endpoints': parsed.get('endpoints', []),
+        'permissions': parsed.get('permissions', []),
+    }
+
+
+def to_entities_edges(extracted: dict, prd_id: str, project_id: str, repo_id: str = ''):
+    """把 LLM 抽取结果转成 entities + edges。
+
+    entities: FEATURE / CONSTRAINT / PERMISSION
+    edges: 全部锚到 prd_id (强关联, confidence=1.0)
+    """
+    entities, edges = [], []
+    base = 'FEAT:%s' % prd_id[:12] if prd_id else 'FEAT:unknown'
+
+    # 功能点 → FEATURE
+    for f in extracted.get('features', []):
+        fname = f.get('name', '').strip()
+        if not fname:
+            continue
+        fid = '%s:%s' % (base, fname[:20])
+        entities.append({
+            'id': fid, 'repo_id': repo_id, 'type': 'FEATURE',
+            'canonical': fname, 'cn': f.get('desc', ''),
+            'props': {'source': 'prd_llm', 'prd_id': prd_id},
+        })
+        edges.append(_anchor_edge(fid, prd_id, repo_id))
+
+    # 业务规则 → CONSTRAINT
+    for r in extracted.get('rules', []):
+        rule = r.get('rule', '').strip()
+        if not rule:
+            continue
+        cid = 'CONST:%s:%s' % (prd_id[:8], rule[:20])
+        entities.append({
+            'id': cid, 'repo_id': repo_id, 'type': 'CONSTRAINT',
+            'canonical': rule[:50], 'cn': r.get('type', ''),
+            'props': {'source': 'prd_llm', 'prd_id': prd_id, 'rule': rule},
+        })
+        edges.append(_anchor_edge(cid, prd_id, repo_id))
+
+    # 权限 → PERMISSION
+    for p in extracted.get('permissions', []):
+        code = p.get('code', '').strip()
+        if not code:
+            continue
+        pid = 'PERM:PRD:%s:%s' % (prd_id[:8], code[:20])
+        entities.append({
+            'id': pid, 'repo_id': repo_id, 'type': 'PERMISSION',
+            'canonical': code, 'cn': p.get('desc', ''),
+            'props': {'source': 'prd_llm', 'prd_id': prd_id},
+        })
+        edges.append(_anchor_edge(pid, prd_id, repo_id))
+
+    return entities, edges
+
+
+def _anchor_edge(from_id: str, prd_id: str, repo_id: str) -> dict:
+    """实体 → PRD 的强关联边。"""
+    return {
+        'from_id': from_id, 'to_id': 'prd:%s' % prd_id,
+        'edge_type': 'anchors', 'from_repo': repo_id, 'to_repo': repo_id,
+        'confidence': 1.0, 'source': 'explicit',
+        'props': {'via': 'prd_llm'},
+    }

package/templates/qoder/scripts/domain/kg/extract/prd/req_anchor.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+"""REQ-ID 强关联: PRD/commit/代码注释 三种锚定来源。
+
+复用 archive_prd.py 的正则 (REQ-2026-005 / REQ-012 两种格式)。
+锚点写 anchors 表, 关联写 edges (confidence=1.0, source=explicit)。
+"""
+from __future__ import annotations
+import os
+import sys
+import re
+
+_THIS = os.path.dirname(os.path.abspath(__file__))
+for _i in range(8):
+    _p = os.path.dirname(_THIS)
+    if os.path.isfile(os.path.join(_p, 'foundation', 'bootstrap.py')):
+        sys.path.insert(0, _p); break
+    _THIS = _p
+
+# 复用 archive_prd.py 的正则 (兼容 REQ-2026-005 标准格式 + REQ-012 老格式)
+_REQ_RE = re.compile(r'REQ-(\d{2,4})(?:-(\d{2,4}))?', re.IGNORECASE)
+
+
+def format_req_id(match) -> str:
+    """格式化 REQ-ID (标准化)。"""
+    g1, g2 = match.group(1), match.group(2)
+    if g2:
+        return 'REQ-%s-%s' % (g1, g2)
+    return 'REQ-%s' % g1
+
+
+def find_req_ids(text: str) -> list:
+    """从文本提取所有 REQ-ID (去重保序)。"""
+    seen = set()
+    out = []
+    for m in _REQ_RE.finditer(text or ''):
+        rid = format_req_id(m)
+        if rid not in seen:
+            seen.add(rid)
+            out.append(rid)
+    return out
+
+
+# ============ 三种锚定 ============
+
+def anchor_prd(prd_id: str, title: str, project_id: str, repo_id: str = ''):
+    """时机1: PRD 创建时锚定。
+
+    PRD 标题/内容含 REQ-ID → 锚到 anchors 表。
+    没有 REQ-ID → 用 PRD id 生成锚点 (REQ-PRD-xxx)。
+    Returns: (anchor_entities, anchor_edges)
+    """
+    entities, edges = [], []
+    req_ids = find_req_ids(title)
+    if not req_ids:
+        # 无 REQ-ID, 用 PRD 生成一个锚点
+        req_ids = ['REQ-PRD-%s' % prd_id[:8]]
+
+    for rid in req_ids:
+        # anchors 表实体 (type=ANCHOR)
+        entities.append({
+            'id': rid, 'repo_id': repo_id, 'type': 'ANCHOR',
+            'canonical': rid, 'cn': title[:60],
+            'props': {'kind': 'feature', 'prd_id': prd_id, 'project_id': project_id},
+        })
+        # PRD → 锚点 强关联边
+        edges.append({
+            'from_id': 'prd:%s' % prd_id, 'to_id': rid,
+            'edge_type': 'anchors', 'from_repo': repo_id, 'to_repo': repo_id,
+            'confidence': 1.0, 'source': 'explicit',
+            'props': {'via': 'prd_create'},
+        })
+    return entities, edges
+
+
+def anchor_commit(commit_msg: str, changed_files: list, repo_id: str = ''):
+    """时机2: commit message 带 REQ-ID → 代码文件锚定。
+
+    feat(REQ-2026-005): 资产列表分页
+    → changed_files 里的代码 → 锚到 REQ-2026-005
+    Returns: edges (不产生新实体, 只关联已有代码实体到锚点)
+    """
+    edges = []
+    req_ids = find_req_ids(commit_msg)
+    if not req_ids:
+        return edges
+
+    for rid in req_ids:
+        for filepath in changed_files:
+            # 文件级锚定 (文件可能对应多个实体, 这里先建文件→锚点边)
+            file_entity_id = 'FILE:%s:%s' % (repo_id, filepath.replace('\\', '/'))
+            edges.append({
+                'from_id': file_entity_id, 'to_id': rid,
+                'edge_type': 'implements_req', 'from_repo': repo_id, 'to_repo': repo_id,
+                'confidence': 1.0, 'source': 'commit',
+                'props': {'commit_msg': commit_msg[:100]},
+            })
+    return edges
+
+
+def anchor_inline_code(content: str, filepath: str, repo_id: str = ''):
+    """时机3: 代码注释里的 REQ-ID 锚定。
+
+    // REQ-2026-005 资产列表分页
+    /* REQ-047 老逻辑 */
+    Returns: edges
+    """
+    edges = []
+    req_ids = find_req_ids(content)
+    if not req_ids:
+        return edges
+
+    file_entity_id = 'FILE:%s:%s' % (repo_id, filepath.replace('\\', '/'))
+    for rid in req_ids:
+        edges.append({
+            'from_id': file_entity_id, 'to_id': rid,
+            'edge_type': 'implements_req', 'from_repo': repo_id, 'to_repo': repo_id,
+            'confidence': 1.0, 'source': 'code_comment',
+            'props': {'via': 'inline_comment'},
+        })
+    return edges