@hupan56/wlkj 3.0.0 → 3.1.2

package/templates/qoder/scripts/{kg → domain/kg/search}/search_index.py

@@ -2,9 +2,15 @@
 # -*- coding: utf-8 -*-
 # v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
 import os as _o, sys as _s
-_cp = _o.path.join(_o.path.dirname(_o.path.dirname(_o.path.abspath(__file__))), 'common')
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
 if _cp not in _s.path: _s.path.insert(0, _cp)
 from bootstrap import setup; setup()
+from foundation.core.paths import get_repo_root
 
 """
 Search knowledge index with platform filtering and PRD search.
@@ -33,10 +39,10 @@ except (AttributeError, TypeError, OSError, IOError):
         pass
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from common.terms import CN_MAP, BUSINESS_PATH_MAP, CN_TO_EN, get_platform_map, expand_chinese_query
+from foundation.integrations.terms import CN_MAP, BUSINESS_PATH_MAP, CN_TO_EN, get_platform_map, expand_chinese_query
 PLATFORM_MAP = get_platform_map()
 
-BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+BASE = get_repo_root()
 INDEX_DIR = os.path.join(BASE, 'data', 'index')
 
 # Substring matching against keys shorter than this produces too much noise
@@ -109,17 +115,24 @@ def _get_kg_db():
         return None
 
 
-def _load_keyword_with_cache(con):
-    """加载 code-keyword (11836 行), 带 pickle 缓存层。
+def _load_with_pickle_cache(con, cache_name, loader_fn, empty_sentinel=None):
+    """通用 pickle/mtime 缓存层 (抽自 _load_keyword_with_cache, 推广到全部索引)。
 
-    性能: DuckDB SELECT + Python 循环重建 = ~180ms;
-          pickle.load = ~37ms (提速 5x, 实测数据一致)。
+    性能: ui-style 等索引每次从 DuckDB 全量重建 (3 次查询 + 逐行 json.loads)
+          耗时显著; pickle.load 直接读回对象快数倍。code-keyword 实测 180ms→37ms。
     缓存失效: kg.duckdb 的 mtime 变了就重建 (和 context_pack 缓存逻辑一致)。
     降级: pickle 损坏/版本不兼容 → 静默回退 DuckDB 重建, 不报错。
+    线程/进程安全: 只读连接共享, pickle 写用 temp+replace (原子), 失败不影响返回。
+
+    Args:
+        con: DuckDB 只读连接。
+        cache_name: 缓存文件名 (如 'code-keyword' → .runtime/code-keyword.pkl)。
+        loader_fn: 接收 con, 返回重建后的对象; 返回 None / 空视为无数据。
+        empty_sentinel: loader 返回此值时也视为无数据、不缓存 (默认 None)。
     """
     import pickle
     duckdb_path = os.path.join(INDEX_DIR, 'kg.duckdb')
-    cache_path = os.path.join(BASE, '.qoder', '.runtime', 'code-keyword.pkl')
+    cache_path = os.path.join(BASE, '.qoder', '.runtime', cache_name + '.pkl')
     db_mtime = os.path.getmtime(duckdb_path) if os.path.isfile(duckdb_path) else 0
 
     # ① 缓存有效 (存在 + mtime 匹配) → 直接 load, 跳过 DuckDB
@@ -132,18 +145,17 @@ def _load_keyword_with_cache(con):
             pass  # 缓存损坏 → 回退重建
 
     # ② 缓存无效/缺失 → 从 DuckDB 重建
-    rows = con.execute("SELECT keyword, file_path FROM keywords").fetchall()
-    result = {}
-    for kw, fp in rows:
-        result.setdefault(kw, []).append(fp)
-    if not result:
-        return None
+    result = loader_fn(con)
+    if result is None or result == empty_sentinel:
+        return None  # 无数据, 不缓存 (避免缓存空结果误导后续)
 
-    # ③ 写缓存 (best-effort, 失败不影响返回结果)
+    # ③ 写缓存 (best-effort, 失败不影响返回结果; 用原子写防止半文件)
     try:
         os.makedirs(os.path.dirname(cache_path), exist_ok=True)
-        with open(cache_path, 'wb') as f:
+        tmp = cache_path + '.tmp'
+        with open(tmp, 'wb') as f:
             pickle.dump(result, f, protocol=pickle.HIGHEST_PROTOCOL)
+        os.replace(tmp, cache_path)
     except OSError:
         pass
     return result
@@ -162,58 +174,75 @@ def _duckdb_load(filename):
     try:
         if filename == 'code-keyword.json':
             # {keyword: [files]} — 11836 行重建耗时 180ms, 用 pickle 缓存提速到 37ms
-            return _load_keyword_with_cache(con)
+            def _kw_loader(con):
+                rows = con.execute("SELECT keyword, file_path FROM keywords").fetchall()
+                result = {}
+                for kw, fp in rows:
+                    result.setdefault(kw, []).append(fp)
+                return result if result else None
+            return _load_with_pickle_cache(con, 'code-keyword', _kw_loader)
 
         elif filename == 'code-api.json':
             # {endpoint: controller_file}
-            rows = con.execute("SELECT endpoint, controller_file FROM endpoints").fetchall()
-            return dict(rows) if rows else None
+            def _api_loader(con):
+                rows = con.execute("SELECT endpoint, controller_file FROM endpoints").fetchall()
+                return dict(rows) if rows else None
+            return _load_with_pickle_cache(con, 'code-api', _api_loader)
 
         elif filename == 'ui-style.json':
             # Reconstruct the full ui-style structure from pages/fields/style_meta
-            result = {'version': '1.0', 'projects': {}, 'common_components': {}, 'form_components': {}, 'field_map': {}}
-            # Pages
-            page_rows = con.execute("SELECT file, project, page_type, components, table_columns, form_fields, layout FROM pages").fetchall()
-            for f, proj, ptype, comps, cols, fields, layout in page_rows:
-                proj_data = result['projects'].setdefault(proj, {'page_types': {}, 'page_examples': {}})
-                proj_data['page_types'][ptype] = proj_data['page_types'].get(ptype, 0) + 1
-                examples = proj_data['page_examples'].setdefault(ptype, [])
-                examples.append({'file': f, 'components': json.loads(comps) if comps else [],
-                                 'table_columns': json.loads(cols) if cols else [],
-                                 'form_fields': json.loads(fields) if fields else [],
-                                 'layout': json.loads(layout) if layout else {}})
-            # Fields
-            field_rows = con.execute("SELECT field, title, count, sample_files FROM fields").fetchall()
-            for field, title, count, samples in field_rows:
-                titles = title.split('/') if title else []
-                result['field_map'][field] = {'field': field, 'titles': titles, 'count': count,
-                                              'sample_files': json.loads(samples) if samples else []}
-            # Style meta (layout_fingerprint per project)
-            meta_rows = con.execute("SELECT project, stack, layout, access_mode, sidebar_width, color_primary, color_success, color_warning, color_destructive, radius, vue_files, common_components, form_components FROM style_meta").fetchall()
-            for row in meta_rows:
-                proj = row[0]
-                proj_data = result['projects'].setdefault(proj, {'page_types': {}, 'page_examples': {}})
-                proj_data['layout_fingerprint'] = {
-                    'stack': row[1], 'layout': row[2], 'access_mode': row[3],
-                    'sidebar_width': row[4], 'color_primary': row[5], 'color_success': row[6],
-                    'color_warning': row[7], 'color_destructive': row[8], 'radius': row[9],
-                }
-                proj_data['vue_files'] = row[10]
-                if row[11]:
-                    result['common_components'] = json.loads(row[11])
-                if row[12]:
-                    result['form_components'] = json.loads(row[12])
-            return result if result['projects'] or result['field_map'] else None
+            # 最贵的索引: 3 次 DuckDB 查询 + 对 components/table_columns/
+            # form_fields/layout 逐行 json.loads。加 pickle 缓存后二次调用秒回。
+            def _style_loader(con):
+                result = {'version': '1.0', 'projects': {}, 'common_components': {}, 'form_components': {}, 'field_map': {}}
+                # Pages
+                page_rows = con.execute("SELECT file, project, page_type, components, table_columns, form_fields, layout FROM pages").fetchall()
+                for f, proj, ptype, comps, cols, fields, layout in page_rows:
+                    proj_data = result['projects'].setdefault(proj, {'page_types': {}, 'page_examples': {}})
+                    proj_data['page_types'][ptype] = proj_data['page_types'].get(ptype, 0) + 1
+                    examples = proj_data['page_examples'].setdefault(ptype, [])
+                    examples.append({'file': f, 'components': json.loads(comps) if comps else [],
+                                     'table_columns': json.loads(cols) if cols else [],
+                                     'form_fields': json.loads(fields) if fields else [],
+                                     'layout': json.loads(layout) if layout else {}})
+                # Fields
+                field_rows = con.execute("SELECT field, title, count, sample_files FROM fields").fetchall()
+                for field, title, count, samples in field_rows:
+                    titles = title.split('/') if title else []
+                    result['field_map'][field] = {'field': field, 'titles': titles, 'count': count,
+                                                  'sample_files': json.loads(samples) if samples else []}
+                # Style meta (layout_fingerprint per project)
+                meta_rows = con.execute("SELECT project, stack, layout, access_mode, sidebar_width, color_primary, color_success, color_warning, color_destructive, radius, vue_files, common_components, form_components FROM style_meta").fetchall()
+                for row in meta_rows:
+                    proj = row[0]
+                    proj_data = result['projects'].setdefault(proj, {'page_types': {}, 'page_examples': {}})
+                    proj_data['layout_fingerprint'] = {
+                        'stack': row[1], 'layout': row[2], 'access_mode': row[3],
+                        'sidebar_width': row[4], 'color_primary': row[5], 'color_success': row[6],
+                        'color_warning': row[7], 'color_destructive': row[8], 'radius': row[9],
+                    }
+                    proj_data['vue_files'] = row[10]
+                    if row[11]:
+                        result['common_components'] = json.loads(row[11])
+                    if row[12]:
+                        result['form_components'] = json.loads(row[12])
+                return result if result['projects'] or result['field_map'] else None
+            # empty_sentinel: 全空结构 (version 是固定键, 不算数据)
+            return _load_with_pickle_cache(
+                con, 'ui-style', _style_loader,
+                empty_sentinel={'version': '1.0', 'projects': {}, 'common_components': {}, 'form_components': {}, 'field_map': {}})
 
         elif filename == 'prd-index.json':
-            rows = con.execute("SELECT file, title, keywords, features, cn_terms FROM prds").fetchall()
-            result = {}
-            for f, title, kws, feats, cn in rows:
-                result[f] = {'title': title,
-                             'keywords': json.loads(kws) if kws else [],
-                             'features': json.loads(feats) if feats else [],
-                             'cn_terms': json.loads(cn) if cn else []}
-            return result if result else None
+            def _prd_loader(con):
+                rows = con.execute("SELECT file, title, keywords, features, cn_terms FROM prds").fetchall()
+                result = {}
+                for f, title, kws, feats, cn in rows:
+                    result[f] = {'title': title,
+                                 'keywords': json.loads(kws) if kws else [],
+                                 'features': json.loads(feats) if feats else [],
+                                 'cn_terms': json.loads(cn) if cn else []}
+                return result if result else None
+            return _load_with_pickle_cache(con, 'prd-index', _prd_loader)
 
         elif filename == 'ui-modules.json':
             rows = con.execute("SELECT dir, cn, project, path FROM modules").fetchall()
@@ -337,7 +366,7 @@ def _check_index_freshness():
             if warn_mtime > meta_mtime:
                 return  # 已经提示过了 (在本次索引周期内)
             print('[提示] 代码有更新但知识图谱未刷新 (上次索引: %s)' % last_sync_str)
-            print('  刷新: python .qoder/scripts/git_sync.py --index-only')
+            print('  刷新: python .qoder/scripts/domain/task/git_sync.py --index-only')
             try:
                 os.makedirs(os.path.dirname(warned_path), exist_ok=True)
                 open(warned_path, 'w').close()
@@ -415,12 +444,12 @@ def _cleanup_cache_dir(cache_dir, prefix, max_files=50):
 
 
 # _expand_chinese_query 已移至 common/terms.py (单一数据源, context_pack 共用)
-# import 见文件头部: from common.terms import expand_chinese_query
+# import 见文件头部: from foundation.integrations.terms import expand_chinese_query
 
 
 def _search_keywords_impl(query, platform=None):
     """实际的搜索逻辑 (原 search_keywords 内容)。"""
-    ki = load_index('code-keyword.json', hint='Run: python git_sync.py')
+    ki = load_index('code-keyword.json', hint='Run: python .qoder/scripts/domain/task/git_sync.py')
     if ki is None:
         return
 
@@ -437,8 +466,8 @@ def _search_keywords_impl(query, platform=None):
 
     # 性能优化 A2: 用倒排索引缩小候选集 (从 O(all_keys) 降到 O(命中数))
     try:
-        from common.search_engine import get_inverted_index, search_keywords_fast
-        from common.paths import get_repo_root
+        from search_engine import get_inverted_index, search_keywords_fast
+        from foundation.core.paths import get_repo_root
         inverted = get_inverted_index(str(get_repo_root()))
         kw_matches = search_keywords_fast(words, inverted, ki)
     except Exception:
@@ -460,6 +489,7 @@ def _search_keywords_impl(query, platform=None):
 
     if not matches:
         print('No matches for: {}'.format(query))
+        _semantic_fallback(query, platform)
         return
 
     total = 0
@@ -478,8 +508,44 @@ def _search_keywords_impl(query, platform=None):
     print('Total: {} matches'.format(total))
 
 
+def _semantic_fallback(query, platform=None):
+    """关键词 0 命中时, 用语义检索 (bge-small-zh embedding) 补一轮。
+
+    解决"中文查英文代码"的漏召: 用户查"报销"时关键词匹配不到 reimburse/expense,
+    但语义向量能跨语言命中。把命中的实体解析成文件, 按 platform 过滤后列出。
+
+    模型未装/无 embedding → 静默跳过, 不报错 (符合优雅降级原则)。
+    """
+    try:
+        from domain.kg.graph.kg_semantic import semantic_search_files
+    except ImportError:
+        return
+    results = semantic_search_files(query, top_k=8)
+    if not results:
+        return
+    # 按 platform 过滤 + 补全项目前缀 (相对 vue 路径用存在性兜底)
+    filtered = []
+    try:
+        from domain.kg.graph.kg_semantic import filter_semantic_by_platform
+        filtered = filter_semantic_by_platform(results, platform, BASE)
+    except Exception:
+        filtered = [(fp.replace('\\', '/'), cn, sc) for fp, eid, cn, sc in results]
+    if not filtered:
+        return
+    print()
+    print('[语义补充] 关键词未命中, 用语义检索找到 (中→英跨语言):')
+    for fp, cn, score in filtered[:6]:
+        short = fp
+        if len(short.split('/')) > 5:
+            parts = short.split('/')
+            short = '/'.join(parts[:2]) + '/.../' + '/'.join(parts[-2:])
+        full = 'data/code/' + fp
+        print('  %.2f  %s  ->  %s' % (score, cn[:20], full))
+
+
+
 def search_prds(query):
-    prd_index = load_index('prd-index.json', hint='Run: python git_sync.py --prd-only')
+    prd_index = load_index('prd-index.json', hint='Run: python .qoder/scripts/domain/task/git_sync.py --prd-only')
     if prd_index is None:
         return
     q = query.lower()
@@ -580,7 +646,7 @@ def search_field(field_name):
 
 def search_api(query):
     """Search API endpoints in code-api.json."""
-    ai = load_index('code-api.json', hint='Run: python git_sync.py --index-only')
+    ai = load_index('code-api.json', hint='Run: python .qoder/scripts/domain/task/git_sync.py --index-only')
     if ai is None:
         return
     q = query.lower()
@@ -597,7 +663,7 @@ def search_api(query):
 
 def show_modules():
     """Show project overview from code-stats.json."""
-    mm = load_index('code-stats.json', hint='Run: python git_sync.py --index-only')
+    mm = load_index('code-stats.json', hint='Run: python .qoder/scripts/domain/task/git_sync.py --index-only')
     if mm is None:
         return
     print('=== Project Modules ===')
@@ -611,7 +677,7 @@ def show_modules():
 
 def show_top_keywords():
     """Show top keywords by file count."""
-    ki = load_index('code-keyword.json', hint='Run: python git_sync.py')
+    ki = load_index('code-keyword.json', hint='Run: python .qoder/scripts/domain/task/git_sync.py')
     if ki is None:
         return
     top = sorted(ki.items(), key=lambda x: len(x[1]), reverse=True)[:50]
@@ -666,8 +732,8 @@ def show_vben():
 def search_wiki_cli(query):
     """搜 Repo Wiki (Qoder IDE 生成的语义级模块文档)。"""
     try:
-        from common.repowiki import search_wiki, build_wiki_index
-        from common.terms import get_cn_map_with_auto
+        from repowiki import search_wiki, build_wiki_index
+        from foundation.integrations.terms import get_cn_map_with_auto
     except ImportError:
         print('Wiki 搜索不可用 (common.repowiki 模块缺失)')
         return
@@ -712,7 +778,7 @@ def search_wiki_cli(query):
 
 def search_impact(query):
     """改这个接口/符号影响哪些页面/按钮/模块? (U1: 内存图遍历)"""
-    from common.graph_traverse import CodeGraph
+    from domain.kg.graph.graph_traverse import CodeGraph
     g = CodeGraph(INDEX_DIR)
     result = g.impact(query)
     if result:
@@ -724,7 +790,7 @@ def search_impact(query):
 
 def search_context360(query):
     """符号的360度视图: 端点/函数/处理器/PRD (U1: 内存图遍历)"""
-    from common.graph_traverse import CodeGraph
+    from domain.kg.graph.graph_traverse import CodeGraph
     g = CodeGraph(INDEX_DIR)
     result = g.context360(query)
     if result:
@@ -810,8 +876,8 @@ def _graph_cli(method, *args):
     try:
         if method == 'get_workflow':
             # DuckDB 路径 (v3.0: kg_duckdb 已迁入 kg/ 子包)
-            from kg.kg_duckdb import get_db as _kg_get_db
-            import kg.kg_duckdb as K
+            from domain.kg.storage.kg_duckdb import get_db as _kg_get_db
+            import domain.kg.storage.kg_duckdb as K
             q = (args[0] if args else '').strip()
             if not q:
                 print('请提供模块名, 如: assets / cost / hr / maintenance')
@@ -830,8 +896,8 @@ def _graph_cli(method, *args):
                 print('  %d. [%s] %s — %s' % (r[0], r[1] or '?', r[2] or '', r[3] or ''))
             return
         # JSON 索引路径 (CodeGraph)
-        from common.graph_traverse import CodeGraph
-        from common.paths import DATA_INDEX_DIR
+        from domain.kg.graph.graph_traverse import CodeGraph
+        from foundation.core.paths import DATA_INDEX_DIR
         g = CodeGraph(str(DATA_INDEX_DIR))
         fn = getattr(g, method, None)
         if not fn: