@lorrylurui/code-intelligence-mcp 1.1.14 → 1.2.0

package/dist/tools/searchSymbols.js

@@ -2,12 +2,16 @@ import { z } from 'zod';
 import { rankSemanticHits, rankSymbols } from '../services/ranking.js';
 export const searchSymbolsInput = z.object({
     query: z.string().min(1),
-    type: z.enum(['component', 'util', 'selector', 'type']).optional(),
+    type: z
+        .enum(['component', 'function', 'hook', 'type', 'interface', 'class'])
+        .optional(),
     ranked: z.boolean().optional().default(true),
     /** Phase 5：自然语言 / 描述句检索（需 EMBEDDING_SERVICE_URL + 索引已写入 embedding） */
     semantic: z.boolean().optional().default(false),
     limit: z.number().int().min(1).max(100).optional().default(20),
 });
+const THREADHOLD_SIMILARITY_FOR_FINAL = 0.6;
+const TOP_K_FOR_FINAL_RESULTS = 20; // 结果上限,返回相似度高的，保证数据质量
 export function createSearchSymbolsTool(repository) {
     return {
         name: 'search_symbols',
@@ -21,7 +25,8 @@ export function createSearchSymbolsTool(repository) {
                 });
                 const simById = new Map(hits.map((h) => [h.symbol.id, h.similarity]));
                 const resultRows = input.ranked
-                    ? rankSemanticHits(hits).map((item) => ({
+                    ? rankSemanticHits(hits)
+                        .map((item) => ({
                         id: item.symbol.id,
                         name: item.symbol.name,
                         type: item.symbol.type,
@@ -33,6 +38,9 @@ export function createSearchSymbolsTool(repository) {
                         reasonDetail: item.reason,
                         semanticSimilarity: Number((simById.get(item.symbol.id) ?? 0).toFixed(4)),
                     }))
+                        .filter((x) => x.semanticSimilarity >=
+                        THREADHOLD_SIMILARITY_FOR_FINAL) // 阈值过滤，去掉明显不相关的结果
+                        .slice(0, TOP_K_FOR_FINAL_RESULTS)
                     : hits.map((h) => ({
                         id: h.symbol.id,
                         name: h.symbol.name,
@@ -53,7 +61,8 @@ export function createSearchSymbolsTool(repository) {
             }
             const rows = await repository.search(input.query, input.type);
             const resultRows = input.ranked
-                ? rankSymbols(input.query, rows).map((item) => ({
+                ? rankSymbols(input.query, rows)
+                    .map((item) => ({
                     id: item.symbol.id,
                     name: item.symbol.name,
                     type: item.symbol.type,
@@ -64,6 +73,8 @@ export function createSearchSymbolsTool(repository) {
                     reason: item.reason.summary,
                     reasonDetail: item.reason,
                 }))
+                    .filter((x) => x.score >= THREADHOLD_SIMILARITY_FOR_FINAL) // 阈值过滤，去掉明显不相关的结果
+                    .slice(0, TOP_K_FOR_FINAL_RESULTS)
                 : rows.map((r) => ({
                     id: r.id,
                     name: r.name,

package/dist/workers/embeddingWorker.js

@@ -0,0 +1,100 @@
+/**
+ * BullMQ embedding worker（常驻消费进程）。
+ *
+ * 流程：
+ * 1. 收到 job { semanticHash }
+ * 2. 查 semantic_hash 缓存：若已有 status=online 的符号带 embedding → 直接复用（0 次 API 调用）
+ * 3. 缓存未命中 → 取一条 pending 行构建语义文本 → 调 embedding API
+ * 4. 批量 UPDATE：所有 semantic_hash 相同且 status=pending 的行一次性写入向量并置 online
+ *
+ * 并发/限流：
+ * - concurrency 控制同时处理的 job 数（默认 5）
+ * - BullMQ limiter 控制全局 RPM（默认 100/min，留 buffer 低于 OpenAI 3000 RPM）
+ *
+ * 大仓分片：
+ * - 直接启动多个 worker 进程（同一 Redis）即可水平扩展，BullMQ 原生分布式协调
+ */
+import { Worker } from 'bullmq';
+import Redis from 'ioredis';
+import { env } from '../config/env.js';
+import { getMySqlPool } from '../db/mysql.js';
+import { createEmbeddingClient } from './embeddingClient.js';
+import { indexedRowToEmbedText } from '../indexer/embedText.js';
+import { SYMBOL_STATUS } from '../config/symbolStatus.js';
+async function processEmbedJob(job, pool) {
+    const { semanticHash } = job.data;
+    const table = env.mysqlSymbolsTable;
+    const embedClient = createEmbeddingClient(env.embeddingServiceUrl);
+    // Step 1: 缓存命中检查 —— 相同 semantic_hash 已有 online 向量
+    const [cached] = await pool.query(`SELECT embedding FROM ${table}
+         WHERE semantic_hash = ? AND status = ? AND embedding IS NOT NULL
+         LIMIT 1`, [semanticHash, SYMBOL_STATUS.ONLINE]);
+    let vector;
+    if (cached.length > 0) {
+        // Cache hit: 直接复用已有向量，0 次 API 调用
+        vector =
+            typeof cached[0].embedding === 'string'
+                ? JSON.parse(cached[0].embedding)
+                : cached[0].embedding;
+        console.error(`[worker] cache hit  hash=${semanticHash.slice(0, 10)}…`);
+    }
+    else {
+        // Cache miss: 取一条 pending 行做 embedding
+        const [pending] = await pool.query(`SELECT name, type, category, path, description, content, meta
+             FROM ${table}
+             WHERE semantic_hash = ? AND status = ?
+             LIMIT 1`, [semanticHash, SYMBOL_STATUS.PENDING]);
+        if (pending.length === 0) {
+            // 所有行已被并发 worker 处理，幂等退出
+            return;
+        }
+        const row = pending[0];
+        const meta = typeof row.meta === 'string'
+            ? JSON.parse(row.meta)
+            : (row.meta ?? {});
+        const doc = indexedRowToEmbedText({ ...row, meta });
+        const vectors = await embedClient.embed([doc]);
+        vector = vectors[0];
+        console.error(`[worker] embedded   hash=${semanticHash.slice(0, 10)}… path=${row.path}:${row.name}`);
+    }
+    // Step 2: 批量写入 —— 批量更新所有拥有相同 semantic_hash 的 pending 行
+    // 一次 API 调用覆盖 N 个同义符号（大仓重复代码/多文件同函数场景收益明显）
+    await pool.query(`UPDATE ${table}
+         SET embedding = CAST(? AS JSON), status = ?
+         WHERE semantic_hash = ? AND status = ?`, [
+        JSON.stringify(vector),
+        SYMBOL_STATUS.ONLINE,
+        semanticHash,
+        SYMBOL_STATUS.PENDING,
+    ]);
+}
+/**
+ * 启动 embedding worker，返回 Worker 实例（可用于优雅关闭）。
+ */
+export function startEmbeddingWorker(opts = {}) {
+    const { concurrency = 5, rpmLimit = 100 } = opts;
+    const connection = new Redis(env.redisUrl, {
+        maxRetriesPerRequest: null,
+        enableReadyCheck: false,
+    });
+    const pool = getMySqlPool();
+    if (!pool) {
+        throw new Error('[embeddingWorker] MySQL pool unavailable — check env vars');
+    }
+    const worker = new Worker('embedding', (job) => processEmbedJob(job, pool), {
+        connection,
+        concurrency,
+        // 全局限流：所有 worker 进程共享，防止触发 OpenAI rate limit
+        limiter: { max: rpmLimit, duration: 60_000 },
+    });
+    worker.on('completed', (job) => {
+        console.error(`[worker] ✓ job done  hash=${job.data.semanticHash.slice(0, 10)}…`);
+    });
+    worker.on('failed', (job, err) => {
+        console.error(`[worker] ✗ job fail  hash=${job?.data?.semanticHash?.slice(0, 10)}… err=${err.message}`);
+    });
+    worker.on('error', (err) => {
+        console.error(`[worker] error: ${err.message}`);
+    });
+    return worker;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@lorrylurui/code-intelligence-mcp",
-    "version": "1.1.14",
+    "version": "1.2.0",
     "private": false,
     "description": "MCP server 提供仓库内可复用代码块（ts/tsx/js/jsx/css/less）的索引和查询能力，支持基于代码上下文的智能推荐。",
     "type": "module",
@@ -10,15 +10,16 @@
     ],
     "bin": {
         "code-intelligence-mcp": "./dist/index.js",
-        "code-intelligence-index": "./dist/cli/index-codebase.js"
+        "code-intelligence-index": "./dist/cli/index-codebase-cli.js"
     },
     "scripts": {
         "dev": "tsx watch --clear-screen=false --exclude node_modules --exclude dist src/index.ts",
         "dev:mcp": "node ./scripts/mcp-dev-watch.mjs",
         "build": "tsc -p tsconfig.json",
         "start": "node dist/index.js",
-        "index": "tsx src/cli/index-codebase.ts",
-        "detect-duplicates": "tsx src/cli/detect-duplicates.ts",
+        "index": "tsx src/cli/index-codebase-cli.ts",
+        "ci-index": "tsx src/cli/ci-index-cli.ts",
+        "worker:embedding": "tsx src/cli/embedding-worker-cli.ts",
         "embedding:dev": "cd embedding-service && python3 -m uvicorn app:app --host 127.0.0.1 --port 8765",
         "docker:up": "docker compose up -d",
         "docker:down": "docker compose down",
@@ -29,8 +30,10 @@
         "@babel/types": "^7.29.0",
         "@modelcontextprotocol/sdk": "^1.12.3",
         "@types/react": "^19.2.14",
+        "bullmq": "^5.74.1",
         "dotenv": "^16.4.5",
         "fast-glob": "^3.3.2",
+        "ioredis": "^5.10.1",
         "mysql2": "^3.11.3",
         "react": "^19.2.4",
         "ts-morph": "^25.0.0",