@lorrylurui/code-intelligence-mcp 2.0.7 → 2.0.9

package/README.md

@@ -45,3 +45,22 @@
 
 MYSQL\*SYMBOLS_TABLE=frontend_collections_symbols
 INDEX_GLOB=xxx/\*\*/\_.{js,jsx,ts,tsx}
+
+## 4) 离线测评
+
+```javascript
+npx tsx src/cli/eval-recommendation-cli.ts
+# 或指定 limit
+npx tsx src/cli/eval-recommendation-cli.ts --limit 10
+```
+
+## 5）分析离线测评结果
+
+```javascript
+npm run analyze                                          # 自动读最新结果文件
+npm run analyze -- offline_eval/results/2026-05-27.jsonl  # 指定文件
+npm run analyze -- --baseline offline_eval/results/2026-05-26.jsonl  # 与 baseline 对比 delta
+
+```
+
+或者直接引用eval-analysis.prompt.md 对最新结果分析

package/dist/cli/ci-index.js

@@ -2,75 +2,64 @@
 import { env } from '../config/env.js';
 import { getPool } from '../db/postgres.js';
 import { indexProject } from '../indexer/indexProject.js';
-import { DEFAULT_STATUS_ON_UPSERT, SYMBOL_STATUS, } from '../config/symbolStatus.js';
+import { upsertSymbols } from '../indexer/persistSymbols.js';
 import { enqueueEmbeddingBatch, closeEmbeddingQueue, } from '../services/embeddingQueue.js';
+import { markRemovedSymbolsOffline } from '../services/reconcileIndexedSymbols.js';
 export async function runIncrementalIndex(opts) {
     const { projectRoot, changedFiles, deletedFiles, renamedFiles = [] } = opts;
     const pool = getPool();
     const tableName = env.symbolsTable;
-    // 1. 删除文件：标记 offline
-    for (const file of deletedFiles) {
-        await pool.query(`UPDATE ${tableName} SET status = $1 WHERE path = $2`, [SYMBOL_STATUS.OFFLINE, file]);
-        console.error(`[ci-index] marked offline: ${file}`);
-    }
-    // 2. 重命名文件：更新path
-    for (const { from, to } of renamedFiles) {
-        await pool.query(`UPDATE ${tableName} SET path = $1 WHERE path = $2`, [
-            to,
-            from,
-        ]);
-        console.error(`[ci-index] renamed: ${from} -> ${to}`);
-    }
-    // 3. 变更/新增文件：重新索引并标记 pending
-    if (changedFiles.length > 0) {
-        const rows = await indexProject({
+    const rows = changedFiles.length > 0
+        ? await indexProject({
             projectRoot,
             globPatterns: changedFiles,
-        });
-        for (const row of rows) {
-            // 写入结构化数据
-            // status 逻辑：新行写 pending；已有行仅在 semantic_hash 发生变化时才重置为 pending，
-            // hash 未变说明语义未变，保留原 status（online → 缓存命中，不重复 embedding）
-            await pool.query(`INSERT INTO ${tableName}
-                   (name, type, category, path, description, content, meta,
-                    file_hash, semantic_hash, status,
-                    usage_count, created_at, updated_at)
-                 VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, $10, 0, NOW(), NOW())
-                 ON CONFLICT (path, name) DO UPDATE SET
-                   type             = EXCLUDED.type,
-                   category         = EXCLUDED.category,
-                   description      = EXCLUDED.description,
-                   content          = EXCLUDED.content,
-                   meta             = EXCLUDED.meta,
-                   file_hash        = EXCLUDED.file_hash,
-                   semantic_hash    = EXCLUDED.semantic_hash,
-                   status           = CASE WHEN ${tableName}.semantic_hash = EXCLUDED.semantic_hash
-                                          THEN ${tableName}.status
-                                          ELSE EXCLUDED.status END,
-                   updated_at       = NOW()`, [
-                row.name,
-                row.type,
-                row.category ?? null,
-                row.path,
-                row.description ?? null,
-                row.content ?? null,
-                JSON.stringify(row.meta),
-                row.file_hash,
-                row.semantic_hash,
-                DEFAULT_STATUS_ON_UPSERT,
-            ]);
-            console.error(`[ci-index] upserted: ${row.path}:${row.name}`);
+        })
+        : [];
+    const nullPayload = rows.map(() => null);
+    const hashes = [
+        ...new Set(rows.map((r) => r.semantic_hash).filter(Boolean)),
+    ];
+    const client = await pool.connect();
+    try {
+        await client.query('BEGIN');
+        // 1. 删除文件：复用 reconcile 的整文件 offline 语义
+        if (deletedFiles.length > 0) {
+            await markRemovedSymbolsOffline(client, deletedFiles, []);
+            for (const file of deletedFiles) {
+                console.error(`[ci-index] marked offline: ${file}`);
+            }
+        }
+        // 2. 重命名文件：更新path
+        for (const { from, to } of renamedFiles) {
+            await client.query(`UPDATE ${tableName} SET path = $1 WHERE path = $2`, [to, from]);
+            console.error(`[ci-index] renamed: ${from} -> ${to}`);
+        }
+        // 3. 变更/新增文件：重新索引并标记 pending
+        if (rows.length > 0) {
+            await upsertSymbols(client, rows, nullPayload);
+            for (const row of rows) {
+                console.error(`[ci-index] upserted: ${row.path}:${row.name}`);
+            }
+            await markRemovedSymbolsOffline(client, changedFiles, rows);
         }
-        // 批量入队：jobId = semanticHash，相同 hash 自动去重，1000 个符号可能只产生 N 个唯一 job
-        const hashes = [
-            ...new Set(rows.map((r) => r.semantic_hash).filter(Boolean)),
-        ];
+        await client.query('COMMIT');
+    }
+    catch (error) {
+        await client.query('ROLLBACK');
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+    try {
         if (hashes.length > 0) {
             await enqueueEmbeddingBatch(hashes, env.symbolsTable);
             console.error(`[ci-index] enqueued ${hashes.length} unique semantic hashes for embedding`);
         }
+        await closeEmbeddingQueue();
+    }
+    finally {
+        await pool.end();
     }
-    await closeEmbeddingQueue();
-    await pool.end();
     console.error(`[ci-index] processed ${deletedFiles.length} deletions, ${renamedFiles.length} renames, ${changedFiles.length} changes`);
 }

package/dist/cli/eval-analyze-cli.js

@@ -0,0 +1,308 @@
+/**
+ * eval-analyze-cli.ts — 离线评测结果分析工具
+ *
+ * 用法：
+ *   npm run analyze                             # 自动读取 offline_eval/results/ 最新文件
+ *   npm run analyze -- offline_eval/results/2026-05-27.jsonl
+ *   npm run analyze -- --dir offline_eval/results --baseline offline_eval/results/2026-05-26.jsonl
+ *
+ * 输出：
+ *   - 关键指标汇总（含与 baseline 对比 delta）
+ *   - 按语言/符号类型分组 Recall@10
+ *   - 失败归因分布
+ *   - 主要发现（自动检测中英文差距、类型推断问题、误触等）
+ *   - 建议优先级列表
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as readline from 'node:readline';
+// ─── CLI 参数 ────────────────────────────────────────────────────────────────
+const args = process.argv.slice(2);
+function getArg(flag, fallback) {
+    const idx = args.indexOf(flag);
+    return idx !== -1 && args[idx + 1] ? args[idx + 1] : fallback;
+}
+const RESULTS_DIR = getArg('--dir', 'offline_eval/results');
+const BASELINE_PATH = getArg('--baseline', '');
+// ─── 分析阈值（可按实际结果微调） ────────────────────────────────────────────
+const THRESHOLDS = {
+    /** 中英文 Recall@10 差距超过此值时触发"中文召回偏弱"发现 */
+    ZH_EN_RECALL_GAP: 0.1,
+    /** 函数类 Recall@10 低于此值时触发"函数类型推断"发现 */
+    FUNC_RECALL_LOW: 0.5,
+    /** no_semantic_recall 占比超过此值时触发"归因主导"发现 */
+    NO_SEMANTIC_DOMINANCE: 0.5,
+    /** no_semantic_recall 占比超过此值时输出归因修正建议 */
+    NO_SEMANTIC_REC_TRIGGER: 0.3,
+    /** quality_gate_rejected 条数超过此值时输出质量门控建议 */
+    QG_COUNT_MIN: 2,
+    /** ranked_below_topk 条数超过此值时输出排名调整建议 */
+    RANKED_COUNT_MIN: 1,
+    /** 终端横向进度条宽度（字符数） */
+    BAR_WIDTH: 20,
+    /** 中文零召回示例最多展示条数 */
+    ZH_ZERO_EXAMPLE_LIMIT: 4,
+    /** 误触示例最多展示条数 */
+    FP_EXAMPLE_LIMIT: 3,
+};
+function findLatestResultsFile() {
+    // 支持直接传路径（不带 flag）
+    const explicit = args.find((a) => a.endsWith('.jsonl') && !a.startsWith('--'));
+    if (explicit)
+        return explicit;
+    if (!fs.existsSync(RESULTS_DIR)) {
+        throw new Error(`Results directory not found: ${RESULTS_DIR}`);
+    }
+    const files = fs
+        .readdirSync(RESULTS_DIR)
+        .filter((f) => f.endsWith('.jsonl'))
+        .sort()
+        .reverse();
+    if (files.length === 0) {
+        throw new Error(`No .jsonl result files found in ${RESULTS_DIR}`);
+    }
+    return path.join(RESULTS_DIR, files[0]);
+}
+// ─── 文件加载 ─────────────────────────────────────────────────────────────────
+async function loadResults(filePath) {
+    const results = [];
+    const rl = readline.createInterface({
+        input: fs.createReadStream(filePath),
+        crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (trimmed)
+            results.push(JSON.parse(trimmed));
+    }
+    return results;
+}
+// ─── 工具函数 ─────────────────────────────────────────────────────────────────
+function avg(nums) {
+    if (nums.length === 0)
+        return 0;
+    return nums.reduce((s, n) => s + n, 0) / nums.length;
+}
+function pct(n) {
+    return (n * 100).toFixed(1) + '%';
+}
+function delta(curr, base) {
+    if (base === undefined)
+        return '';
+    const d = curr - base;
+    const sign = d >= 0 ? '+' : '';
+    return ` (${sign}${pct(d)})`;
+}
+function recallByTag(results, tag) {
+    const tagged = results.filter((r) => !r.isNegativeSample && r.tags.includes(tag));
+    return {
+        recall: avg(tagged.map((r) => r.recallMain ?? 0)),
+        count: tagged.length,
+    };
+}
+function computeMetrics(positive, negative) {
+    return {
+        recallMain: avg(positive.map((r) => r.recallMain ?? 0)),
+        recall50: avg(positive.map((r) => r.recall50 ?? 0)),
+        mrr: avg(positive.map((r) => r.mrrMain ?? 0)),
+        ndcg: avg(positive.map((r) => r.ndcgMain ?? 0)),
+        top1Acc: positive.filter((r) => r.top1Correct === true).length /
+            (positive.length || 1),
+        fpRate: negative.filter((r) => r.falsePositive).length /
+            (negative.length || 1),
+    };
+}
+// ─── 主分析逻辑 ───────────────────────────────────────────────────────────────
+async function analyze() {
+    const filePath = findLatestResultsFile();
+    const fileName = path.basename(filePath);
+    const results = await loadResults(filePath);
+    const positive = results.filter((r) => !r.isNegativeSample);
+    const negative = results.filter((r) => r.isNegativeSample);
+    const metrics = computeMetrics(positive, negative);
+    // Baseline（用于 delta 对比）
+    let baseMetrics;
+    if (BASELINE_PATH && fs.existsSync(BASELINE_PATH)) {
+        const baseResults = await loadResults(BASELINE_PATH);
+        const basePos = baseResults.filter((r) => !r.isNegativeSample);
+        const baseNeg = baseResults.filter((r) => r.isNegativeSample);
+        baseMetrics = computeMetrics(basePos, baseNeg);
+    }
+    // 失败归因统计
+    const allFailures = positive.flatMap((r) => r.failures);
+    const failureCounts = {};
+    for (const f of allFailures) {
+        failureCounts[f.type] = (failureCounts[f.type] ?? 0) + 1;
+    }
+    const totalFailures = allFailures.length;
+    // 分组 Recall（按语言 + 类型标签）
+    const tagGroups = [
+        'en',
+        'zh',
+        'zh-en',
+        'component',
+        'hook',
+        'function',
+        'util',
+        'form',
+    ];
+    const tagRecalls = new Map();
+    for (const tag of tagGroups) {
+        const stat = recallByTag(positive, tag);
+        if (stat.count > 0)
+            tagRecalls.set(tag, stat);
+    }
+    // 零召回 query
+    const zeroRecall = positive.filter((r) => r.recallMain === 0);
+    // 误触（false positive）示例
+    const fpExamples = negative.filter((r) => r.falsePositive);
+    // ─── 输出报告 ──────────────────────────────────────────────────────────────
+    const sep = '='.repeat(60);
+    const sub = '─'.repeat(60);
+    console.log('\n' + sep);
+    console.log(`数据来源：${fileName}`);
+    console.log(sep);
+    // ── 关键指标 ──
+    console.log('\n关键指标\n');
+    console.log(`  Recall@10:      ${pct(metrics.recallMain).padStart(7)}${delta(metrics.recallMain, baseMetrics?.recallMain)}`);
+    console.log(`  Recall@50:      ${pct(metrics.recall50).padStart(7)}${delta(metrics.recall50, baseMetrics?.recall50)}`);
+    console.log(`  MRR@10:         ${pct(metrics.mrr).padStart(7)}${delta(metrics.mrr, baseMetrics?.mrr)}`);
+    console.log(`  nDCG@10:        ${pct(metrics.ndcg).padStart(7)}${delta(metrics.ndcg, baseMetrics?.ndcg)}`);
+    console.log(`  Top1 Acc:       ${pct(metrics.top1Acc).padStart(7)}${delta(metrics.top1Acc, baseMetrics?.top1Acc)}`);
+    console.log(`  False Positive: ${pct(metrics.fpRate).padStart(7)}${delta(metrics.fpRate, baseMetrics?.fpRate)}`);
+    console.log(`\n  总 query 数：${results.length}（正例 ${positive.length}，负例 ${negative.length}）`);
+    // ── 分组 Recall ──
+    console.log('\n' + sub);
+    console.log('按语言/符号类型 Recall@10\n');
+    for (const [tag, stat] of tagRecalls) {
+        const bar = '█'
+            .repeat(Math.round(stat.recall * THRESHOLDS.BAR_WIDTH))
+            .padEnd(THRESHOLDS.BAR_WIDTH);
+        console.log(`  ${tag.padEnd(12)} ${bar} ${pct(stat.recall).padStart(7)}  (${stat.count} queries)`);
+    }
+    // ── 失败归因 ──
+    console.log('\n' + sub);
+    console.log('失败归因分布\n');
+    const failureActionHints = {
+        no_semantic_recall: '→ 扩展 queryVariants / 中文同义词映射 / 调大 SYMBOL_TOP_K',
+        quality_gate_rejected: '→ 按 type 降低质量门控阈值',
+        ranked_below_topk: '→ 调整 RANK_WEIGHTS / LITERAL_MATCH_PRIORITY_BOOST',
+        reusability_filtered: '→ 检查 isReusableCandidate 路径规则',
+        structure_filtered: '→ 检查 category 过滤条件',
+    };
+    const sortedFailures = Object.entries(failureCounts).sort((a, b) => b[1] - a[1]);
+    for (const [type, count] of sortedFailures) {
+        const p = totalFailures > 0
+            ? ((count / totalFailures) * 100).toFixed(1)
+            : '0.0';
+        const hint = failureActionHints[type] ?? '';
+        console.log(`  ${type.padEnd(28)} ${String(count).padStart(3)} (${p}%)  ${hint}`);
+    }
+    // ── 主要发现 ──
+    console.log('\n' + sub);
+    console.log('主要发现\n');
+    const findings = [];
+    // 发现1：中英文召回差距
+    const zhStat = tagRecalls.get('zh');
+    const enStat = tagRecalls.get('en');
+    if (zhStat &&
+        enStat &&
+        enStat.recall - zhStat.recall > THRESHOLDS.ZH_EN_RECALL_GAP) {
+        const zhZero = positive
+            .filter((r) => r.tags.includes('zh') && r.recallMain === 0)
+            .map((r) => `"${r.query}"`)
+            .slice(0, THRESHOLDS.ZH_ZERO_EXAMPLE_LIMIT);
+        findings.push(`中文 query 召回明显弱于英文\n` +
+            `   中文 Recall@10 = ${pct(zhStat.recall)}，英文 = ${pct(enStat.recall)}，差距 ${pct(enStat.recall - zhStat.recall)}\n` +
+            `   零召回中文 query 示例：${zhZero.join('、')}`);
+    }
+    // 发现2：函数类符号类型推断
+    const funcStat = tagRecalls.get('function');
+    if (funcStat && funcStat.recall < THRESHOLDS.FUNC_RECALL_LOW) {
+        findings.push(`函数类 query 召回偏低（Recall@10 = ${pct(funcStat.recall)}）\n` +
+            `   可能原因：类型推断关键词缺少 function/formatter/validate，导致回退到 component 候选池`);
+    }
+    // 发现3：no_semantic_recall 主导
+    const noSemanticCount = failureCounts['no_semantic_recall'] ?? 0;
+    if (noSemanticCount / (totalFailures || 1) >
+        THRESHOLDS.NO_SEMANTIC_DOMINANCE) {
+        findings.push(`失败以 no_semantic_recall 为主（${noSemanticCount}/${totalFailures} 条，${((noSemanticCount / (totalFailures || 1)) * 100).toFixed(0)}%）\n` +
+            `   注意：该分类会掩盖 quality_gate_rejected 等更具体原因（当 DB 中无对应 id 时降级记录）`);
+    }
+    // 发现4：误触
+    if (fpExamples.length > 0) {
+        findings.push(`负例误触 ${fpExamples.length} 条（False Positive = ${pct(metrics.fpRate)}）\n` +
+            `   示例：${fpExamples
+                .slice(0, THRESHOLDS.FP_EXAMPLE_LIMIT)
+                .map((r) => `"${r.query}"`)
+                .join('、')}`);
+    }
+    // 发现5：零召回 query 总数
+    if (zeroRecall.length > 0) {
+        const byTag = (tag) => zeroRecall.filter((r) => r.tags.includes(tag)).length;
+        findings.push(`${zeroRecall.length} 条正例 Recall@10 = 0\n` +
+            `   其中 zh:${byTag('zh')}，en:${byTag('en')}，component:${byTag('component')}，hook:${byTag('hook')}，function:${byTag('function')}`);
+    }
+    findings.forEach((f, i) => console.log(`${i + 1}. ${f}\n`));
+    // ── 建议优先级 ──
+    console.log(sub);
+    console.log('建议优先级（按预期收益排序）\n');
+    const recs = [];
+    if (zhStat &&
+        enStat &&
+        enStat.recall - zhStat.recall > THRESHOLDS.ZH_EN_RECALL_GAP) {
+        recs.push({
+            priority: '高',
+            action: '强化中文同义词扩展（导航栏→navigation bar、日期格式化→format date 等）',
+            reason: `中英召回差距 ${pct(enStat.recall - zhStat.recall)}，影响 ${zhStat.count} 条 query`,
+        });
+    }
+    if (funcStat && funcStat.recall < THRESHOLDS.FUNC_RECALL_LOW) {
+        recs.push({
+            priority: '高',
+            action: '补类型推断关键词（category: function、query: formatter/validate/format）',
+            reason: `函数类 Recall@10 = ${pct(funcStat.recall)}，类型误判导致错误候选池`,
+        });
+    }
+    if (noSemanticCount / (totalFailures || 1) >
+        THRESHOLDS.NO_SEMANTIC_REC_TRIGGER) {
+        recs.push({
+            priority: '中高',
+            action: '修正 eval 失败归因逻辑（name+path 查 DB id，避免 no_semantic_recall 兜底掩盖）',
+            reason: `${noSemanticCount} 条记为 no_semantic_recall，诊断精度不足`,
+        });
+    }
+    const qgCount = failureCounts['quality_gate_rejected'] ?? 0;
+    if (qgCount > THRESHOLDS.QG_COUNT_MIN) {
+        recs.push({
+            priority: '中高',
+            action: '按 symbol type 分阈值降低质量门控（函数/Hook 可比组件更宽松）',
+            reason: `${qgCount} 条被 quality gate 拦截`,
+        });
+    }
+    const rankedCount = failureCounts['ranked_below_topk'] ?? 0;
+    if (rankedCount > THRESHOLDS.RANKED_COUNT_MIN) {
+        recs.push({
+            priority: '中',
+            action: '加强 index-priority tie-break（同目录 index/menu/panel 命中时强制 index 优先）',
+            reason: `${rankedCount} 条有正确候选但排名未进 Top-K`,
+        });
+    }
+    if (recs.length === 0) {
+        recs.push({
+            priority: '中',
+            action: '持续扩充 query_set.jsonl 覆盖更多边界场景',
+            reason: '当前指标已较好，可增加测试覆盖度',
+        });
+    }
+    recs.forEach((r, i) => {
+        console.log(`${i + 1}. [${r.priority}] ${r.action}`);
+        console.log(`   原因：${r.reason}`);
+        console.log();
+    });
+    console.log(sep + '\n');
+}
+analyze().catch((e) => {
+    console.error(e instanceof Error ? e.message : e);
+    process.exit(1);
+});