@lorrylurui/code-intelligence-mcp 2.0.9 → 2.1.0

package/README.md

@@ -52,6 +52,8 @@ INDEX_GLOB=xxx/\*\*/\_.{js,jsx,ts,tsx}
 npx tsx src/cli/eval-recommendation-cli.ts
 # 或指定 limit
 npx tsx src/cli/eval-recommendation-cli.ts --limit 10
+
+npm run eval
 ```
 
 ## 5）分析离线测评结果

package/dist/cli/eval-analyze-cli.js

@@ -78,6 +78,9 @@ async function loadResults(filePath) {
     return results;
 }
 // ─── 工具函数 ─────────────────────────────────────────────────────────────────
+/**
+ * 计算平均值，空数组时返回 0。
+ */
 function avg(nums) {
     if (nums.length === 0)
         return 0;
@@ -100,12 +103,17 @@ function recallByTag(results, tag) {
         count: tagged.length,
     };
 }
+/**
+ * 计算各项指标的平均值，返回一个汇总对象。
+ */
 function computeMetrics(positive, negative) {
     return {
         recallMain: avg(positive.map((r) => r.recallMain ?? 0)),
         recall50: avg(positive.map((r) => r.recall50 ?? 0)),
-        mrr: avg(positive.map((r) => r.mrrMain ?? 0)),
-        ndcg: avg(positive.map((r) => r.ndcgMain ?? 0)),
+        firstHitScore: avg(positive.map((r) => r.firstHitScore ?? 0)),
+        rankingQuality: avg(positive.map((r) => r.rankingQuality ?? 0)),
+        coverage: positive.filter((r) => (r.recallMain ?? 0) > 0).length /
+            (positive.length || 1),
         top1Acc: positive.filter((r) => r.top1Correct === true).length /
             (positive.length || 1),
         fpRate: negative.filter((r) => r.falsePositive).length /
@@ -120,7 +128,7 @@ async function analyze() {
     const positive = results.filter((r) => !r.isNegativeSample);
     const negative = results.filter((r) => r.isNegativeSample);
     const metrics = computeMetrics(positive, negative);
-    // Baseline（用于 delta 对比）
+    // 如果不传 --baseline，baseMetrics 就是 undefined，delta() 函数返回空字符串，指标后面不显示涨跌。
     let baseMetrics;
     if (BASELINE_PATH && fs.existsSync(BASELINE_PATH)) {
         const baseResults = await loadResults(BASELINE_PATH);
@@ -164,12 +172,10 @@ async function analyze() {
     console.log(sep);
     // ── 关键指标 ──
     console.log('\n关键指标\n');
-    console.log(`  Recall@10:      ${pct(metrics.recallMain).padStart(7)}${delta(metrics.recallMain, baseMetrics?.recallMain)}`);
-    console.log(`  Recall@50:      ${pct(metrics.recall50).padStart(7)}${delta(metrics.recall50, baseMetrics?.recall50)}`);
-    console.log(`  MRR@10:         ${pct(metrics.mrr).padStart(7)}${delta(metrics.mrr, baseMetrics?.mrr)}`);
-    console.log(`  nDCG@10:        ${pct(metrics.ndcg).padStart(7)}${delta(metrics.ndcg, baseMetrics?.ndcg)}`);
-    console.log(`  Top1 Acc:       ${pct(metrics.top1Acc).padStart(7)}${delta(metrics.top1Acc, baseMetrics?.top1Acc)}`);
-    console.log(`  False Positive: ${pct(metrics.fpRate).padStart(7)}${delta(metrics.fpRate, baseMetrics?.fpRate)}`);
+    console.log(`  召回率(Recall@10):             ${pct(metrics.recallMain).padStart(7)}${delta(metrics.recallMain, baseMetrics?.recallMain)}`);
+    console.log(`  首位命中分(MRR@10):             ${pct(metrics.firstHitScore).padStart(7)}${delta(metrics.firstHitScore, baseMetrics?.firstHitScore)}`);
+    console.log(`  首条准确率(Top-1):              ${pct(metrics.top1Acc).padStart(7)}${delta(metrics.top1Acc, baseMetrics?.top1Acc)}`);
+    console.log(`  误触率(FP):                    ${pct(metrics.fpRate).padStart(7)}${delta(metrics.fpRate, baseMetrics?.fpRate)}`);
     console.log(`\n  总 query 数：${results.length}（正例 ${positive.length}，负例 ${negative.length}）`);
     // ── 分组 Recall ──
     console.log('\n' + sub);
@@ -216,7 +222,7 @@ async function analyze() {
             `   中文 Recall@10 = ${pct(zhStat.recall)}，英文 = ${pct(enStat.recall)}，差距 ${pct(enStat.recall - zhStat.recall)}\n` +
             `   零召回中文 query 示例：${zhZero.join('、')}`);
     }
-    // 发现2：函数类符号类型推断
+    // 发现2：函数类类型推断
     const funcStat = tagRecalls.get('function');
     if (funcStat && funcStat.recall < THRESHOLDS.FUNC_RECALL_LOW) {
         findings.push(`函数类 query 召回偏低（Recall@10 = ${pct(funcStat.recall)}）\n` +

package/dist/cli/eval-recommendation-cli.js

@@ -23,7 +23,9 @@ function getArg(flag, fallback) {
 const QUERY_SET_PATH = getArg('--query-set', 'offline_eval/query_set.jsonl');
 const OUTPUT_DIR = getArg('--output', 'offline_eval/results');
 const TOP_K_MAIN = Number(getArg('--limit', '10')); // Recall@K_MAIN / MRR@K / nDCG@K
-const TOP_K_WIDE = 50; // Recall@50（宽口径）
+const RECALL_WIDE_K = 50; // 宽口径召回深度（用于 Recall@50），不是测试集数量
+const REL_RELEVANT_MIN = 1; // rel >= 1 计入相关结果
+const REL_PRIMARY = 2; // rel = 2 表示主答案/最高相关度
 // ─── 指标计算 ─────────────────────────────────────────────────────────────────
 /**
  * 覆盖率 Recall@K：前 K 条结果中命中的相关条目占全部相关条目的比例。
@@ -31,11 +33,12 @@ const TOP_K_WIDE = 50; // Recall@50（宽口径）
  * 负例（expected 全为 rel=0）视为完全命中，返回 1。
  */
 function recallAtK(returnedNames, expected, k) {
-    const relevant = expected.filter((e) => e.rel >= 1);
+    const relevant = expected.filter((e) => e.rel >= REL_RELEVANT_MIN);
     if (relevant.length === 0)
         return 1;
     const topK = returnedNames.slice(0, k);
     const hits = relevant.filter((e) => topK.includes(e.name));
+    // 召回率@k = 真实召回的 / 所有相关的
     return hits.length / relevant.length;
 }
 /**
@@ -43,9 +46,10 @@ function recallAtK(returnedNames, expected, k) {
  * 衡量「最佳结果排多靠前」；未命中则返回 0。
  */
 function mrrAtK(returnedNames, expected, k) {
-    const relevantNames = new Set(expected.filter((e) => e.rel >= 1).map((e) => e.name));
+    const relevantNames = new Set(expected.filter((e) => e.rel >= REL_RELEVANT_MIN).map((e) => e.name));
     const topK = returnedNames.slice(0, k);
     for (let i = 0; i < topK.length; i++) {
+        // 有一个命中的 就返回对应的 MRR 分数，越靠前分数越高；如果都没命中，最后返回 0。
         if (relevantNames.has(topK[i]))
             return 1 / (i + 1);
     }
@@ -72,9 +76,11 @@ function ndcgAtK(returnedNames, expected, k) {
     }, 0);
     return idcg === 0 ? 1 : dcg / idcg;
 }
-// ─── 失败分类（无 ID 时按名称降级处理） ─────────────────────────────────────
+/**
+ * 返回失败阶段原因数组（无 ID 时按名称降级处理）
+ */
 function classifyFailuresFromTrace(expected, returnedNames, evalTrace, idByName) {
-    const relevant = expected.filter((e) => e.rel >= 1);
+    const relevant = expected.filter((e) => e.rel >= REL_RELEVANT_MIN);
     const failures = [];
     for (const exp of relevant) {
         if (returnedNames.includes(exp.name))
@@ -123,8 +129,10 @@ function printSummary(results, kMain, baseline) {
     const negative = results.filter((r) => r.isNegativeSample);
     const recallMain = avg(positive.map((r) => r.recallMain ?? 0));
     const recall50 = avg(positive.map((r) => r.recall50 ?? 0));
-    const mrr = avg(positive.map((r) => r.mrrMain ?? 0));
-    const ndcg = avg(positive.map((r) => r.ndcgMain ?? 0));
+    const firstHitScore = avg(positive.map((r) => r.firstHitScore ?? 0));
+    // const rankingQuality = avg(positive.map((r) => r.rankingQuality ?? 0));
+    const coverage = positive.filter((r) => (r.recallMain ?? 0) > 0).length /
+        (positive.length || 1);
     const top1Acc = positive.filter((r) => r.top1Correct === true).length /
         (positive.length || 1);
     const fpRate = negative.filter((r) => r.falsePositive).length / (negative.length || 1);
@@ -141,12 +149,10 @@ function printSummary(results, kMain, baseline) {
     console.log('='.repeat(60));
     console.log(`Queries total:  ${results.length}  (positive: ${positive.length}, negative: ${negative.length})`);
     console.log('');
-    console.log(`Recall@${kMain}:    ${formatPct(recallMain)}${diff('recallMain', recallMain)}`);
-    console.log(`Recall@50:    ${formatPct(recall50)}${diff('recall50', recall50)}`);
-    console.log(`MRR@${kMain}:       ${formatPct(mrr)}${diff('mrr', mrr)}`);
-    console.log(`nDCG@${kMain}:      ${formatPct(ndcg)}${diff('ndcg', ndcg)}`);
-    console.log(`Top1 Acc:     ${formatPct(top1Acc)}${diff('top1Acc', top1Acc)}`);
-    console.log(`False Pos:    ${formatPct(fpRate)}  (negative samples incorrectly returned results)`);
+    console.log(`召回率(Recall@${kMain}):         ${formatPct(recallMain)}${diff('recallMain', recallMain)}`);
+    console.log(`首位命中分(MRR@${kMain}):         ${formatPct(firstHitScore)}${diff('firstHitScore', firstHitScore)}`);
+    console.log(`首条准确率(Top-1):               ${formatPct(top1Acc)}${diff('top1Acc', top1Acc)}`);
+    console.log(`误触率(FP):                      ${formatPct(fpRate)}  (负例被错误推荐)`);
     console.log('');
     // ── Failure breakdown ──
     const allFailures = positive.flatMap((r) => r.failures);
@@ -192,6 +198,8 @@ async function loadQuerySet(filePath) {
         const trimmed = line.trim();
         if (!trimmed)
             continue;
+        if (trimmed.startsWith('#') || trimmed.startsWith('//'))
+            continue;
         cases.push(JSON.parse(trimmed));
     }
     return cases;
@@ -199,47 +207,43 @@ async function loadQuerySet(filePath) {
 async function runEval() {
     console.log(`Loading query set: ${QUERY_SET_PATH}`);
     const cases = await loadQuerySet(QUERY_SET_PATH);
-    console.log(`Loaded ${cases.length} queries. Running eval with limit=${TOP_K_MAIN}/${TOP_K_WIDE}...\n`);
+    console.log(`Loaded ${cases.length} queries. Running eval with limit=${TOP_K_MAIN}/${RECALL_WIDE_K}...\n`);
     const repository = new SymbolRepository();
     const service = new RecommendationService(repository);
     const results = [];
     for (const queryCase of cases) {
         const isNegative = queryCase.expected.length === 0;
-        // Run with wide limit (Recall@50)
         const wideResult = await service.recommendComponent({
             ...queryCase.input,
-            limit: TOP_K_WIDE,
+            limit: RECALL_WIDE_K,
             evalMode: true,
         });
         const wideNames = [
             ...(wideResult.recommended ? [wideResult.recommended.name] : []),
             ...wideResult.alternatives.map((a) => a.name),
         ];
-        // Run with main limit for MRR/nDCG (or reuse wide result slice)
         const mainNames = wideNames.slice(0, TOP_K_MAIN);
-        // Build id map from returned results
         const allReturned = [
             ...(wideResult.recommended ? [wideResult.recommended] : []),
             ...wideResult.alternatives,
         ];
         const idByName = buildIdMapFromResult(wideResult.recommended, wideResult.alternatives);
-        // Metrics (skip for negative samples)
         const recallMain = isNegative
             ? null
             : recallAtK(mainNames, queryCase.expected, TOP_K_MAIN);
         const recall50 = isNegative
             ? null
-            : recallAtK(wideNames, queryCase.expected, TOP_K_WIDE);
-        const mrrMain = isNegative
+            : recallAtK(wideNames, queryCase.expected, RECALL_WIDE_K);
+        const firstHitRank = isNegative
             ? null
             : mrrAtK(mainNames, queryCase.expected, TOP_K_MAIN);
-        const ndcgMain = isNegative
-            ? null
-            : ndcgAtK(mainNames, queryCase.expected, TOP_K_MAIN);
+        // const rankingQuality = isNegative
+        //     ? null
+        //     : ndcgAtK(mainNames, queryCase.expected, TOP_K_MAIN);
         const top1Correct = isNegative
             ? null
-            : queryCase.expected.some((e) => e.rel === 2 && wideResult.recommended?.name === e.name);
-        // Failure classification
+            : queryCase.expected.some((e) => e.rel === REL_PRIMARY &&
+                wideResult.recommended?.name === e.name);
         const failures = isNegative
             ? []
             : classifyFailuresFromTrace(queryCase.expected, wideNames, wideResult.evalTrace, idByName);
@@ -250,8 +254,8 @@ async function runEval() {
             tags: queryCase.tags,
             recallMain,
             recall50,
-            mrrMain,
-            ndcgMain,
+            firstHitScore: firstHitRank,
+            // rankingQuality,
             top1Correct,
             returnedNames: mainNames,
             failures,
@@ -259,19 +263,16 @@ async function runEval() {
             falsePositive,
         };
         results.push(qr);
-        // Progress
         const status = isNegative
             ? falsePositive
-                ? '✗ FP'
-                : '✓ TN'
+                ? '✗ False Positive）' // 负例，但系统返回了结果 → 误触发（False Positive）
+                : '✓ True Negative' // 负例，系统正确返回空   → 真负例（True Negative）
             : recallMain === 1
-                ? `✓ R@${TOP_K_MAIN}=1.0`
-                : `✗ R@${TOP_K_MAIN}=${(recallMain ?? 0).toFixed(2)}`;
+                ? `✓ R@${TOP_K_MAIN}=1.0 完全召回`
+                : `✗ R@${TOP_K_MAIN}=${(recallMain ?? 0).toFixed(2)} 不完全召回`;
         console.log(`  [${queryCase.id}] ${queryCase.input.query.slice(0, 40).padEnd(40)}  ${status}`);
     }
-    // Print summary
     printSummary(results, TOP_K_MAIN, null);
-    // Write JSONL report
     if (OUTPUT_DIR) {
         fs.mkdirSync(OUTPUT_DIR, { recursive: true });
         const dateStr = new Date().toISOString().slice(0, 10);
@@ -280,7 +281,7 @@ async function runEval() {
         fs.writeFileSync(outPath, lines + '\n', 'utf8');
         console.log(`Report written to: ${outPath}`);
     }
-    // Exit with non-zero if any positive query has recall=0
+    // 如果有正例查询完全没有召回任何相关结果，视为严重问题，输出警告并退出非 0 状态码以示 CI 失败。
     const zeroRecall = results.filter((r) => !r.isNegativeSample && r.recallMain === 0);
     if (zeroRecall.length > 0) {
         console.log(`\nWARN: ${zeroRecall.length} positive queries have Recall@${TOP_K_MAIN}=0:`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@lorrylurui/code-intelligence-mcp",
-    "version": "2.0.9",
+    "version": "2.1.0",
     "private": false,
     "description": "MCP server 提供仓库内可复用代码块（ts/tsx/js/jsx/css/less）的索引和查询能力，支持基于代码上下文的智能推荐。",
     "type": "module",