modelfunction-cli 0.1.2 → 0.2.0

package/README.md

@@ -5,7 +5,7 @@
 ## 安装
 
 ```bash
-npx -y modelfunction-cli@0.1.2 --version
+npx -y modelfunction-cli@0.2.0 --version
 # 或
 npm i -D modelfunction-cli
 npx mf --version
@@ -64,18 +64,43 @@ INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
 
 | 命令 | 作用 |
 | --- | --- |
-| `mf scan` | 扫描工具定义；`--probe` 连 MCP server；`--from` 导入运行时导出的工具数组 |
-| `mf eval` | 跑评分卡；`--fail-under` 阈值；`--baseline` + `--fail-on-regression` 回归门禁 |
+| `mf scan` | 扫描工具定义（function.json / OpenAPI / MCP 配置 / Python `@tool`）；`--probe` 连 MCP server；`--from` 导入运行时工具数组；草稿按来源分组（`profiles`）并列出未覆盖的 skill |
+| `mf eval` | 跑评分卡；`--fail-under` 阈值；`--baseline` + `--fail-on-regression` 回归门禁；`--profile` 按挂载组分别评分 |
 | `mf case add` | 写入用例（默认 `source=agent`、`reviewed=false`） |
-| `mf coverage` | 每工具用例数、参数覆盖、负例比例、疑似混淆对 |
-| `mf lint` | 静态检查描述与参数 schema |
-| `mf compare` | 跨版本比较、干扰归因、混淆矩阵 |
-| `mf ablate` | 留一法：去掉某个工具再跑一遍，量化它是否干扰其他工具 |
+| `mf coverage` | 每工具用例数、参数覆盖、负例比例、疑似混淆对；有 `profiles` 时按组统计 |
+| `mf lint` | 静态检查描述与参数 schema；有 `profiles` 时给出「这些组不要合并挂载」 |
+| `mf compare` | 跨版本比较、干扰归因、混淆矩阵；两份报告都有分组时按组对比 |
+| `mf ablate` | 留一法：去掉某个工具再跑一遍，量化它是否干扰其他工具；`--profile` 限定在某个挂载组内 |
+| `mf cluster` | 聚类工具，建议挂载分组；`--out` 把分组写成套件的 `profiles` |
 | `mf exec` | 单工具执行 + 指标（Node / Python），支持 `--allow-read/write/net`、超时、sideEffects 准入 |
 | `mf eval --execute` | 一次跑完选择 + 参数 + 运行断言（`toolInput` / `expect`） |
 | `mf sandbox check` | 探测本机可用隔离引擎与降级建议 |
 | `mf mcp` | stdio MCP server，供 Claude Code / Cursor / Kimi Code 调用 |
 
+## 单次结果会抖：用 `--repeat` 看稳定性
+
+实测 DeepSeek-V4-Flash：同一条用例、temperature=0，一次选对、下一次干脆不调用。单次评分卡里 ±1 条用例就是 ±8pp，小套件上足以把"分组有没有用"看反。
+
+```bash
+mf eval --models dsv4-flash --repeat 3    # 每条跑 3 次取多数，并列出结果不稳定的用例
+```
+
+`--repeat` 会让调用次数和成本乘以 n；正式基线建议 3 次，日常回归 1 次够用。
+
+## 挂载分组：验证"少挂一点"值不值
+
+平台通常按业务把工具拆给不同 agent（网优 / 故障 / 运维各挂各的）。分组到底降了多少误选，用同一批用例比一次就知道：
+
+```bash
+mf cluster --suite mf.suite.json --out mf.suite.json    # 写入建议分组（profiles），可手工修改
+mf eval --suite mf.suite.json --models internal-ds \
+  --profile all --out report.md                        # 每组 vs 全量，一页 markdown
+```
+
+每组只跑"本组工具 + 本组用例"，负例（不该调用工具）每组都跑；`all-in-one` 列是同一批用例在全部工具都挂时的结果。两列之差就是分组收益，按组给出，可直接拿去和平台方对账。工具很少时分组通常没有收益（模型本来就选得对），差异小于 10pp 请先加 `--repeat 3` 复核再下结论。
+
+配套的三个检查：`mf lint` 列出组间高风险对（相似工具分属不同组，合并挂载会互抢）；`mf coverage` 按组看谁还缺用例；两次 `--profile` 报告之间用 `mf compare` 看各组升降（分组定义变了会明确标注不可比）。组内还有干扰时，用 `mf ablate --profile <组> --tool <工具>` 做组内留一法，看拆掉谁最划算。
+
 ## MCP 配置
 
 ```json
@@ -83,13 +108,13 @@ INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
   "mcpServers": {
     "modelfunction": {
       "command": "npx",
-      "args": ["-y", "modelfunction-cli@0.1.2", "mcp"]
+      "args": ["-y", "modelfunction-cli@0.2.0", "mcp"]
     }
   }
 }
 ```
 
-暴露：`scan_tools`、`list_suite`、`coverage`、`add_cases`、`run_eval`。
+暴露：`scan_tools`、`list_suite`、`coverage`、`add_cases`、`run_eval`（`run_eval` 支持 `profile` 参数，传 `all` 或组名即可按挂载组评分）。
 
 ## 数据边界
 
@@ -105,7 +130,7 @@ INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
 
 ```bash
 # 1. 把 mf 挂成 MCP server
-claude mcp add modelfunction -- npx -y modelfunction-cli@0.1.2 mcp
+claude mcp add modelfunction -- npx -y modelfunction-cli@0.2.0 mcp
 ```
 
 ```text

package/fixtures/suite.json

@@ -146,5 +146,9 @@
       },
       "note": "有“简历”字样，但真实意图是薪资"
     }
-  ]
+  ],
+  "profiles": {
+    "screening": ["resume_score", "interview_question"],
+    "offer": ["salary_band", "jd_rewrite"]
+  }
 }

package/lib/ablate.js

@@ -64,8 +64,9 @@ export function analyzeAblation(model, full, reduced) {
 }
 
 export function formatAblation(payload) {
+  const scope = payload.profile ? `，组 ${payload.profile.name}，${payload.profile.tools} 个工具` : "";
   const lines = [
-    `ablate: ${payload.tool}（suite ${payload.suite}，去掉后期望该工具的用例 ${payload.droppedCases} 条）`
+    `ablate: ${payload.tool}（suite ${payload.suite}${scope}，去掉后期望该工具的用例 ${payload.droppedCases} 条）`
   ];
 
   for (const analysis of payload.models) {

package/lib/cases.js

@@ -2,11 +2,30 @@ import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
 import { loadSuite } from "./suite.js";
 import { writeFileAtomic } from "./fsutil.js";
+import { normalizeProfiles, profileCaseCoverage } from "./profile.js";
 
 const MIN_CASES_PER_TOOL = 3;
 const MIN_NEGATIVE_RATIO = 0.2;
 const PAIR_SIMILARITY = 0.35;
 
+// `get_stock_data` and `get_news` share only the verb; that is not a family.
+// These tokens carry no domain meaning, so they never count as a shared name
+// signal (neither for confusability nor for clustering).
+export const GENERIC_NAME_TOKENS = new Set(["tool", "tools", "api", "get", "set", "do"]);
+
+// English boilerplate: function words plus the verbs every tool description
+// starts with. Left in, "Retrieve data for a given ticker" makes any two tools
+// look half-identical.
+const STOPWORDS = new Set([
+  "the", "and", "for", "with", "from", "this", "that", "these", "those", "are", "was", "were",
+  "has", "have", "you", "your", "can", "will", "given", "based", "using", "into", "over", "per",
+  "via", "not", "all", "any", "each", "its", "their", "data",
+  "get", "gets", "retrieve", "retrieves", "retrieved", "fetch", "fetches", "query", "queries",
+  "search", "searches", "find", "finds", "read", "reads", "write", "writes", "create", "creates",
+  "update", "updates", "delete", "deletes", "list", "lists", "run", "runs", "call", "calls",
+  "use", "uses", "return", "returns", "tool", "tools", "api", "apis"
+]);
+
 export async function addCases(suitePath, incoming, { source = "agent", reviewed = false, dryRun = false } = {}) {
   const abs = resolve(suitePath);
   const raw = JSON.parse(await readFile(abs, "utf8"));
@@ -121,12 +140,30 @@ export function computeCoverage(suite) {
   }
   if (unreviewed > 0) warnings.push(`${unreviewed} 条未复核用例不计入正式分数`);
 
+  const skills = suite.skills?.length ?? 0;
+  if (skills > 0) warnings.push(`${skills} 个 skill 不参与工具评测（当前只覆盖 function / MCP 工具）`);
+
+  const profileList = suite.profiles
+    ? normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name))
+    : null;
+  const profiles = profileList ? profileCaseCoverage(profileList, suite.cases) : null;
+  if (profiles) {
+    for (const row of profiles) {
+      if (row.cases === 0) warnings.push(`组 ${row.name}: 0 条正例（只有负例），该组工具没被真正测试`);
+      else if (row.missing.length > 0) {
+        warnings.push(`组 ${row.name}: 工具 ${row.missing.join(", ")} 没有对应用例`);
+      }
+    }
+  }
+
   return {
     total,
     negative,
     negativeRatio,
     unreviewed,
+    skills,
     tools,
+    profiles,
     pairs: suspectedPairs(suite.tools),
     warnings
   };
@@ -134,16 +171,29 @@ export function computeCoverage(suite) {
 
 export function formatCoverage(report) {
   const lines = [
-    `cases: ${report.total}（负例 ${report.negative}，${Math.round(report.negativeRatio * 100)}%；未复核 ${report.unreviewed}）`,
-    "",
-    "tool                    cases  args  reviewed",
-    "----------------------  -----  ----  --------"
+    `cases: ${report.total}（负例 ${report.negative}，${Math.round(report.negativeRatio * 100)}%；未复核 ${report.unreviewed}）`
   ];
+  if (report.skills > 0) lines.push(`skills: ${report.skills}（不参与工具评测）`);
+  lines.push(
+    "",
+    "tool                            cases  args  reviewed",
+    "------------------------------  -----  ----  --------"
+  );
   for (const tool of report.tools) {
     lines.push(
-      `${tool.name.padEnd(22)}  ${String(tool.total).padStart(5)}  ${String(tool.withArgs).padStart(4)}  ${String(tool.reviewed).padStart(8)}`
+      `${tool.name.padEnd(30)}  ${String(tool.total).padStart(5)}  ${String(tool.withArgs).padStart(4)}  ${String(tool.reviewed).padStart(8)}`
     );
   }
+  if (report.profiles && report.profiles.length > 0) {
+    lines.push("", "profiles（分组后可评分的用例；负例每组都跑）:");
+    lines.push("group                   tools  cases  neg  tools-without-cases");
+    lines.push("----------------------  -----  -----  ---  -------------------");
+    for (const row of report.profiles) {
+      lines.push(
+        `${row.name.padEnd(22)}  ${String(row.tools).padStart(5)}  ${String(row.cases).padStart(5)}  ${String(row.negatives).padStart(3)}  ${row.missing.length ? row.missing.join(", ") : "-"}`
+      );
+    }
+  }
   if (report.pairs.length > 0) {
     lines.push("", "suspected confusing pairs（建议各补 1 条歧义用例）:");
     for (const pair of report.pairs) {
@@ -157,29 +207,100 @@ export function formatCoverage(report) {
   return lines.join("\n");
 }
 
+export function pairSimilarity(x, y) {
+  const a = tokens(`${x.name} ${x.description}`);
+  const b = tokens(`${y.name} ${y.description}`);
+  const union = new Set([...a, ...b]);
+  let lexical = 0;
+  if (union.size > 0) {
+    let shared = 0;
+    for (const token of union) if (a.has(token) && b.has(token)) shared += 1;
+    lexical = shared / union.size;
+  }
+  const structural = structuralScore(x, y);
+  return {
+    similarity: Math.max(lexical, structural),
+    basis: structural > lexical ? "structural" : "lexical"
+  };
+}
+
+// Clustering wants "same family" (alarm_query / alarm_ack belong to one agent),
+// which is looser than "confusable" (lint's question). Shared name token plus
+// any parameter overlap is enough to belong together; two shared tokens are
+// enough on their own (fault_alarm_query / fault_alarm_ack take different ids).
+// One shared token with disjoint params is not: cross-domain tools often share
+// a bare verb (query / list) and nothing else.
+export function familyScore(x, y) {
+  const left = nameTokens(x.name).filter((token) => !GENERIC_NAME_TOKENS.has(token));
+  const right = nameTokens(y.name).filter((token) => !GENERIC_NAME_TOKENS.has(token));
+  const shared = left.filter((token) => right.includes(token));
+  if (shared.length === 0) return 0;
+  const paramOverlap = jaccard(paramNames(x), paramNames(y));
+  if (paramOverlap > 0) return 0.8;
+  if (shared.length >= 2) return 0.7;
+  return paramNames(x).size === 0 && paramNames(y).size === 0 ? 0.6 : 0.2;
+}
+
 export function suspectedPairs(tools) {
   const pairs = [];
   for (let i = 0; i < tools.length; i += 1) {
     for (let j = i + 1; j < tools.length; j += 1) {
-      const a = tokens(`${tools[i].name} ${tools[i].description}`);
-      const b = tokens(`${tools[j].name} ${tools[j].description}`);
-      const union = new Set([...a, ...b]);
-      if (union.size === 0) continue;
-      let shared = 0;
-      for (const token of union) if (a.has(token) && b.has(token)) shared += 1;
-      const similarity = shared / union.size;
+      const { similarity, basis } = pairSimilarity(tools[i], tools[j]);
       if (similarity >= PAIR_SIMILARITY) {
-        pairs.push({ a: tools[i].name, b: tools[j].name, similarity });
+        pairs.push({ a: tools[i].name, b: tools[j].name, similarity, basis });
       }
     }
   }
   return pairs.sort((x, y) => y.similarity - x.similarity);
 }
 
+// Tools that share part of the name (search / file / read...) and the same
+// parameter shape confuse the model even when the wording differs, which the
+// lexical score alone misses. Name order is unreliable: `tavily_search` leads
+// with the vendor, `search_docs` with the action.
+function structuralScore(x, y) {
+  const left = nameTokens(x.name).filter((token) => !GENERIC_NAME_TOKENS.has(token));
+  const right = nameTokens(y.name).filter((token) => !GENERIC_NAME_TOKENS.has(token));
+  const sharedName = left.some((token) => right.includes(token));
+  if (!sharedName) return 0;
+  const params = paramNames(x);
+  const otherParams = paramNames(y);
+  if (params.size === 0 && otherParams.size === 0) return 0;
+  const paramOverlap = jaccard(params, otherParams);
+  if (paramOverlap < 0.34) return 0;
+  const requiredOverlap = jaccard(
+    new Set(x.parameters?.required ?? []),
+    new Set(y.parameters?.required ?? [])
+  );
+  return 0.5 + 0.5 * (0.6 * paramOverlap + 0.4 * requiredOverlap);
+}
+
+function nameTokens(name) {
+  return String(name ?? "")
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+}
+
+function paramNames(tool) {
+  return new Set(Object.keys(tool.parameters?.properties ?? {}));
+}
+
+function jaccard(a, b) {
+  if (a.size === 0 && b.size === 0) return 1;
+  const union = new Set([...a, ...b]);
+  let shared = 0;
+  for (const item of union) if (a.has(item) && b.has(item)) shared += 1;
+  return shared / union.size;
+}
+
 function tokens(text) {
   const set = new Set();
   const lowered = String(text ?? "").toLowerCase();
-  for (const word of lowered.match(/[a-z][a-z0-9_]{1,}/g) ?? []) set.add(word);
+  for (const word of lowered.match(/[a-z][a-z0-9_]{1,}/g) ?? []) {
+    if (!STOPWORDS.has(word)) set.add(word);
+  }
   for (const chunk of lowered.match(/[一-鿿]+/g) ?? []) {
     if (chunk.length === 1) set.add(chunk);
     for (let i = 0; i + 1 < chunk.length; i += 1) set.add(chunk.slice(i, i + 2));

package/lib/cluster.js

@@ -0,0 +1,103 @@
+import { pairSimilarity, familyScore, GENERIC_NAME_TOKENS } from "./cases.js";
+
+export function clusterTools(tools, { threshold = 0.5, maxSize = 10 } = {}) {
+  const scores = tools.map(() => new Array(tools.length).fill(0));
+  const pairs = [];
+  for (let i = 0; i < tools.length; i += 1) {
+    for (let j = i + 1; j < tools.length; j += 1) {
+      const { similarity, basis } = pairSimilarity(tools[i], tools[j]);
+      const family = familyScore(tools[i], tools[j]);
+      const link = Math.max(similarity, family);
+      scores[i][j] = link;
+      scores[j][i] = link;
+      if (link >= threshold) {
+        pairs.push({ i, j, similarity, basis: family > similarity ? "family" : basis });
+      }
+    }
+  }
+
+  const degree = tools.map((_, index) =>
+    pairs.reduce((sum, pair) => sum + (pair.i === index || pair.j === index ? 1 : 0), 0)
+  );
+  const order = tools
+    .map((tool, index) => index)
+    .sort((a, b) => degree[b] - degree[a] || tools[a].name.localeCompare(tools[b].name));
+
+  const raw = [];
+  for (const index of order) {
+    let best = null;
+    for (const cluster of raw) {
+      if (cluster.members.length >= maxSize) continue;
+      const average =
+        cluster.members.reduce((sum, member) => sum + scores[index][member], 0) / cluster.members.length;
+      if (average >= threshold && (!best || average > best.average)) best = { cluster, average };
+    }
+    if (best) best.cluster.members.push(index);
+    else raw.push({ members: [index] });
+  }
+
+  const clusterOf = new Map();
+  raw.forEach((cluster, position) => cluster.members.forEach((member) => clusterOf.set(member, position)));
+
+  const usedLabels = new Set();
+  const clusters = raw.map((cluster, position) => {
+    const members = [...cluster.members].sort((a, b) => tools[a].name.localeCompare(tools[b].name));
+    const label = uniqueLabel(labelFor(members.map((index) => tools[index].name)), usedLabels, position);
+    return { label, tools: members.map((index) => tools[index].name) };
+  });
+
+  const riskyPairs = pairs
+    .filter((pair) => clusterOf.get(pair.i) !== clusterOf.get(pair.j))
+    .map((pair) => ({
+      a: tools[pair.i].name,
+      b: tools[pair.j].name,
+      similarity: Number(pair.similarity.toFixed(2)),
+      basis: pair.basis
+    }))
+    .sort((x, y) => y.similarity - x.similarity);
+
+  return { threshold, total: tools.length, clusters, riskyPairs };
+}
+
+function labelFor(names) {
+  const counts = new Map();
+  for (const name of names) {
+    for (const token of String(name).toLowerCase().split(/[^a-z0-9]+/)) {
+      if (!token || GENERIC_NAME_TOKENS.has(token)) continue;
+      counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+  }
+  let best = null;
+  for (const [token, count] of counts) {
+    if (!best || count > best.count || (count === best.count && token < best.token)) best = { token, count };
+  }
+  return best && best.count > 1 ? best.token : "";
+}
+
+function uniqueLabel(base, used, position) {
+  let label = base || `cluster-${position + 1}`;
+  let suffix = 2;
+  while (used.has(label)) {
+    label = `${base || `cluster-${position + 1}`}-${suffix}`;
+    suffix += 1;
+  }
+  used.add(label);
+  return label;
+}
+
+export function formatClusters(result) {
+  const lines = [
+    `tools ${result.total} → clusters ${result.clusters.length}（threshold ${result.threshold}）`
+  ];
+  for (const cluster of result.clusters) {
+    lines.push(`  [${cluster.label}] ${cluster.tools.length}: ${cluster.tools.join(", ")}`);
+  }
+  if (result.riskyPairs.length > 0) {
+    lines.push("", "跨组高风险（相似但被分到不同组，合并挂载会互抢）：");
+    for (const pair of result.riskyPairs) {
+      const reason = pair.basis === "structural" ? "形状接近" : "描述相近";
+      lines.push(`  ${pair.a} × ${pair.b}  ${pair.similarity.toFixed(2)}（${reason}）`);
+    }
+  }
+  return lines.join("\n");
+}

package/lib/compare.js

@@ -129,6 +129,55 @@ export function compareScorecards(before, after) {
     executionChanged,
     executionBefore,
     executionAfter,
+    models,
+    profiles: compareProfiles(before, after)
+  };
+}
+
+function compareProfiles(before, after) {
+  const beforeReport = before.profiles ?? null;
+  const afterReport = after.profiles ?? null;
+  if (!beforeReport && !afterReport) return null;
+  if (!beforeReport || !afterReport) {
+    return { comparable: false, reason: "只有一份报告包含分组结果", models: [] };
+  }
+  const comparable = beforeReport.hash === afterReport.hash;
+  const modelsAfter = new Map((afterReport.models ?? []).map((model) => [model.model, model]));
+  const models = [];
+  for (const modelBefore of beforeReport.models ?? []) {
+    const modelAfter = modelsAfter.get(modelBefore.model);
+    if (!modelAfter) {
+      models.push({ model: modelBefore.model, reason: "模型不在新报告中", profiles: [] });
+      continue;
+    }
+    const entriesAfter = new Map((modelAfter.profiles ?? []).map((entry) => [entry.profile, entry]));
+    const entries = [];
+    for (const entryBefore of modelBefore.profiles ?? []) {
+      const entryAfter = entriesAfter.get(entryBefore.profile);
+      if (!entryAfter) {
+        entries.push({ name: entryBefore.profile, reason: "新报告没有这一组" });
+        continue;
+      }
+      entries.push({
+        name: entryBefore.profile,
+        reviewed: entryAfter.reviewed,
+        mounted: {
+          before: entryBefore.mounted?.selectionAccuracy ?? null,
+          after: entryAfter.mounted?.selectionAccuracy ?? null
+        },
+        allInOne: {
+          before: entryBefore.allInOne?.selectionAccuracy ?? null,
+          after: entryAfter.allInOne?.selectionAccuracy ?? null
+        }
+      });
+    }
+    models.push({ model: modelBefore.model, profiles: entries });
+  }
+  return {
+    comparable,
+    reason: comparable ? null : "分组定义已变更（profiles hash 不同）",
+    beforeHash: beforeReport.hash,
+    afterHash: afterReport.hash,
     models
   };
 }
@@ -194,9 +243,44 @@ export function formatCompare(report) {
     }
   }
 
+  if (report.profiles) {
+    lines.push("");
+    if (!report.profiles.comparable) {
+      lines.push(`挂载分组: ${report.profiles.reason}`);
+    } else {
+      lines.push(`挂载分组（同一定义 ${report.profiles.beforeHash}，组内准确率）:`);
+    }
+    for (const model of report.profiles.models) {
+      if (model.reason) {
+        lines.push(`  [${model.model}] ${model.reason}`);
+        continue;
+      }
+      lines.push(`  [${model.model}]`);
+      for (const entry of model.profiles) {
+        if (entry.reason) {
+          lines.push(`    ${entry.name}: ${entry.reason}`);
+          continue;
+        }
+        const delta =
+          entry.mounted.before === null || entry.mounted.after === null
+            ? ""
+            : `（${signed(entry.mounted.after - entry.mounted.before)}）`;
+        lines.push(
+          `    ${entry.name}  ${percent(entry.mounted.before)} → ${percent(entry.mounted.after)}${delta}` +
+            `  全量 ${percent(entry.allInOne.before)} → ${percent(entry.allInOne.after)}`
+        );
+      }
+    }
+  }
+
   return lines.join("\n");
 }
 
+function signed(value) {
+  const points = Math.round(value * 100);
+  return `${points > 0 ? "+" : ""}${points}pp`;
+}
+
 function diffEntries(beforeList, afterList, key) {
   const before = new Map((beforeList ?? []).map((entry) => [entry[key], entry.hash]));
   const after = new Map((afterList ?? []).map((entry) => [entry[key], entry.hash]));

package/lib/mcp-server.js

@@ -32,7 +32,7 @@ export async function startMcpServer() {
     {
       title: "Scan tools",
       description:
-        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核）。",
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核；来源 ≥2 时草稿带按来源分组的 profiles，并列出不参与评测的 skills）。",
       inputSchema: {
         root: z.string().optional().describe("仓库目录，默认进程当前目录"),
         probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
@@ -142,13 +142,26 @@ export async function startMcpServer() {
         suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json"),
         models: z.string().optional().describe("逗号分隔的模型 id，如 mock,dsv4-flash"),
         cases: z.string().optional().describe("只跑这些用例 id（逗号分隔）"),
+        profile: z
+          .string()
+          .optional()
+          .describe("按挂载组评分：all 或组名（逗号分隔）；套件需含 profiles，结果里带 groups/lifted/jammed"),
+        repeat: z
+          .number()
+          .int()
+          .min(1)
+          .max(10)
+          .optional()
+          .describe("每条用例重复次数，取多数结果；建议正式基线用 3"),
         failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2")
       }
     },
-    async ({ suite, models, cases, failUnder }) => {
+    async ({ suite, models, cases, profile, repeat, failUnder }) => {
       const args = ["eval", "--json", "--suite", suitePath(suite)];
       if (models) args.push("--models", models);
       if (cases) args.push("--case", cases);
+      if (profile) args.push("--profile", profile);
+      if (repeat !== undefined) args.push("--repeat", String(repeat));
       if (failUnder !== undefined) args.push("--fail-under", String(failUnder));
 
       const result = await runCli(args);