modelfunction-cli 0.2.0 → 0.3.0

package/README.md

@@ -65,13 +65,13 @@ INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
 | 命令 | 作用 |
 | --- | --- |
 | `mf scan` | 扫描工具定义（function.json / OpenAPI / MCP 配置 / Python `@tool`）；`--probe` 连 MCP server；`--from` 导入运行时工具数组；草稿按来源分组（`profiles`）并列出未覆盖的 skill |
-| `mf eval` | 跑评分卡；`--fail-under` 阈值；`--baseline` + `--fail-on-regression` 回归门禁；`--profile` 按挂载组分别评分 |
+| `mf eval` | 跑评分卡；`--repeat` 多次取多数并报不稳定用例；`--fail-under` 阈值；`--baseline` + `--fail-on-regression` 回归门禁；`--profile` 按挂载组分别评分 |
 | `mf case add` | 写入用例（默认 `source=agent`、`reviewed=false`） |
 | `mf coverage` | 每工具用例数、参数覆盖、负例比例、疑似混淆对；有 `profiles` 时按组统计 |
-| `mf lint` | 静态检查描述与参数 schema；有 `profiles` 时给出「这些组不要合并挂载」 |
+| `mf lint` | 静态检查工具描述与参数 schema，以及 skills 的 name/description（不调用模型）；重复规则自动合并成一条；有 `profiles` 时给出「这些组不要合并挂载」 |
 | `mf compare` | 跨版本比较、干扰归因、混淆矩阵；两份报告都有分组时按组对比 |
 | `mf ablate` | 留一法：去掉某个工具再跑一遍，量化它是否干扰其他工具；`--profile` 限定在某个挂载组内 |
-| `mf cluster` | 聚类工具，建议挂载分组；`--out` 把分组写成套件的 `profiles` |
+| `mf cluster` | 聚类工具，建议挂载分组；参数形状拆散的同一族（如 `sessions_*`）列为「同族候选」；`--out` 把分组写成套件的 `profiles` |
 | `mf exec` | 单工具执行 + 指标（Node / Python），支持 `--allow-read/write/net`、超时、sideEffects 准入 |
 | `mf eval --execute` | 一次跑完选择 + 参数 + 运行断言（`toolInput` / `expect`） |
 | `mf sandbox check` | 探测本机可用隔离引擎与降级建议 |
@@ -114,7 +114,7 @@ mf eval --suite mf.suite.json --models internal-ds \
 }
 ```
 
-暴露：`scan_tools`、`list_suite`、`coverage`、`add_cases`、`run_eval`（`run_eval` 支持 `profile` 参数，传 `all` 或组名即可按挂载组评分）。
+暴露：`scan_tools`、`list_suite`、`coverage`、`add_cases`、`run_eval`（`run_eval` 支持 `profile` 与 `repeat` 参数：传 `all` 或组名按挂载组评分，`repeat` 取多数结果）。
 
 ## 数据边界
 

package/lib/cases.js

@@ -141,7 +141,7 @@ export function computeCoverage(suite) {
   if (unreviewed > 0) warnings.push(`${unreviewed} 条未复核用例不计入正式分数`);
 
   const skills = suite.skills?.length ?? 0;
-  if (skills > 0) warnings.push(`${skills} 个 skill 不参与工具评测（当前只覆盖 function / MCP 工具）`);
+  if (skills > 0) warnings.push(`${skills} 个 skill 只做静态检查，不参与模型评测`);
 
   const profileList = suite.profiles
     ? normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name))
@@ -173,7 +173,7 @@ export function formatCoverage(report) {
   const lines = [
     `cases: ${report.total}（负例 ${report.negative}，${Math.round(report.negativeRatio * 100)}%；未复核 ${report.unreviewed}）`
   ];
-  if (report.skills > 0) lines.push(`skills: ${report.skills}（不参与工具评测）`);
+  if (report.skills > 0) lines.push(`skills: ${report.skills}（仅静态检查，不参与模型评测）`);
   lines.push(
     "",
     "tool                            cases  args  reviewed",
@@ -207,9 +207,9 @@ export function formatCoverage(report) {
   return lines.join("\n");
 }
 
-export function pairSimilarity(x, y) {
-  const a = tokens(`${x.name} ${x.description}`);
-  const b = tokens(`${y.name} ${y.description}`);
+export function pairSimilarity(x, y, { extraStop } = {}) {
+  const a = tokens(`${x.name} ${x.description}`, extraStop);
+  const b = tokens(`${y.name} ${y.description}`, extraStop);
   const union = new Set([...a, ...b]);
   let lexical = 0;
   if (union.size > 0) {
@@ -241,12 +241,12 @@ export function familyScore(x, y) {
   return paramNames(x).size === 0 && paramNames(y).size === 0 ? 0.6 : 0.2;
 }
 
-export function suspectedPairs(tools) {
+export function suspectedPairs(tools, { threshold = PAIR_SIMILARITY, extraStop } = {}) {
   const pairs = [];
   for (let i = 0; i < tools.length; i += 1) {
     for (let j = i + 1; j < tools.length; j += 1) {
-      const { similarity, basis } = pairSimilarity(tools[i], tools[j]);
-      if (similarity >= PAIR_SIMILARITY) {
+      const { similarity, basis } = pairSimilarity(tools[i], tools[j], { extraStop });
+      if (similarity >= threshold) {
         pairs.push({ a: tools[i].name, b: tools[j].name, similarity, basis });
       }
     }
@@ -275,12 +275,25 @@ function structuralScore(x, y) {
   return 0.5 + 0.5 * (0.6 * paramOverlap + 0.4 * requiredOverlap);
 }
 
-function nameTokens(name) {
-  return String(name ?? "")
+// Plural folds matter on real tool sets: sessions_list / session_status and
+// issue / issues describe the same thing, but naive token matching misses both.
+// Only the safe cases are folded; "class"/"process" already end in ss and pass
+// through untouched.
+export function normalizeWord(word) {
+  const value = String(word ?? "").toLowerCase();
+  if (value.length < 3) return value;
+  if (/ies$/.test(value)) return `${value.slice(0, -3)}y`;
+  if (/s$/.test(value) && !/(ss|us|is)$/.test(value)) return value.slice(0, -1);
+  return value;
+}
+
+export function nameTokens(name) {
+  return String(name)
     .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
     .toLowerCase()
     .split(/[^a-z0-9]+/)
-    .filter(Boolean);
+    .filter(Boolean)
+    .map(normalizeWord);
 }
 
 function paramNames(tool) {
@@ -295,11 +308,17 @@ function jaccard(a, b) {
   return shared / union.size;
 }
 
-function tokens(text) {
+function tokens(text, extraStop) {
   const set = new Set();
   const lowered = String(text ?? "").toLowerCase();
   for (const word of lowered.match(/[a-z][a-z0-9_]{1,}/g) ?? []) {
-    if (!STOPWORDS.has(word)) set.add(word);
+    for (const part of word.split("_")) {
+      const normalized = normalizeWord(part);
+      if (!normalized) continue;
+      if (STOPWORDS.has(part) || STOPWORDS.has(normalized)) continue;
+      if (extraStop?.has(part) || extraStop?.has(normalized)) continue;
+      set.add(normalized);
+    }
   }
   for (const chunk of lowered.match(/[一-鿿]+/g) ?? []) {
     if (chunk.length === 1) set.add(chunk);

package/lib/cluster.js

@@ -1,4 +1,16 @@
-import { pairSimilarity, familyScore, GENERIC_NAME_TOKENS } from "./cases.js";
+import { pairSimilarity, familyScore, nameTokens, GENERIC_NAME_TOKENS } from "./cases.js";
+
+// A shared domain noun ("session", "goal", "tool") says these tools serve one
+// purpose even when their verbs and parameter shapes differ — exactly what
+// familyScore refuses to merge. Surface it as a hint instead of guessing: the
+// platform already knows its real mounting map.
+const FAMILY_HINT_STOP = new Set([
+  ...GENERIC_NAME_TOKENS,
+  "run", "call", "use", "search", "find", "query", "fetch", "create", "update",
+  "list", "send", "read", "write", "add", "remove", "delete", "start", "stop",
+  "open", "close", "poll", "log", "status", "data", "info", "result", "generate",
+  "save", "load", "merge", "sync", "scan", "check", "compare", "exec", "probe"
+]);
 
 export function clusterTools(tools, { threshold = 0.5, maxSize = 10 } = {}) {
   const scores = tools.map(() => new Array(tools.length).fill(0));
@@ -38,6 +50,8 @@ export function clusterTools(tools, { threshold = 0.5, maxSize = 10 } = {}) {
 
   const clusterOf = new Map();
   raw.forEach((cluster, position) => cluster.members.forEach((member) => clusterOf.set(member, position)));
+  const clusterSize = raw.map((cluster) => cluster.members.length);
+  const familyHints = buildFamilyHints(tools, clusterOf, clusterSize);
 
   const usedLabels = new Set();
   const clusters = raw.map((cluster, position) => {
@@ -56,7 +70,39 @@ export function clusterTools(tools, { threshold = 0.5, maxSize = 10 } = {}) {
     }))
     .sort((x, y) => y.similarity - x.similarity);
 
-  return { threshold, total: tools.length, clusters, riskyPairs };
+  return { threshold, total: tools.length, clusters, riskyPairs, familyHints };
+}
+
+function buildFamilyHints(tools, clusterOf, clusterSize) {
+  const byToken = new Map();
+  tools.forEach((tool, index) => {
+    for (const token of new Set(nameTokens(tool.name))) {
+      if (token.length < 3 || FAMILY_HINT_STOP.has(token)) continue;
+      const list = byToken.get(token) ?? [];
+      list.push(index);
+      byToken.set(token, list);
+    }
+  });
+
+  const hints = [];
+  const seen = new Set();
+  for (const [token, members] of byToken) {
+    if (members.length < 2) continue;
+    const clusterIds = new Set(members.map((index) => clusterOf.get(index)));
+    if (clusterIds.size < 2) continue;
+    const names = members.map((index) => tools[index].name).sort();
+    const signature = names.join("\n");
+    if (seen.has(signature)) continue;
+    seen.add(signature);
+    const isLoose = (index) => clusterSize[clusterOf.get(index)] === 1;
+    hints.push({
+      token,
+      size: names.length,
+      merged: members.filter((index) => !isLoose(index)).map((index) => tools[index].name).sort(),
+      loose: members.filter(isLoose).map((index) => tools[index].name).sort()
+    });
+  }
+  return hints.sort((a, b) => b.size - a.size || a.token.localeCompare(b.token)).slice(0, 8);
 }
 
 function labelFor(names) {
@@ -99,5 +145,14 @@ export function formatClusters(result) {
       lines.push(`  ${pair.a} × ${pair.b}  ${pair.similarity.toFixed(2)}（${reason}）`);
     }
   }
+  if ((result.familyHints ?? []).length > 0) {
+    lines.push("", "同族候选（共享标识词但参数形状不同，未自动合并；以平台侧真实挂载映射为准）:");
+    for (const hint of result.familyHints) {
+      const parts = [];
+      if (hint.merged.length > 0) parts.push(`已合并 ${hint.merged.join(", ")}`);
+      if (hint.loose.length > 0) parts.push(`未合并 ${hint.loose.join(", ")}`);
+      lines.push(`  [${hint.token}] ${hint.size} 个：${parts.join(" | ")}`);
+    }
+  }
   return lines.join("\n");
 }

package/lib/scan.js

@@ -294,7 +294,9 @@ export function buildSuiteDraft(report) {
     })),
     cases: []
   };
-  if (report.skills.length > 0) draft.skills = report.skills.map((skill) => skill.name);
+  if (report.skills.length > 0) {
+    draft.skills = report.skills.map((skill) => ({ name: skill.name, description: skill.description }));
+  }
   // Where a tool came from is the only mount hint the scan can observe without
   // guessing: one MCP server or one spec file is normally one thing agents mount.
   const profiles = groupBySource(report.tools);
@@ -354,7 +356,7 @@ export function formatScanSummary(report) {
   lines.push(
     "",
     `tools:       ${report.tools.length}`,
-    `skills:      ${report.skills.length}（不进入工具评测）`,
+    `skills:      ${report.skills.length}（仅静态检查）`,
     `mcp servers: ${report.mcpServers.length}${probed > 0 ? `（已 probe ${probed} 个）` : "（需 --probe 才能拿到工具清单）"}`,
     `code match:  ${report.codeMatches.length}（low confidence，需人工确认）`
   );

package/lib/suite.js

@@ -29,7 +29,7 @@ export async function loadSuite(suitePath, { allowEmptyCases = false } = {}) {
     tools,
     cases,
     profiles: raw.profiles ?? null,
-    skills: Array.isArray(raw.skills) ? raw.skills.map(String) : [],
+    skills: Array.isArray(raw.skills) ? raw.skills.map(normalizeSkill) : [],
     execution:
       raw.execution && typeof raw.execution === "object" && !Array.isArray(raw.execution)
         ? { ...raw.execution }
@@ -58,6 +58,23 @@ async function normalizeTool(entry, baseDir) {
   };
 }
 
+// Skills carry no parameter schema in the prompt-injected form (SKILL.md
+// frontmatter), so the only checkable surface is name + description. Both the
+// bare-name shorthand and the full form are accepted.
+function normalizeSkill(entry, index) {
+  if (typeof entry === "string") {
+    const name = entry.trim();
+    if (!name) throw new Error(`skill #${index + 1}: name must be a non-empty string`);
+    return { name, description: "" };
+  }
+  if (entry && typeof entry === "object") {
+    const name = typeof entry.name === "string" ? entry.name.trim() : "";
+    if (!name) throw new Error(`skill #${index + 1}: "name" must be a non-empty string`);
+    return { name, description: typeof entry.description === "string" ? entry.description : "" };
+  }
+  throw new Error(`skill #${index + 1}: must be a name or {name, description}`);
+}
+
 function normalizeCase(entry, index, toolNames) {
   const id = entry.id ?? `case_${index + 1}`;
   if (typeof entry.input !== "string" || entry.input.trim() === "") {

package/mf.js

@@ -50,10 +50,10 @@ Usage:
   mf scan [options]
   mf case add --suite <file> (--input <file> | stdin) [--source agent] [--reviewed] [--dry-run]
   mf coverage [--suite <file>] [--json]
-  mf lint [--suite <file>] [--json]
+  mf lint [--suite <file>] [--json]  # 静态检查 tools 与 skills，不调用模型
   mf compare <before.json> <after.json> [--json]
   mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--json]
-  mf cluster [--suite <file>] [--threshold <0.5>] [--max-size <10>] [--out <file>] [--json]  # 建议挂载分组
+  mf cluster [--suite <file>] [--threshold <0.5>] [--max-size <10>] [--out <file>] [--json]  # 建议挂载分组 + 同族候选
   mf exec --entry <file> (--input <json> | --input-json <json>) [--runtime node|python] [--python <cmd>] [--export run] [--allow-read <paths>] [--allow-write <paths>] [--allow-net <hosts>] [--timeout <ms>] [--warmup <n>] [--json]
   mf sandbox check [--json]   # probe available isolation engines on this machine
   mf mcp                # start stdio MCP server for coding agents

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "type": "module",
   "license": "MIT",
   "description": "mf — function-calling accuracy scorecard for AI agents",