modelfunction-cli 0.1.3 → 0.3.0

package/mf.js

@@ -3,18 +3,30 @@ import { readFile } from "node:fs/promises";
 import { basename, dirname, extname, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { loadSuite, toModelTools } from "./lib/suite.js";
-import { scoreModel, pct } from "./lib/score.js";
+import { scoreModel, pickRepresentative, pct } from "./lib/score.js";
 import { formatTable, formatCaseLines, formatMarkdown } from "./lib/report.js";
 import { scanProject, buildSuiteDraft, formatScanSummary, addTools, redactReport } from "./lib/scan.js";
 import { probeStdioServer } from "./lib/mcp-probe.js";
 import { loadAdapterTools } from "./lib/adapter-import.js";
 import { addCases, computeCoverage, formatCoverage } from "./lib/cases.js";
 import { lintSuite, formatLint } from "./lib/lint.js";
-import { suiteFingerprint, hashText } from "./lib/fingerprint.js";
+import { suiteFingerprint, hashText, hashOf } from "./lib/fingerprint.js";
 import { runExecutionChecks } from "./lib/eval-exec.js";
 import { checkSandbox, formatSandboxReport } from "./lib/sandbox-check.js";
 import { compareScorecards, formatCompare } from "./lib/compare.js";
 import { analyzeAblation, formatAblation } from "./lib/ablate.js";
+import { clusterTools, formatClusters } from "./lib/cluster.js";
+import {
+  normalizeProfiles,
+  selectProfiles,
+  selectProfileCases,
+  profileWarnings,
+  analyzeMount,
+  modelTotals,
+  profileCaseCoverage,
+  formatProfileReport,
+  formatProfileMarkdown
+} from "./lib/profile.js";
 import { runExec } from "./lib/exec.js";
 import { writeFileAtomic } from "./lib/fsutil.js";
 import { createMockAdapter } from "./lib/adapters/mock.js";
@@ -38,9 +50,10 @@ Usage:
   mf scan [options]
   mf case add --suite <file> (--input <file> | stdin) [--source agent] [--reviewed] [--dry-run]
   mf coverage [--suite <file>] [--json]
-  mf lint [--suite <file>] [--json]
+  mf lint [--suite <file>] [--json]  # 静态检查 tools 与 skills，不调用模型
   mf compare <before.json> <after.json> [--json]
-  mf ablate --tool <name> [--suite <file>] [--models <ids>] [--json]
+  mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--json]
+  mf cluster [--suite <file>] [--threshold <0.5>] [--max-size <10>] [--out <file>] [--json]  # 建议挂载分组 + 同族候选
   mf exec --entry <file> (--input <json> | --input-json <json>) [--runtime node|python] [--python <cmd>] [--export run] [--allow-read <paths>] [--allow-write <paths>] [--allow-net <hosts>] [--timeout <ms>] [--warmup <n>] [--json]
   mf sandbox check [--json]   # probe available isolation engines on this machine
   mf mcp                # start stdio MCP server for coding agents
@@ -50,6 +63,8 @@ Eval options:
   --models <ids>        comma-separated ids from models.json (default: models with API keys set)
   --models-file <path>  model registry (default: built-in registry)
   --case <ids>          run only these case ids (comma-separated, repeatable)
+  --profile <names>     score each mounted group separately ("all" or names from suite.profiles)
+  --repeat <n>          run each case n times and score the majority; reports unstable cases (default 1)
   --show-cases          print per-case results
   --json                print JSON instead of the table
   --out <file>          write scorecard (.md => markdown, otherwise JSON)
@@ -81,23 +96,27 @@ Examples:
 `;
 
 export async function runEval(options, io = console) {
+  if (options.profiles && options.execute) {
+    throw new Error("--profile 暂不支持与 --execute 组合");
+  }
   const suite = await loadSuite(options.suite);
   const cases = selectCases(suite.cases, options.cases);
   const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
   const tools = toModelTools(suite.tools);
 
-  const summaries = [];
+  const runs = [];
   for (const adapter of adapters) {
-    const results = await evaluate(adapter, tools, cases);
-    summaries.push(
-      scoreModel({
-        model: adapter.id,
-        cases,
-        results,
-        pricing: registry[adapter.id]?.pricing ?? null
-      })
-    );
+    const evaluated = await evaluate(adapter, tools, cases, options.repeat ?? 1);
+    const summary = scoreModel({
+      model: adapter.id,
+      cases,
+      results: evaluated.results,
+      pricing: registry[adapter.id]?.pricing ?? null
+    });
+    if (evaluated.stability) summary.stability = evaluated.stability;
+    runs.push({ adapter, results: evaluated.results, summary });
   }
+  const summaries = runs.map((run) => run.summary);
 
   let executionInfo = null;
   if (options.execute) {
@@ -124,18 +143,30 @@ export async function runEval(options, io = console) {
     };
   }
 
+  const profileReport = options.profiles
+    ? await evaluateProfiles({
+        suite,
+        cases,
+        runs,
+        registry,
+        selection: options.profiles,
+        repeat: options.repeat ?? 1
+      })
+    : null;
+
   if (options.out) {
     const target = resolve(options.out);
     const markdown = extname(target).toLowerCase() === ".md";
     const payload = markdown
-      ? formatMarkdown(suite, cases, summaries)
-      : JSON.stringify(serialize(suite, cases, summaries, executionInfo), null, 2);
+      ? formatMarkdown(suite, cases, summaries) +
+        (profileReport ? `\n\n${formatProfileMarkdown(profileReport)}` : "")
+      : JSON.stringify(serialize(suite, cases, summaries, executionInfo, profileReport), null, 2);
     await writeFileAtomic(target, payload);
     io.log(`wrote ${target}`);
   }
 
   if (options.json) {
-    io.log(JSON.stringify(serialize(suite, cases, summaries, executionInfo), null, 2));
+    io.log(JSON.stringify(serialize(suite, cases, summaries, executionInfo, profileReport), null, 2));
   } else {
     io.log(`suite: ${suite.name} — ${cases.length} cases, ${suite.tools.length} tools`);
     io.log(`file:  ${suite.path}`);
@@ -147,6 +178,14 @@ export async function runEval(options, io = console) {
         io.log("");
       }
       io.log(formatTable([summary]));
+      if (summary.stability) {
+        const unstable = summary.stability.unstable;
+        const list = unstable.slice(0, 8).map((item) => item.id).join(", ");
+        io.log(
+          `stability: ${summary.stability.runs} 次/用例，${unstable.length} 条结果不稳定` +
+            (unstable.length > 0 ? `（${list}${unstable.length > 8 ? "…" : ""}）` : "")
+        );
+      }
       if (summary.unreviewedTotal > 0) {
         io.log(`note: ${summary.unreviewedTotal} unreviewed case(s) excluded from select/args/pass`);
       }
@@ -170,10 +209,18 @@ export async function runEval(options, io = console) {
       io.log("combined");
       io.log(formatTable(summaries));
     }
+    if (profileReport) {
+      io.log("");
+      io.log(formatProfileReport(profileReport, { showCases: options.showCases }));
+    }
     if (summaries.some((summary) => summary.rows.some((row) => row.usage?.estimated))) {
       io.log("");
       io.log("note: tokens marked with ~ are estimated (mock adapter); pricing in models.json is indicative.");
     }
+    if (summaries.some((summary) => summary.model === "mock")) {
+      io.log("");
+      io.log("note: mock 是规则路由器，只用于验证流程；它在你自己的工具上得低分不代表真实模型表现，请用 --models <id> 测真实模型。");
+    }
   }
 
   let regressions = [];
@@ -210,6 +257,17 @@ export async function runEval(options, io = console) {
     );
     return 2;
   }
+  if (profileReport && options.failUnder !== null) {
+    const below = profileReport.models.flatMap((model) =>
+      model.profiles
+        .filter((entry) => entry.reviewed > 0 && entry.mounted.selectionAccuracy < options.failUnder)
+        .map((entry) => `${model.model}/${entry.profile} (${pct(entry.mounted.selectionAccuracy)})`)
+    );
+    if (below.length > 0) {
+      io.error(`profile below threshold ${pct(options.failUnder)}: ${below.join(", ")}`);
+      return 2;
+    }
+  }
   if (regressions.length > 0) return 2;
   return 0;
 }
@@ -220,25 +278,40 @@ export async function runAblate(options, io = console) {
   const ablated = suite.tools.find((tool) => tool.name === options.tool);
   if (!ablated) throw new Error(`unknown tool "${options.tool}"`);
 
-  const baseCases = selectCases(suite.cases, options.cases);
+  const scope = options.profiles && options.profiles.length > 0 ? ablationScope(suite, options.profiles) : null;
+  if (scope && !scope.tools.some((tool) => tool.name === ablated.name)) {
+    throw new Error(`tool "${ablated.name}" 不在组 "${scope.name}" 里`);
+  }
+
+  const selectedCases = selectCases(suite.cases, options.cases);
+  const baseTools = scope ? scope.tools : suite.tools;
+  const baseCases = scope
+    ? selectProfileCases(selectedCases, new Set(scope.tools.map((tool) => tool.name)))
+    : selectedCases;
   const reducedCases = baseCases.filter((testCase) => testCase.expectedTool !== ablated.name);
   const droppedCases = baseCases.length - reducedCases.length;
   const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
 
-  const fullTools = toModelTools(suite.tools);
-  const reducedTools = toModelTools(suite.tools.filter((tool) => tool.name !== ablated.name));
+  const fullTools = toModelTools(baseTools);
+  const reducedTools = toModelTools(baseTools.filter((tool) => tool.name !== ablated.name));
   const models = [];
 
   for (const adapter of adapters) {
     const pricing = registry[adapter.id]?.pricing ?? null;
-    const fullResults = await evaluate(adapter, fullTools, baseCases);
+    const fullResults = (await evaluate(adapter, fullTools, baseCases)).results;
     const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
-    const reducedResults = await evaluate(adapter, reducedTools, reducedCases);
+    const reducedResults = (await evaluate(adapter, reducedTools, reducedCases)).results;
     const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
     models.push(analyzeAblation(adapter.id, full, reduced));
   }
 
-  const payload = { tool: ablated.name, suite: suite.name, droppedCases, models };
+  const payload = {
+    tool: ablated.name,
+    suite: suite.name,
+    droppedCases,
+    models,
+    ...(scope ? { profile: { name: scope.name, tools: baseTools.length } } : {})
+  };
   io.log(options.json ? JSON.stringify(payload, null, 2) : formatAblation(payload));
   return 0;
 }
@@ -271,7 +344,8 @@ export async function runScan(options, io = console) {
         source: server.source,
         origin: "mcp-probe",
         confidence: "high",
-        note: `server ${server.name}`
+        note: `server ${server.name}`,
+        meta: { provider: server.name }
       });
       report.probes.push({ server: server.name, ok: true, toolCount: added.length, tools: added });
     }
@@ -333,7 +407,9 @@ export async function runCaseAdd(options, io = console) {
 }
 
 export async function runCoverage(options, io = console) {
-  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
+  // A scan draft has no cases yet; "what is still missing" is exactly the view
+  // a user needs right after scanning.
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE), { allowEmptyCases: true });
   const report = computeCoverage(suite);
   io.log(options.json ? JSON.stringify(report, null, 2) : formatCoverage(report));
   return 0;
@@ -346,7 +422,7 @@ export async function runSandboxCheck(options, io = console) {
 }
 
 export async function runLint(options, io = console) {
-  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE), { allowEmptyCases: true });
   const report = lintSuite(suite);
   io.log(options.json ? JSON.stringify(report, null, 2) : formatLint(report));
   return report.counts.error > 0 ? 1 : 0;
@@ -416,29 +492,44 @@ function autoSelect(registry) {
     .map(([id]) => id);
 }
 
-async function evaluate(adapter, tools, cases) {
+async function evaluate(adapter, tools, cases, repeat = 1) {
   const results = [];
+  const unstable = [];
   for (const testCase of cases) {
-    const messages = [
-      { role: "system", content: SYSTEM_PROMPT },
-      { role: "user", content: testCase.input }
-    ];
-    try {
-      results.push(await adapter.call({ input: testCase.input, messages, tools }));
-    } catch (error) {
-      results.push({
-        ok: false,
-        error: String(error?.message ?? error),
-        toolCalls: [],
-        usage: { input: 0, output: 0 },
-        latencyMs: 0
-      });
+    const runs = [];
+    for (let attempt = 0; attempt < repeat; attempt += 1) {
+      runs.push(await callOnce(adapter, tools, testCase));
     }
+    if (repeat === 1) {
+      results.push(runs[0]);
+      continue;
+    }
+    const picked = pickRepresentative(runs);
+    results.push(picked.result);
+    if (picked.unstable) unstable.push({ id: testCase.id, outcomes: picked.outcomes });
+  }
+  return { results, stability: repeat > 1 ? { runs: repeat, unstable } : null };
+}
+
+async function callOnce(adapter, tools, testCase) {
+  const messages = [
+    { role: "system", content: SYSTEM_PROMPT },
+    { role: "user", content: testCase.input }
+  ];
+  try {
+    return await adapter.call({ input: testCase.input, messages, tools });
+  } catch (error) {
+    return {
+      ok: false,
+      error: String(error?.message ?? error),
+      toolCalls: [],
+      usage: { input: 0, output: 0 },
+      latencyMs: 0
+    };
   }
-  return results;
 }
 
-function serialize(suite, cases, summaries, executionInfo = null) {
+function serialize(suite, cases, summaries, executionInfo = null, profileReport = null) {
   return {
     suite: {
       name: suite.name,
@@ -448,10 +539,105 @@ function serialize(suite, cases, summaries, executionInfo = null) {
       fingerprint: suiteFingerprint(suite, cases),
       ...(executionInfo ? { execution: executionInfo } : {})
     },
-    models: summaries
+    models: summaries,
+    ...(profileReport ? { profiles: profileReport } : {})
+  };
+}
+
+function ablationScope(suite, names) {
+  if (names.length !== 1) throw new Error("mf ablate --profile 一次只能指定一个组");
+  const profiles = normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name));
+  if (!profiles) {
+    throw new Error(`suite ${suite.name} 没有 profiles；先运行 mf cluster --out <suite> 生成分组`);
+  }
+  const [profile] = selectProfiles(profiles, names);
+  const mounted = new Set(profile.tools);
+  return { name: profile.name, tools: suite.tools.filter((tool) => mounted.has(tool.name)) };
+}
+
+async function evaluateProfiles({ suite, cases, runs, registry, selection, repeat = 1 }) {
+  const profiles = normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name));
+  if (!profiles) {
+    throw new Error(`suite ${suite.name} 没有 profiles；先运行 mf cluster --out <suite> 生成分组`);
+  }
+  const selected = selectProfiles(profiles, selection);
+  const indexById = new Map(cases.map((testCase, index) => [testCase.id, index]));
+  const warnings = profileWarnings(selected, suite);
+
+  const groups = [];
+  const caseSets = selected.map((profile) => {
+    const profileCases = selectProfileCases(cases, new Set(profile.tools));
+    if (profileCases.length === 0) warnings.push(`profile "${profile.name}" 没有可评分的用例，已跳过`);
+    groups.push({ name: profile.name, tools: profile.tools.length, cases: profileCases.length });
+    return profileCases;
+  });
+  for (const row of profileCaseCoverage(selected, cases)) {
+    if (row.cases === 0) {
+      warnings.push(`组 ${row.name} 只有负例，mounted 分数只反映"不该调用"，不反映工具选择`);
+    }
+  }
+
+  const models = [];
+  for (const run of runs) {
+    const pricing = registry[run.adapter.id]?.pricing ?? null;
+    const entries = [];
+    for (const [index, profile] of selected.entries()) {
+      const profileCases = caseSets[index];
+      if (profileCases.length === 0) continue;
+      const mountedTools = suite.tools.filter((tool) => profile.tools.includes(tool.name));
+      const mountedEval = await evaluate(run.adapter, toModelTools(mountedTools), profileCases, repeat);
+      const mounted = scoreModel({
+        model: run.adapter.id,
+        cases: profileCases,
+        results: mountedEval.results,
+        pricing
+      });
+      const allInOne = scoreModel({
+        model: run.adapter.id,
+        cases: profileCases,
+        results: profileCases.map((testCase) => run.results[indexById.get(testCase.id)]),
+        pricing
+      });
+      const entry = analyzeMount(profile, mounted, allInOne);
+      if (mountedEval.stability) entry.stability = mountedEval.stability;
+      entries.push(entry);
+    }
+    if (entries.length > 0 && entries.every((entry) => entry.reviewed === 0)) {
+      warnings.push("所有用例都未复核，分组分数不可用；把认可的用例 reviewed 改为 true 后再看");
+    }
+    models.push({ model: run.adapter.id, profiles: entries, totals: modelTotals(entries) });
+  }
+
+  return {
+    hash: hashOf(
+      selected.map((profile) => [profile.name, [...profile.tools].sort()]).sort((a, b) => (a[0] < b[0] ? -1 : 1))
+    ),
+    groups,
+    warnings,
+    models
   };
 }
 
+export async function runCluster(options, io = console) {
+  const suite = await loadSuite(options.suite, { allowEmptyCases: true });
+  const result = clusterTools(suite.tools, { threshold: options.threshold, maxSize: options.maxSize });
+  if (options.out) {
+    const target = resolve(options.out);
+    const profiles = Object.fromEntries(result.clusters.map((cluster) => [cluster.label, cluster.tools]));
+    // Keep the suite file as the author wrote it (refs, execution, extra keys)
+    // and only add profiles.
+    const raw = JSON.parse(await readFile(suite.path, "utf8"));
+    await writeFileAtomic(target, `${JSON.stringify({ ...raw, profiles }, null, 2)}\n`);
+    if (raw.profiles && Object.keys(raw.profiles).length > 0) {
+      io.log(`note: 覆盖原有 ${Object.keys(raw.profiles).length} 个分组，原 profiles 不保留`);
+    }
+    io.log(`profiles written: ${target}`);
+  }
+  if (options.json) io.log(JSON.stringify(result, null, 2));
+  else io.log(formatClusters(result));
+  return 0;
+}
+
 function parseArgs(argv) {
   const options = {
     command: argv[0],
@@ -459,6 +645,8 @@ function parseArgs(argv) {
     models: null,
     modelsFile: DEFAULT_MODELS,
     cases: null,
+    profiles: null,
+    repeat: 1,
     showCases: false,
     json: false,
     out: null,
@@ -474,6 +662,8 @@ function parseArgs(argv) {
     reviewed: false,
     dryRun: false,
     tool: null,
+    threshold: 0.5,
+    maxSize: 10,
     baseline: null,
     failOnRegression: false,
     execute: false,
@@ -505,6 +695,13 @@ function parseArgs(argv) {
       case "--models": options.models = splitList(value()); break;
       case "--models-file": options.modelsFile = resolve(value()); break;
       case "--case": options.cases = (options.cases ?? []).concat(splitList(value())); break;
+      case "--profile": options.profiles = splitList(value()); break;
+      case "--repeat": {
+        const raw = Number(value());
+        if (!Number.isInteger(raw) || raw < 1 || raw > 10) throw new Error("--repeat expects an integer between 1 and 10");
+        options.repeat = raw;
+        break;
+      }
       case "--show-cases": options.showCases = true; break;
       case "--json": options.json = true; break;
       case "--out": options.out = value(); break;
@@ -524,6 +721,18 @@ function parseArgs(argv) {
       case "--reviewed": options.reviewed = true; break;
       case "--dry-run": options.dryRun = true; break;
       case "--tool": options.tool = value(); break;
+      case "--threshold": {
+        const raw = Number(value());
+        if (!Number.isFinite(raw) || raw <= 0 || raw > 1) throw new Error("--threshold expects a ratio between 0 and 1");
+        options.threshold = raw;
+        break;
+      }
+      case "--max-size": {
+        const raw = Number(value());
+        if (!Number.isInteger(raw) || raw <= 0) throw new Error("--max-size expects a positive integer");
+        options.maxSize = raw;
+        break;
+      }
       case "--baseline": options.baseline = resolve(value()); break;
       case "--fail-on-regression": options.failOnRegression = true; break;
       case "--entry": options.entry = value(); break;
@@ -605,7 +814,7 @@ if (isMain) {
     }
   } else if (argv.length === 0 || argv[0] === "help" || argv[0] === "-h" || argv[0] === "--help") {
     process.stdout.write(USAGE);
-  } else if (!["eval", "scan", "case", "coverage", "lint", "ablate", "exec"].includes(argv[0])) {
+  } else if (!["eval", "scan", "case", "coverage", "lint", "ablate", "cluster", "exec"].includes(argv[0])) {
     process.stderr.write(`unknown command "${argv[0]}"\n\n${USAGE}`);
     process.exit(1);
   } else if (argv[0] === "case" && argv[1] !== "add") {
@@ -615,6 +824,10 @@ if (isMain) {
     try {
       const args = argv[0] === "case" ? [argv[0], ...argv.slice(2)] : argv;
       const options = parseArgs(args);
+      if (options.help) {
+        process.stdout.write(USAGE);
+        process.exit(0);
+      }
       const code =
         argv[0] === "scan"
           ? await runScan(options)
@@ -626,7 +839,9 @@ if (isMain) {
                 ? await runLint(options)
                 : argv[0] === "ablate"
                   ? await runAblate(options)
-                  : argv[0] === "exec"
+                  : argv[0] === "cluster"
+                    ? await runCluster(options)
+                    : argv[0] === "exec"
                     ? await runExec(options)
                     : await runEval(options);
       process.exit(code);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.1.3",
+  "version": "0.3.0",
   "type": "module",
   "license": "MIT",
   "description": "mf — function-calling accuracy scorecard for AI agents",