modelfunction-cli 0.5.0 → 0.5.2

package/mf.js

@@ -4,14 +4,15 @@ import { mkdir, readFile } from "node:fs/promises";
 import { basename, dirname, extname, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { loadSuite, toModelTools } from "./lib/suite.js";
-import { scoreModel, pickRepresentative, pct } from "./lib/score.js";
+import { scoreModel, pickRepresentative, pct, costCnyOf } from "./lib/score.js";
 import { formatTable, formatCaseLines, formatMarkdown } from "./lib/report.js";
 import { scanProject, buildSuiteDraft, formatScanSummary, addTools, redactReport } from "./lib/scan.js";
-import { probeStdioServer } from "./lib/mcp-probe.js";
+import { probeStdioServer, probeHttpServer } from "./lib/mcp-probe.js";
 import { loadAdapterTools } from "./lib/adapter-import.js";
 import { addCases, computeCoverage, formatCoverage } from "./lib/cases.js";
 import { lintSuite, formatLint } from "./lib/lint.js";
-import { check, formatCheck } from "./lib/check.js";
+import { check, checkMany, formatCheck, formatCheckMany } from "./lib/check.js";
+import { auditSuite, formatAudit } from "./lib/audit.js";
 import { suiteFingerprint, hashText, hashOf } from "./lib/fingerprint.js";
 import { runExecutionChecks } from "./lib/eval-exec.js";
 import { checkSandbox, formatSandboxReport } from "./lib/sandbox-check.js";
@@ -30,6 +31,7 @@ import {
   formatProfileMarkdown
 } from "./lib/profile.js";
 import { runExec } from "./lib/exec.js";
+import { createBudget, CostLimitError } from "./lib/budget.js";
 import { writeFileAtomic } from "./lib/fsutil.js";
 import {
   applyModelCredentials,
@@ -71,10 +73,11 @@ Usage:
   mf scan [options]
   mf case add --suite <file> (--input <file> | stdin) [--source agent] [--reviewed] [--dry-run]
   mf coverage [--suite <file>] [--json]
+  mf audit [--suite <file>] [--out <file>] [--json]  # 静态审计：lint + 分组边界 + 用例覆盖 + 挂载
   mf lint [--suite <file>] [--json]  # 静态检查 tools 与 skills，不调用模型
-  mf check --manifest <file> [--installed <file>] [--json]  # 安装门禁：新能力 × 已装能力
+  mf check (--manifest <file> | --manifests <file>) [--installed <file>] [--json]  # 安装门禁：新能力 × 已装能力（--manifests 支持一包）
   mf compare <before.json> <after.json> [--json]
-  mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--json]
+  mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--max-cost <cny>] [--progress] [--json]
   mf cluster [--suite <file>] [--threshold <0.5>] [--max-size <10>] [--out <file>] [--json]  # 建议挂载分组 + 同族候选
   mf exec --entry <file> (--input <json> | --input-json <json>) [--runtime node|python] [--python <cmd>] [--export run] [--allow-read <paths>] [--allow-write <paths>] [--allow-net <hosts>] [--timeout <ms>] [--warmup <n>] [--json]
   mf sandbox check [--json]   # probe available isolation engines on this machine
@@ -93,6 +96,8 @@ Eval options:
   --case <ids>          run only these case ids (comma-separated, repeatable)
   --profile <names>     score each mounted group separately ("all" or names from suite.profiles)
   --repeat <n>          run each case n times and score the majority; reports unstable cases (default 1)
+  --max-cost <cny>      stop after the estimated spend crosses this limit (exit 2; no partial scorecard)
+  --progress            print per-case progress to stderr (stdout stays clean for --json)
   --show-cases          print per-case results
   --json                print JSON instead of the table
   --out <file>          write scorecard (.md => markdown, otherwise JSON)
@@ -106,7 +111,7 @@ Eval options:
 Scan options:
   --root <dir>          repository to scan (default: .)
   --from <file>         import tools from a module (export default [...] or export const tools)
-  --probe               start stdio MCP servers and read tools/list
+  --probe               connect stdio or streamable-HTTP MCP servers and read tools/list
   --timeout <ms>        probe timeout per server (default: 5000)
   --out <file>          suite draft to write (default: mf.suite.json)
   --report <file>       scan report to write (default: mf.scan.json)
@@ -134,17 +139,31 @@ export async function runEval(options, io = console) {
   const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
   const tools = toModelTools(suite.tools);
 
+  const budget = createBudget(options.maxCost ?? null);
   const runs = [];
-  for (const adapter of adapters) {
-    const evaluated = await evaluate(adapter, tools, cases, options.repeat ?? 1);
-    const summary = scoreModel({
-      model: adapter.id,
-      cases,
-      results: evaluated.results,
-      pricing: registry[adapter.id]?.pricing ?? null
-    });
-    if (evaluated.stability) summary.stability = evaluated.stability;
-    runs.push({ adapter, results: evaluated.results, summary });
+  try {
+    for (const adapter of adapters) {
+      const pricing = registry[adapter.id]?.pricing ?? null;
+      const evaluated = await evaluate(adapter, tools, cases, options.repeat ?? 1, {
+        pricing,
+        budget,
+        onProgress: makeProgress(options, io, adapter.id, cases.length)
+      });
+      const summary = scoreModel({
+        model: adapter.id,
+        cases,
+        results: evaluated.results,
+        pricing
+      });
+      if (evaluated.stability) summary.stability = evaluated.stability;
+      runs.push({ adapter, results: evaluated.results, summary });
+    }
+  } catch (error) {
+    if (error instanceof CostLimitError) {
+      io.error(costLimitMessage(error));
+      return 2;
+    }
+    throw error;
   }
   const summaries = runs.map((run) => run.summary);
 
@@ -173,16 +192,28 @@ export async function runEval(options, io = console) {
     };
   }
 
-  const profileReport = options.profiles
-    ? await evaluateProfiles({
+  let profileReport = null;
+  if (options.profiles) {
+    try {
+      profileReport = await evaluateProfiles({
         suite,
         cases,
         runs,
         registry,
         selection: options.profiles,
-        repeat: options.repeat ?? 1
-      })
-    : null;
+        repeat: options.repeat ?? 1,
+        budget,
+        progress: options.progress,
+        io
+      });
+    } catch (error) {
+      if (error instanceof CostLimitError) {
+        io.error(costLimitMessage(error));
+        return 2;
+      }
+      throw error;
+    }
+  }
 
   if (options.out) {
     const target = resolve(options.out);
@@ -357,14 +388,35 @@ export async function runAblate(options, io = console) {
   const fullTools = toModelTools(baseTools);
   const reducedTools = toModelTools(baseTools.filter((tool) => tool.name !== ablated.name));
   const models = [];
+  const budget = createBudget(options.maxCost ?? null);
 
-  for (const adapter of adapters) {
-    const pricing = registry[adapter.id]?.pricing ?? null;
-    const fullResults = (await evaluate(adapter, fullTools, baseCases)).results;
-    const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
-    const reducedResults = (await evaluate(adapter, reducedTools, reducedCases)).results;
-    const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
-    models.push(analyzeAblation(adapter.id, full, reduced));
+  try {
+    for (const adapter of adapters) {
+      const pricing = registry[adapter.id]?.pricing ?? null;
+      const fullResults = (
+        await evaluate(adapter, fullTools, baseCases, 1, {
+          pricing,
+          budget,
+          onProgress: makeProgress(options, io, `${adapter.id}/full`, baseCases.length)
+        })
+      ).results;
+      const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
+      const reducedResults = (
+        await evaluate(adapter, reducedTools, reducedCases, 1, {
+          pricing,
+          budget,
+          onProgress: makeProgress(options, io, `${adapter.id}/ablated`, reducedCases.length)
+        })
+      ).results;
+      const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
+      models.push(analyzeAblation(adapter.id, full, reduced));
+    }
+  } catch (error) {
+    if (error instanceof CostLimitError) {
+      io.error(costLimitMessage(error));
+      return 2;
+    }
+    throw error;
   }
 
   const payload = {
@@ -399,11 +451,10 @@ export async function runScan(options, io = console) {
 
   if (options.probe) {
     for (const server of [...report.mcpServers]) {
-      if (server.transport !== "stdio") {
-        report.probes.push({ server: server.name, ok: false, error: "http transport 暂不支持" });
-        continue;
-      }
-      const result = await probeStdioServer(server, { timeoutMs: options.timeoutMs });
+      const result =
+        server.transport === "stdio"
+          ? await probeStdioServer(server, { timeoutMs: options.timeoutMs })
+          : await probeHttpServer(server, { timeoutMs: options.timeoutMs });
       if (!result.ok) {
         report.probes.push({ server: server.name, ok: false, error: result.error, stderr: result.stderr });
         continue;
@@ -495,6 +546,35 @@ export async function runCoverage(options, io = console) {
   return 0;
 }
 
+export async function runAudit(options, io = console) {
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE), {
+    allowEmptyCases: true,
+    allowEmptyTools: true
+  });
+  const report = auditSuite(suite);
+  const rules = {};
+  for (const finding of report.lint.findings) rules[finding.code] = (rules[finding.code] ?? 0) + 1;
+  noteUsage({
+    toolCount: report.suite.tools,
+    caseCount: report.suite.cases,
+    lintErrors: report.lint.counts.error,
+    lintWarns: report.lint.counts.warn,
+    lintInfos: report.lint.counts.info,
+    lintRules: rules,
+    clusters: report.clusters.clusters.length,
+    profileGroups: report.profiles?.groups.length ?? 0
+  });
+
+  if (options.out) {
+    const target = resolve(options.out);
+    const markdown = extname(target).toLowerCase() === ".md";
+    await writeFileAtomic(target, markdown ? `${formatAudit(report)}\n` : `${JSON.stringify(report, null, 2)}\n`);
+    io.log(`wrote ${target}`);
+  }
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatAudit(report));
+  return 0;
+}
+
 export async function runSandboxCheck(options, io = console) {
   const report = checkSandbox();
   io.log(options.json ? JSON.stringify(report, null, 2) : formatSandboxReport(report));
@@ -504,11 +584,24 @@ export async function runSandboxCheck(options, io = console) {
 // Advisory install gate: static only, never blocks. Exit code stays 0 for a
 // warn verdict so a platform can log it without failing the install.
 export async function runCheck(options, io = console) {
-  if (!options.manifest) throw new Error("--manifest <file> 必填（新能力的 [name, description, parameters?]）");
-  const manifest = JSON.parse(await readFile(resolve(options.manifest), "utf8"));
   const installed = options.installed
     ? JSON.parse(await readFile(resolve(options.installed), "utf8"))
     : [];
+
+  if (options.manifests) {
+    const manifests = JSON.parse(await readFile(resolve(options.manifests), "utf8"));
+    const result = checkMany({ manifests, installed });
+    noteUsage({
+      checkVerdict: result.verdict,
+      checkFindings: Object.values(result.summary).reduce((sum, count) => sum + count, 0)
+    });
+    io.log(options.json ? JSON.stringify(result, null, 2) : formatCheckMany(result));
+    return 0;
+  }
+  if (!options.manifest) {
+    throw new Error("--manifest 或 --manifests <file> 必填（[name, description, parameters?]）");
+  }
+  const manifest = JSON.parse(await readFile(resolve(options.manifest), "utf8"));
   const result = check({ manifest, installed });
   noteUsage({ checkVerdict: result.verdict, checkFindings: result.findings.length });
   io.log(options.json ? JSON.stringify(result, null, 2) : formatCheck(result));
@@ -719,25 +812,47 @@ function autoSelect(registry) {
     .map(([id]) => id);
 }
 
-async function evaluate(adapter, tools, cases, repeat = 1) {
+async function evaluate(
+  adapter,
+  tools,
+  cases,
+  repeat = 1,
+  { pricing = null, budget = null, onProgress = null } = {}
+) {
   const results = [];
   const unstable = [];
-  for (const testCase of cases) {
+  for (const [index, testCase] of cases.entries()) {
     const runs = [];
     for (let attempt = 0; attempt < repeat; attempt += 1) {
-      runs.push(await callOnce(adapter, tools, testCase));
+      const result = await callOnce(adapter, tools, testCase);
+      budget?.spend(costCnyOf(result.usage, pricing));
+      runs.push(result);
     }
     if (repeat === 1) {
       results.push(runs[0]);
-      continue;
+    } else {
+      const picked = pickRepresentative(runs);
+      results.push(picked.result);
+      if (picked.unstable) unstable.push({ id: testCase.id, outcomes: picked.outcomes });
     }
-    const picked = pickRepresentative(runs);
-    results.push(picked.result);
-    if (picked.unstable) unstable.push({ id: testCase.id, outcomes: picked.outcomes });
+    onProgress?.(index + 1, cases.length);
   }
   return { results, stability: repeat > 1 ? { runs: repeat, unstable } : null };
 }
 
+function makeProgress(options, io, label, total) {
+  if (!options.progress) return null;
+  const step = Math.max(1, Math.ceil(total / 10));
+  return (done, count) => {
+    if (done !== count && done % step !== 0) return;
+    io.error(`[${label}] ${done}/${count}`);
+  };
+}
+
+function costLimitMessage(error) {
+  return `${error.message}；已发出的请求已计费，本次不计分。减少 --case/--models/--repeat 或提高 --max-cost 后重跑。`;
+}
+
 async function callOnce(adapter, tools, testCase) {
   const messages = [
     { role: "system", content: SYSTEM_PROMPT },
@@ -782,7 +897,17 @@ function ablationScope(suite, names) {
   return { name: profile.name, tools: suite.tools.filter((tool) => mounted.has(tool.name)) };
 }
 
-async function evaluateProfiles({ suite, cases, runs, registry, selection, repeat = 1 }) {
+async function evaluateProfiles({
+  suite,
+  cases,
+  runs,
+  registry,
+  selection,
+  repeat = 1,
+  budget = null,
+  progress = false,
+  io = console
+}) {
   const profiles = normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name));
   if (!profiles) {
     throw new Error(`suite ${suite.name} 没有 profiles；先运行 mf cluster --out <suite> 生成分组`);
@@ -812,7 +937,11 @@ async function evaluateProfiles({ suite, cases, runs, registry, selection, repea
       const profileCases = caseSets[index];
       if (profileCases.length === 0) continue;
       const mountedTools = suite.tools.filter((tool) => profile.tools.includes(tool.name));
-      const mountedEval = await evaluate(run.adapter, toModelTools(mountedTools), profileCases, repeat);
+      const mountedEval = await evaluate(run.adapter, toModelTools(mountedTools), profileCases, repeat, {
+        pricing,
+        budget,
+        onProgress: makeProgress({ progress }, io, `${run.adapter.id}/${profile.name}`, profileCases.length)
+      });
       const mounted = scoreModel({
         model: run.adapter.id,
         cases: profileCases,
@@ -875,6 +1004,8 @@ function parseArgs(argv) {
     cases: null,
     profiles: null,
     repeat: 1,
+    maxCost: null,
+    progress: false,
     showCases: false,
     json: false,
     out: null,
@@ -891,6 +1022,7 @@ function parseArgs(argv) {
     dryRun: false,
     tool: null,
     manifest: null,
+    manifests: null,
     installed: null,
     threshold: 0.5,
     maxSize: 10,
@@ -932,6 +1064,13 @@ function parseArgs(argv) {
         options.repeat = raw;
         break;
       }
+      case "--max-cost": {
+        const raw = Number(value());
+        if (!Number.isFinite(raw) || raw <= 0) throw new Error("--max-cost expects a positive CNY amount");
+        options.maxCost = raw;
+        break;
+      }
+      case "--progress": options.progress = true; break;
       case "--show-cases": options.showCases = true; break;
       case "--json": options.json = true; break;
       case "--out": options.out = value(); break;
@@ -952,6 +1091,7 @@ function parseArgs(argv) {
       case "--dry-run": options.dryRun = true; break;
       case "--tool": options.tool = value(); break;
       case "--manifest": options.manifest = resolve(value()); break;
+      case "--manifests": options.manifests = resolve(value()); break;
       case "--installed": options.installed = resolve(value()); break;
       case "--threshold": {
         const raw = Number(value());
@@ -1104,7 +1244,7 @@ if (isMain) {
     }
   } else if (argv.length === 0 || argv[0] === "help" || argv[0] === "-h" || argv[0] === "--help") {
     process.stdout.write(USAGE);
-  } else if (!["eval", "scan", "case", "coverage", "lint", "check", "ablate", "cluster", "exec"].includes(argv[0])) {
+  } else if (!["eval", "scan", "case", "coverage", "audit", "lint", "check", "ablate", "cluster", "exec"].includes(argv[0])) {
     process.stderr.write(`unknown command "${argv[0]}"\n\n${USAGE}`);
     process.exit(1);
   } else if (argv[0] === "case" && argv[1] !== "add") {
@@ -1126,6 +1266,8 @@ if (isMain) {
             ? await runCaseAdd(options)
             : argv[0] === "coverage"
               ? await runCoverage(options)
+              : argv[0] === "audit"
+                ? await runAudit(options)
               : argv[0] === "lint"
                 ? await runLint(options)
                 : argv[0] === "check"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "type": "module",
   "license": "MIT",
   "description": "mf — function-calling accuracy scorecard for AI agents",