modelfunction-cli 0.4.4 → 0.5.1

package/mf.js

@@ -1,17 +1,18 @@
 #!/usr/bin/env node
 import { realpathSync } from "node:fs";
-import { readFile } from "node:fs/promises";
+import { mkdir, readFile } from "node:fs/promises";
 import { basename, dirname, extname, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { loadSuite, toModelTools } from "./lib/suite.js";
-import { scoreModel, pickRepresentative, pct } from "./lib/score.js";
+import { scoreModel, pickRepresentative, pct, costCnyOf } from "./lib/score.js";
 import { formatTable, formatCaseLines, formatMarkdown } from "./lib/report.js";
 import { scanProject, buildSuiteDraft, formatScanSummary, addTools, redactReport } from "./lib/scan.js";
-import { probeStdioServer } from "./lib/mcp-probe.js";
+import { probeStdioServer, probeHttpServer } from "./lib/mcp-probe.js";
 import { loadAdapterTools } from "./lib/adapter-import.js";
 import { addCases, computeCoverage, formatCoverage } from "./lib/cases.js";
 import { lintSuite, formatLint } from "./lib/lint.js";
 import { check, formatCheck } from "./lib/check.js";
+import { auditSuite, formatAudit } from "./lib/audit.js";
 import { suiteFingerprint, hashText, hashOf } from "./lib/fingerprint.js";
 import { runExecutionChecks } from "./lib/eval-exec.js";
 import { checkSandbox, formatSandboxReport } from "./lib/sandbox-check.js";
@@ -30,7 +31,27 @@ import {
   formatProfileMarkdown
 } from "./lib/profile.js";
 import { runExec } from "./lib/exec.js";
+import { createBudget, CostLimitError } from "./lib/budget.js";
 import { writeFileAtomic } from "./lib/fsutil.js";
+import {
+  applyModelCredentials,
+  credentialsPath,
+  listCredentials,
+  redactText,
+  registerSecrets,
+  removeCredential,
+  setCredential
+} from "./lib/credentials.js";
+import {
+  buildFeedbackBundle,
+  defaultFeedbackPath,
+  formatStats,
+  noteUsage,
+  readUsageEvents,
+  recordRunSync,
+  summarizeUsage,
+  usageDir
+} from "./lib/usage.js";
 import { createMockAdapter } from "./lib/adapters/mock.js";
 import { createOpenAiAdapter } from "./lib/adapters/openai.js";
 
@@ -52,13 +73,20 @@ Usage:
   mf scan [options]
   mf case add --suite <file> (--input <file> | stdin) [--source agent] [--reviewed] [--dry-run]
   mf coverage [--suite <file>] [--json]
+  mf audit [--suite <file>] [--out <file>] [--json]  # 静态审计：lint + 分组边界 + 用例覆盖 + 挂载
   mf lint [--suite <file>] [--json]  # 静态检查 tools 与 skills，不调用模型
   mf check --manifest <file> [--installed <file>] [--json]  # 安装门禁：新能力 × 已装能力
   mf compare <before.json> <after.json> [--json]
-  mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--json]
+  mf ablate --tool <name> [--suite <file>] [--profile <group>] [--models <ids>] [--max-cost <cny>] [--progress] [--json]
   mf cluster [--suite <file>] [--threshold <0.5>] [--max-size <10>] [--out <file>] [--json]  # 建议挂载分组 + 同族候选
   mf exec --entry <file> (--input <json> | --input-json <json>) [--runtime node|python] [--python <cmd>] [--export run] [--allow-read <paths>] [--allow-write <paths>] [--allow-net <hosts>] [--timeout <ms>] [--warmup <n>] [--json]
   mf sandbox check [--json]   # probe available isolation engines on this machine
+  mf config set-key <ENV_NAME>  # store a model key in ~/.modelfunction/credentials.json (hidden input)
+  mf config remove-key <ENV_NAME>  # delete a stored key
+  mf config list [--json]       # names + masked values only
+  mf config path                # print the credentials file path
+  mf stats [--json]             # local usage summary (never uploaded)
+  mf feedback [--out <file>] [--json]  # export a redacted feedback bundle; you decide whether to send it
   mf mcp                # start stdio MCP server for coding agents
 
 Eval options:
@@ -68,6 +96,8 @@ Eval options:
   --case <ids>          run only these case ids (comma-separated, repeatable)
   --profile <names>     score each mounted group separately ("all" or names from suite.profiles)
   --repeat <n>          run each case n times and score the majority; reports unstable cases (default 1)
+  --max-cost <cny>      stop after the estimated spend crosses this limit (exit 2; no partial scorecard)
+  --progress            print per-case progress to stderr (stdout stays clean for --json)
   --show-cases          print per-case results
   --json                print JSON instead of the table
   --out <file>          write scorecard (.md => markdown, otherwise JSON)
@@ -81,13 +111,15 @@ Eval options:
 Scan options:
   --root <dir>          repository to scan (default: .)
   --from <file>         import tools from a module (export default [...] or export const tools)
-  --probe               start stdio MCP servers and read tools/list
+  --probe               connect stdio or streamable-HTTP MCP servers and read tools/list
   --timeout <ms>        probe timeout per server (default: 5000)
   --out <file>          suite draft to write (default: mf.suite.json)
   --report <file>       scan report to write (default: mf.scan.json)
   --no-write            print only, do not write files
 
 Env keys: DEEPSEEK_API_KEY, DASHSCOPE_API_KEY, ZHIPU_API_KEY, OPENAI_API_KEY, SILICONFLOW_API_KEY
+Credentials: env vars win; missing model keys are read from ~/.modelfunction/credentials.json (MF_CONFIG_DIR overrides).
+Usage: recorded locally under ~/.modelfunction/usage (counts only, never uploaded); MF_NO_USAGE=1 opts out.
 
 Examples:
   mf eval --models mock --show-cases
@@ -107,17 +139,31 @@ export async function runEval(options, io = console) {
   const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
   const tools = toModelTools(suite.tools);
 
+  const budget = createBudget(options.maxCost ?? null);
   const runs = [];
-  for (const adapter of adapters) {
-    const evaluated = await evaluate(adapter, tools, cases, options.repeat ?? 1);
-    const summary = scoreModel({
-      model: adapter.id,
-      cases,
-      results: evaluated.results,
-      pricing: registry[adapter.id]?.pricing ?? null
-    });
-    if (evaluated.stability) summary.stability = evaluated.stability;
-    runs.push({ adapter, results: evaluated.results, summary });
+  try {
+    for (const adapter of adapters) {
+      const pricing = registry[adapter.id]?.pricing ?? null;
+      const evaluated = await evaluate(adapter, tools, cases, options.repeat ?? 1, {
+        pricing,
+        budget,
+        onProgress: makeProgress(options, io, adapter.id, cases.length)
+      });
+      const summary = scoreModel({
+        model: adapter.id,
+        cases,
+        results: evaluated.results,
+        pricing
+      });
+      if (evaluated.stability) summary.stability = evaluated.stability;
+      runs.push({ adapter, results: evaluated.results, summary });
+    }
+  } catch (error) {
+    if (error instanceof CostLimitError) {
+      io.error(costLimitMessage(error));
+      return 2;
+    }
+    throw error;
   }
   const summaries = runs.map((run) => run.summary);
 
@@ -146,16 +192,28 @@ export async function runEval(options, io = console) {
     };
   }
 
-  const profileReport = options.profiles
-    ? await evaluateProfiles({
+  let profileReport = null;
+  if (options.profiles) {
+    try {
+      profileReport = await evaluateProfiles({
         suite,
         cases,
         runs,
         registry,
         selection: options.profiles,
-        repeat: options.repeat ?? 1
-      })
-    : null;
+        repeat: options.repeat ?? 1,
+        budget,
+        progress: options.progress,
+        io
+      });
+    } catch (error) {
+      if (error instanceof CostLimitError) {
+        io.error(costLimitMessage(error));
+        return 2;
+      }
+      throw error;
+    }
+  }
 
   if (options.out) {
     const target = resolve(options.out);
@@ -252,6 +310,35 @@ export async function runEval(options, io = console) {
   const failed = options.failUnder === null
     ? []
     : summaries.filter((summary) => summary.selectionAccuracy < options.failUnder);
+  const profileBelow =
+    profileReport && options.failUnder !== null
+      ? profileReport.models.flatMap((model) =>
+          model.profiles
+            .filter((entry) => entry.reviewed > 0 && entry.mounted.selectionAccuracy < options.failUnder)
+            .map((entry) => `${model.model}/${entry.profile} (${pct(entry.mounted.selectionAccuracy)})`)
+        )
+      : [];
+
+  noteUsage({
+    models: summaries.map((summary) => summary.model),
+    caseCount: cases.length,
+    toolCount: suite.tools.length,
+    reviewedCount: cases.filter((testCase) => testCase.reviewed !== false).length,
+    repeat: options.repeat ?? 1,
+    profileGroups: profileReport ? profileReport.groups.length : 0,
+    selectionAccuracy: average(summaries.map((summary) => summary.selectionAccuracy)),
+    argAccuracy: average(summaries.map((summary) => summary.argAccuracy)),
+    passRate: average(summaries.map((summary) => summary.passRate)),
+    unstableCases: summaries.reduce((total, summary) => total + (summary.stability?.unstable.length ?? 0), 0),
+    execEngine: executionInfo?.engine,
+    execExecuted: sumExec(summaries, "executed"),
+    execPassed: sumExec(summaries, "passed"),
+    execBlocked: sumExec(summaries, "blocked"),
+    execSkipped: sumExec(summaries, "skipped"),
+    failedThreshold: failed.length > 0 || profileBelow.length > 0,
+    regressions: regressions.length > 0
+  });
+
   if (failed.length > 0) {
     io.error(
       `below threshold ${pct(options.failUnder)}: ${failed
@@ -260,21 +347,24 @@ export async function runEval(options, io = console) {
     );
     return 2;
   }
-  if (profileReport && options.failUnder !== null) {
-    const below = profileReport.models.flatMap((model) =>
-      model.profiles
-        .filter((entry) => entry.reviewed > 0 && entry.mounted.selectionAccuracy < options.failUnder)
-        .map((entry) => `${model.model}/${entry.profile} (${pct(entry.mounted.selectionAccuracy)})`)
-    );
-    if (below.length > 0) {
-      io.error(`profile below threshold ${pct(options.failUnder)}: ${below.join(", ")}`);
-      return 2;
-    }
+  if (profileBelow.length > 0) {
+    io.error(`profile below threshold ${pct(options.failUnder)}: ${profileBelow.join(", ")}`);
+    return 2;
   }
   if (regressions.length > 0) return 2;
   return 0;
 }
 
+function average(values) {
+  const usable = values.filter((value) => typeof value === "number" && Number.isFinite(value));
+  if (usable.length === 0) return undefined;
+  return usable.reduce((sum, value) => sum + value, 0) / usable.length;
+}
+
+function sumExec(summaries, field) {
+  return summaries.reduce((total, summary) => total + (summary.exec?.[field] ?? 0), 0);
+}
+
 export async function runAblate(options, io = console) {
   const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
   if (!options.tool) throw new Error("--tool is required");
@@ -298,14 +388,35 @@ export async function runAblate(options, io = console) {
   const fullTools = toModelTools(baseTools);
   const reducedTools = toModelTools(baseTools.filter((tool) => tool.name !== ablated.name));
   const models = [];
+  const budget = createBudget(options.maxCost ?? null);
 
-  for (const adapter of adapters) {
-    const pricing = registry[adapter.id]?.pricing ?? null;
-    const fullResults = (await evaluate(adapter, fullTools, baseCases)).results;
-    const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
-    const reducedResults = (await evaluate(adapter, reducedTools, reducedCases)).results;
-    const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
-    models.push(analyzeAblation(adapter.id, full, reduced));
+  try {
+    for (const adapter of adapters) {
+      const pricing = registry[adapter.id]?.pricing ?? null;
+      const fullResults = (
+        await evaluate(adapter, fullTools, baseCases, 1, {
+          pricing,
+          budget,
+          onProgress: makeProgress(options, io, `${adapter.id}/full`, baseCases.length)
+        })
+      ).results;
+      const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
+      const reducedResults = (
+        await evaluate(adapter, reducedTools, reducedCases, 1, {
+          pricing,
+          budget,
+          onProgress: makeProgress(options, io, `${adapter.id}/ablated`, reducedCases.length)
+        })
+      ).results;
+      const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
+      models.push(analyzeAblation(adapter.id, full, reduced));
+    }
+  } catch (error) {
+    if (error instanceof CostLimitError) {
+      io.error(costLimitMessage(error));
+      return 2;
+    }
+    throw error;
   }
 
   const payload = {
@@ -315,6 +426,12 @@ export async function runAblate(options, io = console) {
     models,
     ...(scope ? { profile: { name: scope.name, tools: baseTools.length } } : {})
   };
+  noteUsage({
+    models: models.map((model) => model.model),
+    caseCount: baseCases.length,
+    toolCount: baseTools.length,
+    profileGroups: scope ? 1 : 0
+  });
   io.log(options.json ? JSON.stringify(payload, null, 2) : formatAblation(payload));
   return 0;
 }
@@ -334,11 +451,10 @@ export async function runScan(options, io = console) {
 
   if (options.probe) {
     for (const server of [...report.mcpServers]) {
-      if (server.transport !== "stdio") {
-        report.probes.push({ server: server.name, ok: false, error: "http transport 暂不支持" });
-        continue;
-      }
-      const result = await probeStdioServer(server, { timeoutMs: options.timeoutMs });
+      const result =
+        server.transport === "stdio"
+          ? await probeStdioServer(server, { timeoutMs: options.timeoutMs })
+          : await probeHttpServer(server, { timeoutMs: options.timeoutMs });
       if (!result.ok) {
         report.probes.push({ server: server.name, ok: false, error: result.error, stderr: result.stderr });
         continue;
@@ -355,6 +471,13 @@ export async function runScan(options, io = console) {
     }
   }
 
+  noteUsage({
+    scanTools: report.tools.length,
+    scanSkills: report.skills.length,
+    scanMcpServers: report.mcpServers.length,
+    scanProbes: report.probes.length
+  });
+
   const draft = buildSuiteDraft(report);
 
   if (options.json) {
@@ -418,10 +541,40 @@ export async function runCoverage(options, io = console) {
     allowEmptyTools: true
   });
   const report = computeCoverage(suite);
+  noteUsage({ toolCount: suite.tools.length, caseCount: suite.cases.length });
   io.log(options.json ? JSON.stringify(report, null, 2) : formatCoverage(report));
   return 0;
 }
 
+export async function runAudit(options, io = console) {
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE), {
+    allowEmptyCases: true,
+    allowEmptyTools: true
+  });
+  const report = auditSuite(suite);
+  const rules = {};
+  for (const finding of report.lint.findings) rules[finding.code] = (rules[finding.code] ?? 0) + 1;
+  noteUsage({
+    toolCount: report.suite.tools,
+    caseCount: report.suite.cases,
+    lintErrors: report.lint.counts.error,
+    lintWarns: report.lint.counts.warn,
+    lintInfos: report.lint.counts.info,
+    lintRules: rules,
+    clusters: report.clusters.clusters.length,
+    profileGroups: report.profiles?.groups.length ?? 0
+  });
+
+  if (options.out) {
+    const target = resolve(options.out);
+    const markdown = extname(target).toLowerCase() === ".md";
+    await writeFileAtomic(target, markdown ? `${formatAudit(report)}\n` : `${JSON.stringify(report, null, 2)}\n`);
+    io.log(`wrote ${target}`);
+  }
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatAudit(report));
+  return 0;
+}
+
 export async function runSandboxCheck(options, io = console) {
   const report = checkSandbox();
   io.log(options.json ? JSON.stringify(report, null, 2) : formatSandboxReport(report));
@@ -437,6 +590,7 @@ export async function runCheck(options, io = console) {
     ? JSON.parse(await readFile(resolve(options.installed), "utf8"))
     : [];
   const result = check({ manifest, installed });
+  noteUsage({ checkVerdict: result.verdict, checkFindings: result.findings.length });
   io.log(options.json ? JSON.stringify(result, null, 2) : formatCheck(result));
   return 0;
 }
@@ -447,6 +601,15 @@ export async function runLint(options, io = console) {
     allowEmptyTools: true
   });
   const report = lintSuite(suite);
+  const rules = {};
+  for (const finding of report.findings) rules[finding.code] = (rules[finding.code] ?? 0) + 1;
+  noteUsage({
+    toolCount: report.tools,
+    lintErrors: report.counts.error,
+    lintWarns: report.counts.warn,
+    lintInfos: report.counts.info,
+    lintRules: rules
+  });
   io.log(options.json ? JSON.stringify(report, null, 2) : formatLint(report));
   return report.counts.error > 0 ? 1 : 0;
 }
@@ -461,6 +624,123 @@ export async function runCompare(options, io = console) {
   return changed ? 1 : 0;
 }
 
+export async function runConfig(options, io = console) {
+  const [sub, name] = options.args ?? [];
+  if (sub === "set-key") {
+    if (!name) throw new Error("usage: mf config set-key <ENV_NAME>");
+    const value = await (options.readSecret ?? readSecret)(`${name}: `);
+    const result = await setCredential(name, value);
+    io.log(`saved ${result.name} → ${result.path}`);
+    io.log("permissions: 0600 where supported. env vars still win; CI should use env or a secret manager.");
+    return 0;
+  }
+  if (sub === "remove-key") {
+    if (!name) throw new Error("usage: mf config remove-key <ENV_NAME>");
+    const result = await removeCredential(name);
+    io.log(
+      result.removed
+        ? `removed ${result.name} from ${result.path}`
+        : `${result.name} is not stored (${result.path})`
+    );
+    return 0;
+  }
+  if (sub === "list") {
+    const entries = await listCredentials();
+    if (options.json) {
+      io.log(JSON.stringify({ path: credentialsPath(), keys: entries }, null, 2));
+      return 0;
+    }
+    if (entries.length === 0) {
+      io.log(`no stored keys (${credentialsPath()})`);
+      return 0;
+    }
+    for (const entry of entries) io.log(`${entry.name}  ${entry.masked}`);
+    io.log(`file: ${credentialsPath()}`);
+    return 0;
+  }
+  if (sub === "path") {
+    io.log(credentialsPath());
+    return 0;
+  }
+  throw new Error("usage: mf config <set-key <ENV_NAME> | remove-key <ENV_NAME> | list | path>");
+}
+
+export async function runStats(options, io = console) {
+  const { events, files, skipped } = await readUsageEvents();
+  const summary = summarizeUsage(events);
+  if (options.json) {
+    io.log(JSON.stringify({ path: usageDir(), files: files.length, skipped, ...summary }, null, 2));
+  } else {
+    io.log(formatStats(summary, { events: events.length, files: files.length, skipped, path: usageDir() }));
+  }
+  return 0;
+}
+
+export async function runFeedback(options, io = console) {
+  const { events } = await readUsageEvents();
+  const version = await readVersion();
+  const bundle = buildFeedbackBundle({ events, version });
+  const target = resolve(options.out ?? defaultFeedbackPath());
+  await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+  await writeFileAtomic(target, `${JSON.stringify(bundle, null, 2)}\n`, { mode: 0o600 });
+  if (options.json) io.log(JSON.stringify(bundle, null, 2));
+  io.log(`wrote ${target}`);
+  io.log(bundle.privacy);
+  io.log("send it to hi@yunzerui.com if you want us to look（可选，不发送不影响使用）。");
+  return 0;
+}
+
+// Hidden input: a key must never appear in argv, shell history, or logs.
+async function readSecret(prompt) {
+  const stdin = process.stdin;
+  if (!stdin.isTTY) {
+    const chunks = [];
+    for await (const chunk of stdin) chunks.push(chunk);
+    const value = Buffer.concat(chunks).toString("utf8").trim();
+    if (!value) throw new Error("empty input; nothing written");
+    return value;
+  }
+  process.stdout.write(prompt);
+  return new Promise((resolvePromise, reject) => {
+    let value = "";
+    const cleanup = () => {
+      stdin.removeListener("data", onData);
+      stdin.setRawMode(false);
+      stdin.pause();
+    };
+    const onData = (chunk) => {
+      for (const char of String(chunk)) {
+        if (char === "\u0003") {
+          cleanup();
+          process.stdout.write("\n");
+          reject(new Error("cancelled"));
+          return;
+        }
+        if (char === "\r" || char === "\n") {
+          cleanup();
+          process.stdout.write("\n");
+          resolvePromise(value.trim());
+          return;
+        }
+        if (char === "\u007f" || char === "\b") {
+          value = value.slice(0, -1);
+          continue;
+        }
+        value += char;
+      }
+    };
+    stdin.setRawMode(true);
+    stdin.setEncoding("utf8");
+    stdin.resume();
+    stdin.on("data", onData);
+  });
+}
+
+async function readVersion() {
+  const pkg = JSON.parse(await readFile(new URL("./package.json", import.meta.url), "utf8"));
+  return pkg.version;
+}
+
 async function readStdin() {
   const chunks = [];
   for await (const chunk of process.stdin) chunks.push(chunk);
@@ -477,6 +757,8 @@ function selectCases(cases, ids) {
 
 async function resolveModels(ids, modelsFile, io) {
   const registry = JSON.parse(await readFile(resolve(modelsFile), "utf8"));
+  // Env wins; the credentials file only fills what is missing.
+  await applyModelCredentials(registry);
   const explicit = Boolean(ids && ids.length > 0);
   const chosen = explicit ? ids : autoSelect(registry);
   const adapters = [];
@@ -493,11 +775,13 @@ async function resolveModels(ids, modelsFile, io) {
     if (config.type !== "openai") {
       throw new Error(`model "${id}": unsupported type "${config.type}"`);
     }
-    if (!process.env[config.apiKeyEnv]) {
+    const apiKey = process.env[config.apiKeyEnv];
+    if (!apiKey) {
       if (explicit) throw new Error(`model "${id}": environment variable ${config.apiKeyEnv} is not set`);
       io.error(`skip ${id}: ${config.apiKeyEnv} is not set`);
       continue;
     }
+    registerSecrets([apiKey]);
     adapters.push({ id, ...createOpenAiAdapter({ id, ...config }) });
   }
 
@@ -515,25 +799,47 @@ function autoSelect(registry) {
     .map(([id]) => id);
 }
 
-async function evaluate(adapter, tools, cases, repeat = 1) {
+async function evaluate(
+  adapter,
+  tools,
+  cases,
+  repeat = 1,
+  { pricing = null, budget = null, onProgress = null } = {}
+) {
   const results = [];
   const unstable = [];
-  for (const testCase of cases) {
+  for (const [index, testCase] of cases.entries()) {
     const runs = [];
     for (let attempt = 0; attempt < repeat; attempt += 1) {
-      runs.push(await callOnce(adapter, tools, testCase));
+      const result = await callOnce(adapter, tools, testCase);
+      budget?.spend(costCnyOf(result.usage, pricing));
+      runs.push(result);
     }
     if (repeat === 1) {
       results.push(runs[0]);
-      continue;
+    } else {
+      const picked = pickRepresentative(runs);
+      results.push(picked.result);
+      if (picked.unstable) unstable.push({ id: testCase.id, outcomes: picked.outcomes });
     }
-    const picked = pickRepresentative(runs);
-    results.push(picked.result);
-    if (picked.unstable) unstable.push({ id: testCase.id, outcomes: picked.outcomes });
+    onProgress?.(index + 1, cases.length);
   }
   return { results, stability: repeat > 1 ? { runs: repeat, unstable } : null };
 }
 
+function makeProgress(options, io, label, total) {
+  if (!options.progress) return null;
+  const step = Math.max(1, Math.ceil(total / 10));
+  return (done, count) => {
+    if (done !== count && done % step !== 0) return;
+    io.error(`[${label}] ${done}/${count}`);
+  };
+}
+
+function costLimitMessage(error) {
+  return `${error.message}；已发出的请求已计费，本次不计分。减少 --case/--models/--repeat 或提高 --max-cost 后重跑。`;
+}
+
 async function callOnce(adapter, tools, testCase) {
   const messages = [
     { role: "system", content: SYSTEM_PROMPT },
@@ -578,7 +884,17 @@ function ablationScope(suite, names) {
   return { name: profile.name, tools: suite.tools.filter((tool) => mounted.has(tool.name)) };
 }
 
-async function evaluateProfiles({ suite, cases, runs, registry, selection, repeat = 1 }) {
+async function evaluateProfiles({
+  suite,
+  cases,
+  runs,
+  registry,
+  selection,
+  repeat = 1,
+  budget = null,
+  progress = false,
+  io = console
+}) {
   const profiles = normalizeProfiles(suite.profiles, suite.tools.map((tool) => tool.name));
   if (!profiles) {
     throw new Error(`suite ${suite.name} 没有 profiles；先运行 mf cluster --out <suite> 生成分组`);
@@ -608,7 +924,11 @@ async function evaluateProfiles({ suite, cases, runs, registry, selection, repea
       const profileCases = caseSets[index];
       if (profileCases.length === 0) continue;
       const mountedTools = suite.tools.filter((tool) => profile.tools.includes(tool.name));
-      const mountedEval = await evaluate(run.adapter, toModelTools(mountedTools), profileCases, repeat);
+      const mountedEval = await evaluate(run.adapter, toModelTools(mountedTools), profileCases, repeat, {
+        pricing,
+        budget,
+        onProgress: makeProgress({ progress }, io, `${run.adapter.id}/${profile.name}`, profileCases.length)
+      });
       const mounted = scoreModel({
         model: run.adapter.id,
         cases: profileCases,
@@ -644,6 +964,7 @@ async function evaluateProfiles({ suite, cases, runs, registry, selection, repea
 export async function runCluster(options, io = console) {
   const suite = await loadSuite(options.suite, { allowEmptyCases: true });
   const result = clusterTools(suite.tools, { threshold: options.threshold, maxSize: options.maxSize });
+  noteUsage({ toolCount: suite.tools.length, clusters: result.clusters.length });
   if (options.out) {
     const target = resolve(options.out);
     const profiles = Object.fromEntries(result.clusters.map((cluster) => [cluster.label, cluster.tools]));
@@ -670,6 +991,8 @@ function parseArgs(argv) {
     cases: null,
     profiles: null,
     repeat: 1,
+    maxCost: null,
+    progress: false,
     showCases: false,
     json: false,
     out: null,
@@ -727,6 +1050,13 @@ function parseArgs(argv) {
         options.repeat = raw;
         break;
       }
+      case "--max-cost": {
+        const raw = Number(value());
+        if (!Number.isFinite(raw) || raw <= 0) throw new Error("--max-cost expects a positive CNY amount");
+        options.maxCost = raw;
+        break;
+      }
+      case "--progress": options.progress = true; break;
       case "--show-cases": options.showCases = true; break;
       case "--json": options.json = true; break;
       case "--out": options.out = value(); break;
@@ -811,20 +1141,36 @@ if (isMain) {
   let argv = process.argv.slice(2);
   // `npx modelfunction-cli mf scan` passes the bin name through; accept it.
   if (argv[0] === "mf") argv = argv.slice(1);
+
+  const version = await readVersion();
+  const startedAt = Date.now();
+  let commandName = null;
+  const skipUsage = new Set(["mcp", "config", "stats", "feedback"]);
+  // Usage is local and count-only; the exit hook keeps every branch covered.
+  process.on("exit", (code) => {
+    if (!commandName || skipUsage.has(commandName)) return;
+    recordRunSync({
+      command: commandName,
+      exitCode: typeof code === "number" ? code : 0,
+      durationMs: Date.now() - startedAt,
+      version
+    });
+  });
+
   if (argv[0] === "--version" || argv[0] === "-v") {
-    const pkg = JSON.parse(await readFile(new URL("./package.json", import.meta.url), "utf8"));
-    process.stdout.write(`mf ${pkg.version}\n`);
+    process.stdout.write(`mf ${version}\n`);
     process.exit(0);
   } else if (argv[0] === "sandbox") {
     if (argv[1] !== "check") {
       process.stderr.write("usage: mf sandbox check [--json]\n");
       process.exit(1);
     }
+    commandName = "sandbox";
     try {
       const code = await runSandboxCheck({ json: argv.includes("--json") });
       process.exit(code);
     } catch (error) {
-      process.stderr.write(`${error?.message ?? error}\n`);
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
       process.exit(1);
     }
   } else if (argv[0] === "compare") {
@@ -833,11 +1179,44 @@ if (isMain) {
       process.stderr.write("usage: mf compare <before.json> <after.json> [--json]\n");
       process.exit(1);
     }
+    commandName = "compare";
     try {
       const code = await runCompare({ paths, json: argv.includes("--json") });
       process.exit(code);
     } catch (error) {
-      process.stderr.write(`${error?.message ?? error}\n`);
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
+      process.exit(1);
+    }
+  } else if (argv[0] === "config") {
+    try {
+      const code = await runConfig({ args: argv.slice(1), json: argv.includes("--json") });
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
+      process.exit(1);
+    }
+  } else if (argv[0] === "stats") {
+    try {
+      const code = await runStats({ json: argv.includes("--json") });
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
+      process.exit(1);
+    }
+  } else if (argv[0] === "feedback") {
+    try {
+      const outIndex = argv.indexOf("--out");
+      if (outIndex >= 0 && !argv[outIndex + 1]) {
+        process.stderr.write("usage: mf feedback [--out <file>] [--json]\n");
+        process.exit(1);
+      }
+      const code = await runFeedback({
+        out: outIndex >= 0 ? argv[outIndex + 1] : null,
+        json: argv.includes("--json")
+      });
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
       process.exit(1);
     }
   } else if (argv[0] === "mcp") {
@@ -845,12 +1224,12 @@ if (isMain) {
       const { startMcpServer } = await import("./lib/mcp-server.js");
       await startMcpServer();
     } catch (error) {
-      process.stderr.write(`${error?.message ?? error}\n`);
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
       process.exit(1);
     }
   } else if (argv.length === 0 || argv[0] === "help" || argv[0] === "-h" || argv[0] === "--help") {
     process.stdout.write(USAGE);
-  } else if (!["eval", "scan", "case", "coverage", "lint", "check", "ablate", "cluster", "exec"].includes(argv[0])) {
+  } else if (!["eval", "scan", "case", "coverage", "audit", "lint", "check", "ablate", "cluster", "exec"].includes(argv[0])) {
     process.stderr.write(`unknown command "${argv[0]}"\n\n${USAGE}`);
     process.exit(1);
   } else if (argv[0] === "case" && argv[1] !== "add") {
@@ -864,6 +1243,7 @@ if (isMain) {
         process.stdout.write(USAGE);
         process.exit(0);
       }
+      commandName = argv[0] === "case" ? "case" : argv[0];
       const code =
         argv[0] === "scan"
           ? await runScan(options)
@@ -871,6 +1251,8 @@ if (isMain) {
             ? await runCaseAdd(options)
             : argv[0] === "coverage"
               ? await runCoverage(options)
+              : argv[0] === "audit"
+                ? await runAudit(options)
               : argv[0] === "lint"
                 ? await runLint(options)
                 : argv[0] === "check"
@@ -884,7 +1266,7 @@ if (isMain) {
                     : await runEval(options);
       process.exit(code);
     } catch (error) {
-      process.stderr.write(`${error?.message ?? error}\n`);
+      process.stderr.write(`${redactText(error?.message ?? error)}\n`);
       process.exit(1);
     }
   }