modelfunction-cli 0.4.4 → 0.5.1

package/lib/mcp-server.js

@@ -8,8 +8,9 @@ import { z } from "zod";
 import { loadSuite } from "./suite.js";
 import { scanProject, addTools, buildSuiteDraft, redactReport } from "./scan.js";
 import { writeFileAtomic } from "./fsutil.js";
-import { probeStdioServer } from "./mcp-probe.js";
+import { probeStdioServer, probeHttpServer } from "./mcp-probe.js";
 import { addCases, computeCoverage } from "./cases.js";
+import { auditSuite } from "./audit.js";
 
 const MF_PATH = fileURLToPath(new URL("../mf.js", import.meta.url));
 const PACKAGE_JSON = new URL("../package.json", import.meta.url);
@@ -32,7 +33,7 @@ export async function startMcpServer() {
     {
       title: "Scan tools",
       description:
-        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核；来源 ≥2 时草稿带按来源分组的 profiles，并列出不参与评测的 skills）。",
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio 或 streamable-HTTP MCP server 拉取 tools/list（HTTP headers 只用于连接，不写入报告）。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核；来源 ≥2 时草稿带按来源分组的 profiles，并列出不参与评测的 skills）。",
       inputSchema: {
         root: z.string().optional().describe("仓库目录，默认进程当前目录"),
         probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
@@ -44,11 +45,10 @@ export async function startMcpServer() {
       const report = await scanProject(root ?? process.cwd());
       if (probe) {
         for (const mcpServer of [...report.mcpServers]) {
-          if (mcpServer.transport !== "stdio") {
-            report.probes.push({ server: mcpServer.name, ok: false, error: "http transport 暂不支持" });
-            continue;
-          }
-          const result = await probeStdioServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 });
+          const result =
+            mcpServer.transport === "stdio"
+              ? await probeStdioServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 })
+              : await probeHttpServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 });
           if (!result.ok) {
             report.probes.push({ server: mcpServer.name, ok: false, error: result.error });
             continue;
@@ -109,6 +109,22 @@ export async function startMcpServer() {
     async ({ suite }) => json(computeCoverage(await loadSuite(suitePath(suite))))
   );
 
+  server.registerTool(
+    "audit",
+    {
+      title: "Audit tool set",
+      description:
+        "一次静态审计整个工具集：lint 发现、混淆族与边界建议、用例覆盖、挂载检查。不调用模型、不执行工具、无网络。返回 JSON。",
+      inputSchema: { suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json") }
+    },
+    async ({ suite }) =>
+      json(
+        auditSuite(
+          await loadSuite(suitePath(suite), { allowEmptyCases: true, allowEmptyTools: true })
+        )
+      )
+  );
+
   server.registerTool(
     "add_cases",
     {
@@ -154,16 +170,20 @@ export async function startMcpServer() {
           .max(10)
           .optional()
           .describe("每条用例重复次数，取多数结果；建议正式基线用 3"),
-        failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2")
+        failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2"),
+        maxCost: z.number().positive().optional().describe("累计估算成本上限（CNY），超限退出 2 且不返回部分评分卡"),
+        progress: z.boolean().optional().describe("把逐用例进度写到 stderr（stdout 仍为纯 JSON）")
       }
     },
-    async ({ suite, models, cases, profile, repeat, failUnder }) => {
+    async ({ suite, models, cases, profile, repeat, failUnder, maxCost, progress }) => {
       const args = ["eval", "--json", "--suite", suitePath(suite)];
       if (models) args.push("--models", models);
       if (cases) args.push("--case", cases);
       if (profile) args.push("--profile", profile);
       if (repeat !== undefined) args.push("--repeat", String(repeat));
       if (failUnder !== undefined) args.push("--fail-under", String(failUnder));
+      if (maxCost !== undefined) args.push("--max-cost", String(maxCost));
+      if (progress) args.push("--progress");
 
       const result = await runCli(args);
       let scorecard = null;

package/lib/sandbox-check.js

@@ -1,7 +1,9 @@
 import { spawnSync } from "node:child_process";
 import { permissionFlag } from "./node-permission.js";
 
-function run(command, args, timeout = 4000) {
+// Shared CI runners can be slow to start a process; a 4s probe that times out
+// under load turns into a false "engine unavailable" and flakes the suite.
+function run(command, args, timeout = 10_000) {
   const result = spawnSync(command, args, { encoding: "utf8", timeout, windowsHide: true });
   return {
     ok: result.status === 0,

package/lib/scan.js

@@ -337,9 +337,11 @@ export function addTools(
 export function redactReport(report) {
   return {
     ...report,
-    mcpServers: report.mcpServers.map(({ env, ...rest }) => ({
+    mcpServers: report.mcpServers.map(({ env, headers, ...rest }) => ({
       ...rest,
-      envKeys: env ? Object.keys(env) : []
+      url: rest.url ? sanitizeUrl(rest.url) : null,
+      envKeys: env ? Object.keys(env) : [],
+      headerKeys: headers ? Object.keys(headers).sort() : []
     }))
   };
 }
@@ -526,10 +528,26 @@ function mcpServersFrom(config) {
     command: value.command ?? null,
     args: Array.isArray(value.args) ? value.args : [],
     env: value.env && typeof value.env === "object" ? value.env : null,
+    headers: value.headers && typeof value.headers === "object" ? value.headers : null,
     url: value.url ?? null
   }));
 }
 
+// Remote MCP configs often carry tokens in the query string; keep the report
+// safe to share while the live server object still probes the full URL.
+function sanitizeUrl(raw) {
+  try {
+    const url = new URL(raw);
+    url.username = "";
+    url.password = "";
+    url.search = "";
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
 function toolsFromOpenApi(spec) {
   const tools = [];
   const paths = spec.paths ?? {};

package/lib/score.js

@@ -76,6 +76,19 @@ export function pickRepresentative(runs) {
   };
 }
 
+export function costCnyOf(usage, pricing) {
+  if (!pricing) return 0;
+  const input = usage?.input ?? 0;
+  const cached = Math.min(input, usage?.cached ?? 0);
+  const output = usage?.output ?? 0;
+  return (
+    ((input - cached) * (pricing.input ?? 0) +
+      cached * (pricing.cachedInput ?? pricing.input ?? 0) +
+      output * (pricing.output ?? 0)) /
+    1e6
+  );
+}
+
 export function scoreModel({ model, cases, results, pricing }) {
   const rows = cases.map((testCase, index) => {
     const result = results[index];
@@ -115,13 +128,7 @@ export function scoreModel({ model, cases, results, pricing }) {
     inputTokens,
     rows.reduce((sum, row) => sum + (row.usage.cached ?? 0), 0)
   );
-  const uncachedTokens = inputTokens - cachedTokens;
-  const costCny = pricing
-    ? (uncachedTokens * (pricing.input ?? 0) +
-        cachedTokens * (pricing.cachedInput ?? pricing.input ?? 0) +
-        outputTokens * (pricing.output ?? 0)) /
-      1e6
-    : 0;
+  const costCny = rows.reduce((sum, row) => sum + costCnyOf(row.usage, pricing), 0);
   const reviewedTotal = reviewedRows.length;
 
   return {

package/lib/usage.js

@@ -0,0 +1,341 @@
+import { appendFile, mkdir, readdir, readFile } from "node:fs/promises";
+import { appendFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { configDir } from "./credentials.js";
+
+// Events are redacted by construction: only counts, ids and ratios leave the
+// process. Never add free-form text (inputs, descriptions, paths, errors).
+const OPT_OUT_ENV = "MF_NO_USAGE";
+const CI_ENV_KEYS = [
+  "GITHUB_ACTIONS",
+  "GITLAB_CI",
+  "CIRCLECI",
+  "TRAVIS",
+  "BUILDKITE",
+  "TF_BUILD",
+  "JENKINS_URL",
+  "CNB_BUILD_ID"
+];
+
+let pendingDetails = null;
+
+export function usageDir() {
+  return join(configDir(), "usage");
+}
+
+export function usageFile(date = new Date()) {
+  const month = date.toISOString().slice(0, 7);
+  return join(usageDir(), `${month}.jsonl`);
+}
+
+export function feedbackDir() {
+  return join(configDir(), "feedback");
+}
+
+export function defaultFeedbackPath(date = new Date()) {
+  const stamp = date.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
+  return join(feedbackDir(), `mf-feedback-${stamp}.json`);
+}
+
+export function noteUsage(details) {
+  pendingDetails = { ...(pendingDetails ?? {}), ...sanitizeDetails(details) };
+}
+
+export async function recordRun({ command, exitCode = 0, durationMs = 0, version }) {
+  if (process.env[OPT_OUT_ENV] === "1") {
+    pendingDetails = null;
+    return;
+  }
+  const record = buildRecord({ command, exitCode, durationMs, version });
+  try {
+    await mkdir(usageDir(), { recursive: true, mode: 0o700 });
+    await appendFile(usageFile(), `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+  } catch {
+    // Usage logging must never fail a command.
+  } finally {
+    pendingDetails = null;
+  }
+}
+
+// The process "exit" hook is synchronous, so the CLI uses this variant.
+export function recordRunSync({ command, exitCode = 0, durationMs = 0, version }) {
+  if (process.env[OPT_OUT_ENV] === "1") {
+    pendingDetails = null;
+    return;
+  }
+  const record = buildRecord({ command, exitCode, durationMs, version });
+  try {
+    mkdirSync(usageDir(), { recursive: true, mode: 0o700 });
+    appendFileSync(usageFile(), `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+  } catch {
+    // ignore
+  } finally {
+    pendingDetails = null;
+  }
+}
+
+export function buildRecord({ command, exitCode, durationMs, version }) {
+  return {
+    schema: 1,
+    ts: new Date().toISOString(),
+    command: typeof command === "string" ? command.slice(0, 32) : "unknown",
+    exitCode: Number.isInteger(exitCode) ? exitCode : 1,
+    durationMs: Math.max(0, Math.round(Number(durationMs) || 0)),
+    mf: typeof version === "string" ? version : "unknown",
+    platform: process.platform,
+    arch: process.arch,
+    node: process.versions.node,
+    ci: detectCi(),
+    ...(pendingDetails ?? {})
+  };
+}
+
+function detectCi() {
+  if (process.env.CI === "true" || process.env.CI === "1") return true;
+  return CI_ENV_KEYS.some((key) => Boolean(process.env[key]));
+}
+
+export async function readUsageEvents() {
+  let files = [];
+  try {
+    files = (await readdir(usageDir())).filter((name) => /^\d{4}-\d{2}\.jsonl$/.test(name)).sort();
+  } catch (error) {
+    if (error.code === "ENOENT") return { events: [], files: [], skipped: 0 };
+    throw error;
+  }
+  const events = [];
+  let skipped = 0;
+  for (const file of files) {
+    const text = await readFile(join(usageDir(), file), "utf8");
+    for (const line of text.split(/\r?\n/)) {
+      if (!line.trim()) continue;
+      try {
+        const parsed = JSON.parse(line);
+        if (parsed && typeof parsed === "object" && typeof parsed.ts === "string") events.push(parsed);
+        else skipped += 1;
+      } catch {
+        skipped += 1;
+      }
+    }
+  }
+  events.sort((a, b) => a.ts.localeCompare(b.ts));
+  return { events, files, skipped };
+}
+
+export function summarizeUsage(events) {
+  const summary = {
+    runs: events.length,
+    firstRun: events[0]?.ts ?? null,
+    lastRun: events.at(-1)?.ts ?? null,
+    activeDays: new Set(events.map((event) => event.ts.slice(0, 10))).size,
+    ciRuns: 0,
+    localRuns: 0,
+    exitCodes: {},
+    commands: {},
+    platforms: {},
+    nodeVersions: {},
+    mfVersions: {},
+    models: {},
+    eval: { runs: 0, executed: 0, meanSelection: null, meanPass: null, meanArgs: null, unstableCases: 0 },
+    lint: { runs: 0, errors: 0, warns: 0, infos: 0, rules: {} },
+    check: { runs: 0, verdicts: {} },
+    scan: { runs: 0, tools: 0, skills: 0 }
+  };
+  const selection = [];
+  const pass = [];
+  const args = [];
+
+  for (const event of events) {
+    if (event.ci) summary.ciRuns += 1;
+    else summary.localRuns += 1;
+    bump(summary.exitCodes, String(event.exitCode ?? 0));
+    bump(summary.commands, event.command ?? "unknown");
+    if (event.platform) bump(summary.platforms, event.platform);
+    if (event.node) bump(summary.nodeVersions, event.node);
+    if (event.mf) bump(summary.mfVersions, event.mf);
+    for (const model of event.models ?? []) bump(summary.models, model);
+
+    if (event.command === "eval") {
+      summary.eval.runs += 1;
+      if (event.execEngine) summary.eval.executed += 1;
+      if (event.unstableCases) summary.eval.unstableCases += event.unstableCases;
+      if (typeof event.selectionAccuracy === "number") selection.push(event.selectionAccuracy);
+      if (typeof event.passRate === "number") pass.push(event.passRate);
+      if (typeof event.argAccuracy === "number") args.push(event.argAccuracy);
+    }
+    if (event.command === "lint") {
+      summary.lint.runs += 1;
+      summary.lint.errors += event.lintErrors ?? 0;
+      summary.lint.warns += event.lintWarns ?? 0;
+      summary.lint.infos += event.lintInfos ?? 0;
+      for (const [rule, count] of Object.entries(event.lintRules ?? {})) {
+        summary.lint.rules[rule] = (summary.lint.rules[rule] ?? 0) + count;
+      }
+    }
+    if (event.command === "check") {
+      summary.check.runs += 1;
+      bump(summary.check.verdicts, event.checkVerdict ?? "unknown");
+    }
+    if (event.command === "scan") {
+      summary.scan.runs += 1;
+      summary.scan.tools += event.scanTools ?? 0;
+      summary.scan.skills += event.scanSkills ?? 0;
+    }
+  }
+
+  summary.eval.meanSelection = mean(selection);
+  summary.eval.meanPass = mean(pass);
+  summary.eval.meanArgs = mean(args);
+  return summary;
+}
+
+export function buildFeedbackBundle({ events, version, generatedAt = new Date() }) {
+  return {
+    schema: 1,
+    kind: "modelfunction-feedback",
+    generatedAt: generatedAt.toISOString(),
+    mf: {
+      version,
+      node: process.versions.node,
+      platform: process.platform,
+      arch: process.arch
+    },
+    usage: summarizeUsage(events),
+    privacy:
+      "仅包含命令名、次数、模型 id、计数与比率。不含用例原文、工具描述、源码、文件路径或密钥。",
+    request:
+      "我们想了解：哪些命令被真实使用、在哪些平台上跑、评测通过率大概什么水平、lint/check 主要命中哪些规则。用于决定下一步做什么，不需要你的项目细节。"
+  };
+}
+
+export function formatStats(summary, { events = 0, files = 0, skipped = 0, path = usageDir() } = {}) {
+  if (summary.runs === 0) {
+    return [
+      `local usage: ${path}`,
+      "还没有本地记录。运行任意 mf 命令后会自动记录（仅计数，不上传）。",
+      "opt out: set MF_NO_USAGE=1"
+    ].join("\n");
+  }
+  const lines = [
+    `local usage: ${path} (${events} events, ${files} file(s)${skipped > 0 ? `, ${skipped} unreadable line(s)` : ""})`,
+    `runs: ${summary.runs} (local ${summary.localRuns}, ci ${summary.ciRuns})   first: ${summary.firstRun}   last: ${summary.lastRun}   active days: ${summary.activeDays}`,
+    `exit codes: ${pairs(summary.exitCodes)}`,
+    `commands:   ${pairs(summary.commands)}`
+  ];
+  if (Object.keys(summary.models).length > 0) lines.push(`models:     ${pairs(summary.models)}`);
+  if (summary.eval.runs > 0) {
+    const parts = [`${summary.eval.runs} run(s)`];
+    if (summary.eval.meanSelection !== null) parts.push(`select ${percent(summary.eval.meanSelection)}`);
+    if (summary.eval.meanPass !== null) parts.push(`pass ${percent(summary.eval.meanPass)}`);
+    if (summary.eval.executed > 0) parts.push(`${summary.eval.executed} with execution`);
+    if (summary.eval.unstableCases > 0) parts.push(`${summary.eval.unstableCases} unstable case(s)`);
+    lines.push(`eval:       ${parts.join(", ")}`);
+  }
+  if (summary.lint.runs > 0) {
+    const top = Object.entries(summary.lint.rules)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5)
+      .map(([rule, count]) => `${rule} ${count}`);
+    lines.push(
+      `lint:       ${summary.lint.runs} run(s), error ${summary.lint.errors} / warn ${summary.lint.warns} / info ${summary.lint.infos}` +
+        (top.length > 0 ? `; top: ${top.join(", ")}` : "")
+    );
+  }
+  if (summary.check.runs > 0) lines.push(`check:      ${summary.check.runs} run(s), ${pairs(summary.check.verdicts)}`);
+  if (summary.scan.runs > 0) lines.push(`scan:       ${summary.scan.runs} run(s), ${summary.scan.tools} tools, ${summary.scan.skills} skills`);
+  lines.push(`versions:   mf ${keys(summary.mfVersions)}   node ${keys(summary.nodeVersions)}   os ${keys(summary.platforms)}`);
+  lines.push(`no network, no telemetry. opt out: set MF_NO_USAGE=1. clear: delete ${path}`);
+  return lines.join("\n");
+}
+
+function bump(target, key) {
+  target[key] = (target[key] ?? 0) + 1;
+}
+
+function mean(values) {
+  if (values.length === 0) return null;
+  return Math.round((values.reduce((sum, value) => sum + value, 0) / values.length) * 10000) / 10000;
+}
+
+function percent(value) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+function keys(counts) {
+  return Object.keys(counts).sort().slice(0, 5).join(", ") || "-";
+}
+
+function pairs(counts) {
+  return (
+    Object.entries(counts)
+      .sort((a, b) => b[1] - a[1])
+      .map(([key, value]) => `${key} ${value}`)
+      .join(", ") || "-"
+  );
+}
+
+const INT_FIELDS = new Set([
+  "caseCount",
+  "toolCount",
+  "reviewedCount",
+  "profileGroups",
+  "repeat",
+  "unstableCases",
+  "execExecuted",
+  "execPassed",
+  "execBlocked",
+  "execSkipped",
+  "scanTools",
+  "scanSkills",
+  "scanMcpServers",
+  "scanProbes",
+  "lintErrors",
+  "lintWarns",
+  "lintInfos",
+  "checkFindings",
+  "clusters"
+]);
+const RATIO_FIELDS = new Set(["selectionAccuracy", "argAccuracy", "passRate"]);
+const BOOL_FIELDS = new Set(["failedThreshold", "regressions"]);
+const STRING_FIELDS = new Set(["execEngine", "checkVerdict"]);
+const STRING_ARRAY_FIELDS = new Set(["models"]);
+const COUNT_OBJECT_FIELDS = new Set(["lintRules"]);
+
+export function sanitizeDetails(details) {
+  if (!details || typeof details !== "object") return {};
+  const out = {};
+  for (const [key, value] of Object.entries(details)) {
+    if (INT_FIELDS.has(key)) {
+      if (Number.isInteger(value) && value >= 0) out[key] = value;
+      continue;
+    }
+    if (RATIO_FIELDS.has(key)) {
+      if (typeof value === "number" && Number.isFinite(value)) out[key] = Math.round(value * 10000) / 10000;
+      continue;
+    }
+    if (BOOL_FIELDS.has(key)) {
+      if (typeof value === "boolean") out[key] = value;
+      continue;
+    }
+    if (STRING_FIELDS.has(key)) {
+      if (typeof value === "string" && /^[A-Za-z0-9_-]{1,32}$/.test(value)) out[key] = value;
+      continue;
+    }
+    if (STRING_ARRAY_FIELDS.has(key)) {
+      if (Array.isArray(value)) {
+        out[key] = value.filter((item) => typeof item === "string" && item.length <= 64).slice(0, 20);
+      }
+      continue;
+    }
+    if (COUNT_OBJECT_FIELDS.has(key)) {
+      if (value && typeof value === "object" && !Array.isArray(value)) {
+        out[key] = Object.fromEntries(
+          Object.entries(value)
+            .filter(([rule, count]) => /^[a-z0-9-]{1,40}$/.test(rule) && Number.isInteger(count) && count >= 0)
+            .slice(0, 40)
+        );
+      }
+    }
+  }
+  return out;
+}