modelfunction-cli 0.5.0 → 0.5.2

package/lib/mcp-probe.js

@@ -1,10 +1,67 @@
 import { spawn } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
 
 const MAX_BYTES = 1_000_000;
 const MAX_STDERR = 4_000;
-const CLIENT_INFO = { name: "mf", version: "0.1.0" };
+const PACKAGE = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+const CLIENT_INFO = { name: "mf", version: PACKAGE.version };
 const PROTOCOL_VERSION = "2025-06-18";
 
+// Streamable HTTP is the current remote MCP transport. Headers come from the
+// user's own config and are never copied into reports or logs.
+export async function probeHttpServer(server, { timeoutMs = 5_000 } = {}) {
+  if (!server?.url) return { ok: false, error: "missing url" };
+  let url;
+  try {
+    url = new URL(server.url);
+  } catch {
+    return { ok: false, error: "invalid url" };
+  }
+
+  const client = new Client(CLIENT_INFO);
+  const headers = server.headers && Object.keys(server.headers).length > 0 ? server.headers : null;
+  const transport = new StreamableHTTPClientTransport(url, {
+    requestInit: headers ? { headers } : undefined
+  });
+
+  try {
+    await withTimeout(client.connect(transport), timeoutMs, "connect");
+    const result = await withTimeout(client.listTools(), timeoutMs, "tools/list");
+    return { ok: true, tools: normalizeTools(result?.tools ?? []) };
+  } catch (error) {
+    return { ok: false, error: `http: ${scrubUrl(error?.message ?? error, url)}` };
+  } finally {
+    await client.close().catch(() => {});
+  }
+}
+
+function withTimeout(promise, ms, label) {
+  let id;
+  const timeout = new Promise((_, reject) => {
+    id = setTimeout(() => reject(new Error(`${label} timeout after ${ms}ms`)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(id));
+}
+
+function scrubUrl(message, url) {
+  let text = String(message ?? "");
+  text = text.split(url.href).join(`${url.origin}${url.pathname}`);
+  if (url.search) text = text.split(url.search).join("");
+  return text;
+}
+
+function normalizeTools(tools) {
+  return tools
+    .filter((tool) => typeof tool?.name === "string" && tool.name.trim() !== "")
+    .map((tool) => ({
+      name: tool.name.trim(),
+      description: tool.description ?? "",
+      parameters: tool.inputSchema ?? { type: "object", properties: {} }
+    }));
+}
+
 export async function probeStdioServer(server, { timeoutMs = 5_000, cwd } = {}) {
   if (!server?.command) {
     return { ok: false, error: "missing command" };
@@ -129,14 +186,7 @@ export async function probeStdioServer(server, { timeoutMs = 5_000, cwd } = {})
     });
     send({ jsonrpc: "2.0", method: "notifications/initialized" });
     const result = await request("tools/list", {});
-    const tools = (result?.tools ?? [])
-      .filter((tool) => typeof tool?.name === "string" && tool.name.trim() !== "")
-      .map((tool) => ({
-        name: tool.name.trim(),
-        description: tool.description ?? "",
-        parameters: tool.inputSchema ?? { type: "object", properties: {} }
-      }));
-    return { ok: true, tools };
+    return { ok: true, tools: normalizeTools(result?.tools ?? []) };
   } catch (error) {
     return {
       ok: false,

package/lib/mcp-server.js

@@ -8,8 +8,13 @@ import { z } from "zod";
 import { loadSuite } from "./suite.js";
 import { scanProject, addTools, buildSuiteDraft, redactReport } from "./scan.js";
 import { writeFileAtomic } from "./fsutil.js";
-import { probeStdioServer } from "./mcp-probe.js";
+import { probeStdioServer, probeHttpServer } from "./mcp-probe.js";
 import { addCases, computeCoverage } from "./cases.js";
+import { auditSuite } from "./audit.js";
+import { check, checkMany } from "./check.js";
+import { lintSuite } from "./lint.js";
+import { clusterTools } from "./cluster.js";
+import { compareScorecards } from "./compare.js";
 
 const MF_PATH = fileURLToPath(new URL("../mf.js", import.meta.url));
 const PACKAGE_JSON = new URL("../package.json", import.meta.url);
@@ -32,7 +37,7 @@ export async function startMcpServer() {
     {
       title: "Scan tools",
       description:
-        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核；来源 ≥2 时草稿带按来源分组的 profiles，并列出不参与评测的 skills）。",
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio 或 streamable-HTTP MCP server 拉取 tools/list（HTTP headers 只用于连接，不写入报告）。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核；来源 ≥2 时草稿带按来源分组的 profiles，并列出不参与评测的 skills）。",
       inputSchema: {
         root: z.string().optional().describe("仓库目录，默认进程当前目录"),
         probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
@@ -44,11 +49,10 @@ export async function startMcpServer() {
       const report = await scanProject(root ?? process.cwd());
       if (probe) {
         for (const mcpServer of [...report.mcpServers]) {
-          if (mcpServer.transport !== "stdio") {
-            report.probes.push({ server: mcpServer.name, ok: false, error: "http transport 暂不支持" });
-            continue;
-          }
-          const result = await probeStdioServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 });
+          const result =
+            mcpServer.transport === "stdio"
+              ? await probeStdioServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 })
+              : await probeHttpServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 });
           if (!result.ok) {
             report.probes.push({ server: mcpServer.name, ok: false, error: result.error });
             continue;
@@ -109,6 +113,94 @@ export async function startMcpServer() {
     async ({ suite }) => json(computeCoverage(await loadSuite(suitePath(suite))))
   );
 
+  server.registerTool(
+    "audit",
+    {
+      title: "Audit tool set",
+      description:
+        "一次静态审计整个工具集：lint 发现、混淆族与边界建议、用例覆盖、挂载检查。不调用模型、不执行工具、无网络。返回 JSON。",
+      inputSchema: { suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json") }
+    },
+    async ({ suite }) =>
+      json(
+        auditSuite(
+          await loadSuite(suitePath(suite), { allowEmptyCases: true, allowEmptyTools: true })
+        )
+      )
+  );
+
+  server.registerTool(
+    "lint",
+    {
+      title: "Lint suite",
+      description:
+        "静态检查 tools 与 skills 的描述、参数与工具对边界（不调用模型）。返回 counts / findings / profilePairs。",
+      inputSchema: { suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json") }
+    },
+    async ({ suite }) =>
+      json(
+        lintSuite(
+          await loadSuite(suitePath(suite), { allowEmptyCases: true, allowEmptyTools: true })
+        )
+      )
+  );
+
+  server.registerTool(
+    "check",
+    {
+      title: "Install gate",
+      description:
+        "安装门禁：把新能力与已装能力做静态比较，返回 pass/warn、findings 与竞争族。传 manifest 检查单个；传 manifests 检查一包（新能力之间也会互比）。不阻断、不调用模型、无网络。",
+      inputSchema: {
+        manifest: z.any().optional().describe("新能力 {name, description, parameters?} 或 tool schema"),
+        manifests: z.array(z.any()).optional().describe("一次安装一包能力时的数组"),
+        installed: z.any().optional().describe("已装能力数组，或 {tools: [], skills: []}")
+      }
+    },
+    async ({ manifest, manifests, installed }) => {
+      if (manifests) return json(checkMany({ manifests, installed }));
+      return json(check({ manifest, installed }));
+    }
+  );
+
+  server.registerTool(
+    "cluster",
+    {
+      title: "Cluster tools",
+      description:
+        "聚类工具，给出挂载分组、跨组高风险与族内边界清单（每族要补哪句选择条件）。不调用模型。",
+      inputSchema: {
+        suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json"),
+        threshold: z.number().min(0).max(1).optional().describe("聚类阈值，默认 0.5"),
+        maxSize: z.number().int().positive().optional().describe("单组最大工具数，默认 10")
+      }
+    },
+    async ({ suite, threshold, maxSize }) => {
+      const loaded = await loadSuite(suitePath(suite), { allowEmptyCases: true });
+      return json(clusterTools(loaded.tools, { threshold, maxSize }));
+    }
+  );
+
+  server.registerTool(
+    "compare",
+    {
+      title: "Compare scorecards",
+      description:
+        "比较两份评分卡 JSON：可比性、升降、干扰归因与混淆矩阵。只读文件，不调用模型。",
+      inputSchema: {
+        before: z.string().describe("旧评分卡 JSON 路径"),
+        after: z.string().describe("新评分卡 JSON 路径")
+      }
+    },
+    async ({ before, after }) => {
+      const [oldText, newText] = await Promise.all([
+        readFile(resolve(before), "utf8"),
+        readFile(resolve(after), "utf8")
+      ]);
+      return json(compareScorecards(JSON.parse(oldText), JSON.parse(newText)));
+    }
+  );
+
   server.registerTool(
     "add_cases",
     {
@@ -154,16 +246,20 @@ export async function startMcpServer() {
           .max(10)
           .optional()
           .describe("每条用例重复次数，取多数结果；建议正式基线用 3"),
-        failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2")
+        failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2"),
+        maxCost: z.number().positive().optional().describe("累计估算成本上限（CNY），超限退出 2 且不返回部分评分卡"),
+        progress: z.boolean().optional().describe("把逐用例进度写到 stderr（stdout 仍为纯 JSON）")
       }
     },
-    async ({ suite, models, cases, profile, repeat, failUnder }) => {
+    async ({ suite, models, cases, profile, repeat, failUnder, maxCost, progress }) => {
       const args = ["eval", "--json", "--suite", suitePath(suite)];
       if (models) args.push("--models", models);
       if (cases) args.push("--case", cases);
       if (profile) args.push("--profile", profile);
       if (repeat !== undefined) args.push("--repeat", String(repeat));
       if (failUnder !== undefined) args.push("--fail-under", String(failUnder));
+      if (maxCost !== undefined) args.push("--max-cost", String(maxCost));
+      if (progress) args.push("--progress");
 
       const result = await runCli(args);
       let scorecard = null;

package/lib/scan.js

@@ -337,9 +337,11 @@ export function addTools(
 export function redactReport(report) {
   return {
     ...report,
-    mcpServers: report.mcpServers.map(({ env, ...rest }) => ({
+    mcpServers: report.mcpServers.map(({ env, headers, ...rest }) => ({
       ...rest,
-      envKeys: env ? Object.keys(env) : []
+      url: rest.url ? sanitizeUrl(rest.url) : null,
+      envKeys: env ? Object.keys(env) : [],
+      headerKeys: headers ? Object.keys(headers).sort() : []
     }))
   };
 }
@@ -526,10 +528,26 @@ function mcpServersFrom(config) {
     command: value.command ?? null,
     args: Array.isArray(value.args) ? value.args : [],
     env: value.env && typeof value.env === "object" ? value.env : null,
+    headers: value.headers && typeof value.headers === "object" ? value.headers : null,
     url: value.url ?? null
   }));
 }
 
+// Remote MCP configs often carry tokens in the query string; keep the report
+// safe to share while the live server object still probes the full URL.
+function sanitizeUrl(raw) {
+  try {
+    const url = new URL(raw);
+    url.username = "";
+    url.password = "";
+    url.search = "";
+    url.hash = "";
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
 function toolsFromOpenApi(spec) {
   const tools = [];
   const paths = spec.paths ?? {};

package/lib/score.js

@@ -76,6 +76,19 @@ export function pickRepresentative(runs) {
   };
 }
 
+export function costCnyOf(usage, pricing) {
+  if (!pricing) return 0;
+  const input = usage?.input ?? 0;
+  const cached = Math.min(input, usage?.cached ?? 0);
+  const output = usage?.output ?? 0;
+  return (
+    ((input - cached) * (pricing.input ?? 0) +
+      cached * (pricing.cachedInput ?? pricing.input ?? 0) +
+      output * (pricing.output ?? 0)) /
+    1e6
+  );
+}
+
 export function scoreModel({ model, cases, results, pricing }) {
   const rows = cases.map((testCase, index) => {
     const result = results[index];
@@ -115,13 +128,7 @@ export function scoreModel({ model, cases, results, pricing }) {
     inputTokens,
     rows.reduce((sum, row) => sum + (row.usage.cached ?? 0), 0)
   );
-  const uncachedTokens = inputTokens - cachedTokens;
-  const costCny = pricing
-    ? (uncachedTokens * (pricing.input ?? 0) +
-        cachedTokens * (pricing.cachedInput ?? pricing.input ?? 0) +
-        outputTokens * (pricing.output ?? 0)) /
-      1e6
-    : 0;
+  const costCny = rows.reduce((sum, row) => sum + costCnyOf(row.usage, pricing), 0);
   const reviewedTotal = reviewedRows.length;
 
   return {

package/lib/usage.js

@@ -6,6 +6,16 @@ import { configDir } from "./credentials.js";
 // Events are redacted by construction: only counts, ids and ratios leave the
 // process. Never add free-form text (inputs, descriptions, paths, errors).
 const OPT_OUT_ENV = "MF_NO_USAGE";
+const CI_ENV_KEYS = [
+  "GITHUB_ACTIONS",
+  "GITLAB_CI",
+  "CIRCLECI",
+  "TRAVIS",
+  "BUILDKITE",
+  "TF_BUILD",
+  "JENKINS_URL",
+  "CNB_BUILD_ID"
+];
 
 let pendingDetails = null;
 
@@ -75,10 +85,16 @@ export function buildRecord({ command, exitCode, durationMs, version }) {
     platform: process.platform,
     arch: process.arch,
     node: process.versions.node,
+    ci: detectCi(),
     ...(pendingDetails ?? {})
   };
 }
 
+function detectCi() {
+  if (process.env.CI === "true" || process.env.CI === "1") return true;
+  return CI_ENV_KEYS.some((key) => Boolean(process.env[key]));
+}
+
 export async function readUsageEvents() {
   let files = [];
   try {
@@ -112,6 +128,8 @@ export function summarizeUsage(events) {
     firstRun: events[0]?.ts ?? null,
     lastRun: events.at(-1)?.ts ?? null,
     activeDays: new Set(events.map((event) => event.ts.slice(0, 10))).size,
+    ciRuns: 0,
+    localRuns: 0,
     exitCodes: {},
     commands: {},
     platforms: {},
@@ -128,6 +146,8 @@ export function summarizeUsage(events) {
   const args = [];
 
   for (const event of events) {
+    if (event.ci) summary.ciRuns += 1;
+    else summary.localRuns += 1;
     bump(summary.exitCodes, String(event.exitCode ?? 0));
     bump(summary.commands, event.command ?? "unknown");
     if (event.platform) bump(summary.platforms, event.platform);
@@ -198,7 +218,7 @@ export function formatStats(summary, { events = 0, files = 0, skipped = 0, path
   }
   const lines = [
     `local usage: ${path} (${events} events, ${files} file(s)${skipped > 0 ? `, ${skipped} unreadable line(s)` : ""})`,
-    `runs: ${summary.runs}   first: ${summary.firstRun}   last: ${summary.lastRun}   active days: ${summary.activeDays}`,
+    `runs: ${summary.runs} (local ${summary.localRuns}, ci ${summary.ciRuns})   first: ${summary.firstRun}   last: ${summary.lastRun}   active days: ${summary.activeDays}`,
     `exit codes: ${pairs(summary.exitCodes)}`,
     `commands:   ${pairs(summary.commands)}`
   ];