modelfunction-cli 0.0.0-stage → 0.1.1

package/lib/mcp-server.js

@@ -0,0 +1,222 @@
+import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+import { loadSuite } from "./suite.js";
+import { scanProject, addTools, buildSuiteDraft, redactReport } from "./scan.js";
+import { writeFileAtomic } from "./fsutil.js";
+import { probeStdioServer } from "./mcp-probe.js";
+import { addCases, computeCoverage } from "./cases.js";
+
+const MF_PATH = fileURLToPath(new URL("../mf.js", import.meta.url));
+const PACKAGE_JSON = new URL("../package.json", import.meta.url);
+const MAX_CHILD_OUTPUT = 5_000_000;
+
+function json(result) {
+  return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+}
+
+function suitePath(input) {
+  return resolve(input ?? "mf.suite.json");
+}
+
+export async function startMcpServer() {
+  const pkg = JSON.parse(await readFile(PACKAGE_JSON, "utf8"));
+  const server = new McpServer({ name: "modelfunction", version: pkg.version });
+
+  server.registerTool(
+    "scan_tools",
+    {
+      title: "Scan tools",
+      description:
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核）。",
+      inputSchema: {
+        root: z.string().optional().describe("仓库目录，默认进程当前目录"),
+        probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
+        timeoutMs: z.number().optional().describe("单个 MCP server 超时（毫秒，默认 5000）"),
+        writeSuite: z.string().optional().describe("套件草稿输出路径，如 mf.suite.json")
+      }
+    },
+    async ({ root, probe, timeoutMs, writeSuite }) => {
+      const report = await scanProject(root ?? process.cwd());
+      if (probe) {
+        for (const mcpServer of [...report.mcpServers]) {
+          if (mcpServer.transport !== "stdio") {
+            report.probes.push({ server: mcpServer.name, ok: false, error: "http transport 暂不支持" });
+            continue;
+          }
+          const result = await probeStdioServer(mcpServer, { timeoutMs: timeoutMs ?? 5000 });
+          if (!result.ok) {
+            report.probes.push({ server: mcpServer.name, ok: false, error: result.error });
+            continue;
+          }
+          const added = addTools(report, result.tools, {
+            source: mcpServer.source,
+            origin: "mcp-probe",
+            confidence: "high",
+            note: `server ${mcpServer.name}`
+          });
+          report.probes.push({ server: mcpServer.name, ok: true, toolCount: added.length, tools: added });
+        }
+      }
+      const payload = redactReport(report);
+      if (!writeSuite) return json(payload);
+      const target = resolve(writeSuite);
+      await writeFileAtomic(target, JSON.stringify(buildSuiteDraft(report), null, 2));
+      return json({ ...payload, suiteDraftPath: target });
+    }
+  );
+
+  server.registerTool(
+    "list_suite",
+    {
+      title: "List suite",
+      description: "读取套件概览：工具、用例数、复核状态分布。",
+      inputSchema: { suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json") }
+    },
+    async ({ suite }) => {
+      const loaded = await loadSuite(suitePath(suite));
+      const sources = {};
+      for (const testCase of loaded.cases) {
+        sources[testCase.source] = (sources[testCase.source] ?? 0) + 1;
+      }
+      return json({
+        name: loaded.name,
+        path: loaded.path,
+        tools: loaded.tools.map((tool) => ({
+          name: tool.name,
+          description: tool.description,
+          parameters: Object.keys(tool.parameters?.properties ?? {})
+        })),
+        cases: loaded.cases.length,
+        reviewed: loaded.cases.filter((testCase) => testCase.reviewed !== false).length,
+        sources
+      });
+    }
+  );
+
+  server.registerTool(
+    "coverage",
+    {
+      title: "Case coverage",
+      description: "报告每个工具的用例数、参数覆盖、负例比例、疑似混淆的工具对与缺口。",
+      inputSchema: { suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json") }
+    },
+    async ({ suite }) => json(computeCoverage(await loadSuite(suitePath(suite))))
+  );
+
+  server.registerTool(
+    "add_cases",
+    {
+      title: "Add cases",
+      description:
+        "把编码 Agent 撰写的用例写入套件。默认 source=agent、reviewed=false；未复核用例不计入正式分数。",
+      inputSchema: {
+        suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json"),
+        cases: z.array(z.any()).describe("用例数组，字段 id/input/expectedTool/expectedArgs/note/tags"),
+        source: z.string().optional().describe("来源标记，默认 agent"),
+        reviewed: z.boolean().optional().describe("是否已复核，默认 false"),
+        dryRun: z.boolean().optional().describe("只校验不写入")
+      }
+    },
+    async ({ suite, cases, source, reviewed, dryRun }) =>
+      json(
+        await addCases(suitePath(suite), cases, {
+          source: source ?? "agent",
+          reviewed: reviewed ?? false,
+          dryRun: dryRun ?? false
+        })
+      )
+  );
+
+  server.registerTool(
+    "run_eval",
+    {
+      title: "Run evaluation",
+      description:
+        "对套件跑评分卡（调用被测模型）。返回 JSON 评分卡；耗时约 用例数 × 模型数 × 单次延迟。",
+      inputSchema: {
+        suite: z.string().optional().describe("套件路径，默认 ./mf.suite.json"),
+        models: z.string().optional().describe("逗号分隔的模型 id，如 mock,dsv4-flash"),
+        cases: z.string().optional().describe("只跑这些用例 id（逗号分隔）"),
+        failUnder: z.number().optional().describe("选择准确率阈值，低于则退出码为 2")
+      }
+    },
+    async ({ suite, models, cases, failUnder }) => {
+      const args = ["eval", "--json", "--suite", suitePath(suite)];
+      if (models) args.push("--models", models);
+      if (cases) args.push("--case", cases);
+      if (failUnder !== undefined) args.push("--fail-under", String(failUnder));
+
+      const result = await runCli(args);
+      let scorecard = null;
+      try {
+        scorecard = JSON.parse(result.stdout);
+      } catch {
+        scorecard = null;
+      }
+      return json({
+        exitCode: result.code,
+        scorecard,
+        ...(result.error ? { error: result.error } : {}),
+        ...(scorecard ? {} : { stdout: result.stdout.slice(-4000) }),
+        stderr: result.stderr.slice(-4000)
+      });
+    }
+  );
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  const shutdown = () => process.exit(0);
+  process.stdin.on("end", shutdown);
+  process.stdin.on("close", shutdown);
+  process.on("SIGINT", shutdown);
+  process.on("SIGTERM", shutdown);
+  console.error("modelfunction mcp server running on stdio");
+}
+
+function runCli(args, timeoutMs = 300_000) {
+  return new Promise((resolvePromise) => {
+    const child = spawn(process.execPath, [MF_PATH, ...args], {
+      stdio: ["ignore", "pipe", "pipe"],
+      windowsHide: true
+    });
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    let error = null;
+
+    const finish = (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      try {
+        child.kill();
+      } catch {
+        // ignore
+      }
+      resolvePromise({ code, stdout, stderr, error });
+    };
+
+    const timer = setTimeout(() => {
+      error = `timeout after ${timeoutMs}ms`;
+      finish(-1);
+    }, timeoutMs);
+
+    child.stdout.on("data", (chunk) => {
+      if (stdout.length < MAX_CHILD_OUTPUT) stdout += chunk.toString("utf8");
+    });
+    child.stderr.on("data", (chunk) => {
+      if (stderr.length < MAX_CHILD_OUTPUT) stderr += chunk.toString("utf8");
+    });
+    child.on("error", (err) => {
+      error = `spawn failed: ${err.message}`;
+      finish(-1);
+    });
+    child.on("exit", (code) => finish(code ?? -1));
+  });
+}

package/lib/report.js

@@ -0,0 +1,119 @@
+import { pct } from "./score.js";
+import { suiteFingerprint } from "./fingerprint.js";
+
+function money(value, estimated) {
+  if (value === 0) return "0";
+  const text = value >= 0.01 ? value.toFixed(3) : value.toFixed(4);
+  return `${estimated ? "~" : ""}${text}`;
+}
+
+function isEstimated(summary) {
+  return summary.rows.some((row) => row.usage?.estimated);
+}
+
+export function formatCaseLines(summary, cases) {
+  const lines = [];
+  const byId = new Map(cases.map((testCase) => [testCase.id, testCase]));
+  for (const row of summary.rows) {
+    const testCase = byId.get(row.id);
+    const expected = row.expectedTool ?? "(no tool)";
+    if (row.error) {
+      lines.push(`FAIL ${row.id}  error: ${row.error}`);
+    } else if (!row.selectionCorrect) {
+      lines.push(`FAIL ${row.id}  expected ${expected}, got ${row.actualTool ?? "(no tool)"}`);
+    } else if (row.args.failures.length > 0) {
+      lines.push(`FAIL ${row.id}  ${row.actualTool}  args mismatch: ${row.args.failures.join(", ")}`);
+    } else {
+      lines.push(`OK   ${row.id}  ${row.actualTool ?? "(no tool)"}`);
+    }
+    if (testCase?.note) lines.push(`     note: ${testCase.note}`);
+  }
+  return lines;
+}
+
+export function formatTable(summaries) {
+  const headers = ["model", "select", "args", "pass", "p50ms", "p95ms", "cny/call", "errors"];
+  const rows = summaries.map((summary) => [
+    summary.model,
+    pct(summary.selectionAccuracy),
+    pct(summary.argAccuracy),
+    pct(summary.passRate),
+    String(summary.p50LatencyMs),
+    String(summary.p95LatencyMs),
+    money(summary.avgCostCny, isEstimated(summary)),
+    String(summary.errors)
+  ]);
+
+  const widths = headers.map((header, index) =>
+    Math.max(header.length, ...rows.map((row) => row[index].length))
+  );
+  const pad = (value, index) => (index === 0 ? value.padEnd(widths[index]) : value.padStart(widths[index]));
+  const line = (cells) => cells.map(pad).join("  ");
+  const divider = widths.map((width) => "-".repeat(width)).join("  ");
+
+  return [line(headers), divider, ...rows.map(line)].join("\n");
+}
+
+export function formatMarkdown(suite, cases, summaries) {
+  const fingerprint = suiteFingerprint(suite, cases);
+  const lines = [
+    `# ${suite.name} — function-calling scorecard`,
+    "",
+    (suite.description || "").trim(),
+    "",
+    `- Suite: \`${suite.path}\``,
+    `- Cases: ${cases.length}`,
+    `- Tools: ${suite.tools.map((tool) => tool.name).join(", ")}`,
+    `- Fingerprint: \`${fingerprint.hash}\``,
+    `- Tool versions: ${fingerprint.tools.map((tool) => `${tool.name}@${tool.hash}`).join(", ")}`,
+    ""
+  ];
+
+  const withExec = summaries.some((summary) => summary.exec);
+  const headers = withExec
+    ? ["model", "select", "args", "pass", "exec", "run p50 ms", "max RSS MB", "p50 ms", "p95 ms", "CNY/call", "errors"]
+    : ["model", "select", "args", "pass", "p50 ms", "p95 ms", "CNY/call", "errors"];
+  lines.push(`| ${headers.join(" | ")} |`, `| ${headers.map(() => "---").join(" | ")} |`);
+
+  for (const summary of summaries) {
+    const cells = [
+      summary.model,
+      pct(summary.selectionAccuracy),
+      pct(summary.argAccuracy),
+      pct(summary.passRate)
+    ];
+    if (withExec) {
+      cells.push(summary.exec ? `${summary.exec.passed}/${summary.exec.executed}` : "-");
+      cells.push(summary.exec?.p50DurationMs ?? "-");
+      cells.push(summary.exec?.maxPeakRssMB ?? "-");
+    }
+    cells.push(
+      String(summary.p50LatencyMs),
+      String(summary.p95LatencyMs),
+      money(summary.avgCostCny, isEstimated(summary)),
+      String(summary.errors)
+    );
+    lines.push(`| ${cells.join(" | ")} |`);
+  }
+
+  for (const summary of summaries) {
+    lines.push(
+      "",
+      `## ${summary.model}（reviewed ${summary.reviewedTotal}/${summary.total}）`,
+      "",
+      "| case | expected | got | args | result |",
+      "| --- | --- | --- | --- | --- |"
+    );
+    for (const row of summary.rows) {
+      const result = row.error ? `error: ${row.error}` : row.passed ? "pass" : "fail";
+      const args = row.args.total
+        ? `${row.args.matched}/${row.args.total}${row.args.failures.length ? ` (miss: ${row.args.failures.join(", ")})` : ""}`
+        : "-";
+      lines.push(
+        `| ${row.id} | ${row.expectedTool ?? "(none)"} | ${row.actualTool ?? "(none)"} | ${args} | ${result} |`
+      );
+    }
+  }
+
+  return lines.join("\n");
+}

package/lib/runtime/harness.mjs

@@ -0,0 +1,145 @@
+import { readFileSync, writeSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+
+const args = parseArgs(process.argv.slice(2));
+
+function parseArgs(argv) {
+  const parsed = { entry: null, input: null, exportName: "run", warmup: 0 };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const next = () => {
+      const value = argv[i + 1];
+      if (value === undefined) throw new Error(`missing value for ${arg}`);
+      i += 1;
+      return value;
+    };
+    if (arg === "--entry") parsed.entry = next();
+    else if (arg === "--input") parsed.input = next();
+    else if (arg === "--export") parsed.exportName = next();
+    else if (arg === "--warmup") parsed.warmup = Number(next()) || 0;
+  }
+  return parsed;
+}
+
+function emit(payload) {
+  const line = `${JSON.stringify(payload)}\n`;
+  try {
+    writeSync(3, line);
+  } catch {
+    process.stdout.write(line);
+  }
+}
+
+function metrics({ durationMs, importMs, ruBefore, muBefore, peakRss, startRss }) {
+  const ru = process.resourceUsage();
+  const mu = process.memoryUsage();
+  return {
+    durationMs: Math.round(durationMs * 100) / 100,
+    importMs: Math.round(importMs * 100) / 100,
+    cpuUserMs: Math.round((ru.userCPUTime - ruBefore.userCPUTime) / 1000),
+    cpuSystemMs: Math.round((ru.systemCPUTime - ruBefore.systemCPUTime) / 1000),
+    fsReadDelta: ru.fsRead - ruBefore.fsRead,
+    fsWriteDelta: ru.fsWrite - ruBefore.fsWrite,
+    startRssMB: Math.round((startRss / 1048576) * 10) / 10,
+    peakRssMB: Math.round((Math.max(peakRss, mu.rss) / 1048576) * 10) / 10,
+    heapUsedDeltaMB: Math.round(((mu.heapUsed - muBefore.heapUsed) / 1048576) * 100) / 100
+  };
+}
+
+if (!args.entry || !args.input) {
+  emit({ ok: false, error: { name: "HarnessError", code: "BAD_ARGS", message: "--entry and --input are required" } });
+  process.exit(2);
+}
+
+const startRss = process.memoryUsage().rss;
+const importStarted = performance.now();
+let fn;
+try {
+  const moduleUrl = pathToFileURL(args.entry).href;
+  const mod = await import(moduleUrl);
+  const candidate = mod[args.exportName] ?? mod.default ?? mod.run;
+  if (typeof candidate !== "function") {
+    emit({
+      ok: false,
+      error: {
+        name: "HarnessError",
+        code: "EXPORT_NOT_FOUND",
+        message: `${args.entry} 需要导出 ${args.exportName}、default 或 run 函数`
+      },
+      metrics: { importMs: Math.round((performance.now() - importStarted) * 100) / 100 }
+    });
+    process.exit(2);
+  }
+  fn = candidate;
+} catch (error) {
+  emit({
+    ok: false,
+    error: { name: error?.name, code: error?.code, message: error?.message, phase: "import" },
+    metrics: { importMs: Math.round((performance.now() - importStarted) * 100) / 100 }
+  });
+  process.exit(1);
+}
+const importMs = performance.now() - importStarted;
+
+let input;
+try {
+  input = JSON.parse(readFileSync(args.input, "utf8"));
+} catch (error) {
+  emit({
+    ok: false,
+    error: { name: error?.name, code: error?.code, message: `input 读取失败: ${error?.message}`, phase: "input" },
+    metrics: { importMs }
+  });
+  process.exit(2);
+}
+
+let peakRss = process.memoryUsage().rss;
+const sampler = setInterval(() => {
+  peakRss = Math.max(peakRss, process.memoryUsage().rss);
+}, 10);
+if (typeof sampler.unref === "function") sampler.unref();
+
+try {
+  for (let i = 0; i < args.warmup; i += 1) {
+    await fn(input);
+  }
+} catch (error) {
+  clearInterval(sampler);
+  emit({
+    ok: false,
+    error: { name: error?.name, code: error?.code, message: error?.message, phase: "warmup" },
+    metrics: { importMs, durationMs: 0, startRssMB: Math.round((startRss / 1048576) * 10) / 10 }
+  });
+  process.exit(1);
+}
+
+const ruBefore = process.resourceUsage();
+const muBefore = process.memoryUsage();
+const started = performance.now();
+
+try {
+  const result = await fn(input);
+  const durationMs = performance.now() - started;
+  clearInterval(sampler);
+  emit({
+    ok: true,
+    result,
+    metrics: metrics({ durationMs, importMs, ruBefore, muBefore, peakRss, startRss })
+  });
+  process.exit(0);
+} catch (error) {
+  const durationMs = performance.now() - started;
+  clearInterval(sampler);
+  emit({
+    ok: false,
+    error: {
+      name: error?.name,
+      code: error?.code,
+      message: error?.message,
+      stack: String(error?.stack ?? "").split("\n").slice(0, 10).join("\n"),
+      phase: "call"
+    },
+    metrics: metrics({ durationMs, importMs, ruBefore, muBefore, peakRss, startRss })
+  });
+  process.exit(1);
+}