modelfunction-cli 0.0.0-stage → 0.1.1

package/mf.js

@@ -0,0 +1,638 @@
+#!/usr/bin/env node
+import { readFile } from "node:fs/promises";
+import { basename, dirname, extname, relative, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadSuite, toModelTools } from "./lib/suite.js";
+import { scoreModel, pct } from "./lib/score.js";
+import { formatTable, formatCaseLines, formatMarkdown } from "./lib/report.js";
+import { scanProject, buildSuiteDraft, formatScanSummary, addTools, redactReport } from "./lib/scan.js";
+import { probeStdioServer } from "./lib/mcp-probe.js";
+import { loadAdapterTools } from "./lib/adapter-import.js";
+import { addCases, computeCoverage, formatCoverage } from "./lib/cases.js";
+import { lintSuite, formatLint } from "./lib/lint.js";
+import { suiteFingerprint, hashText } from "./lib/fingerprint.js";
+import { runExecutionChecks } from "./lib/eval-exec.js";
+import { checkSandbox, formatSandboxReport } from "./lib/sandbox-check.js";
+import { compareScorecards, formatCompare } from "./lib/compare.js";
+import { analyzeAblation, formatAblation } from "./lib/ablate.js";
+import { runExec } from "./lib/exec.js";
+import { writeFileAtomic } from "./lib/fsutil.js";
+import { createMockAdapter } from "./lib/adapters/mock.js";
+import { createOpenAiAdapter } from "./lib/adapters/openai.js";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_SUITE = resolve(HERE, "fixtures/suite.json");
+const DEFAULT_MODELS = resolve(HERE, "fixtures/models.json");
+
+const SYSTEM_PROMPT = [
+  "You are a tool-calling assistant.",
+  "Call exactly one tool when the user request clearly matches it.",
+  "Do not call any tool when no tool applies; ask a clarifying question instead of inventing arguments.",
+  "Only use values that appear in the user message."
+].join(" ");
+
+const USAGE = `mf — function-calling accuracy scorecard
+
+Usage:
+  mf eval [options]
+  mf scan [options]
+  mf case add --suite <file> (--input <file> | stdin) [--source agent] [--reviewed] [--dry-run]
+  mf coverage [--suite <file>] [--json]
+  mf lint [--suite <file>] [--json]
+  mf compare <before.json> <after.json> [--json]
+  mf ablate --tool <name> [--suite <file>] [--models <ids>] [--json]
+  mf exec --entry <file> (--input <json> | --input-json <json>) [--runtime node|python] [--python <cmd>] [--export run] [--allow-read <paths>] [--allow-write <paths>] [--allow-net <hosts>] [--timeout <ms>] [--warmup <n>] [--json]
+  mf sandbox check [--json]   # probe available isolation engines on this machine
+  mf mcp                # start stdio MCP server for coding agents
+
+Eval options:
+  --suite <path>        eval suite (default: built-in demo suite)
+  --models <ids>        comma-separated ids from models.json (default: models with API keys set)
+  --models-file <path>  model registry (default: built-in registry)
+  --case <ids>          run only these case ids (comma-separated, repeatable)
+  --show-cases          print per-case results
+  --json                print JSON instead of the table
+  --out <file>          write scorecard (.md => markdown, otherwise JSON)
+  --fail-under <ratio>  exit 2 if any model's selection accuracy is below ratio (e.g. 0.8)
+  --baseline <file>     previous scorecard JSON (used with --fail-on-regression)
+  --fail-on-regression  exit 2 if any reviewed case regressed vs --baseline
+  --execute             run toolInput and assert expect (needs --entry or suite.execution.entry)
+  --exec-timeout <ms>   per execution timeout (suite.execution.timeoutMs or 10000)
+  --allow-side-effects  allow executing tools declaring write/external/unknown sideEffects
+
+Scan options:
+  --root <dir>          repository to scan (default: .)
+  --from <file>         import tools from a module (export default [...] or export const tools)
+  --probe               start stdio MCP servers and read tools/list
+  --timeout <ms>        probe timeout per server (default: 5000)
+  --out <file>          suite draft to write (default: mf.suite.json)
+  --report <file>       scan report to write (default: mf.scan.json)
+  --no-write            print only, do not write files
+
+Env keys: DEEPSEEK_API_KEY, DASHSCOPE_API_KEY, ZHIPU_API_KEY, OPENAI_API_KEY, SILICONFLOW_API_KEY
+
+Examples:
+  mf eval --models mock --show-cases
+  mf eval --models deepseek,qwen --out scorecard.md
+  mf scan
+  mf scan --root ../my-agent --no-write
+  mf case add --suite mf.suite.json --input cases.json
+  mf coverage --suite mf.suite.json
+`;
+
+export async function runEval(options, io = console) {
+  const suite = await loadSuite(options.suite);
+  const cases = selectCases(suite.cases, options.cases);
+  const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
+  const tools = toModelTools(suite.tools);
+
+  const summaries = [];
+  for (const adapter of adapters) {
+    const results = await evaluate(adapter, tools, cases);
+    summaries.push(
+      scoreModel({
+        model: adapter.id,
+        cases,
+        results,
+        pricing: registry[adapter.id]?.pricing ?? null
+      })
+    );
+  }
+
+  let executionInfo = null;
+  if (options.execute) {
+    const entry = options.execEntry ?? options.entry ?? suite.execution?.entry;
+    if (!entry) throw new Error("--execute 需要 --entry，或套件中的 execution.entry");
+    await runExecutionChecks({
+      suite,
+      cases,
+      summaries,
+      options: {
+        ...options,
+        execEntry: entry,
+        execRuntime: options.runtime,
+        execPython: options.python
+      }
+    });
+    const entryPath = resolve(entry);
+    const runtime = suite.execution?.runtime ?? options.runtime ?? "node";
+    executionInfo = {
+      runtime,
+      engine: runtime === "python" ? "python-audit" : "node-permission",
+      entry: basename(entryPath),
+      hash: hashText(await readFile(entryPath, "utf8"))
+    };
+  }
+
+  if (options.out) {
+    const target = resolve(options.out);
+    const markdown = extname(target).toLowerCase() === ".md";
+    const payload = markdown
+      ? formatMarkdown(suite, cases, summaries)
+      : JSON.stringify(serialize(suite, cases, summaries, executionInfo), null, 2);
+    await writeFileAtomic(target, payload);
+    io.log(`wrote ${target}`);
+  }
+
+  if (options.json) {
+    io.log(JSON.stringify(serialize(suite, cases, summaries, executionInfo), null, 2));
+  } else {
+    io.log(`suite: ${suite.name} — ${cases.length} cases, ${suite.tools.length} tools`);
+    io.log(`file:  ${suite.path}`);
+    for (const summary of summaries) {
+      io.log("");
+      if (options.showCases) {
+        io.log(`[${summary.model}]`);
+        for (const line of formatCaseLines(summary, cases)) io.log(`  ${line}`);
+        io.log("");
+      }
+      io.log(formatTable([summary]));
+      if (summary.unreviewedTotal > 0) {
+        io.log(`note: ${summary.unreviewedTotal} unreviewed case(s) excluded from select/args/pass`);
+      }
+      if (summary.exec) {
+        const parts = [
+          `exec: ${summary.exec.passed}/${summary.exec.executed} passed`,
+          `engine ${summary.exec.engine}`
+        ];
+        if (summary.exec.skipped > 0) {
+          parts.push(
+            `${summary.exec.skipped} skipped${summary.exec.blocked > 0 ? `（${summary.exec.blocked} 因 sideEffects 被拦截，需 --allow-side-effects）` : ""}`
+          );
+        }
+        if (summary.exec.p50DurationMs !== null) parts.push(`p50 ${summary.exec.p50DurationMs}ms`);
+        if (summary.exec.maxPeakRssMB !== null) parts.push(`peak ${summary.exec.maxPeakRssMB}MB`);
+        io.log(parts.join(", "));
+      }
+    }
+    if (summaries.length > 1) {
+      io.log("");
+      io.log("combined");
+      io.log(formatTable(summaries));
+    }
+    if (summaries.some((summary) => summary.rows.some((row) => row.usage?.estimated))) {
+      io.log("");
+      io.log("note: tokens marked with ~ are estimated (mock adapter); pricing in models.json is indicative.");
+    }
+  }
+
+  let regressions = [];
+  if (options.failOnRegression) {
+    if (!options.baseline) {
+      io.error("--fail-on-regression requires --baseline <scorecard.json>");
+      return 2;
+    }
+    const baseline = JSON.parse(await readFile(resolve(options.baseline), "utf8"));
+    const comparison = compareScorecards(baseline, serialize(suite, cases, summaries, executionInfo));
+    if (!options.json) {
+      io.log("");
+      io.log(formatCompare(comparison));
+    }
+    const comparable = comparison.models.filter((model) => model.comparable && model.reviewed > 0);
+    if (comparable.length === 0) {
+      io.error("定义或用例已变更，无法判定回归（门禁跳过）");
+    } else {
+      regressions = comparable.flatMap((model) =>
+        model.regressions.map((item) => `${model.model}/${item.id}`)
+      );
+      if (regressions.length > 0) io.error(`regressions: ${regressions.join(", ")}`);
+    }
+  }
+
+  const failed = options.failUnder === null
+    ? []
+    : summaries.filter((summary) => summary.selectionAccuracy < options.failUnder);
+  if (failed.length > 0) {
+    io.error(
+      `below threshold ${pct(options.failUnder)}: ${failed
+        .map((summary) => `${summary.model} (${pct(summary.selectionAccuracy)})`)
+        .join(", ")}`
+    );
+    return 2;
+  }
+  if (regressions.length > 0) return 2;
+  return 0;
+}
+
+export async function runAblate(options, io = console) {
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
+  if (!options.tool) throw new Error("--tool is required");
+  const ablated = suite.tools.find((tool) => tool.name === options.tool);
+  if (!ablated) throw new Error(`unknown tool "${options.tool}"`);
+
+  const baseCases = selectCases(suite.cases, options.cases);
+  const reducedCases = baseCases.filter((testCase) => testCase.expectedTool !== ablated.name);
+  const droppedCases = baseCases.length - reducedCases.length;
+  const { registry, adapters } = await resolveModels(options.models, options.modelsFile, io);
+
+  const fullTools = toModelTools(suite.tools);
+  const reducedTools = toModelTools(suite.tools.filter((tool) => tool.name !== ablated.name));
+  const models = [];
+
+  for (const adapter of adapters) {
+    const pricing = registry[adapter.id]?.pricing ?? null;
+    const fullResults = await evaluate(adapter, fullTools, baseCases);
+    const full = scoreModel({ model: adapter.id, cases: baseCases, results: fullResults, pricing });
+    const reducedResults = await evaluate(adapter, reducedTools, reducedCases);
+    const reduced = scoreModel({ model: adapter.id, cases: reducedCases, results: reducedResults, pricing });
+    models.push(analyzeAblation(adapter.id, full, reduced));
+  }
+
+  const payload = { tool: ablated.name, suite: suite.name, droppedCases, models };
+  io.log(options.json ? JSON.stringify(payload, null, 2) : formatAblation(payload));
+  return 0;
+}
+
+export async function runScan(options, io = console) {
+  const report = await scanProject(options.root ?? ".");
+
+  if (options.from) {
+    const tools = await loadAdapterTools(options.from);
+    addTools(report, tools, {
+      source: relative(process.cwd(), resolve(options.from)) || options.from,
+      origin: "adapter",
+      confidence: "high",
+      note: "从用户模块动态导入，schema 为运行时真实值"
+    });
+  }
+
+  if (options.probe) {
+    for (const server of [...report.mcpServers]) {
+      if (server.transport !== "stdio") {
+        report.probes.push({ server: server.name, ok: false, error: "http transport 暂不支持" });
+        continue;
+      }
+      const result = await probeStdioServer(server, { timeoutMs: options.timeoutMs });
+      if (!result.ok) {
+        report.probes.push({ server: server.name, ok: false, error: result.error, stderr: result.stderr });
+        continue;
+      }
+      const added = addTools(report, result.tools, {
+        source: server.source,
+        origin: "mcp-probe",
+        confidence: "high",
+        note: `server ${server.name}`
+      });
+      report.probes.push({ server: server.name, ok: true, toolCount: added.length, tools: added });
+    }
+  }
+
+  const draft = buildSuiteDraft(report);
+
+  if (options.json) {
+    io.log(JSON.stringify({ report: redactReport(report), draft }, null, 2));
+    return report.tools.length > 0 ? 0 : 3;
+  }
+
+  io.log(formatScanSummary(report));
+
+  if (!options.noWrite) {
+    const suitePath = resolve(options.out ?? "mf.suite.json");
+    const reportPath = resolve(options.report ?? "mf.scan.json");
+    await writeFileAtomic(suitePath, JSON.stringify(draft, null, 2));
+    await writeFileAtomic(reportPath, JSON.stringify(redactReport(report), null, 2));
+    io.log("");
+    io.log(`wrote ${suitePath}`);
+    io.log(`wrote ${reportPath}`);
+  }
+
+  return report.tools.length > 0 ? 0 : 3;
+}
+
+export async function runCaseAdd(options, io = console) {
+  const suitePath = resolve(options.suite ?? DEFAULT_SUITE);
+  let payload;
+  if (options.inputPath) {
+    payload = JSON.parse(await readFile(resolve(options.inputPath), "utf8"));
+  } else {
+    payload = JSON.parse(await readStdin());
+  }
+
+  const result = await addCases(suitePath, payload, {
+    source: options.source,
+    reviewed: options.reviewed,
+    dryRun: options.dryRun
+  });
+
+  io.log(`suite: ${result.path}`);
+  io.log(`added: ${result.added.length}${result.dryRun ? " (dry-run)" : ""}`);
+  for (const testCase of result.added) {
+    io.log(
+      `  + ${testCase.id} → ${testCase.expectedTool ?? "(no tool)"} [${testCase.source}${testCase.reviewed ? ", reviewed" : ""}]`
+    );
+  }
+  if (result.warnings.length > 0) {
+    for (const warning of result.warnings) io.error(`warn: ${warning}`);
+  }
+  if (result.rejected.length > 0) {
+    io.error(`rejected: ${result.rejected.length}`);
+    for (const item of result.rejected) io.error(`  - ${item.id}: ${item.reason}`);
+    return 1;
+  }
+  return 0;
+}
+
+export async function runCoverage(options, io = console) {
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
+  const report = computeCoverage(suite);
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatCoverage(report));
+  return 0;
+}
+
+export async function runSandboxCheck(options, io = console) {
+  const report = checkSandbox();
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatSandboxReport(report));
+  return 0;
+}
+
+export async function runLint(options, io = console) {
+  const suite = await loadSuite(resolve(options.suite ?? DEFAULT_SUITE));
+  const report = lintSuite(suite);
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatLint(report));
+  return report.counts.error > 0 ? 1 : 0;
+}
+
+export async function runCompare(options, io = console) {
+  const [beforePath, afterPath] = options.paths;
+  const before = JSON.parse(await readFile(resolve(beforePath), "utf8"));
+  const after = JSON.parse(await readFile(resolve(afterPath), "utf8"));
+  const report = compareScorecards(before, after);
+  io.log(options.json ? JSON.stringify(report, null, 2) : formatCompare(report));
+  const changed = report.toolChanges.length > 0 || report.caseChanges.length > 0;
+  return changed ? 1 : 0;
+}
+
+async function readStdin() {
+  const chunks = [];
+  for await (const chunk of process.stdin) chunks.push(chunk);
+  return Buffer.concat(chunks).toString("utf8");
+}
+
+function selectCases(cases, ids) {
+  if (!ids || ids.length === 0) return cases;
+  const wanted = new Set(ids);
+  const missing = ids.filter((id) => !cases.some((testCase) => testCase.id === id));
+  if (missing.length > 0) throw new Error(`unknown case ids: ${missing.join(", ")}`);
+  return cases.filter((testCase) => wanted.has(testCase.id));
+}
+
+async function resolveModels(ids, modelsFile, io) {
+  const registry = JSON.parse(await readFile(resolve(modelsFile), "utf8"));
+  const explicit = Boolean(ids && ids.length > 0);
+  const chosen = explicit ? ids : autoSelect(registry);
+  const adapters = [];
+
+  for (const id of chosen) {
+    const config = registry[id];
+    if (!config) {
+      throw new Error(`unknown model "${id}" (available: ${Object.keys(registry).join(", ")})`);
+    }
+    if (config.type === "mock") {
+      adapters.push({ id, ...createMockAdapter() });
+      continue;
+    }
+    if (config.type !== "openai") {
+      throw new Error(`model "${id}": unsupported type "${config.type}"`);
+    }
+    if (!process.env[config.apiKeyEnv]) {
+      if (explicit) throw new Error(`model "${id}": environment variable ${config.apiKeyEnv} is not set`);
+      io.error(`skip ${id}: ${config.apiKeyEnv} is not set`);
+      continue;
+    }
+    adapters.push({ id, ...createOpenAiAdapter({ id, ...config }) });
+  }
+
+  if (adapters.length === 0) {
+    if (explicit) throw new Error("no runnable models");
+    io.error("no API keys found; falling back to the mock adapter");
+    adapters.push({ id: "mock", ...createMockAdapter() });
+  }
+  return { registry, adapters };
+}
+
+function autoSelect(registry) {
+  return Object.entries(registry)
+    .filter(([, config]) => config.type === "openai" && process.env[config.apiKeyEnv])
+    .map(([id]) => id);
+}
+
+async function evaluate(adapter, tools, cases) {
+  const results = [];
+  for (const testCase of cases) {
+    const messages = [
+      { role: "system", content: SYSTEM_PROMPT },
+      { role: "user", content: testCase.input }
+    ];
+    try {
+      results.push(await adapter.call({ input: testCase.input, messages, tools }));
+    } catch (error) {
+      results.push({
+        ok: false,
+        error: String(error?.message ?? error),
+        toolCalls: [],
+        usage: { input: 0, output: 0 },
+        latencyMs: 0
+      });
+    }
+  }
+  return results;
+}
+
+function serialize(suite, cases, summaries, executionInfo = null) {
+  return {
+    suite: {
+      name: suite.name,
+      path: suite.path,
+      tools: suite.tools.map((tool) => tool.name),
+      cases: cases.length,
+      fingerprint: suiteFingerprint(suite, cases),
+      ...(executionInfo ? { execution: executionInfo } : {})
+    },
+    models: summaries
+  };
+}
+
+function parseArgs(argv) {
+  const options = {
+    command: argv[0],
+    suite: DEFAULT_SUITE,
+    models: null,
+    modelsFile: DEFAULT_MODELS,
+    cases: null,
+    showCases: false,
+    json: false,
+    out: null,
+    failUnder: null,
+    root: ".",
+    report: null,
+    noWrite: false,
+    from: null,
+    probe: false,
+    timeoutMs: 5000,
+    inputPath: null,
+    source: "agent",
+    reviewed: false,
+    dryRun: false,
+    tool: null,
+    baseline: null,
+    failOnRegression: false,
+    execute: false,
+    execEntry: null,
+    execTimeoutMs: 10_000,
+    allowSideEffects: false,
+    entry: null,
+    runtime: "node",
+    python: "python",
+    exportName: "run",
+    inputJson: null,
+    allowRead: [],
+    allowWrite: [],
+    allowNet: [],
+    passEnv: [],
+    warmup: 0
+  };
+
+  for (let i = 1; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const value = () => {
+      const next = argv[i + 1];
+      if (next === undefined || next.startsWith("--")) throw new Error(`missing value for ${arg}`);
+      i += 1;
+      return next;
+    };
+    switch (arg) {
+      case "--suite": options.suite = resolve(value()); break;
+      case "--models": options.models = splitList(value()); break;
+      case "--models-file": options.modelsFile = resolve(value()); break;
+      case "--case": options.cases = (options.cases ?? []).concat(splitList(value())); break;
+      case "--show-cases": options.showCases = true; break;
+      case "--json": options.json = true; break;
+      case "--out": options.out = value(); break;
+      case "--root": options.root = resolve(value()); break;
+      case "--from": options.from = value(); break;
+      case "--probe": options.probe = true; break;
+      case "--timeout": {
+        const raw = Number(value());
+        if (!Number.isFinite(raw) || raw <= 0) throw new Error("--timeout expects a positive number of milliseconds");
+        options.timeoutMs = raw;
+        break;
+      }
+      case "--report": options.report = value(); break;
+      case "--no-write": options.noWrite = true; break;
+      case "--input": options.inputPath = value(); break;
+      case "--source": options.source = value(); break;
+      case "--reviewed": options.reviewed = true; break;
+      case "--dry-run": options.dryRun = true; break;
+      case "--tool": options.tool = value(); break;
+      case "--baseline": options.baseline = resolve(value()); break;
+      case "--fail-on-regression": options.failOnRegression = true; break;
+      case "--entry": options.entry = value(); break;
+      case "--execute": options.execute = true; break;
+      case "--exec-timeout": options.execTimeoutMs = Number(value()) || 10_000; break;
+      case "--allow-side-effects": options.allowSideEffects = true; break;
+      case "--runtime": options.runtime = value(); break;
+      case "--python": options.python = value(); break;
+      case "--export": options.exportName = value(); break;
+      case "--input-json": options.inputJson = value(); break;
+      case "--allow-read": options.allowRead = splitList(value()); break;
+      case "--allow-write": options.allowWrite = splitList(value()); break;
+      case "--allow-net": options.allowNet = splitList(value()); break;
+      case "--pass-env": options.passEnv = splitList(value()); break;
+      case "--warmup": options.warmup = Number(value()) || 0; break;
+      case "--fail-under": {
+        const raw = Number(value());
+        if (!Number.isFinite(raw) || raw < 0 || raw > 1) throw new Error("--fail-under expects a ratio between 0 and 1");
+        options.failUnder = raw;
+        break;
+      }
+      case "-h":
+      case "--help":
+        options.help = true;
+        break;
+      default:
+        throw new Error(`unknown option "${arg}"`);
+    }
+  }
+  return options;
+}
+
+function splitList(value) {
+  return value.split(",").map((item) => item.trim()).filter(Boolean);
+}
+
+const isMain = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (isMain) {
+  let argv = process.argv.slice(2);
+  // `npx modelfunction-cli mf scan` passes the bin name through; accept it.
+  if (argv[0] === "mf") argv = argv.slice(1);
+  if (argv[0] === "--version" || argv[0] === "-v") {
+    const pkg = JSON.parse(await readFile(new URL("./package.json", import.meta.url), "utf8"));
+    process.stdout.write(`mf ${pkg.version}\n`);
+    process.exit(0);
+  } else if (argv[0] === "sandbox") {
+    if (argv[1] !== "check") {
+      process.stderr.write("usage: mf sandbox check [--json]\n");
+      process.exit(1);
+    }
+    try {
+      const code = await runSandboxCheck({ json: argv.includes("--json") });
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${error?.message ?? error}\n`);
+      process.exit(1);
+    }
+  } else if (argv[0] === "compare") {
+    const paths = argv.slice(1).filter((arg) => !arg.startsWith("--"));
+    if (paths.length !== 2) {
+      process.stderr.write("usage: mf compare <before.json> <after.json> [--json]\n");
+      process.exit(1);
+    }
+    try {
+      const code = await runCompare({ paths, json: argv.includes("--json") });
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${error?.message ?? error}\n`);
+      process.exit(1);
+    }
+  } else if (argv[0] === "mcp") {
+    try {
+      const { startMcpServer } = await import("./lib/mcp-server.js");
+      await startMcpServer();
+    } catch (error) {
+      process.stderr.write(`${error?.message ?? error}\n`);
+      process.exit(1);
+    }
+  } else if (argv.length === 0 || argv[0] === "help" || argv[0] === "-h" || argv[0] === "--help") {
+    process.stdout.write(USAGE);
+  } else if (!["eval", "scan", "case", "coverage", "lint", "ablate", "exec"].includes(argv[0])) {
+    process.stderr.write(`unknown command "${argv[0]}"\n\n${USAGE}`);
+    process.exit(1);
+  } else if (argv[0] === "case" && argv[1] !== "add") {
+    process.stderr.write(`usage: mf case add --suite <file> (--input <file> | stdin)\n`);
+    process.exit(1);
+  } else {
+    try {
+      const args = argv[0] === "case" ? [argv[0], ...argv.slice(2)] : argv;
+      const options = parseArgs(args);
+      const code =
+        argv[0] === "scan"
+          ? await runScan(options)
+          : argv[0] === "case"
+            ? await runCaseAdd(options)
+            : argv[0] === "coverage"
+              ? await runCoverage(options)
+              : argv[0] === "lint"
+                ? await runLint(options)
+                : argv[0] === "ablate"
+                  ? await runAblate(options)
+                  : argv[0] === "exec"
+                    ? await runExec(options)
+                    : await runEval(options);
+      process.exit(code);
+    } catch (error) {
+      process.stderr.write(`${error?.message ?? error}\n`);
+      process.exit(1);
+    }
+  }
+}

package/package.json

@@ -1,6 +1,40 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "type": "module",
+  "license": "MIT",
+  "description": "mf — function-calling accuracy scorecard for AI agents",
+  "keywords": [
+    "llm",
+    "function-calling",
+    "tool-use",
+    "mcp",
+    "eval",
+    "agent"
+  ],
+  "bin": {
+    "mf": "mf.js"
+  },
+  "files": [
+    "mf.js",
+    "lib",
+    "fixtures",
+    "skills",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "node --test \"test/**/*.test.js\"",
+    "eval": "node mf.js eval",
+    "eval:mock": "node mf.js eval --models mock --show-cases",
+    "pack:check": "npm pack --dry-run",
+    "prepublishOnly": "npm test"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.32.2",
+    "zod": "^4.6.7"
+  }
+}