modelfunction-cli 0.1.0 → 0.1.1

package/README.md

@@ -5,7 +5,7 @@
 ## 安装
 
 ```bash
-npx -y modelfunction-cli@0.1.0 --version
+npx -y modelfunction-cli@0.1.1 --version
 # 或
 npm i -D modelfunction-cli
 npx mf --version
@@ -31,6 +31,28 @@ SILICONFLOW_API_KEY=... mf eval --models dsv4-flash
 OPENAI_API_KEY=... mf eval --models openai
 ```
 
+### 自建 / 内网网关
+
+企业内网基本都是 OpenAI 兼容协议，写一个 models-file 即可，Key 只走环境变量、不落盘：
+
+```json
+{
+  "internal-ds": {
+    "type": "openai",
+    "baseUrl": "https://llm.example.internal/v1",
+    "model": "deepseek-v4-flash",
+    "apiKeyEnv": "INTERNAL_LLM_KEY"
+  }
+}
+```
+
+```bash
+INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
+  --models-file my-models.json --models internal-ds
+```
+
+`pricing` 可省（省了就不输出成本）；要算成本请按自家价目填写，默认 registry 里的价格只是参考值。
+
 ## 命令
 
 | 命令 | 作用 |
@@ -54,7 +76,7 @@ OPENAI_API_KEY=... mf eval --models openai
   "mcpServers": {
     "modelfunction": {
       "command": "npx",
-      "args": ["-y", "modelfunction-cli@0.1.0", "mcp"]
+      "args": ["-y", "modelfunction-cli@0.1.1", "mcp"]
     }
   }
 }
@@ -72,4 +94,26 @@ OPENAI_API_KEY=... mf eval --models openai
 
 由你或你的编码 Agent 撰写，规则模板见包内 `skills/mf-cases/SKILL.md`。未复核用例（`reviewed:false`）不计入正式分数。
 
+工具一多就别手工誊抄，直接让编码 Agent 驱动 `mf`：
+
+```bash
+# 1. 把 mf 挂成 MCP server
+claude mcp add modelfunction -- npx -y modelfunction-cli@0.1.1 mcp
+```
+
+```text
+# 2. 让 Agent 扫仓库并写出套件草稿
+用 modelfunction 的 scan_tools 扫 .，probe=true，writeSuite=mf.suite.json
+
+# 3. 让 Agent 补用例
+按 skills/mf-cases/SKILL.md 的规则，为每个工具补 3 条正例和 1 条负例，用 add_cases
+写入 mf.suite.json，然后跑 coverage 看还缺哪些参数和负例
+```
+
+```bash
+# 4. 人工复核：把认可的用例 reviewed 改为 true，再跑正式评分
+mf coverage --suite mf.suite.json
+mf eval --suite mf.suite.json --models internal-ds
+```
+
 License: MIT

package/fixtures/suite.json

@@ -2,7 +2,7 @@
   "name": "recruiting-agent",
   "description": "招聘 Agent 的 function-calling 评测集：简历评分、面试题、薪资区间、JD 改写。",
   "tools": [
-    { "ref": "../../demo/function.json" },
+    { "ref": "./tools/resume_score.json" },
     {
       "name": "interview_question",
       "description": "根据岗位和候选人背景生成针对性的模拟面试题。",

package/fixtures/tools/resume_score.json

@@ -0,0 +1,37 @@
+{
+  "name": "resume_score",
+  "description": "简历与 JD 匹配评分",
+  "version": "0.1.0",
+  "input": {
+    "type": "object",
+    "required": ["resume", "jd"],
+    "properties": {
+      "resume": {
+        "type": "string",
+        "minLength": 20,
+        "description": "候选人简历文本"
+      },
+      "jd": {
+        "type": "string",
+        "minLength": 20,
+        "description": "职位描述文本"
+      }
+    }
+  },
+  "output": {
+    "type": "object",
+    "properties": {
+      "score": { "type": "number", "minimum": 0, "maximum": 100 },
+      "reasons": { "type": "array", "items": { "type": "string" } },
+      "matched": { "type": "array", "items": { "type": "string" } },
+      "missing": { "type": "array", "items": { "type": "string" } },
+      "years": { "type": "number" }
+    }
+  },
+  "sideEffects": "none",
+  "model": "mock-keyword-v1",
+  "price": { "amount": 0.02, "currency": "CNY", "unit": "call" },
+  "mcp": true,
+  "auth": { "type": "api_key" },
+  "rateLimit": { "requests": 60, "window": "1m" }
+}

package/lib/mcp-server.js

@@ -6,7 +6,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { loadSuite } from "./suite.js";
-import { scanProject, addTools, redactReport } from "./scan.js";
+import { scanProject, addTools, buildSuiteDraft, redactReport } from "./scan.js";
+import { writeFileAtomic } from "./fsutil.js";
 import { probeStdioServer } from "./mcp-probe.js";
 import { addCases, computeCoverage } from "./cases.js";
 
@@ -31,14 +32,15 @@ export async function startMcpServer() {
     {
       title: "Scan tools",
       description:
-        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。只读，不执行工具。",
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核）。",
       inputSchema: {
         root: z.string().optional().describe("仓库目录，默认进程当前目录"),
         probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
-        timeoutMs: z.number().optional().describe("单个 MCP server 超时（毫秒，默认 5000）")
+        timeoutMs: z.number().optional().describe("单个 MCP server 超时（毫秒，默认 5000）"),
+        writeSuite: z.string().optional().describe("套件草稿输出路径，如 mf.suite.json")
       }
     },
-    async ({ root, probe, timeoutMs }) => {
+    async ({ root, probe, timeoutMs, writeSuite }) => {
       const report = await scanProject(root ?? process.cwd());
       if (probe) {
         for (const mcpServer of [...report.mcpServers]) {
@@ -60,7 +62,11 @@ export async function startMcpServer() {
           report.probes.push({ server: mcpServer.name, ok: true, toolCount: added.length, tools: added });
         }
       }
-      return json(redactReport(report));
+      const payload = redactReport(report);
+      if (!writeSuite) return json(payload);
+      const target = resolve(writeSuite);
+      await writeFileAtomic(target, JSON.stringify(buildSuiteDraft(report), null, 2));
+      return json({ ...payload, suiteDraftPath: target });
     }
   );
 

package/mf.js

@@ -46,9 +46,9 @@ Usage:
   mf mcp                # start stdio MCP server for coding agents
 
 Eval options:
-  --suite <path>        eval suite (default: cli/fixtures/suite.json)
+  --suite <path>        eval suite (default: built-in demo suite)
   --models <ids>        comma-separated ids from models.json (default: models with API keys set)
-  --models-file <path>  model registry (default: cli/fixtures/models.json)
+  --models-file <path>  model registry (default: built-in registry)
   --case <ids>          run only these case ids (comma-separated, repeatable)
   --show-cases          print per-case results
   --json                print JSON instead of the table
@@ -563,7 +563,9 @@ function splitList(value) {
 const isMain = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
 
 if (isMain) {
-  const argv = process.argv.slice(2);
+  let argv = process.argv.slice(2);
+  // `npx modelfunction-cli mf scan` passes the bin name through; accept it.
+  if (argv[0] === "mf") argv = argv.slice(1);
   if (argv[0] === "--version" || argv[0] === "-v") {
     const pkg = JSON.parse(await readFile(new URL("./package.json", import.meta.url), "utf8"));
     process.stdout.write(`mf ${pkg.version}\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "license": "MIT",
   "description": "mf — function-calling accuracy scorecard for AI agents",
@@ -13,7 +13,7 @@
     "agent"
   ],
   "bin": {
-    "mf": "./mf.js"
+    "mf": "mf.js"
   },
   "files": [
     "mf.js",

package/skills/mf-cases/SKILL.md

@@ -9,6 +9,7 @@ description: 为 ModelFunction 评测套件撰写 function-calling 用例。当
 
 # 输入
 
+0. 还没有套件时：先调 `scan_tools`（`probe=true` 可连 MCP server）并传 `writeSuite: "mf.suite.json"`，得到工具清单草稿，再补用例。
 1. 工具定义：`mf.scan.json` 或 `mf.suite.json`（只看名字、description、parameters）
 2. 覆盖率现状：`mf coverage --suite <file> --json`
 3. 可选：用户提供的真实日志、投诉记录、产品文档