modelfunction-cli 0.1.0 → 0.1.2

package/README.md

@@ -5,12 +5,19 @@
 ## 安装
 
 ```bash
-npx -y modelfunction-cli@0.1.0 --version
+npx -y modelfunction-cli@0.1.2 --version
 # 或
 npm i -D modelfunction-cli
 npx mf --version
 ```
 
+全局安装（可选）后命令为 `mf`，也可以用 `modelfunction`——避开装了 TeX 的机器上 `mf` 与 Metafont 同名：
+
+```bash
+npm i -g modelfunction-cli
+modelfunction --version
+```
+
 ## 五步上手
 
 ```bash
@@ -31,6 +38,28 @@ SILICONFLOW_API_KEY=... mf eval --models dsv4-flash
 OPENAI_API_KEY=... mf eval --models openai
 ```
 
+### 自建 / 内网网关
+
+企业内网基本都是 OpenAI 兼容协议，写一个 models-file 即可，Key 只走环境变量、不落盘：
+
+```json
+{
+  "internal-ds": {
+    "type": "openai",
+    "baseUrl": "https://llm.example.internal/v1",
+    "model": "deepseek-v4-flash",
+    "apiKeyEnv": "INTERNAL_LLM_KEY"
+  }
+}
+```
+
+```bash
+INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
+  --models-file my-models.json --models internal-ds
+```
+
+`pricing` 可省（省了就不输出成本）；要算成本请按自家价目填写，默认 registry 里的价格只是参考值。
+
 ## 命令
 
 | 命令 | 作用 |
@@ -54,7 +83,7 @@ OPENAI_API_KEY=... mf eval --models openai
   "mcpServers": {
     "modelfunction": {
       "command": "npx",
-      "args": ["-y", "modelfunction-cli@0.1.0", "mcp"]
+      "args": ["-y", "modelfunction-cli@0.1.2", "mcp"]
     }
   }
 }
@@ -72,4 +101,26 @@ OPENAI_API_KEY=... mf eval --models openai
 
 由你或你的编码 Agent 撰写，规则模板见包内 `skills/mf-cases/SKILL.md`。未复核用例（`reviewed:false`）不计入正式分数。
 
+工具一多就别手工誊抄，直接让编码 Agent 驱动 `mf`：
+
+```bash
+# 1. 把 mf 挂成 MCP server
+claude mcp add modelfunction -- npx -y modelfunction-cli@0.1.2 mcp
+```
+
+```text
+# 2. 让 Agent 扫仓库并写出套件草稿
+用 modelfunction 的 scan_tools 扫 .，probe=true，writeSuite=mf.suite.json
+
+# 3. 让 Agent 补用例
+按 skills/mf-cases/SKILL.md 的规则，为每个工具补 3 条正例和 1 条负例，用 add_cases
+写入 mf.suite.json，然后跑 coverage 看还缺哪些参数和负例
+```
+
+```bash
+# 4. 人工复核：把认可的用例 reviewed 改为 true，再跑正式评分
+mf coverage --suite mf.suite.json
+mf eval --suite mf.suite.json --models internal-ds
+```
+
 License: MIT

package/fixtures/suite.json

@@ -2,7 +2,7 @@
   "name": "recruiting-agent",
   "description": "招聘 Agent 的 function-calling 评测集：简历评分、面试题、薪资区间、JD 改写。",
   "tools": [
-    { "ref": "../../demo/function.json" },
+    { "ref": "./tools/resume_score.json" },
     {
       "name": "interview_question",
       "description": "根据岗位和候选人背景生成针对性的模拟面试题。",

package/fixtures/tools/resume_score.json

@@ -0,0 +1,37 @@
+{
+  "name": "resume_score",
+  "description": "简历与 JD 匹配评分",
+  "version": "0.1.0",
+  "input": {
+    "type": "object",
+    "required": ["resume", "jd"],
+    "properties": {
+      "resume": {
+        "type": "string",
+        "minLength": 20,
+        "description": "候选人简历文本"
+      },
+      "jd": {
+        "type": "string",
+        "minLength": 20,
+        "description": "职位描述文本"
+      }
+    }
+  },
+  "output": {
+    "type": "object",
+    "properties": {
+      "score": { "type": "number", "minimum": 0, "maximum": 100 },
+      "reasons": { "type": "array", "items": { "type": "string" } },
+      "matched": { "type": "array", "items": { "type": "string" } },
+      "missing": { "type": "array", "items": { "type": "string" } },
+      "years": { "type": "number" }
+    }
+  },
+  "sideEffects": "none",
+  "model": "mock-keyword-v1",
+  "price": { "amount": 0.02, "currency": "CNY", "unit": "call" },
+  "mcp": true,
+  "auth": { "type": "api_key" },
+  "rateLimit": { "requests": 60, "window": "1m" }
+}

package/lib/lint.js

@@ -15,6 +15,42 @@ const GENERIC_NAMES = new Set([
 
 const WHEN_HINT = /(用于|用来|当|如果|适用|适合|use when|use this|helps? )/i;
 
+// Identifier tokens, matched on token boundaries only: `request_text` is text,
+// `request_id` is an identifier. Strong = platform-injected (session/tenant/trace
+// class); weak = business id the user may or may not supply.
+const STRONG_ID_TOKENS = new Set([
+  "uuid", "guid", "sid", "nonce", "token", "signature",
+  "sessionid", "tenantid", "traceid", "threadid", "conversationid", "correlationid", "requestid",
+  "session", "tenant", "trace", "thread", "conversation", "correlation"
+]);
+const WEAK_ID_TOKENS = new Set(["id", "ids"]);
+const OPAQUE_ID_SCHEMA = /uuid|[0-9a-f]{8}-[0-9a-f]{4}/i;
+const DESCRIPTION_ID_HINT = /(会话\s*id|session\s*id|租户|tenant|trace|请求\s*id|request\s*id|签名|signature|\btoken\b|nonce|uuid|标识|编号)/i;
+const ID_SOURCE_HINT = /(来源|取自|来自|传入|注入|平台|系统|上下文|请求头|header|无需|自动|会提供|由调用方|上一步)/i;
+const FIXED_ID_LENGTHS = new Set([8, 16, 24, 32, 36, 40, 64]);
+
+function nameTokens(name) {
+  return String(name)
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .split(/[^A-Za-z0-9]+/)
+    .map((token) => token.toLowerCase())
+    .filter(Boolean);
+}
+
+function idStrength(name) {
+  const tokens = nameTokens(name);
+  if (tokens.length === 0) return null;
+  const last = tokens[tokens.length - 1];
+  const joinedTail = tokens.slice(-2).join("");
+  if (STRONG_ID_TOKENS.has(last) || STRONG_ID_TOKENS.has(joinedTail)) return "strong";
+  if (WEAK_ID_TOKENS.has(last)) return "weak";
+  return null;
+}
+
+const BOUNDARY_HINT = /(不适用|不要用于|不用于|不能用于|而非|区别于|not for|do not use)/i;
+// Chinese descriptions are compared as bigrams, so mid-0.3s similarity is noise.
+const NO_BOUNDARY_SIMILARITY = 0.5;
+
 export function lintSuite(suite) {
   const findings = [];
   const add = (severity, code, tool, message, extra = {}) =>
@@ -63,6 +99,55 @@ export function lintSuite(suite) {
       });
     }
 
+    for (const name of required) {
+      const schema = properties[name];
+      if (!schema) continue;
+      const description = String(schema.description ?? "");
+      const fixedLength =
+        typeof schema.minLength === "number" &&
+        schema.minLength === schema.maxLength &&
+        FIXED_ID_LENGTHS.has(schema.minLength);
+      const schemaOpaque =
+        (typeof schema.format === "string" && OPAQUE_ID_SCHEMA.test(schema.format)) ||
+        (typeof schema.pattern === "string" && OPAQUE_ID_SCHEMA.test(schema.pattern)) ||
+        fixedLength;
+      const nameStrength = idStrength(name);
+      // Name tokens alone are a weak signal: the author picks the names. Schema
+      // shape and the description are what the model actually reasons from.
+      const strong = schemaOpaque || DESCRIPTION_ID_HINT.test(description) || nameStrength === "strong";
+      const weak = nameStrength === "weak";
+      const scalar = schema.type === undefined || schema.type === "string";
+      if ((strong || weak) && scalar && !ID_SOURCE_HINT.test(description)) {
+        add(
+          strong ? "warn" : "info",
+          "required-unsourced",
+          tool.name,
+          strong
+            ? `必填参数 ${name} 像是外部注入的标识，但没说明从哪里获取；模型拿不到它时可能干脆不调用任何工具`
+            : `必填参数 ${name} 是业务 ID；如果用户不会在对话里给出，模型可能编造或干脆不调用`,
+          {
+            param: name,
+            suggestion: `改为平台注入（移出 schema），或在 description 写明来源，例如「由平台在会话开始时注入，模型无需生成」`
+          }
+        );
+      }
+    }
+
+    const toolCases = (suite.cases ?? []).filter((testCase) => testCase.expectedTool === tool.name);
+    if (toolCases.length > 0) {
+      for (const name of required) {
+        const covered = toolCases.some(
+          (testCase) => testCase.expectedArgs && Object.prototype.hasOwnProperty.call(testCase.expectedArgs, name)
+        );
+        if (!covered) {
+          add("info", "required-uncovered", tool.name, `必填参数 ${name} 没有任何用例校验过，模型是否真能填对它还没有证据`, {
+            param: name,
+            suggestion: `补一条用例在 expectedArgs 里断言 ${name}，或把它降为可选 / 改为平台注入`
+          });
+        }
+      }
+    }
+
     for (const [name, schema] of Object.entries(properties)) {
       if (!schema?.description) {
         add("warn", "param-desc-missing", tool.name, `参数 ${name} 没有说明`, {
@@ -83,9 +168,21 @@ export function lintSuite(suite) {
   }
 
   for (const pair of suspectedPairs(suite.tools)) {
-    add("warn", "pair-overlap", pair.a, `与 ${pair.b} 的描述相似度 ${pair.similarity.toFixed(2)}，容易被混淆`, {
-      suggestion: `在 ${pair.a} 和 ${pair.b} 的 description 里各写一句“不适用于什么”，并补一条歧义用例`
+    const similarity = pair.similarity.toFixed(2);
+    const declaresBoundary = [pair.a, pair.b].some((name) => {
+      const tool = suite.tools.find((item) => item.name === name);
+      return BOUNDARY_HINT.test(tool?.description ?? "");
     });
+    if (declaresBoundary) {
+      add("info", "pair-overlap", pair.a, `与 ${pair.b} 的描述相似度 ${similarity}，已写互斥边界`, {
+        pair: [pair.a, pair.b]
+      });
+    } else if (pair.similarity >= NO_BOUNDARY_SIMILARITY) {
+      add("warn", "pair-no-boundary", pair.a, `与 ${pair.b} 的描述相似度 ${similarity}，且两边的 description 都没写“不适用于”边界，模型只能靠相似度硬猜`, {
+        pair: [pair.a, pair.b],
+        suggestion: `在 ${pair.a} 和 ${pair.b} 的 description 里各写一句“不适用于什么”，并补一条歧义用例`
+      });
+    }
   }
 
   const counts = { error: 0, warn: 0, info: 0 };

package/lib/mcp-server.js

@@ -6,7 +6,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { loadSuite } from "./suite.js";
-import { scanProject, addTools, redactReport } from "./scan.js";
+import { scanProject, addTools, buildSuiteDraft, redactReport } from "./scan.js";
+import { writeFileAtomic } from "./fsutil.js";
 import { probeStdioServer } from "./mcp-probe.js";
 import { addCases, computeCoverage } from "./cases.js";
 
@@ -31,14 +32,15 @@ export async function startMcpServer() {
     {
       title: "Scan tools",
       description:
-        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。只读，不执行工具。",
+        "扫描仓库里的 function.json / OpenAPI / MCP 配置 / SKILL.md / 代码内定义。probe=true 时连接 stdio MCP server 拉取 tools/list。不执行工具。默认只读；传 writeSuite 时在该路径写出套件草稿（cases 为空，需补用例并复核）。",
       inputSchema: {
         root: z.string().optional().describe("仓库目录，默认进程当前目录"),
         probe: z.boolean().optional().describe("是否连接 MCP server 拉取工具清单"),
-        timeoutMs: z.number().optional().describe("单个 MCP server 超时（毫秒，默认 5000）")
+        timeoutMs: z.number().optional().describe("单个 MCP server 超时（毫秒，默认 5000）"),
+        writeSuite: z.string().optional().describe("套件草稿输出路径，如 mf.suite.json")
       }
     },
-    async ({ root, probe, timeoutMs }) => {
+    async ({ root, probe, timeoutMs, writeSuite }) => {
       const report = await scanProject(root ?? process.cwd());
       if (probe) {
         for (const mcpServer of [...report.mcpServers]) {
@@ -60,7 +62,11 @@ export async function startMcpServer() {
           report.probes.push({ server: mcpServer.name, ok: true, toolCount: added.length, tools: added });
         }
       }
-      return json(redactReport(report));
+      const payload = redactReport(report);
+      if (!writeSuite) return json(payload);
+      const target = resolve(writeSuite);
+      await writeFileAtomic(target, JSON.stringify(buildSuiteDraft(report), null, 2));
+      return json({ ...payload, suiteDraftPath: target });
     }
   );
 

package/mf.js

@@ -46,9 +46,9 @@ Usage:
   mf mcp                # start stdio MCP server for coding agents
 
 Eval options:
-  --suite <path>        eval suite (default: cli/fixtures/suite.json)
+  --suite <path>        eval suite (default: built-in demo suite)
   --models <ids>        comma-separated ids from models.json (default: models with API keys set)
-  --models-file <path>  model registry (default: cli/fixtures/models.json)
+  --models-file <path>  model registry (default: built-in registry)
   --case <ids>          run only these case ids (comma-separated, repeatable)
   --show-cases          print per-case results
   --json                print JSON instead of the table
@@ -563,7 +563,9 @@ function splitList(value) {
 const isMain = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
 
 if (isMain) {
-  const argv = process.argv.slice(2);
+  let argv = process.argv.slice(2);
+  // `npx modelfunction-cli mf scan` passes the bin name through; accept it.
+  if (argv[0] === "mf") argv = argv.slice(1);
   if (argv[0] === "--version" || argv[0] === "-v") {
     const pkg = JSON.parse(await readFile(new URL("./package.json", import.meta.url), "utf8"));
     process.stdout.write(`mf ${pkg.version}\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modelfunction-cli",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "license": "MIT",
   "description": "mf — function-calling accuracy scorecard for AI agents",
@@ -13,7 +13,8 @@
     "agent"
   ],
   "bin": {
-    "mf": "./mf.js"
+    "mf": "mf.js",
+    "modelfunction": "mf.js"
   },
   "files": [
     "mf.js",

package/skills/mf-cases/SKILL.md

@@ -9,6 +9,7 @@ description: 为 ModelFunction 评测套件撰写 function-calling 用例。当
 
 # 输入
 
+0. 还没有套件时：先调 `scan_tools`（`probe=true` 可连 MCP server）并传 `writeSuite: "mf.suite.json"`，得到工具清单草稿，再补用例。
 1. 工具定义：`mf.scan.json` 或 `mf.suite.json`（只看名字、description、parameters）
 2. 覆盖率现状：`mf coverage --suite <file> --json`
 3. 可选：用户提供的真实日志、投诉记录、产品文档