modelfunction-cli 0.1.3 → 0.2.0

package/lib/profile.js

@@ -0,0 +1,304 @@
+import { pct } from "./score.js";
+
+export function normalizeProfiles(raw, toolNames) {
+  if (raw === undefined || raw === null) return null;
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error('suite "profiles" must be an object of { "group": ["tool", ...] }');
+  }
+  const known = new Set(toolNames);
+  const profiles = [];
+  for (const [key, list] of Object.entries(raw)) {
+    const name = String(key).trim();
+    if (!name) throw new Error('suite "profiles": group name must be non-empty');
+    if (!Array.isArray(list) || list.length === 0) {
+      throw new Error(`profile "${name}": must be a non-empty array of tool names`);
+    }
+    const tools = [];
+    for (const item of list) {
+      if (typeof item !== "string" || item.trim() === "") {
+        throw new Error(`profile "${name}": tool names must be non-empty strings`);
+      }
+      const toolName = item.trim();
+      if (!known.has(toolName)) throw new Error(`profile "${name}": unknown tool "${toolName}"`);
+      if (!tools.includes(toolName)) tools.push(toolName);
+    }
+    profiles.push({ name, tools });
+  }
+  if (profiles.length === 0) throw new Error('suite "profiles" is empty');
+  return profiles;
+}
+
+export function selectProfiles(profiles, wanted) {
+  if (!wanted || wanted.length === 0 || wanted.includes("all")) return profiles;
+  const byName = new Map(profiles.map((profile) => [profile.name, profile]));
+  const missing = wanted.filter((name) => !byName.has(name));
+  if (missing.length > 0) {
+    throw new Error(
+      `unknown profile(s): ${missing.join(", ")}（可用：${profiles.map((profile) => profile.name).join(", ")}）`
+    );
+  }
+  return wanted.map((name) => byName.get(name));
+}
+
+// A case belongs to a group when its expected tool is mounted there. Negative
+// cases ("no tool applies") hold everywhere: the group's agent still gets them.
+export function selectProfileCases(cases, mountedTools) {
+  return cases.filter((testCase) => testCase.expectedTool === null || mountedTools.has(testCase.expectedTool));
+}
+
+export function profileWarnings(profiles, suite) {
+  const mounted = new Set(profiles.flatMap((profile) => profile.tools));
+  const warnings = [];
+  const unmounted = suite.tools.map((tool) => tool.name).filter((name) => !mounted.has(name));
+  if (unmounted.length > 0) warnings.push(`工具不在任何挂载组: ${unmounted.join(", ")}`);
+  const homeless = suite.cases.filter(
+    (testCase) => testCase.expectedTool !== null && !mounted.has(testCase.expectedTool)
+  ).length;
+  if (homeless > 0) warnings.push(`${homeless} 条用例的期望工具不在任何挂载组，不参与分组评分`);
+  const everything = suite.tools.length;
+  for (const profile of profiles) {
+    if (profile.tools.length === everything) {
+      warnings.push(`profile "${profile.name}" 挂载了全部工具，与全量对比没有意义`);
+    }
+  }
+  return warnings;
+}
+
+export function profileCaseCoverage(profiles, cases) {
+  const negatives = cases.filter((testCase) => testCase.expectedTool === null).length;
+  return profiles.map((profile) => {
+    const mounted = new Set(profile.tools);
+    const counts = new Map(profile.tools.map((name) => [name, 0]));
+    for (const testCase of cases) {
+      if (testCase.expectedTool !== null && mounted.has(testCase.expectedTool)) {
+        counts.set(testCase.expectedTool, counts.get(testCase.expectedTool) + 1);
+      }
+    }
+    const missing = [...counts.entries()].filter(([, count]) => count === 0).map(([name]) => name);
+    let total = 0;
+    for (const count of counts.values()) total += count;
+    return { name: profile.name, tools: profile.tools.length, cases: total, negatives, missing };
+  });
+}
+
+export function analyzeMount(profile, mounted, allInOne) {
+  const after = new Map(mounted.rows.map((row) => [row.id, row]));
+  const lifted = [];
+  const jammed = [];
+  for (const before of allInOne.rows) {
+    const row = after.get(before.id);
+    if (!row || before.reviewed === false || row.reviewed === false) continue;
+    if (!before.selectionCorrect && row.selectionCorrect) lifted.push(changeOf(before, row));
+    else if (before.selectionCorrect && !row.selectionCorrect) jammed.push(changeOf(before, row));
+  }
+
+  const mountedMetrics = metricsOf(mounted);
+  const allMetrics = metricsOf(allInOne);
+  const reviewed = mounted.reviewedTotal;
+  return {
+    profile: profile.name,
+    mountedTools: profile.tools.length,
+    cases: mounted.total,
+    reviewed,
+    mounted: mountedMetrics,
+    allInOne: allMetrics,
+    delta: {
+      selection: reviewed ? mountedMetrics.selectionAccuracy - allMetrics.selectionAccuracy : null,
+      args:
+        mountedMetrics.argAccuracy === null || allMetrics.argAccuracy === null
+          ? null
+          : mountedMetrics.argAccuracy - allMetrics.argAccuracy,
+      pass: reviewed ? mountedMetrics.passRate - allMetrics.passRate : null
+    },
+    lifted,
+    jammed
+  };
+}
+
+export function modelTotals(entries) {
+  let reviewed = 0;
+  let mountedCorrect = 0;
+  let allCorrect = 0;
+  for (const entry of entries) {
+    reviewed += entry.reviewed;
+    mountedCorrect += entry.mounted.correct;
+    allCorrect += entry.allInOne.correct;
+  }
+  if (reviewed === 0) {
+    return {
+      reviewed: 0,
+      mountedCorrect: 0,
+      allCorrect: 0,
+      mountedAccuracy: null,
+      allAccuracy: null,
+      delta: null
+    };
+  }
+  const mountedAccuracy = mountedCorrect / reviewed;
+  const allAccuracy = allCorrect / reviewed;
+  return {
+    reviewed,
+    mountedCorrect,
+    allCorrect,
+    mountedAccuracy,
+    allAccuracy,
+    delta: mountedAccuracy - allAccuracy
+  };
+}
+
+export function formatProfileReport(report, { showCases = false } = {}) {
+  const lines = [
+    `mount compare: ${report.groups.length} 组（profiles ${report.hash}）`,
+    "组内用例只在对应挂载下评分；all-in-one 用同一批用例、全部工具都挂。"
+  ];
+
+  for (const model of report.models) {
+    lines.push("", `[${model.model}]`);
+    const rows = model.profiles.map((entry) => [
+      entry.profile,
+      String(entry.mountedTools),
+      String(entry.cases),
+      pct(entry.mounted.selectionAccuracy),
+      pct(entry.allInOne.selectionAccuracy),
+      signedPoints(entry.delta.selection)
+    ]);
+    lines.push(
+      ...renderTable(
+        ["profile", "tools", "cases", "mounted", "all-in-one", "delta"],
+        rows,
+        [1, 2, 3, 4, 5]
+      )
+    );
+    const totals = model.totals;
+    if (totals.reviewed > 0) {
+      lines.push(
+        `  合计: 分组 ${pct(totals.mountedAccuracy)}（${totals.mountedCorrect}/${totals.reviewed}）` +
+          ` / 全量 ${pct(totals.allAccuracy)}（${totals.allCorrect}/${totals.reviewed}）` +
+          `，${signedPoints(totals.delta)}`
+      );
+    }
+    const unstable = unstableOf(model.profiles);
+    if (unstable.length > 0) lines.push(`  组内不稳定: ${unstable.join(", ")}`);
+    if (showCases) {
+      for (const entry of model.profiles) {
+        for (const item of entry.lifted) {
+          lines.push(
+            `  + ${entry.profile}/${item.id}: ${item.before ?? "(no tool)"} → ${item.after ?? "(no tool)"}（期望 ${item.expected ?? "(no tool)"}）`
+          );
+        }
+        for (const item of entry.jammed) {
+          lines.push(
+            `  - ${entry.profile}/${item.id}: ${item.before ?? "(no tool)"} → ${item.after ?? "(no tool)"}（期望 ${item.expected ?? "(no tool)"}）`
+          );
+        }
+      }
+    }
+  }
+
+  if (report.warnings.length > 0) {
+    lines.push("", "warnings:");
+    for (const warning of report.warnings) lines.push(`  - ${warning}`);
+  }
+  return lines.join("\n");
+}
+
+export function formatProfileMarkdown(report) {
+  const lines = [
+    "## 挂载对比（分组 vs 全量）",
+    "",
+    "组内用例只在对应挂载下评分；全量 = 同一批用例、全部工具都挂。",
+    ""
+  ];
+  for (const model of report.models) {
+    lines.push(
+      `### ${model.model}`,
+      "",
+      "| group | tools | cases | mounted | all-in-one | delta |",
+      "| --- | --- | --- | --- | --- | --- |"
+    );
+    for (const entry of model.profiles) {
+      lines.push(
+        `| ${entry.profile} | ${entry.mountedTools} | ${entry.cases} | ${pct(entry.mounted.selectionAccuracy)} | ${pct(entry.allInOne.selectionAccuracy)} | ${signedPoints(entry.delta.selection)} |`
+      );
+    }
+    const totals = model.totals;
+    lines.push("");
+    if (totals.reviewed > 0) {
+      lines.push(
+        `- 合计：分组 ${pct(totals.mountedAccuracy)}（${totals.mountedCorrect}/${totals.reviewed}），` +
+          `全量 ${pct(totals.allAccuracy)}（${totals.allCorrect}/${totals.reviewed}），${signedPoints(totals.delta)}`
+      );
+    } else {
+      lines.push("- 合计：没有已复核用例，无法比较");
+    }
+    const unstable = unstableOf(model.profiles);
+    if (unstable.length > 0) lines.push(`- 组内不稳定：${unstable.join(", ")}`);
+    lines.push("");
+  }
+  if (report.warnings.length > 0) lines.push(`> ${report.warnings.join("；")}`, "");
+  return lines.join("\n").trimEnd();
+}
+
+function unstableOf(entries) {
+  return entries.flatMap((entry) =>
+    (entry.stability?.unstable ?? []).map((item) => `${entry.profile}/${item.id}`)
+  );
+}
+
+function changeOf(before, after) {
+  return {
+    id: before.id,
+    expected: before.expectedTool ?? null,
+    before: before.actualTool ?? null,
+    after: after.actualTool ?? null
+  };
+}
+
+function metricsOf(summary) {
+  const reviewedRows = summary.rows.filter((row) => row.reviewed !== false);
+  const correct = reviewedRows.filter((row) => row.selectionCorrect).length;
+  return {
+    selectionAccuracy: reviewedRows.length ? correct / reviewedRows.length : null,
+    argAccuracy: summary.argAccuracy,
+    passRate: reviewedRows.length
+      ? reviewedRows.filter((row) => row.passed).length / reviewedRows.length
+      : null,
+    correct
+  };
+}
+
+function signedPoints(value) {
+  if (value === null || value === undefined) return "n/a";
+  const points = Math.round(value * 100);
+  return `${points > 0 ? "+" : ""}${points}pp`;
+}
+
+function renderTable(headers, rows, numericColumns) {
+  const widths = headers.map((header, index) =>
+    Math.max(displayWidth(header), ...rows.map((row) => displayWidth(row[index])))
+  );
+  const line = (cells) =>
+    cells
+      .map((cell, index) => padDisplay(cell, widths[index], numericColumns.includes(index)))
+      .join("  ");
+  return [line(headers), widths.map((width) => "-".repeat(width)).join("  "), ...rows.map(line)].map(
+    (text) => `  ${text}`
+  );
+}
+
+function padDisplay(text, width, rightAlign) {
+  const fill = " ".repeat(Math.max(0, width - displayWidth(text)));
+  return rightAlign ? fill + text : text + fill;
+}
+
+export function displayWidth(text) {
+  let width = 0;
+  for (const char of String(text)) {
+    width += /[ᄀ-ᅟ⺀-꓏가-힣豈-﫿︰-﹏＀-｠￠-￦]/.test(
+      char
+    )
+      ? 2
+      : 1;
+  }
+  return width;
+}

package/lib/scan.js

@@ -1,5 +1,5 @@
 import { readdir, readFile, stat } from "node:fs/promises";
-import { basename, join, relative, resolve } from "node:path";
+import { basename, extname, join, relative, resolve } from "node:path";
 
 const SKIP_DIRS = new Set([
   "node_modules",
@@ -43,6 +43,10 @@ const OPENAPI_PATTERN = /^(openapi|swagger)[^/]*\.json$/i;
 const SKILL_FILE = "skill.md";
 const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".mjs", ".cjs", ".py"]);
 
+// Test fixtures and mocks use the same APIs as real tool definitions, so any
+// match there is noise for a tool inventory.
+const TEST_PATH = /(^|[\\/])(__tests__|tests?)([\\/])|\.(test|spec)\.[cm]?[jt]sx?$|(^|[\\/])test_[^\\/]+\.py$|_test\.py$/i;
+
 const CODE_PATTERNS = [
   { label: "registerTool", confidence: "medium", re: /registerTool\s*\(\s*["'`]([\w.-]+)["'`]/ },
   { label: "server.tool", confidence: "medium", re: /\.tool\s*\(\s*["'`]([\w.-]+)["'`]/ },
@@ -209,7 +213,21 @@ export async function scanProject(root = ".") {
       if (!info || info.size > MAX_FILE_BYTES) continue;
       const text = await readText(file, report);
       if (!text) continue;
-      const matches = scanCode(text);
+      if (name.endsWith(".py") && !TEST_PATH.test(rel)) {
+        const pythonTools = scanPythonTools(text);
+        if (pythonTools.length > 0) {
+          addTools(report, pythonTools, {
+            source: rel,
+            origin: "py-tool",
+            confidence: "medium",
+            note: "从 @tool 装饰器与类型注解提取，需人工确认"
+          });
+        }
+      }
+      if (TEST_PATH.test(rel)) continue;
+      // Only matches that carry a tool name are actionable; a bare `input_schema:`
+      // in API plumbing is not a tool and only inflates the report.
+      const matches = scanCode(text).filter((match) => match.name);
       for (const match of matches) {
         report.codeMatches.push({
           source: rel,
@@ -236,13 +254,13 @@ export async function scanProject(root = ".") {
   return report;
 }
 
-export function addTools(report, tools, { source, origin, confidence = "high", note }) {
+export function addTools(report, tools, { source, origin, confidence = "high", note, meta }) {
   const existing = new Set(report.tools.map((tool) => tool.name));
   const added = [];
   for (const tool of tools) {
     if (!tool?.name || existing.has(tool.name)) continue;
     existing.add(tool.name);
-    report.tools.push({ ...tool, source, origin });
+    report.tools.push({ ...tool, source, origin, ...(meta ?? {}) });
     added.push(tool.name);
   }
   report.sources.push({
@@ -266,7 +284,7 @@ export function redactReport(report) {
 }
 
 export function buildSuiteDraft(report) {
-  return {
+  const draft = {
     name: `${basename(report.root)}-draft`,
     description: "由 mf scan 生成。请人工确认工具描述与参数，并补充 cases 后再运行 mf eval。",
     tools: report.tools.map((tool) => ({
@@ -276,6 +294,34 @@ export function buildSuiteDraft(report) {
     })),
     cases: []
   };
+  if (report.skills.length > 0) draft.skills = report.skills.map((skill) => skill.name);
+  // Where a tool came from is the only mount hint the scan can observe without
+  // guessing: one MCP server or one spec file is normally one thing agents mount.
+  const profiles = groupBySource(report.tools);
+  if (profiles) draft.profiles = profiles;
+  return draft;
+}
+
+function groupBySource(tools) {
+  const groups = new Map();
+  const owner = new Map();
+  for (const tool of tools) {
+    const key = tool.provider ?? tool.source ?? "";
+    if (!key) continue;
+    let label = profileLabel(key);
+    if (owner.has(label) && owner.get(label) !== key) label = String(key).replace(/[\\/]+/g, "-");
+    owner.set(label, key);
+    if (!groups.has(label)) groups.set(label, []);
+    groups.get(label).push(tool.name);
+  }
+  if (groups.size < 2) return null;
+  return Object.fromEntries([...groups.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+function profileLabel(key) {
+  const text = String(key);
+  if (!/[\\/]|\.(json|ya?ml|toml|js|mjs|ts|py)$/i.test(text)) return text;
+  return basename(text, extname(text)) || text;
 }
 
 export function formatScanSummary(report) {
@@ -480,6 +526,140 @@ function parseFrontmatter(text) {
   return meta;
 }
 
+// LangChain-style tool definitions: @tool above a typed function. The runtime
+// builds the JSON schema from the same annotations, so this is the real schema,
+// not a guess — only the description can drift (first docstring line vs args_schema).
+export function scanPythonTools(text) {
+  const lines = text.split(/\r?\n/);
+  const tools = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    if (!/^\s*@tool\b/.test(lines[i])) continue;
+    let j = i + 1;
+    while (j < lines.length && j - i <= 4 && !/^\s*(async\s+)?def\s+\w+\s*\(/.test(lines[j])) j += 1;
+    const defMatch = lines[j]?.match(/^\s*(?:async\s+)?def\s+(\w+)\s*\(/);
+    if (!defMatch) continue;
+    const signature = collectSignature(lines, j, defMatch[0].length);
+    if (signature === null) continue;
+    const properties = {};
+    const required = [];
+    for (const part of splitPyParams(signature.text)) {
+      const param = parsePyParam(part);
+      if (!param) continue;
+      properties[param.name] = param.schema;
+      if (param.required) required.push(param.name);
+    }
+    tools.push({
+      name: defMatch[1],
+      description: pythonDocstring(lines, signature.endLine),
+      parameters: {
+        type: "object",
+        properties,
+        ...(required.length > 0 ? { required } : {})
+      }
+    });
+    i = signature.endLine;
+  }
+  return tools;
+}
+
+function collectSignature(lines, startLine, offset) {
+  let depth = 1;
+  let out = "";
+  for (let i = startLine; i < lines.length; i += 1) {
+    const line = lines[i];
+    for (let c = i === startLine ? offset : 0; c < line.length; c += 1) {
+      const ch = line[c];
+      if (ch === "(") depth += 1;
+      else if (ch === ")") {
+        depth -= 1;
+        if (depth === 0) return { text: out, endLine: i };
+      }
+      out += ch;
+    }
+    out += "\n";
+  }
+  return null;
+}
+
+function splitPyParams(text) {
+  const parts = [];
+  let depth = 0;
+  let quote = null;
+  let current = "";
+  for (const ch of text) {
+    if (quote) {
+      current += ch;
+      if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch;
+      current += ch;
+      continue;
+    }
+    if (ch === "(" || ch === "[" || ch === "{") depth += 1;
+    if (ch === ")" || ch === "]" || ch === "}") depth -= 1;
+    if (ch === "," && depth === 0) {
+      parts.push(current);
+      current = "";
+      continue;
+    }
+    current += ch;
+  }
+  if (current.trim()) parts.push(current);
+  return parts.map((part) => part.trim()).filter(Boolean);
+}
+
+const PY_TYPES = {
+  str: "string",
+  int: "number",
+  float: "number",
+  bool: "boolean",
+  list: "array",
+  List: "array",
+  dict: "object",
+  Dict: "object"
+};
+
+function parsePyParam(part) {
+  const eq = part.indexOf("=");
+  const head = eq === -1 ? part : part.slice(0, eq);
+  const hasDefault = eq !== -1;
+  const colon = head.indexOf(":");
+  const name = (colon === -1 ? head : head.slice(0, colon)).trim().replace(/^\*+/, "");
+  if (!name || name === "self" || name === "cls") return null;
+  const typeText = colon === -1 ? "" : head.slice(colon + 1).trim();
+  const description = (typeText.match(/Annotated\s*\[[^,]+,\s*["']([^"']+)["']/) ?? [])[1] ?? "";
+  const schema = { type: pyType(typeText) };
+  if (description) schema.description = description;
+  return { name, schema, required: !hasDefault };
+}
+
+function pyType(typeText) {
+  const core = typeText
+    .replace(/^Annotated\s*\[\s*([^,\]]+).*/s, "$1")
+    .replace(/^Optional\s*\[\s*(.*?)\s*\]$/s, "$1")
+    .trim();
+  return PY_TYPES[core] ?? (core.endsWith("List") || core.startsWith("list") ? "array" : "string");
+}
+
+function pythonDocstring(lines, fromLine) {
+  for (let i = fromLine + 1; i < Math.min(lines.length, fromLine + 6); i += 1) {
+    const match = lines[i].match(/^\s*(?:[rRuU])?("""|''')\s*(.*)$/);
+    if (!match) continue;
+    const quote = match[1];
+    const rest = match[2];
+    const sameLine = rest.includes(quote) ? rest.split(quote)[0] : rest;
+    if (sameLine.trim()) return sameLine.trim();
+    for (let k = i + 1; k < lines.length; k += 1) {
+      const text = lines[k].trim();
+      if (!text) continue;
+      return text.startsWith(quote) ? "" : text;
+    }
+  }
+  return "";
+}
+
 function scanCode(text) {
   const matches = [];
   for (const pattern of CODE_PATTERNS) {

package/lib/score.js

@@ -54,6 +54,28 @@ export function matchArgs(actualArgs, expectedArgs) {
   };
 }
 
+// Even at temperature 0 providers can flip a tool call between identical
+// requests. Repeats exist so the scorecard can show that instead of baking one
+// sample into the number.
+export function pickRepresentative(runs) {
+  const signature = (run) => (run?.ok ? run.toolCalls?.[0]?.name ?? "(no tool)" : `error: ${run?.error ?? ""}`);
+  const counts = new Map();
+  for (const run of runs) {
+    const key = signature(run);
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  let best = null;
+  for (const [key, count] of counts) {
+    if (!best || count > best.count) best = { key, count };
+  }
+  const chosen = runs.find((run) => signature(run) === best.key) ?? runs[0];
+  return {
+    result: chosen,
+    unstable: counts.size > 1,
+    outcomes: [...counts.entries()].map(([outcome, count]) => ({ outcome, count }))
+  };
+}
+
 export function scoreModel({ model, cases, results, pricing }) {
   const rows = cases.map((testCase, index) => {
     const result = results[index];

package/lib/suite.js

@@ -28,6 +28,8 @@ export async function loadSuite(suitePath, { allowEmptyCases = false } = {}) {
     path: abs,
     tools,
     cases,
+    profiles: raw.profiles ?? null,
+    skills: Array.isArray(raw.skills) ? raw.skills.map(String) : [],
     execution:
       raw.execution && typeof raw.execution === "object" && !Array.isArray(raw.execution)
         ? { ...raw.execution }