modelfunction-cli 0.1.3 → 0.3.0

package/lib/scan.js

@@ -1,5 +1,5 @@
 import { readdir, readFile, stat } from "node:fs/promises";
-import { basename, join, relative, resolve } from "node:path";
+import { basename, extname, join, relative, resolve } from "node:path";
 
 const SKIP_DIRS = new Set([
   "node_modules",
@@ -43,6 +43,10 @@ const OPENAPI_PATTERN = /^(openapi|swagger)[^/]*\.json$/i;
 const SKILL_FILE = "skill.md";
 const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".mjs", ".cjs", ".py"]);
 
+// Test fixtures and mocks use the same APIs as real tool definitions, so any
+// match there is noise for a tool inventory.
+const TEST_PATH = /(^|[\\/])(__tests__|tests?)([\\/])|\.(test|spec)\.[cm]?[jt]sx?$|(^|[\\/])test_[^\\/]+\.py$|_test\.py$/i;
+
 const CODE_PATTERNS = [
   { label: "registerTool", confidence: "medium", re: /registerTool\s*\(\s*["'`]([\w.-]+)["'`]/ },
   { label: "server.tool", confidence: "medium", re: /\.tool\s*\(\s*["'`]([\w.-]+)["'`]/ },
@@ -209,7 +213,21 @@ export async function scanProject(root = ".") {
       if (!info || info.size > MAX_FILE_BYTES) continue;
       const text = await readText(file, report);
       if (!text) continue;
-      const matches = scanCode(text);
+      if (name.endsWith(".py") && !TEST_PATH.test(rel)) {
+        const pythonTools = scanPythonTools(text);
+        if (pythonTools.length > 0) {
+          addTools(report, pythonTools, {
+            source: rel,
+            origin: "py-tool",
+            confidence: "medium",
+            note: "从 @tool 装饰器与类型注解提取，需人工确认"
+          });
+        }
+      }
+      if (TEST_PATH.test(rel)) continue;
+      // Only matches that carry a tool name are actionable; a bare `input_schema:`
+      // in API plumbing is not a tool and only inflates the report.
+      const matches = scanCode(text).filter((match) => match.name);
       for (const match of matches) {
         report.codeMatches.push({
           source: rel,
@@ -236,13 +254,13 @@ export async function scanProject(root = ".") {
   return report;
 }
 
-export function addTools(report, tools, { source, origin, confidence = "high", note }) {
+export function addTools(report, tools, { source, origin, confidence = "high", note, meta }) {
   const existing = new Set(report.tools.map((tool) => tool.name));
   const added = [];
   for (const tool of tools) {
     if (!tool?.name || existing.has(tool.name)) continue;
     existing.add(tool.name);
-    report.tools.push({ ...tool, source, origin });
+    report.tools.push({ ...tool, source, origin, ...(meta ?? {}) });
     added.push(tool.name);
   }
   report.sources.push({
@@ -266,7 +284,7 @@ export function redactReport(report) {
 }
 
 export function buildSuiteDraft(report) {
-  return {
+  const draft = {
     name: `${basename(report.root)}-draft`,
     description: "由 mf scan 生成。请人工确认工具描述与参数，并补充 cases 后再运行 mf eval。",
     tools: report.tools.map((tool) => ({
@@ -276,6 +294,36 @@ export function buildSuiteDraft(report) {
     })),
     cases: []
   };
+  if (report.skills.length > 0) {
+    draft.skills = report.skills.map((skill) => ({ name: skill.name, description: skill.description }));
+  }
+  // Where a tool came from is the only mount hint the scan can observe without
+  // guessing: one MCP server or one spec file is normally one thing agents mount.
+  const profiles = groupBySource(report.tools);
+  if (profiles) draft.profiles = profiles;
+  return draft;
+}
+
+function groupBySource(tools) {
+  const groups = new Map();
+  const owner = new Map();
+  for (const tool of tools) {
+    const key = tool.provider ?? tool.source ?? "";
+    if (!key) continue;
+    let label = profileLabel(key);
+    if (owner.has(label) && owner.get(label) !== key) label = String(key).replace(/[\\/]+/g, "-");
+    owner.set(label, key);
+    if (!groups.has(label)) groups.set(label, []);
+    groups.get(label).push(tool.name);
+  }
+  if (groups.size < 2) return null;
+  return Object.fromEntries([...groups.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+function profileLabel(key) {
+  const text = String(key);
+  if (!/[\\/]|\.(json|ya?ml|toml|js|mjs|ts|py)$/i.test(text)) return text;
+  return basename(text, extname(text)) || text;
 }
 
 export function formatScanSummary(report) {
@@ -308,7 +356,7 @@ export function formatScanSummary(report) {
   lines.push(
     "",
     `tools:       ${report.tools.length}`,
-    `skills:      ${report.skills.length}（不进入工具评测）`,
+    `skills:      ${report.skills.length}（仅静态检查）`,
     `mcp servers: ${report.mcpServers.length}${probed > 0 ? `（已 probe ${probed} 个）` : "（需 --probe 才能拿到工具清单）"}`,
     `code match:  ${report.codeMatches.length}（low confidence，需人工确认）`
   );
@@ -480,6 +528,140 @@ function parseFrontmatter(text) {
   return meta;
 }
 
+// LangChain-style tool definitions: @tool above a typed function. The runtime
+// builds the JSON schema from the same annotations, so this is the real schema,
+// not a guess — only the description can drift (first docstring line vs args_schema).
+export function scanPythonTools(text) {
+  const lines = text.split(/\r?\n/);
+  const tools = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    if (!/^\s*@tool\b/.test(lines[i])) continue;
+    let j = i + 1;
+    while (j < lines.length && j - i <= 4 && !/^\s*(async\s+)?def\s+\w+\s*\(/.test(lines[j])) j += 1;
+    const defMatch = lines[j]?.match(/^\s*(?:async\s+)?def\s+(\w+)\s*\(/);
+    if (!defMatch) continue;
+    const signature = collectSignature(lines, j, defMatch[0].length);
+    if (signature === null) continue;
+    const properties = {};
+    const required = [];
+    for (const part of splitPyParams(signature.text)) {
+      const param = parsePyParam(part);
+      if (!param) continue;
+      properties[param.name] = param.schema;
+      if (param.required) required.push(param.name);
+    }
+    tools.push({
+      name: defMatch[1],
+      description: pythonDocstring(lines, signature.endLine),
+      parameters: {
+        type: "object",
+        properties,
+        ...(required.length > 0 ? { required } : {})
+      }
+    });
+    i = signature.endLine;
+  }
+  return tools;
+}
+
+function collectSignature(lines, startLine, offset) {
+  let depth = 1;
+  let out = "";
+  for (let i = startLine; i < lines.length; i += 1) {
+    const line = lines[i];
+    for (let c = i === startLine ? offset : 0; c < line.length; c += 1) {
+      const ch = line[c];
+      if (ch === "(") depth += 1;
+      else if (ch === ")") {
+        depth -= 1;
+        if (depth === 0) return { text: out, endLine: i };
+      }
+      out += ch;
+    }
+    out += "\n";
+  }
+  return null;
+}
+
+function splitPyParams(text) {
+  const parts = [];
+  let depth = 0;
+  let quote = null;
+  let current = "";
+  for (const ch of text) {
+    if (quote) {
+      current += ch;
+      if (ch === quote) quote = null;
+      continue;
+    }
+    if (ch === '"' || ch === "'") {
+      quote = ch;
+      current += ch;
+      continue;
+    }
+    if (ch === "(" || ch === "[" || ch === "{") depth += 1;
+    if (ch === ")" || ch === "]" || ch === "}") depth -= 1;
+    if (ch === "," && depth === 0) {
+      parts.push(current);
+      current = "";
+      continue;
+    }
+    current += ch;
+  }
+  if (current.trim()) parts.push(current);
+  return parts.map((part) => part.trim()).filter(Boolean);
+}
+
+const PY_TYPES = {
+  str: "string",
+  int: "number",
+  float: "number",
+  bool: "boolean",
+  list: "array",
+  List: "array",
+  dict: "object",
+  Dict: "object"
+};
+
+function parsePyParam(part) {
+  const eq = part.indexOf("=");
+  const head = eq === -1 ? part : part.slice(0, eq);
+  const hasDefault = eq !== -1;
+  const colon = head.indexOf(":");
+  const name = (colon === -1 ? head : head.slice(0, colon)).trim().replace(/^\*+/, "");
+  if (!name || name === "self" || name === "cls") return null;
+  const typeText = colon === -1 ? "" : head.slice(colon + 1).trim();
+  const description = (typeText.match(/Annotated\s*\[[^,]+,\s*["']([^"']+)["']/) ?? [])[1] ?? "";
+  const schema = { type: pyType(typeText) };
+  if (description) schema.description = description;
+  return { name, schema, required: !hasDefault };
+}
+
+function pyType(typeText) {
+  const core = typeText
+    .replace(/^Annotated\s*\[\s*([^,\]]+).*/s, "$1")
+    .replace(/^Optional\s*\[\s*(.*?)\s*\]$/s, "$1")
+    .trim();
+  return PY_TYPES[core] ?? (core.endsWith("List") || core.startsWith("list") ? "array" : "string");
+}
+
+function pythonDocstring(lines, fromLine) {
+  for (let i = fromLine + 1; i < Math.min(lines.length, fromLine + 6); i += 1) {
+    const match = lines[i].match(/^\s*(?:[rRuU])?("""|''')\s*(.*)$/);
+    if (!match) continue;
+    const quote = match[1];
+    const rest = match[2];
+    const sameLine = rest.includes(quote) ? rest.split(quote)[0] : rest;
+    if (sameLine.trim()) return sameLine.trim();
+    for (let k = i + 1; k < lines.length; k += 1) {
+      const text = lines[k].trim();
+      if (!text) continue;
+      return text.startsWith(quote) ? "" : text;
+    }
+  }
+  return "";
+}
+
 function scanCode(text) {
   const matches = [];
   for (const pattern of CODE_PATTERNS) {

package/lib/score.js

@@ -54,6 +54,28 @@ export function matchArgs(actualArgs, expectedArgs) {
   };
 }
 
+// Even at temperature 0 providers can flip a tool call between identical
+// requests. Repeats exist so the scorecard can show that instead of baking one
+// sample into the number.
+export function pickRepresentative(runs) {
+  const signature = (run) => (run?.ok ? run.toolCalls?.[0]?.name ?? "(no tool)" : `error: ${run?.error ?? ""}`);
+  const counts = new Map();
+  for (const run of runs) {
+    const key = signature(run);
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+  let best = null;
+  for (const [key, count] of counts) {
+    if (!best || count > best.count) best = { key, count };
+  }
+  const chosen = runs.find((run) => signature(run) === best.key) ?? runs[0];
+  return {
+    result: chosen,
+    unstable: counts.size > 1,
+    outcomes: [...counts.entries()].map(([outcome, count]) => ({ outcome, count }))
+  };
+}
+
 export function scoreModel({ model, cases, results, pricing }) {
   const rows = cases.map((testCase, index) => {
     const result = results[index];

package/lib/suite.js

@@ -28,6 +28,8 @@ export async function loadSuite(suitePath, { allowEmptyCases = false } = {}) {
     path: abs,
     tools,
     cases,
+    profiles: raw.profiles ?? null,
+    skills: Array.isArray(raw.skills) ? raw.skills.map(normalizeSkill) : [],
     execution:
       raw.execution && typeof raw.execution === "object" && !Array.isArray(raw.execution)
         ? { ...raw.execution }
@@ -56,6 +58,23 @@ async function normalizeTool(entry, baseDir) {
   };
 }
 
+// Skills carry no parameter schema in the prompt-injected form (SKILL.md
+// frontmatter), so the only checkable surface is name + description. Both the
+// bare-name shorthand and the full form are accepted.
+function normalizeSkill(entry, index) {
+  if (typeof entry === "string") {
+    const name = entry.trim();
+    if (!name) throw new Error(`skill #${index + 1}: name must be a non-empty string`);
+    return { name, description: "" };
+  }
+  if (entry && typeof entry === "object") {
+    const name = typeof entry.name === "string" ? entry.name.trim() : "";
+    if (!name) throw new Error(`skill #${index + 1}: "name" must be a non-empty string`);
+    return { name, description: typeof entry.description === "string" ? entry.description : "" };
+  }
+  throw new Error(`skill #${index + 1}: must be a name or {name, description}`);
+}
+
 function normalizeCase(entry, index, toolNames) {
   const id = entry.id ?? `case_${index + 1}`;
   if (typeof entry.input !== "string" || entry.input.trim() === "") {