modelfunction-cli 0.0.0-stage → 0.1.0

package/lib/scan.js

@@ -0,0 +1,510 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, join, relative, resolve } from "node:path";
+
+const SKIP_DIRS = new Set([
+  "node_modules",
+  ".git",
+  "dist",
+  "build",
+  ".next",
+  ".nuxt",
+  ".svelte-kit",
+  "coverage",
+  ".venv",
+  "venv",
+  "__pycache__",
+  ".idea",
+  ".turbo",
+  ".cache"
+]);
+
+const MAX_DEPTH = 6;
+const MAX_FILES = 5000;
+const MAX_FILE_BYTES = 200_000;
+
+const FUNCTION_SPEC_NAMES = new Set([
+  "function.json",
+  "tool.json",
+  "tools.json",
+  "functions.json",
+  "mcp-tools.json"
+]);
+
+const MCP_CONFIG_NAMES = new Set([
+  "mcp.json",
+  ".mcp.json",
+  "mcp_config.json",
+  "claude_desktop_config.json",
+  "cline_mcp_settings.json",
+  "mcp_settings.json"
+]);
+
+const OPENAPI_PATTERN = /^(openapi|swagger)[^/]*\.json$/i;
+const SKILL_FILE = "skill.md";
+const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".mjs", ".cjs", ".py"]);
+
+const CODE_PATTERNS = [
+  { label: "registerTool", confidence: "medium", re: /registerTool\s*\(\s*["'`]([\w.-]+)["'`]/ },
+  { label: "server.tool", confidence: "medium", re: /\.tool\s*\(\s*["'`]([\w.-]+)["'`]/ },
+  { label: "py-tool-decorator", confidence: "low", re: /^\s*@tool\b/ },
+  { label: "vercel-tool-call", confidence: "low", re: /\btool\s*\(\s*\{/ },
+  { label: "openai-tools-array", confidence: "low", re: /\btools\s*:\s*\[\s*\{/ },
+  { label: "anthropic-input-schema", confidence: "low", re: /\binput_?schema\s*:/ }
+];
+
+const PY_FRAMEWORKS = {
+  openai: "openai",
+  anthropic: "anthropic",
+  langchain: "langchain",
+  langgraph: "langgraph",
+  crewai: "crewai",
+  litellm: "litellm",
+  mcp: "mcp",
+  "openai-agents": "openai-agents"
+};
+
+const JS_FRAMEWORKS = {
+  openai: "openai",
+  "@anthropic-ai/sdk": "anthropic",
+  ai: "vercel-ai-sdk",
+  langchain: "langchain",
+  "@langchain/core": "langchain",
+  langgraph: "langgraph",
+  "@modelcontextprotocol/sdk": "mcp",
+  "@openai/agents": "openai-agents"
+};
+
+export async function scanProject(root = ".") {
+  const absRoot = resolve(root);
+  const files = [];
+  await walk(absRoot, 0, files);
+
+  const report = {
+    root: absRoot,
+    filesScanned: files.length,
+    sources: [],
+    tools: [],
+    skills: [],
+    mcpServers: [],
+    probes: [],
+    codeMatches: [],
+    dependencies: [],
+    frameworks: [],
+    warnings: []
+  };
+
+  const seenTools = new Set();
+  const dependencies = new Set();
+  const frameworks = new Set();
+
+  for (const file of files.sort()) {
+    const name = basename(file).toLowerCase();
+    const rel = relative(absRoot, file);
+
+    if (name === "package.json") {
+      const pkg = await readJson(file, report);
+      if (pkg) {
+        const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
+        for (const dep of Object.keys(deps)) {
+          dependencies.add(dep);
+          if (JS_FRAMEWORKS[dep]) frameworks.add(JS_FRAMEWORKS[dep]);
+        }
+      }
+      continue;
+    }
+
+    if (name === "requirements.txt" || name === "pyproject.toml") {
+      const text = await readText(file, report);
+      if (text) {
+        for (const [pkg, framework] of Object.entries(PY_FRAMEWORKS)) {
+          if (new RegExp(`(^|[^a-z0-9_-])${pkg}([^a-z0-9_-]|$)`, "i").test(text)) {
+            dependencies.add(pkg);
+            frameworks.add(framework);
+          }
+        }
+      }
+      continue;
+    }
+
+    if (FUNCTION_SPEC_NAMES.has(name)) {
+      const spec = await readJson(file, report);
+      if (!spec) continue;
+      const tools = toolsFromSpec(spec);
+      if (tools.length === 0) {
+        report.warnings.push(`${rel}: 未识别出工具定义（需要 name + input/parameters，或 tools 数组）`);
+        continue;
+      }
+      const added = [];
+      for (const tool of tools) {
+        if (seenTools.has(tool.name)) {
+          report.warnings.push(`${rel}: 工具 "${tool.name}" 重复，已忽略`);
+          continue;
+        }
+        seenTools.add(tool.name);
+        report.tools.push({ ...tool, source: rel, origin: "function-spec" });
+        added.push(tool.name);
+      }
+      report.sources.push({ type: "function-spec", path: rel, confidence: "high", tools: added });
+      continue;
+    }
+
+    if (MCP_CONFIG_NAMES.has(name)) {
+      const config = await readJson(file, report);
+      if (!config) continue;
+      const servers = mcpServersFrom(config);
+      if (servers.length === 0) {
+        report.warnings.push(`${rel}: 未找到 mcpServers/servers 配置`);
+        continue;
+      }
+      report.mcpServers.push(...servers.map((server) => ({ ...server, source: rel })));
+      report.sources.push({
+        type: "mcp-config",
+        path: rel,
+        confidence: "high",
+        servers: servers.map((server) => server.name),
+        note: "工具清单需连接 MCP server 后获取（本轮 scan 不启动进程）"
+      });
+      continue;
+    }
+
+    if (OPENAPI_PATTERN.test(name)) {
+      const spec = await readJson(file, report);
+      if (!spec || !spec.paths) continue;
+      const tools = toolsFromOpenApi(spec);
+      const added = [];
+      for (const tool of tools) {
+        if (seenTools.has(tool.name)) {
+          report.warnings.push(`${rel}: 工具 "${tool.name}" 重复，已忽略`);
+          continue;
+        }
+        seenTools.add(tool.name);
+        report.tools.push({ ...tool, source: rel, origin: "openapi" });
+        added.push(tool.name);
+      }
+      report.sources.push({ type: "openapi", path: rel, confidence: "medium", tools: added });
+      continue;
+    }
+
+    if (name === SKILL_FILE) {
+      const text = await readText(file, report);
+      if (!text) continue;
+      const meta = parseFrontmatter(text);
+      if (!meta.name) {
+        report.warnings.push(`${rel}: SKILL.md 缺少 frontmatter name`);
+        continue;
+      }
+      report.skills.push({ name: meta.name, description: meta.description ?? "", source: rel });
+      report.sources.push({
+        type: "skill",
+        path: rel,
+        confidence: "high",
+        skills: [meta.name],
+        note: "skill 没有参数 schema，不参与工具评测；可作为 skill 选择评测的候选"
+      });
+      continue;
+    }
+
+    if (CODE_EXTENSIONS.has(name.slice(name.lastIndexOf(".")))) {
+      const info = await stat(file).catch(() => null);
+      if (!info || info.size > MAX_FILE_BYTES) continue;
+      const text = await readText(file, report);
+      if (!text) continue;
+      const matches = scanCode(text);
+      for (const match of matches) {
+        report.codeMatches.push({
+          source: rel,
+          line: match.line,
+          label: match.label,
+          name: match.name,
+          confidence: match.confidence
+        });
+      }
+      if (matches.length > 0) {
+        report.sources.push({
+          type: "code",
+          path: rel,
+          confidence: "low",
+          matches: matches.length,
+          note: "代码内定义只能作为草稿，需人工确认"
+        });
+      }
+    }
+  }
+
+  report.dependencies = [...dependencies].sort();
+  report.frameworks = [...frameworks].sort();
+  return report;
+}
+
+export function addTools(report, tools, { source, origin, confidence = "high", note }) {
+  const existing = new Set(report.tools.map((tool) => tool.name));
+  const added = [];
+  for (const tool of tools) {
+    if (!tool?.name || existing.has(tool.name)) continue;
+    existing.add(tool.name);
+    report.tools.push({ ...tool, source, origin });
+    added.push(tool.name);
+  }
+  report.sources.push({
+    type: origin,
+    path: source,
+    confidence,
+    tools: added,
+    ...(note ? { note } : {})
+  });
+  return added;
+}
+
+export function redactReport(report) {
+  return {
+    ...report,
+    mcpServers: report.mcpServers.map(({ env, ...rest }) => ({
+      ...rest,
+      envKeys: env ? Object.keys(env) : []
+    }))
+  };
+}
+
+export function buildSuiteDraft(report) {
+  return {
+    name: `${basename(report.root)}-draft`,
+    description: "由 mf scan 生成。请人工确认工具描述与参数，并补充 cases 后再运行 mf eval。",
+    tools: report.tools.map((tool) => ({
+      name: tool.name,
+      description: tool.description,
+      parameters: tool.parameters
+    })),
+    cases: []
+  };
+}
+
+export function formatScanSummary(report) {
+  const lines = [`scan: ${report.root}（${report.filesScanned} 个文件）`];
+
+  if (report.sources.length === 0) {
+    lines.push("未发现可识别的工具来源");
+  } else {
+    lines.push("", "sources:");
+    for (const source of report.sources) {
+      const detail = source.tools?.length
+        ? `${source.tools.length} tool(s): ${source.tools.join(", ")}`
+        : source.servers?.length
+          ? `${source.servers.length} server(s): ${source.servers.join(", ")}`
+          : source.skills?.length
+            ? `${source.skills.length} skill(s): ${source.skills.join(", ")}`
+            : source.matches
+              ? `${source.matches} code match(es)`
+              : "";
+      lines.push(`  ${source.type.padEnd(13)} ${source.path.padEnd(40)} ${detail} [${source.confidence}]`);
+      if (source.note) lines.push(`  ${" ".repeat(14)}note: ${source.note}`);
+    }
+  }
+
+  if (report.frameworks.length > 0) {
+    lines.push("", `frameworks: ${report.frameworks.join(", ")}`);
+  }
+
+  const probed = (report.probes ?? []).length;
+  lines.push(
+    "",
+    `tools:       ${report.tools.length}`,
+    `skills:      ${report.skills.length}（不进入工具评测）`,
+    `mcp servers: ${report.mcpServers.length}${probed > 0 ? `（已 probe ${probed} 个）` : "（需 --probe 才能拿到工具清单）"}`,
+    `code match:  ${report.codeMatches.length}（low confidence，需人工确认）`
+  );
+
+  if ((report.probes ?? []).length > 0) {
+    lines.push("", "probes:");
+    for (const probe of report.probes) {
+      lines.push(
+        `  ${probe.server.padEnd(20)} ${probe.ok ? `ok, ${probe.toolCount} tool(s)` : `failed: ${probe.error}`}`
+      );
+    }
+  }
+
+  if (report.warnings.length > 0) {
+    lines.push("", "warnings:");
+    for (const warning of report.warnings) lines.push(`  - ${warning}`);
+  }
+
+  const hints = [];
+  if (report.tools.length === 0) hints.push("没有可直接使用的工具定义：请手写 tools 或先连接 MCP server（后续版本支持）");
+  if (report.tools.length > 0) hints.push("cases 为空：补 1 个以上用例后才能运行 mf eval");
+  if (report.codeMatches.length > 0) hints.push("代码内定义请对照源码逐个确认，不要直接采信");
+  if (hints.length > 0) {
+    lines.push("", "next:");
+    for (const hint of hints) lines.push(`  - ${hint}`);
+  }
+
+  return lines.join("\n");
+}
+
+async function walk(dir, depth, files) {
+  if (depth > MAX_DEPTH || files.length >= MAX_FILES) return;
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  entries.sort((a, b) => a.name.localeCompare(b.name));
+  for (const entry of entries) {
+    if (files.length >= MAX_FILES) return;
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      await walk(full, depth + 1, files);
+    } else if (entry.isFile()) {
+      files.push(full);
+    }
+  }
+}
+
+async function readJson(file, report) {
+  const text = await readText(file, report);
+  if (text === null) return null;
+  try {
+    return JSON.parse(text);
+  } catch (error) {
+    report.warnings.push(`${relative(report.root, file)}: JSON 解析失败（${String(error.message).slice(0, 80)}）`);
+    return null;
+  }
+}
+
+async function readText(file, report) {
+  try {
+    return await readFile(file, "utf8");
+  } catch (error) {
+    report.warnings.push(`${relative(report.root, file)}: 读取失败（${String(error.message).slice(0, 80)}）`);
+    return null;
+  }
+}
+
+function toolsFromSpec(spec) {
+  const raw = Array.isArray(spec)
+    ? spec
+    : Array.isArray(spec.tools)
+      ? spec.tools
+      : typeof spec.name === "string"
+        ? [spec]
+        : [];
+  return raw.map(normalizeTool).filter(Boolean);
+}
+
+export function normalizeTool(entry) {
+  const source = entry.function ?? entry;
+  if (!source || typeof source.name !== "string" || source.name.trim() === "") return null;
+  return {
+    name: source.name.trim(),
+    description: source.description ?? "",
+    parameters: source.input ?? source.parameters ?? source.inputSchema ?? { type: "object", properties: {} },
+    sideEffects: typeof source.sideEffects === "string" ? source.sideEffects : null
+  };
+}
+
+function mcpServersFrom(config) {
+  const servers = config.mcpServers ?? config.servers ?? {};
+  return Object.entries(servers).map(([name, value]) => ({
+    name,
+    transport: value.url ? "http" : "stdio",
+    command: value.command ?? null,
+    args: Array.isArray(value.args) ? value.args : [],
+    env: value.env && typeof value.env === "object" ? value.env : null,
+    url: value.url ?? null
+  }));
+}
+
+function toolsFromOpenApi(spec) {
+  const tools = [];
+  const paths = spec.paths ?? {};
+  for (const [path, operations] of Object.entries(paths)) {
+    if (!operations || typeof operations !== "object") continue;
+    const pathParams = Array.isArray(operations.parameters) ? operations.parameters : [];
+    for (const method of ["get", "post", "put", "patch", "delete"]) {
+      const op = operations[method];
+      if (!op || typeof op !== "object") continue;
+      const name = op.operationId ?? `${method}_${path}`.replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "");
+      const properties = {};
+      const required = new Set();
+      const params = [...pathParams, ...(Array.isArray(op.parameters) ? op.parameters : [])];
+      for (const rawParam of params) {
+        const param = deref(spec, rawParam);
+        if (!param?.name) continue;
+        properties[param.name] = param.schema ?? { type: param.type ?? "string" };
+        if (param.description) properties[param.name].description = param.description;
+        if (param.required) required.add(param.name);
+      }
+      const bodySchema = deref(spec, op.requestBody?.content?.["application/json"]?.schema);
+      if (bodySchema?.properties) {
+        for (const [key, value] of Object.entries(bodySchema.properties)) {
+          properties[key] = deref(spec, value);
+        }
+        for (const key of bodySchema.required ?? []) required.add(key);
+      }
+      tools.push({
+        name,
+        description: op.summary ?? op.description ?? "",
+        parameters: {
+          type: "object",
+          properties,
+          ...(required.size > 0 ? { required: [...required] } : {})
+        }
+      });
+    }
+  }
+  return tools;
+}
+
+function deref(spec, node, depth = 0) {
+  if (!node || depth > 5) return node;
+  if (typeof node.$ref === "string" && node.$ref.startsWith("#/")) {
+    let target = spec;
+    for (const part of node.$ref.slice(2).split("/")) {
+      target = target?.[part.replace(/~1/g, "/").replace(/~0/g, "~")];
+      if (!target) return node;
+    }
+    return deref(spec, target, depth + 1);
+  }
+  return node;
+}
+
+function parseFrontmatter(text) {
+  const match = text.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!match) return {};
+  const meta = {};
+  for (const line of match[1].split(/\r?\n/)) {
+    const pair = line.match(/^([A-Za-z0-9_-]+)\s*:\s*(.*)$/);
+    if (!pair) continue;
+    meta[pair[1]] = pair[2].replace(/^["']|["']$/g, "").trim();
+  }
+  return meta;
+}
+
+function scanCode(text) {
+  const matches = [];
+  for (const pattern of CODE_PATTERNS) {
+    const re = new RegExp(pattern.re.source, pattern.re.flags.includes("g") ? pattern.re.flags : `${pattern.re.flags}g`);
+    let match;
+    while ((match = re.exec(text)) !== null) {
+      if (match[0].length <= 500) {
+        matches.push({
+          line: lineAt(text, match.index),
+          label: pattern.label,
+          name: match[1] ?? null,
+          confidence: pattern.confidence
+        });
+      }
+      if (matches.length >= 50) return matches.sort((a, b) => a.line - b.line);
+      if (re.lastIndex === match.index) re.lastIndex += 1;
+    }
+  }
+  return matches.sort((a, b) => a.line - b.line);
+}
+
+function lineAt(text, index) {
+  let line = 1;
+  for (let i = 0; i < index; i += 1) {
+    if (text.charCodeAt(i) === 10) line += 1;
+  }
+  return line;
+}

package/lib/score.js

@@ -0,0 +1,137 @@
+function normalizeText(value) {
+  return String(value).replace(/\s+/g, " ").trim();
+}
+
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (typeof a === "number" && typeof b === "number") return a === b;
+  if (typeof a === "string" && typeof b === "string") {
+    return normalizeText(a) === normalizeText(b);
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    return a.length === b.length && a.every((item, i) => deepEqual(item, b[i]));
+  }
+  if (a && b && typeof a === "object" && typeof b === "object") {
+    const aKeys = Object.keys(a);
+    const bKeys = Object.keys(b);
+    if (aKeys.length !== bKeys.length) return false;
+    return aKeys.every((key) => deepEqual(a[key], b[key]));
+  }
+  return false;
+}
+
+export function matchArg(actual, expected) {
+  if (expected && typeof expected === "object" && !Array.isArray(expected)) {
+    if ("$exists" in expected) {
+      const present = actual !== undefined && actual !== null;
+      return present === Boolean(expected.$exists);
+    }
+    if ("$contains" in expected) {
+      const needle = normalizeText(expected.$contains);
+      if (typeof actual === "string") return normalizeText(actual).includes(needle);
+      if (Array.isArray(actual)) {
+        return actual.some((item) => normalizeText(item).includes(needle));
+      }
+      return false;
+    }
+    if ("$oneOf" in expected) {
+      return expected.$oneOf.some((option) => deepEqual(actual, option));
+    }
+  }
+  return deepEqual(actual, expected);
+}
+
+export function matchArgs(actualArgs, expectedArgs) {
+  const fields = Object.entries(expectedArgs ?? {});
+  const failures = [];
+  for (const [key, expected] of fields) {
+    if (!matchArg(actualArgs?.[key], expected)) failures.push(key);
+  }
+  return {
+    total: fields.length,
+    matched: fields.length - failures.length,
+    failures
+  };
+}
+
+export function scoreModel({ model, cases, results, pricing }) {
+  const rows = cases.map((testCase, index) => {
+    const result = results[index];
+    const actualTool = result.ok ? (result.toolCalls?.[0]?.name ?? null) : null;
+    const selectionCorrect = Boolean(result.ok) && actualTool === testCase.expectedTool;
+    const args = selectionCorrect && testCase.expectedTool
+      ? matchArgs(result.toolCalls?.[0]?.arguments, testCase.expectedArgs)
+      : { total: 0, matched: 0, failures: [] };
+    const passed = Boolean(result.ok) && selectionCorrect && args.failures.length === 0;
+    return {
+      id: testCase.id,
+      expectedTool: testCase.expectedTool,
+      actualTool,
+      selectionCorrect,
+      args,
+      passed,
+      reviewed: testCase.reviewed !== false,
+      source: testCase.source ?? "manual",
+      latencyMs: result.latencyMs ?? 0,
+      usage: result.usage ?? { input: 0, output: 0 },
+      error: result.error ?? null
+    };
+  });
+
+  const total = rows.length;
+  const reviewedRows = rows.filter((row) => row.reviewed);
+  const unreviewedRows = rows.filter((row) => !row.reviewed);
+  const selectionHits = reviewedRows.filter((row) => row.selectionCorrect).length;
+  const argFields = reviewedRows.reduce((sum, row) => sum + row.args.total, 0);
+  const argHits = reviewedRows.reduce((sum, row) => sum + row.args.matched, 0);
+  const passed = reviewedRows.filter((row) => row.passed).length;
+  const errors = rows.filter((row) => row.error).length;
+  const latencies = rows.filter((row) => row.latencyMs > 0).map((row) => row.latencyMs);
+  const inputTokens = rows.reduce((sum, row) => sum + (row.usage.input ?? 0), 0);
+  const outputTokens = rows.reduce((sum, row) => sum + (row.usage.output ?? 0), 0);
+  const cachedTokens = Math.min(
+    inputTokens,
+    rows.reduce((sum, row) => sum + (row.usage.cached ?? 0), 0)
+  );
+  const uncachedTokens = inputTokens - cachedTokens;
+  const costCny = pricing
+    ? (uncachedTokens * (pricing.input ?? 0) +
+        cachedTokens * (pricing.cachedInput ?? pricing.input ?? 0) +
+        outputTokens * (pricing.output ?? 0)) /
+      1e6
+    : 0;
+  const reviewedTotal = reviewedRows.length;
+
+  return {
+    model,
+    total,
+    reviewedTotal,
+    unreviewedTotal: unreviewedRows.length,
+    unreviewedSelectionAccuracy: unreviewedRows.length
+      ? unreviewedRows.filter((row) => row.selectionCorrect).length / unreviewedRows.length
+      : null,
+    selectionAccuracy: reviewedTotal ? selectionHits / reviewedTotal : 0,
+    argAccuracy: argFields ? argHits / argFields : null,
+    passRate: reviewedTotal ? passed / reviewedTotal : 0,
+    errors,
+    p50LatencyMs: percentile(latencies, 0.5),
+    p95LatencyMs: percentile(latencies, 0.95),
+    avgCostCny: total ? costCny / total : 0,
+    totalCostCny: costCny,
+    inputTokens,
+    cachedTokens,
+    outputTokens,
+    rows
+  };
+}
+
+function percentile(values, ratio) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(ratio * sorted.length) - 1));
+  return Math.round(sorted[index]);
+}
+
+export function pct(value) {
+  return value === null ? "n/a" : `${Math.round(value * 100)}%`;
+}