modelfunction-cli 0.0.0-stage → 0.1.0

package/lib/exec.js

@@ -0,0 +1,265 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const HARNESS = fileURLToPath(new URL("./runtime/harness.mjs", import.meta.url));
+const HARNESS_PY = fileURLToPath(new URL("./runtime/harness.py", import.meta.url));
+const MAX_OUTPUT = 1_000_000;
+const ENV_KEEP = ["PATH", "SystemRoot", "SYSTEMROOT", "TEMP", "TMP", "HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA"];
+
+export async function executeTool(options) {
+  const runtime = options.runtime ?? "node";
+  const engine = runtime === "python" ? "python-audit" : "node-permission";
+  const entry = resolve(options.entry);
+  const nodeMajor = Number(process.versions.node.split(".")[0]);
+  if (runtime === "node" && options.allowNet?.length && nodeMajor < 25) {
+    throw new Error("--allow-net 需要 Node >= 25（Node 24 的权限模型不限制网络）");
+  }
+
+  let tempDir = null;
+  let inputPath = options.inputPath ? resolve(options.inputPath) : null;
+  if (!inputPath) {
+    tempDir = await mkdtemp(join(tmpdir(), "mf-exec-"));
+    inputPath = join(tempDir, "input.json");
+    await writeFile(inputPath, options.inputJson ?? "{}", "utf8");
+  }
+
+  try {
+    let args;
+    let command;
+    let resultPath = null;
+    if (runtime === "python") {
+      tempDir = tempDir ?? (await mkdtemp(join(tmpdir(), "mf-exec-")));
+      resultPath = join(tempDir, "result.json");
+      args = [
+        HARNESS_PY,
+        "--entry",
+        entry,
+        "--input",
+        inputPath,
+        "--export",
+        options.exportName ?? "run",
+        "--result",
+        resultPath
+      ];
+      for (const path of options.allowRead ?? []) args.push("--allow-read", resolve(path));
+      for (const path of options.allowWrite ?? []) args.push("--allow-write", resolve(path));
+      for (const host of options.allowNet ?? []) args.push("--allow-net", host);
+      if (options.warmup) args.push("--warmup", String(options.warmup));
+      command = options.python ?? "python";
+    } else {
+      const readPaths = [
+        dirname(HARNESS),
+        dirname(entry),
+        dirname(inputPath),
+        process.cwd(),
+        ...(options.allowRead ?? [])
+      ].map((path) => resolve(path));
+      args = ["--permission"];
+      for (const path of dedupe(readPaths)) args.push(`--allow-fs-read=${path}`);
+      if (options.allowWrite?.length) {
+        for (const path of dedupe(options.allowWrite.map((item) => resolve(item)))) {
+          args.push(`--allow-fs-write=${path}`);
+        }
+      }
+      if (options.allowNet?.length) {
+        for (const host of options.allowNet) args.push(`--allow-net=${host}`);
+      }
+      args.push(HARNESS, "--entry", entry, "--input", inputPath, "--export", options.exportName ?? "run");
+      if (options.warmup) args.push("--warmup", String(options.warmup));
+      command = process.execPath;
+    }
+
+    const env = buildEnv(options.passEnv);
+    if (runtime === "python") {
+      env.PYTHONDONTWRITEBYTECODE = "1";
+      env.PYTHONIOENCODING = "utf-8";
+    }
+
+    const started = performance.now();
+    const run = await spawnHarness(command, args, env, options.timeoutMs ?? 10_000);
+    const wallMs = Math.round(performance.now() - started);
+
+    let report = null;
+    if (resultPath) {
+      try {
+        report = JSON.parse(await readFile(resultPath, "utf8"));
+      } catch {
+        report = null;
+      }
+    }
+    if (!report) {
+      try {
+        report = JSON.parse(run.payload.trim());
+      } catch {
+        const sentinel = run.stdout
+          .split(/\r?\n/)
+          .filter((line) => line.startsWith("__MF_RESULT__"))
+          .pop();
+        if (sentinel) {
+          try {
+            report = JSON.parse(sentinel.slice("__MF_RESULT__".length));
+          } catch {
+            report = null;
+          }
+        }
+      }
+    }
+
+    const permissionDenied =
+      report?.permissionDenied === true ||
+      report?.error?.code === "ERR_ACCESS_DENIED" ||
+      report?.error?.code === "MF_PERMISSION_DENIED" ||
+      /ERR_ACCESS_DENIED/.test(run.stderr);
+    const timedOut = run.timedOut;
+    const ok = Boolean(report?.ok) && run.exitCode === 0 && !timedOut;
+
+    const result = {
+      ok,
+      timedOut,
+      permissionDenied,
+      engine,
+      exitCode: run.exitCode,
+      signal: run.signal,
+      wallMs,
+      stdout: run.stdout.slice(-4000),
+      stderr: run.stderr.slice(-4000),
+      result: report?.result ?? null,
+      error:
+        report?.error ??
+        (report ? null : { name: "HarnessError", code: "NO_REPORT", message: "harness 没有返回结果" }),
+      metrics: report?.metrics ?? null
+    };
+
+    return {
+      result,
+      code: timedOut ? 4 : permissionDenied ? 3 : ok ? 0 : 1
+    };
+  } finally {
+    if (tempDir) await rm(tempDir, { recursive: true, force: true });
+  }
+}
+
+export async function runExec(options, io = console) {
+  const { result, code } = await executeTool(options);
+  io.log(
+    options.json
+      ? JSON.stringify(result, null, 2)
+      : formatExec(resolve(options.entry), result)
+  );
+  return code;
+}
+
+function spawnHarness(command, args, env, timeoutMs) {
+  return new Promise((resolvePromise) => {
+    const child = spawn(command, args, {
+      stdio: ["ignore", "pipe", "pipe", "pipe"],
+      env,
+      windowsHide: true
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let payload = "";
+    let settled = false;
+    let timedOut = false;
+
+    const finish = (exitCode, signal) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      try {
+        child.kill();
+      } catch {
+        // ignore
+      }
+      resolvePromise({ exitCode, signal, timedOut, stdout, stderr, payload });
+    };
+
+    const timer = setTimeout(() => {
+      timedOut = true;
+      finish(null, "SIGKILL");
+    }, timeoutMs);
+
+    child.stdout.on("data", (chunk) => {
+      if (stdout.length < MAX_OUTPUT) stdout += chunk.toString("utf8");
+    });
+    child.stderr.on("data", (chunk) => {
+      if (stderr.length < MAX_OUTPUT) stderr += chunk.toString("utf8");
+    });
+    child.stdio[3].on("data", (chunk) => {
+      if (payload.length < MAX_OUTPUT) payload += chunk.toString("utf8");
+    });
+
+    child.on("error", (error) => {
+      stderr += `\nspawn error: ${error.message}`;
+      finish(null, null);
+    });
+    child.on("exit", (code, signal) => finish(code, signal));
+  });
+}
+
+function buildEnv(passEnv = []) {
+  const env = {};
+  for (const key of ENV_KEEP) {
+    if (process.env[key] !== undefined) env[key] = process.env[key];
+  }
+  for (const key of passEnv) {
+    if (process.env[key] !== undefined) env[key] = process.env[key];
+  }
+  return env;
+}
+
+function dedupe(paths) {
+  return [...new Set(paths)];
+}
+
+function formatExec(entry, result) {
+  const lines = [`exec: ${entry}`];
+  const status = result.timedOut
+    ? "timed out"
+    : result.permissionDenied
+      ? "permission denied"
+      : result.ok
+        ? "ok"
+        : "failed";
+  lines.push(`  status:     ${status}`);
+  if (result.engine) lines.push(`  engine:     ${result.engine}（能力限制，不是安全边界）`);
+  lines.push(`  wall:       ${result.wallMs}ms`);
+  if (result.metrics) {
+    const m = result.metrics;
+    lines.push(`  duration:   ${m.durationMs}ms（import ${m.importMs}ms）`);
+    if (m.cpuUserMs !== undefined) {
+      lines.push(`  cpu:        user ${m.cpuUserMs}ms / sys ${m.cpuSystemMs}ms`);
+      lines.push(`  memory:     peak ${m.peakRssMB}MB（start ${m.startRssMB}MB），heap Δ ${m.heapUsedDeltaMB}MB`);
+      lines.push(`  fs:         read ${m.fsReadDelta} / write ${m.fsWriteDelta}`);
+    } else {
+      lines.push(`  cpu:        ${m.cpuMs ?? "n/a"}ms`);
+      lines.push(
+        `  memory:     peak ${m.peakRssMB ?? "n/a"}MB（start ${m.startRssMB ?? "n/a"}MB），heap peak ${m.heapPeakMB ?? "n/a"}MB`
+      );
+    }
+  }
+  if (result.error) {
+    const error = result.error;
+    lines.push(
+      `  error:      ${error.name ?? "Error"}${error.code ? ` (${error.code})` : ""}: ${error.message ?? ""}`
+    );
+    if (error.stack) {
+      const frames = String(error.stack)
+        .split("\n")
+        .slice(1, 4)
+        .map((line) => `              ${line.trim()}`);
+      lines.push(...frames);
+    }
+  }
+  if (result.ok) {
+    const text = JSON.stringify(result.result);
+    lines.push(`  result:     ${text.length > 800 ? `${text.slice(0, 800)}…` : text}`);
+  }
+  if (result.timedOut) lines.push("  note:       已按超时终止进程");
+  if (result.permissionDenied) lines.push("  note:       权限模型拒绝了文件/网络访问，可用于推断沙箱需求");
+  return lines.join("\n");
+}

package/lib/fingerprint.js

@@ -0,0 +1,52 @@
+import { createHash } from "node:crypto";
+
+export function canonicalize(value) {
+  if (Array.isArray(value)) return `[${value.map(canonicalize).join(",")}]`;
+  if (value && typeof value === "object") {
+    const keys = Object.keys(value).sort();
+    return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalize(value[key])}`).join(",")}}`;
+  }
+  if (typeof value === "string") return JSON.stringify(value.replace(/\s+/g, " ").trim());
+  return JSON.stringify(value ?? null);
+}
+
+export function hashOf(value) {
+  return createHash("sha256").update(canonicalize(value)).digest("hex").slice(0, 16);
+}
+
+export function toolFingerprint(tool) {
+  return hashOf({
+    name: tool.name,
+    description: tool.description ?? "",
+    parameters: tool.parameters ?? {},
+    sideEffects: tool.sideEffects ?? null
+  });
+}
+
+export function caseFingerprint(testCase) {
+  return hashOf({
+    input: testCase.input ?? "",
+    expectedTool: testCase.expectedTool ?? null,
+    expectedArgs: testCase.expectedArgs ?? {},
+    toolInput: testCase.toolInput ?? null,
+    expect: testCase.expect ?? null
+  });
+}
+
+export function hashText(text) {
+  return createHash("sha256").update(String(text ?? "")).digest("hex").slice(0, 16);
+}
+
+export function suiteFingerprint(suite, selectedCases = suite.cases) {
+  const tools = suite.tools.map((tool) => ({ name: tool.name, hash: toolFingerprint(tool) }));
+  const cases = selectedCases.map((testCase) => ({ id: testCase.id, hash: caseFingerprint(testCase) }));
+  return {
+    hash: hashOf({
+      tools: tools.map((tool) => [tool.name, tool.hash]),
+      cases: cases.map((testCase) => [testCase.id, testCase.hash])
+    }),
+    tools,
+    cases,
+    reviewed: selectedCases.filter((testCase) => testCase.reviewed !== false).length
+  };
+}

package/lib/fsutil.js

@@ -0,0 +1,7 @@
+import { rename, writeFile } from "node:fs/promises";
+
+export async function writeFileAtomic(path, data) {
+  const temp = `${path}.${process.pid}.tmp`;
+  await writeFile(temp, data, "utf8");
+  await rename(temp, path);
+}

package/lib/lint.js

@@ -0,0 +1,110 @@
+import { suspectedPairs } from "./cases.js";
+
+const GENERIC_NAMES = new Set([
+  "run",
+  "do",
+  "tool",
+  "helper",
+  "execute",
+  "action",
+  "process",
+  "handle",
+  "call",
+  "use"
+]);
+
+const WHEN_HINT = /(用于|用来|当|如果|适用|适合|use when|use this|helps? )/i;
+
+export function lintSuite(suite) {
+  const findings = [];
+  const add = (severity, code, tool, message, extra = {}) =>
+    findings.push({ severity, code, tool, message, ...extra });
+
+  for (const tool of suite.tools) {
+    const description = (tool.description ?? "").trim();
+    const template = `建议：\`${tool.name}：当用户需要____时使用。输入____，返回____。不适用于____。\``;
+
+    if (!description) {
+      add("error", "desc-missing", tool.name, "工具没有 description，模型无法判断何时调用", { suggestion: template });
+    } else {
+      if (description.length < 20) {
+        add("warn", "desc-short", tool.name, `description 只有 ${description.length} 字，信息不足`, { suggestion: template });
+      }
+      if (description.length > 600) {
+        add("info", "desc-long", tool.name, `description ${description.length} 字，可能挤占上下文`);
+      }
+      if (!WHEN_HINT.test(description)) {
+        add("warn", "desc-no-when", tool.name, "description 没说明触发时机（用于/当/适用…），模型容易误选", {
+          suggestion: template
+        });
+      }
+    }
+
+    if (!/^[a-z][a-z0-9_]*$/.test(tool.name)) {
+      add("warn", "name-style", tool.name, "工具名建议用小写 snake_case");
+    }
+    if (GENERIC_NAMES.has(tool.name)) {
+      add("warn", "name-generic", tool.name, "工具名过于通用，容易和其它工具混淆", {
+        suggestion: "建议用“动词_名词”，例如 search_docs、score_resume"
+      });
+    }
+
+    const parameters = tool.parameters ?? {};
+    const properties = parameters.properties ?? {};
+    const required = parameters.required ?? [];
+    const names = Object.keys(properties);
+
+    if (names.length === 0) {
+      add("info", "schema-empty", tool.name, "工具没有参数；确认确实不需要输入");
+    }
+    if (required.length > 5) {
+      add("warn", "required-many", tool.name, `有 ${required.length} 个必填参数，模型容易漏填或编造`, {
+        suggestion: "把非关键参数改为可选，或在输入里提供默认值"
+      });
+    }
+
+    for (const [name, schema] of Object.entries(properties)) {
+      if (!schema?.description) {
+        add("warn", "param-desc-missing", tool.name, `参数 ${name} 没有说明`, {
+          param: name,
+          suggestion: `建议：\`${name}：____（格式：____；示例：____）\``
+        });
+      }
+      if (!schema?.type) {
+        add("error", "param-type-missing", tool.name, `参数 ${name} 没有 type`, { param: name });
+      }
+      if (schema?.type === "object" && !schema.properties) {
+        add("warn", "param-object-no-props", tool.name, `参数 ${name} 是 object 但未定义 properties`, { param: name });
+      }
+      if (schema?.type === "array" && !schema.items) {
+        add("warn", "param-array-no-items", tool.name, `参数 ${name} 是数组但未定义 items`, { param: name });
+      }
+    }
+  }
+
+  for (const pair of suspectedPairs(suite.tools)) {
+    add("warn", "pair-overlap", pair.a, `与 ${pair.b} 的描述相似度 ${pair.similarity.toFixed(2)}，容易被混淆`, {
+      suggestion: `在 ${pair.a} 和 ${pair.b} 的 description 里各写一句“不适用于什么”，并补一条歧义用例`
+    });
+  }
+
+  const counts = { error: 0, warn: 0, info: 0 };
+  for (const finding of findings) counts[finding.severity] += 1;
+  return { tools: suite.tools.length, counts, findings };
+}
+
+export function formatLint(report) {
+  const lines = [
+    `tools: ${report.tools}  findings: ${report.findings.length}（error ${report.counts.error} / warn ${report.counts.warn} / info ${report.counts.info}）`
+  ];
+  if (report.findings.length === 0) {
+    lines.push("没有发现问题");
+    return lines.join("\n");
+  }
+  for (const finding of report.findings) {
+    const param = finding.param ? `.${finding.param}` : "";
+    lines.push(`[${finding.severity}] ${finding.tool}${param}  ${finding.code}: ${finding.message}`);
+    if (finding.suggestion) lines.push(`         ${finding.suggestion}`);
+  }
+  return lines.join("\n");
+}

package/lib/mcp-probe.js

@@ -0,0 +1,152 @@
+import { spawn } from "node:child_process";
+
+const MAX_BYTES = 1_000_000;
+const MAX_STDERR = 4_000;
+const CLIENT_INFO = { name: "mf", version: "0.1.0" };
+const PROTOCOL_VERSION = "2025-06-18";
+
+export async function probeStdioServer(server, { timeoutMs = 5_000, cwd } = {}) {
+  if (!server?.command) {
+    return { ok: false, error: "missing command" };
+  }
+
+  let child;
+  try {
+    child = spawn(server.command, server.args ?? [], {
+      cwd,
+      env: { ...process.env, ...(server.env ?? {}) },
+      stdio: ["pipe", "pipe", "pipe"],
+      shell: false,
+      windowsHide: true
+    });
+  } catch (error) {
+    return { ok: false, error: `spawn failed: ${error.message}` };
+  }
+
+  const pending = new Map();
+  let nextId = 1;
+  let buffer = "";
+  let stderr = "";
+  let received = 0;
+  let failure = null;
+  let settled = false;
+
+  const fail = (message) => {
+    if (!failure) failure = message;
+  };
+
+  const rejectAll = () => {
+    for (const [, waiter] of pending) waiter.reject(new Error(failure ?? "server closed"));
+    pending.clear();
+  };
+
+  const stop = () => {
+    if (!child || child.killed) return;
+    try {
+      child.kill();
+    } catch {
+      // ignore
+    }
+  };
+
+  const send = (payload) => {
+    if (!child.stdin.writable) throw new Error("stdin closed");
+    child.stdin.write(`${JSON.stringify(payload)}\n`);
+  };
+
+  const request = (method, params) =>
+    new Promise((resolve, reject) => {
+      const id = nextId++;
+      pending.set(id, { resolve, reject });
+      try {
+        send({ jsonrpc: "2.0", id, method, params });
+      } catch (error) {
+        pending.delete(id);
+        reject(error);
+      }
+    });
+
+  const onLine = (line) => {
+    if (!line.trim()) return;
+    let message;
+    try {
+      message = JSON.parse(line);
+    } catch {
+      return;
+    }
+    if (message.id === undefined || !pending.has(message.id)) return;
+    const waiter = pending.get(message.id);
+    pending.delete(message.id);
+    if (message.error) {
+      waiter.reject(new Error(message.error.message ?? "rpc error"));
+    } else {
+      waiter.resolve(message.result);
+    }
+  };
+
+  child.stdout.on("data", (chunk) => {
+    received += chunk.length;
+    if (received > MAX_BYTES) {
+      fail("response exceeded 1MB");
+      stop();
+      return;
+    }
+    buffer += chunk.toString("utf8");
+    let index;
+    while ((index = buffer.indexOf("\n")) >= 0) {
+      const line = buffer.slice(0, index);
+      buffer = buffer.slice(index + 1);
+      onLine(line);
+    }
+  });
+
+  child.stderr.on("data", (chunk) => {
+    if (stderr.length < MAX_STDERR) stderr += chunk.toString("utf8");
+  });
+
+  child.on("error", (error) => {
+    fail(`spawn failed: ${error.message}`);
+    rejectAll();
+  });
+
+  child.on("exit", (code) => {
+    if (settled) return;
+    fail(`server exited (code ${code})`);
+    rejectAll();
+  });
+
+  const timer = setTimeout(() => {
+    fail(`timeout after ${timeoutMs}ms`);
+    stop();
+    rejectAll();
+  }, timeoutMs);
+
+  try {
+    await request("initialize", {
+      protocolVersion: PROTOCOL_VERSION,
+      capabilities: {},
+      clientInfo: CLIENT_INFO
+    });
+    send({ jsonrpc: "2.0", method: "notifications/initialized" });
+    const result = await request("tools/list", {});
+    const tools = (result?.tools ?? [])
+      .filter((tool) => typeof tool?.name === "string" && tool.name.trim() !== "")
+      .map((tool) => ({
+        name: tool.name.trim(),
+        description: tool.description ?? "",
+        parameters: tool.inputSchema ?? { type: "object", properties: {} }
+      }));
+    return { ok: true, tools };
+  } catch (error) {
+    return {
+      ok: false,
+      error: String(error?.message ?? error),
+      stderr: stderr.trim().slice(0, 500)
+    };
+  } finally {
+    settled = true;
+    clearTimeout(timer);
+    rejectAll();
+    stop();
+  }
+}