modelfunction-cli 0.0.0-stage → 0.1.0

package/lib/cases.js

@@ -0,0 +1,188 @@
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { loadSuite } from "./suite.js";
+import { writeFileAtomic } from "./fsutil.js";
+
+const MIN_CASES_PER_TOOL = 3;
+const MIN_NEGATIVE_RATIO = 0.2;
+const PAIR_SIMILARITY = 0.35;
+
+export async function addCases(suitePath, incoming, { source = "agent", reviewed = false, dryRun = false } = {}) {
+  const abs = resolve(suitePath);
+  const raw = JSON.parse(await readFile(abs, "utf8"));
+  const suite = await loadSuite(abs, { allowEmptyCases: true });
+  const tools = new Map(suite.tools.map((tool) => [tool.name, tool]));
+  const ids = new Set(suite.cases.map((testCase) => testCase.id));
+
+  const added = [];
+  const rejected = [];
+  const warnings = [];
+  const list = Array.isArray(incoming) ? incoming : [incoming];
+
+  for (const [index, entry] of list.entries()) {
+    if (!entry || typeof entry !== "object") {
+      rejected.push({ id: `#${index + 1}`, reason: "case must be an object" });
+      continue;
+    }
+    const id =
+      typeof entry.id === "string" && entry.id.trim()
+        ? entry.id.trim()
+        : `case_${ids.size + index + 1}`;
+    if (ids.has(id)) {
+      rejected.push({ id, reason: "duplicate id" });
+      continue;
+    }
+    if (typeof entry.input !== "string" || entry.input.trim() === "") {
+      rejected.push({ id, reason: "input must be a non-empty string" });
+      continue;
+    }
+    const expectedTool = entry.expectedTool ?? null;
+    if (expectedTool !== null && !tools.has(expectedTool)) {
+      rejected.push({ id, reason: `unknown expectedTool "${expectedTool}"` });
+      continue;
+    }
+    const expectedArgs = entry.expectedArgs ?? {};
+    if (expectedTool === null && Object.keys(expectedArgs).length > 0) {
+      rejected.push({ id, reason: "expectedArgs requires expectedTool (omit both for negative cases)" });
+      continue;
+    }
+    if (expectedTool !== null) {
+      const properties = tools.get(expectedTool).parameters?.properties ?? {};
+      for (const key of Object.keys(expectedArgs)) {
+        if (!(key in properties)) {
+          warnings.push(`${id}: expectedArgs.${key} 不在 ${expectedTool} 的 schema 中，请确认`);
+        }
+      }
+    }
+
+    const testCase = {
+      id,
+      input: entry.input,
+      expectedTool,
+      expectedArgs,
+      ...(entry.note ? { note: String(entry.note) } : {}),
+      source: typeof entry.source === "string" && entry.source ? entry.source : source,
+      reviewed: entry.reviewed === undefined ? reviewed : Boolean(entry.reviewed),
+      tags: Array.isArray(entry.tags) ? entry.tags.map(String) : []
+    };
+    ids.add(id);
+    added.push(testCase);
+  }
+
+  if (!dryRun && added.length > 0) {
+    raw.cases = Array.isArray(raw.cases) ? raw.cases : [];
+    raw.cases.push(...added);
+    await writeFileAtomic(abs, `${JSON.stringify(raw, null, 2)}\n`);
+  }
+
+  return { path: abs, added, rejected, warnings, dryRun };
+}
+
+export function computeCoverage(suite) {
+  const perTool = new Map(
+    suite.tools.map((tool) => [tool.name, { total: 0, withArgs: 0, reviewed: 0, unreviewed: 0 }])
+  );
+  let negative = 0;
+  let unreviewed = 0;
+
+  for (const testCase of suite.cases) {
+    const unreviewedCase = testCase.reviewed === false;
+    if (unreviewedCase) unreviewed += 1;
+
+    if (testCase.expectedTool === null) {
+      negative += 1;
+      continue;
+    }
+    const bucket = perTool.get(testCase.expectedTool);
+    if (!bucket) continue;
+    bucket.total += 1;
+    if (Object.keys(testCase.expectedArgs ?? {}).length > 0) bucket.withArgs += 1;
+    if (unreviewedCase) bucket.unreviewed += 1;
+    else bucket.reviewed += 1;
+  }
+
+  const tools = [...perTool.entries()].map(([name, counts]) => ({
+    name,
+    ...counts,
+    missing: Math.max(0, MIN_CASES_PER_TOOL - counts.total)
+  }));
+
+  const total = suite.cases.length;
+  const negativeRatio = total ? negative / total : 0;
+  const warnings = [];
+
+  for (const tool of tools) {
+    if (tool.total === 0) warnings.push(`${tool.name}: 0 条用例`);
+    else if (tool.missing > 0) warnings.push(`${tool.name}: ${tool.total}/${MIN_CASES_PER_TOOL} 条，缺 ${tool.missing} 条`);
+    if (tool.total > 0 && tool.withArgs === 0) warnings.push(`${tool.name}: 没有校验参数的用例`);
+  }
+  if (negativeRatio < MIN_NEGATIVE_RATIO) {
+    warnings.push(`负例占比 ${Math.round(negativeRatio * 100)}% < ${Math.round(MIN_NEGATIVE_RATIO * 100)}%`);
+  }
+  if (unreviewed > 0) warnings.push(`${unreviewed} 条未复核用例不计入正式分数`);
+
+  return {
+    total,
+    negative,
+    negativeRatio,
+    unreviewed,
+    tools,
+    pairs: suspectedPairs(suite.tools),
+    warnings
+  };
+}
+
+export function formatCoverage(report) {
+  const lines = [
+    `cases: ${report.total}（负例 ${report.negative}，${Math.round(report.negativeRatio * 100)}%；未复核 ${report.unreviewed}）`,
+    "",
+    "tool                    cases  args  reviewed",
+    "----------------------  -----  ----  --------"
+  ];
+  for (const tool of report.tools) {
+    lines.push(
+      `${tool.name.padEnd(22)}  ${String(tool.total).padStart(5)}  ${String(tool.withArgs).padStart(4)}  ${String(tool.reviewed).padStart(8)}`
+    );
+  }
+  if (report.pairs.length > 0) {
+    lines.push("", "suspected confusing pairs（建议各补 1 条歧义用例）:");
+    for (const pair of report.pairs) {
+      lines.push(`  ${pair.a} × ${pair.b}  similarity ${pair.similarity.toFixed(2)}`);
+    }
+  }
+  if (report.warnings.length > 0) {
+    lines.push("", "warnings:");
+    for (const warning of report.warnings) lines.push(`  - ${warning}`);
+  }
+  return lines.join("\n");
+}
+
+export function suspectedPairs(tools) {
+  const pairs = [];
+  for (let i = 0; i < tools.length; i += 1) {
+    for (let j = i + 1; j < tools.length; j += 1) {
+      const a = tokens(`${tools[i].name} ${tools[i].description}`);
+      const b = tokens(`${tools[j].name} ${tools[j].description}`);
+      const union = new Set([...a, ...b]);
+      if (union.size === 0) continue;
+      let shared = 0;
+      for (const token of union) if (a.has(token) && b.has(token)) shared += 1;
+      const similarity = shared / union.size;
+      if (similarity >= PAIR_SIMILARITY) {
+        pairs.push({ a: tools[i].name, b: tools[j].name, similarity });
+      }
+    }
+  }
+  return pairs.sort((x, y) => y.similarity - x.similarity);
+}
+
+function tokens(text) {
+  const set = new Set();
+  const lowered = String(text ?? "").toLowerCase();
+  for (const word of lowered.match(/[a-z][a-z0-9_]{1,}/g) ?? []) set.add(word);
+  for (const chunk of lowered.match(/[一-鿿]+/g) ?? []) {
+    if (chunk.length === 1) set.add(chunk);
+    for (let i = 0; i + 1 < chunk.length; i += 1) set.add(chunk.slice(i, i + 2));
+  }
+  return set;
+}

package/lib/compare.js

@@ -0,0 +1,220 @@
+export function compareScorecards(before, after) {
+  const fingerprintBefore = before.suite?.fingerprint ?? null;
+  const fingerprintAfter = after.suite?.fingerprint ?? null;
+
+  const toolChanges = diffEntries(fingerprintBefore?.tools, fingerprintAfter?.tools, "name");
+  const caseChanges = diffEntries(fingerprintBefore?.cases, fingerprintAfter?.cases, "id");
+  const executionBefore = before.suite?.execution?.hash ?? null;
+  const executionAfter = after.suite?.execution?.hash ?? null;
+  const executionChanged = executionBefore !== executionAfter;
+  const casesBefore = new Map((fingerprintBefore?.cases ?? []).map((entry) => [entry.id, entry.hash]));
+  const casesAfter = new Map((fingerprintAfter?.cases ?? []).map((entry) => [entry.id, entry.hash]));
+  const commonCaseIds = [...casesBefore.keys()].filter(
+    (id) => casesAfter.has(id) && casesAfter.get(id) === casesBefore.get(id)
+  );
+
+  const modelsAfter = new Map((after.models ?? []).map((summary) => [summary.model, summary]));
+  const models = [];
+
+  for (const modelBefore of before.models ?? []) {
+    const modelAfter = modelsAfter.get(modelBefore.model);
+    if (!modelAfter) {
+      models.push({ model: modelBefore.model, comparable: false, reason: "模型不在新报告中" });
+      continue;
+    }
+
+    const rowsBefore = new Map((modelBefore.rows ?? []).map((row) => [row.id, row]));
+    const rowsAfter = new Map((modelAfter.rows ?? []).map((row) => [row.id, row]));
+    const comparableIds = commonCaseIds.filter((id) => rowsBefore.has(id) && rowsAfter.has(id));
+
+    const regressions = [];
+    const fixes = [];
+    const argsDrops = [];
+    const confusionCounts = new Map();
+    const interferenceMap = new Map();
+    let reviewed = 0;
+    let selectionBefore = 0;
+    let selectionAfter = 0;
+    let passBefore = 0;
+    let passAfter = 0;
+
+    for (const id of comparableIds) {
+      const rowBefore = rowsBefore.get(id);
+      const rowAfter = rowsAfter.get(id);
+      if (rowBefore.reviewed === false || rowAfter.reviewed === false) continue;
+      reviewed += 1;
+      if (rowBefore.selectionCorrect) selectionBefore += 1;
+      if (rowAfter.selectionCorrect) selectionAfter += 1;
+      if (rowBefore.passed) passBefore += 1;
+      if (rowAfter.passed) passAfter += 1;
+
+      const confusionKey = `${rowAfter.expectedTool ?? "(none)"}→${rowAfter.actualTool ?? "(none)"}`;
+      confusionCounts.set(confusionKey, (confusionCounts.get(confusionKey) ?? 0) + 1);
+
+      if (
+        rowBefore.selectionCorrect &&
+        !rowAfter.selectionCorrect &&
+        rowBefore.expectedTool &&
+        rowAfter.actualTool
+      ) {
+        const interferenceKey = `${rowBefore.expectedTool}→${rowAfter.actualTool}`;
+        const entry = interferenceMap.get(interferenceKey) ?? {
+          expected: rowBefore.expectedTool,
+          stolenBy: rowAfter.actualTool,
+          cases: []
+        };
+        entry.cases.push(id);
+        interferenceMap.set(interferenceKey, entry);
+      }
+
+      if (rowBefore.passed && !rowAfter.passed) {
+        regressions.push({
+          id,
+          before: rowBefore.actualTool ?? null,
+          after: rowAfter.actualTool ?? null,
+          args: rowAfter.args?.failures ?? []
+        });
+      }
+      if (!rowBefore.passed && rowAfter.passed) {
+        fixes.push({ id, before: rowBefore.actualTool ?? null, after: rowAfter.actualTool ?? null });
+      }
+      if (
+        rowBefore.args?.total > 0 &&
+        rowAfter.args?.total > 0 &&
+        (rowAfter.args.matched ?? 0) < (rowBefore.args.matched ?? 0)
+      ) {
+        argsDrops.push({
+          id,
+          before: rowBefore.args.matched,
+          after: rowAfter.args.matched,
+          total: rowBefore.args.total
+        });
+      }
+    }
+
+    const confusion = [...confusionCounts.entries()].map(([key, count]) => {
+      const [expected, actual] = key.split("→");
+      return { expected, actual, count };
+    });
+    const interference = [...interferenceMap.values()].map((entry) => ({
+      ...entry,
+      count: entry.cases.length,
+      stolenByIsNewTool: toolChanges.some(
+        (change) => change.change === "added" && change.name === entry.stolenBy
+      )
+    }));
+
+    models.push({
+      model: modelBefore.model,
+      comparable: toolChanges.length === 0 && caseChanges.length === 0 && !executionChanged,
+      commonCases: comparableIds.length,
+      reviewed,
+      metrics: {
+        selection: { before: ratio(selectionBefore, reviewed), after: ratio(selectionAfter, reviewed) },
+        pass: { before: ratio(passBefore, reviewed), after: ratio(passAfter, reviewed) }
+      },
+      regressions,
+      fixes,
+      argsDrops,
+      confusion,
+      interference
+    });
+  }
+
+  return {
+    beforeHash: fingerprintBefore?.hash ?? null,
+    afterHash: fingerprintAfter?.hash ?? null,
+    toolChanges,
+    caseChanges,
+    executionChanged,
+    executionBefore,
+    executionAfter,
+    models
+  };
+}
+
+export function formatCompare(report) {
+  const lines = [];
+  if (!report.beforeHash || !report.afterHash) {
+    lines.push("warn: 至少一份报告没有 fingerprint（旧版本生成），无法确认定义是否相同");
+  }
+  lines.push(`tool changes: ${report.toolChanges.length}  case changes: ${report.caseChanges.length}`);
+  if (report.executionChanged) {
+    lines.push("execution: 执行入口代码已变更，执行结果不可直接比较");
+  }
+  for (const change of report.toolChanges) {
+    lines.push(`  tool ${change.change}: ${change.name}`);
+  }
+  for (const change of report.caseChanges) {
+    lines.push(`  case ${change.change}: ${change.name}`);
+  }
+
+  for (const model of report.models) {
+    lines.push("");
+    if (model.reviewed === undefined) {
+      lines.push(`[${model.model}] ${model.reason}`);
+      continue;
+    }
+    lines.push(`[${model.model}] ${model.comparable ? "可比较" : "不可直接比较（工具定义或用例已变更）"}`);
+    lines.push(`  共同且未变更用例: ${model.commonCases}（已复核 ${model.reviewed}）`);
+    lines.push(`  选择准确率: ${percent(model.metrics.selection.before)} → ${percent(model.metrics.selection.after)}`);
+    lines.push(`  通过率:     ${percent(model.metrics.pass.before)} → ${percent(model.metrics.pass.after)}`);
+    if (model.interference.length > 0) {
+      lines.push("  工具间干扰（原本选对，现在被别的工具抢走）:");
+      for (const item of model.interference) {
+        const tag = item.stolenByIsNewTool ? "（来自本次新增的工具）" : "";
+        lines.push(`    - ${item.expected} → ${item.stolenBy}: ${item.count} 条${tag}  ${item.cases.join(", ")}`);
+      }
+    }
+    const offDiagonal = (model.confusion ?? []).filter((entry) => entry.expected !== entry.actual);
+    if (offDiagonal.length > 0) {
+      lines.push("  误选分布:");
+      for (const entry of offDiagonal) {
+        lines.push(`    - 期望 ${entry.expected}，实际 ${entry.actual}: ${entry.count}`);
+      }
+    }
+    if (model.regressions.length > 0) {
+      lines.push("  回归:");
+      for (const item of model.regressions) {
+        const args = item.args.length > 0 ? `（参数缺 ${item.args.join(", ")}）` : "";
+        lines.push(`    - ${item.id}: ${item.before ?? "(none)"} → ${item.after ?? "(none)"}${args}`);
+      }
+    }
+    if (model.fixes.length > 0) {
+      lines.push("  修复:");
+      for (const item of model.fixes) {
+        lines.push(`    - ${item.id}: ${item.before ?? "(none)"} → ${item.after ?? "(none)"}`);
+      }
+    }
+    if (model.argsDrops.length > 0) {
+      lines.push("  参数匹配下降:");
+      for (const item of model.argsDrops) {
+        lines.push(`    - ${item.id}: ${item.before}/${item.total} → ${item.after}/${item.total}`);
+      }
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function diffEntries(beforeList, afterList, key) {
+  const before = new Map((beforeList ?? []).map((entry) => [entry[key], entry.hash]));
+  const after = new Map((afterList ?? []).map((entry) => [entry[key], entry.hash]));
+  const changes = [];
+  for (const [name, hash] of before) {
+    if (!after.has(name)) changes.push({ name, change: "removed", before: hash });
+    else if (after.get(name) !== hash) changes.push({ name, change: "changed", before: hash, after: after.get(name) });
+  }
+  for (const [name, hash] of after) {
+    if (!before.has(name)) changes.push({ name, change: "added", after: hash });
+  }
+  return changes;
+}
+
+function ratio(numerator, denominator) {
+  return denominator ? numerator / denominator : null;
+}
+
+function percent(value) {
+  return value === null ? "n/a" : `${Math.round(value * 100)}%`;
+}

package/lib/eval-exec.js

@@ -0,0 +1,140 @@
+import { resolve } from "node:path";
+import { executeTool } from "./exec.js";
+
+export async function runExecutionChecks({ suite, cases, summaries, options }) {
+  const config = suite.execution ?? {};
+  const entry = options.execEntry ?? config.entry;
+  if (!entry) {
+    throw new Error("--execute 需要 --entry，或套件中的 execution.entry");
+  }
+  const entryPath = resolve(entry);
+  const runtime = config.runtime ?? options.execRuntime ?? "node";
+  const engine = runtime === "python" ? "python-audit" : "node-permission";
+  const python = config.python ?? options.execPython ?? "python";
+  const exportName = config.export ?? options.execExport ?? "run";
+  const timeoutMs = config.timeoutMs ?? options.execTimeoutMs ?? 10_000;
+  const allowRead = config.allowRead ?? [];
+  const allowWrite = config.allowWrite ?? [];
+  const allowNet = config.allowNet ?? [];
+
+  const tools = new Map(suite.tools.map((tool) => [tool.name, tool]));
+
+  for (const summary of summaries) {
+    const rows = new Map(summary.rows.map((row) => [row.id, row]));
+    const results = [];
+    let skipped = 0;
+    let blocked = 0;
+
+    for (const testCase of cases) {
+      const row = rows.get(testCase.id);
+      if (!row || row.reviewed === false || !testCase.toolInput || testCase.expectedTool === null) {
+        skipped += 1;
+        continue;
+      }
+      if (!row.selectionCorrect) {
+        skipped += 1;
+        results.push({ id: testCase.id, skipped: true, reason: "selection failed" });
+        continue;
+      }
+
+      const sideEffects = tools.get(testCase.expectedTool)?.sideEffects ?? null;
+      const safe = sideEffects === "none" || sideEffects === "read";
+      if (!safe && !options.allowSideEffects) {
+        blocked += 1;
+        skipped += 1;
+        results.push({
+          id: testCase.id,
+          skipped: true,
+          reason: `sideEffects=${sideEffects ?? "unknown"} requires --allow-side-effects`
+        });
+        continue;
+      }
+
+      const execution = await executeTool({
+        entry: entryPath,
+        runtime,
+        python,
+        exportName,
+        timeoutMs,
+        warmup: 0,
+        inputJson: JSON.stringify({ tool: testCase.expectedTool, arguments: testCase.toolInput }),
+        allowRead,
+        allowWrite,
+        allowNet
+      });
+
+      const checks = evaluateExpect(testCase.expect, execution.result);
+      const entryResult = {
+        id: testCase.id,
+        passed: checks.passed,
+        failures: checks.failures,
+        sideEffects,
+        durationMs: execution.result.metrics?.durationMs ?? null,
+        peakRssMB: execution.result.metrics?.peakRssMB ?? null,
+        exitCode: execution.code
+      };
+      results.push(entryResult);
+      row.exec = entryResult;
+    }
+
+    const executed = results.filter((item) => !item.skipped);
+    const durations = executed.map((item) => item.durationMs).filter((value) => typeof value === "number");
+    const rss = executed.map((item) => item.peakRssMB).filter((value) => typeof value === "number");
+    const passed = executed.filter((item) => item.passed).length;
+    summary.exec = {
+      engine,
+      total: results.length,
+      executed: executed.length,
+      skipped,
+      blocked,
+      passed,
+      passRate: executed.length > 0 ? passed / executed.length : null,
+      p50DurationMs: percentile(durations, 0.5),
+      maxPeakRssMB: rss.length > 0 ? Math.max(...rss) : null,
+      results
+    };
+    summary.execPassRate = summary.exec.passRate;
+  }
+}
+
+function evaluateExpect(expect, execution) {
+  const failures = [];
+  const expected = expect ?? { ok: true };
+  const metrics = execution.metrics ?? {};
+
+  if (expected.ok !== undefined && execution.ok !== expected.ok) {
+    failures.push(`ok: expected ${expected.ok}, got ${execution.ok}`);
+  }
+  if (expected.errorType && execution.error?.name !== expected.errorType) {
+    failures.push(`errorType: expected ${expected.errorType}, got ${execution.error?.name ?? "none"}`);
+  }
+  if (expected.contains !== undefined) {
+    const text = JSON.stringify(execution.result ?? "");
+    for (const needle of [].concat(expected.contains)) {
+      if (!text.includes(String(needle))) failures.push(`contains: 缺少 ${needle}`);
+    }
+  }
+  if (expected.equals !== undefined && JSON.stringify(execution.result) !== JSON.stringify(expected.equals)) {
+    failures.push("equals: 返回结果不一致");
+  }
+  if (typeof expected.maxDurationMs === "number") {
+    const value = metrics.durationMs;
+    if (typeof value !== "number" || value > expected.maxDurationMs) {
+      failures.push(`maxDurationMs: ${value ?? "n/a"} > ${expected.maxDurationMs}`);
+    }
+  }
+  if (typeof expected.maxPeakRssMB === "number") {
+    const value = metrics.peakRssMB;
+    if (typeof value !== "number" || value > expected.maxPeakRssMB) {
+      failures.push(`maxPeakRssMB: ${value ?? "n/a"} > ${expected.maxPeakRssMB}`);
+    }
+  }
+  return { passed: failures.length === 0, failures };
+}
+
+function percentile(values, ratio) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(ratio * sorted.length) - 1));
+  return Math.round(sorted[index]);
+}