niceeval 0.1.0

package/src/runner/discover.ts

@@ -0,0 +1,77 @@
+// 发现:扫 evals/ 找 *.eval.ts(默认导出 EvalDef 或数组),扫 experiments/ 找实验。
+// 路径即身份:id 从相对路径推导,排序保证稳定。
+
+import { readdir } from "node:fs/promises";
+import { dirname, join, relative, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { pad4 } from "../util.ts";
+import type { DiscoveredEval, DiscoveredExperiment, EvalDef, ExperimentDef } from "../types.ts";
+
+const SKIP_DIRS = new Set(["node_modules", ".git", ".niceeval", "dist", ".next"]);
+
+async function walkFiles(dir: string, match: (name: string) => boolean): Promise<string[]> {
+  const out: string[] = [];
+  async function walk(current: string): Promise<void> {
+    let entries;
+    try {
+      entries = await readdir(current, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      const full = join(current, e.name);
+      if (e.isDirectory()) {
+        if (SKIP_DIRS.has(e.name)) continue;
+        await walk(full);
+      } else if (e.isFile() && match(e.name)) {
+        out.push(full);
+      }
+    }
+  }
+  await walk(dir);
+  return out;
+}
+
+export async function discoverEvals(root: string): Promise<DiscoveredEval[]> {
+  const dir = join(root, "evals");
+  const files = (await walkFiles(dir, (n) => n.endsWith(".eval.ts") || n.endsWith(".eval.tsx"))).sort();
+  const out: DiscoveredEval[] = [];
+  for (const file of files) {
+    const mod = (await import(pathToFileURL(file).href)) as { default?: EvalDef | EvalDef[] };
+    const def = mod.default;
+    if (!def) continue;
+    const baseId = relative(dir, file).replace(/\.eval\.tsx?$/, "").split(sep).join("/");
+    const baseDir = dirname(file);
+    if (Array.isArray(def)) {
+      def.forEach((d, i) => out.push({ ...d, id: `${baseId}/${pad4(i)}`, baseDir, sourcePath: file }));
+    } else {
+      out.push({ ...def, id: baseId, baseDir, sourcePath: file });
+    }
+  }
+  return out;
+}
+
+export async function discoverExperiments(root: string): Promise<DiscoveredExperiment[]> {
+  const dir = join(root, "experiments");
+  const files = (await walkFiles(dir, (n) => n.endsWith(".ts") && !n.endsWith(".d.ts"))).sort();
+  const out: DiscoveredExperiment[] = [];
+  for (const file of files) {
+    const mod = (await import(pathToFileURL(file).href)) as { default?: ExperimentDef };
+    const def = mod.default;
+    if (!def || !def.agent) continue;
+    const id = relative(dir, file)
+      .replace(/\.ts$/, "")
+      .replace(/\.experiment$/, "")
+      .split(sep)
+      .join("/");
+    const group = id.includes("/") ? id.split("/")[0]! : "";
+    out.push({ ...def, id, group });
+  }
+  return out;
+}
+
+/** id 前缀过滤:精确匹配或目录前缀(weather 命中 weather 与 weather/*)。 */
+export function makeFilter(patterns: string[]): (id: string) => boolean {
+  if (patterns.length === 0) return () => true;
+  return (id) => patterns.some((p) => id === p || id.startsWith(p + "/"));
+}

package/src/runner/reporters/artifacts.ts

@@ -0,0 +1,81 @@
+// 默认本地工件报告器:给 `niceeval view` 提供稳定的离线输入。
+//
+// 布局(每 eval-attempt 一个文件夹,重数据分文件;summary.json 只留榜单元数据):
+//   .niceeval/<run>/
+//     summary.json                         # run 元数据 + 各 result 的判决/usage/断言/引用(瘦身)
+//     <evalId>/<agent>/<model>/a<attempt>/
+//       events.json  sources.json  trace.json  o11y.json  diff.json
+// view 读 summary.json 渲染榜单,展开某条 trace 时再按需 fetch 它的 trace.json。
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { EvalResult, Reporter, RunSummary } from "../../types.ts";
+
+/** 一个 attempt 的工件子目录(相对 run 根):<evalId>/<agent>/<model>/a<attempt>。 */
+function attemptDir(r: EvalResult): string {
+  const safe = (s: string) => s.replace(/[^\w.@-]/g, "_");
+  // evalId 里的 / 保留作目录层级,其余危险字符替换。
+  const id = r.id.replace(/[^\w./@-]/g, "_");
+  return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}/a${r.attempt}`;
+}
+
+/** summary.json 用的瘦身结果:去掉大数组(events/trace/o11y/diff/sources),换成目录引用 + 存在标记。 */
+function slimResult(r: EvalResult): EvalResult {
+  const { events, sources, o11y, trace, diff, rawTranscript, ...rest } = r;
+  void events;
+  void sources;
+  void o11y;
+  void trace;
+  void diff;
+  void rawTranscript;
+  return {
+    ...rest,
+    artifactsDir: attemptDir(r),
+    hasTrace: !!(trace && trace.length),
+    hasEvents: !!(events && events.length),
+    hasSources: !!(sources && sources.length),
+  };
+}
+
+export function Artifacts(root = ".niceeval"): Reporter {
+  let outputDir = "";
+  const ensureDir = async (): Promise<void> => {
+    if (!outputDir) outputDir = join(root, safeTimestamp(new Date()));
+    await mkdir(outputDir, { recursive: true });
+  };
+
+  return {
+    async onRunStart() {
+      outputDir = join(root, safeTimestamp(new Date()));
+      await mkdir(outputDir, { recursive: true });
+    },
+
+    // 每条结果一出来就把它的重数据落到自己的文件夹(增量、互不影响)。
+    async onEvalComplete(result) {
+      await ensureDir();
+      const dir = join(outputDir, attemptDir(result));
+      await mkdir(dir, { recursive: true });
+      const writes: Promise<unknown>[] = [];
+      if (result.events?.length)
+        writes.push(writeFile(join(dir, "events.json"), JSON.stringify(result.events), "utf-8"));
+      if (result.sources?.length)
+        writes.push(writeFile(join(dir, "sources.json"), JSON.stringify(result.sources), "utf-8"));
+      if (result.trace?.length)
+        writes.push(writeFile(join(dir, "trace.json"), JSON.stringify(result.trace), "utf-8"));
+      if (result.o11y) writes.push(writeFile(join(dir, "o11y.json"), JSON.stringify(result.o11y), "utf-8"));
+      if (result.diff) writes.push(writeFile(join(dir, "diff.json"), JSON.stringify(result.diff), "utf-8"));
+      await Promise.all(writes);
+    },
+
+    // run 结束写瘦身 summary.json(只含榜单要的字段 + 工件引用)。
+    async onRunComplete(summary) {
+      await ensureDir();
+      const slim: RunSummary = { ...summary, outputDir, results: summary.results.map(slimResult) };
+      await writeFile(join(outputDir, "summary.json"), JSON.stringify(slim, null, 2), "utf-8");
+    },
+  };
+}
+
+function safeTimestamp(d: Date): string {
+  return d.toISOString().replace(/[:.]/g, "-");
+}

package/src/runner/reporters/console.ts

@@ -0,0 +1,76 @@
+// 控制台报告器:流式逐行输出,失败断言内联展开,末尾出效率三件套(时间 / token / $)。
+
+import type { EvalResult, Reporter, RunSummary } from "../../types.ts";
+import { t } from "../../i18n/index.ts";
+import { formatDuration, formatTokens, renderRunReport } from "./table.ts";
+
+const SYMBOL: Record<string, string> = {
+  passed: "✓",
+  failed: "✗",
+  errored: "!",
+  skipped: "○",
+};
+
+export function Console(): Reporter {
+  return {
+    onRunStart(evals, _agent, shape) {
+      // compare(多 agent / 多 model)或 runs>1 时,实际 attempt 数 > eval 数。头部如实报清,
+      // 否则「本次运行 5 个 eval」会和末尾「5 passed, 5 failed」(按 attempt 计 10)对不上。
+      const n = shape?.evals ?? evals.length;
+      const extra =
+        shape && shape.totalRuns > n
+          ? t("report.runStartExtra", { configs: shape.configs, totalRuns: shape.totalRuns })
+          : "";
+      process.stdout.write(t("report.runStart", { count: n, extra }));
+    },
+    onEvalComplete(result: EvalResult) {
+      const sym = SYMBOL[result.outcome] ?? "?";
+      const tok = (result.usage?.inputTokens ?? 0) + (result.usage?.outputTokens ?? 0);
+      // requests > 0 但 tokens = 0 → agent 跑了但不上报用量(如 bub);显示 — 而非误导性的 0
+      const tokStr = tok > 0 ? `${formatTokens(tok)} tok` : (result.usage?.requests ?? 0) > 0 ? `— tok` : `0 tok`;
+      const cost = result.estimatedCostUSD !== undefined ? `  $${result.estimatedCostUSD.toFixed(3)}` : "";
+      const who = result.model ? `${result.agent}/${result.model}` : result.agent;
+      const meta = `(${formatDuration(result.durationMs)}  ${tokStr}${cost})`;
+      const label = result.outcome === "passed" ? "" : ` ${formatOutcome(result.outcome)}`;
+      process.stdout.write(`  ${sym} ${result.id}${label}  [${who}]  ${meta}\n`);
+
+      if (result.skipReason) {
+        process.stdout.write(`      ○ ${t("report.skipped")}: ${result.skipReason}\n`);
+      }
+      if (result.error) {
+        process.stdout.write(`      ! ${t("report.error")}: ${truncate(result.error, 400)}\n`);
+      }
+      let lastGroup: string | undefined;
+      for (const a of result.assertions) {
+        if (a.passed) continue;
+        if (a.group !== undefined && a.group !== lastGroup) {
+          process.stdout.write(`      ▸ ${a.group}\n`);
+        }
+        lastGroup = a.group;
+        const sev = a.severity === "gate" ? t("report.gate") : t("report.soft");
+        const thr = a.threshold !== undefined
+          ? t("report.assertionThreshold", { score: a.score.toFixed(2), threshold: a.threshold })
+          : "";
+        const indent = a.group !== undefined ? "        " : "      ";
+        process.stdout.write(`${indent}- ${sev}: ${a.name}${thr}${a.detail ? ` — ${truncate(a.detail, 300)}` : ""}\n`);
+      }
+    },
+    onRunComplete(summary: RunSummary) {
+      process.stdout.write(renderRunReport(summary));
+    },
+  };
+}
+
+function formatOutcome(outcome: string): string {
+  switch (outcome) {
+    case "passed": return t("report.passed");
+    case "failed": return t("report.failed");
+    case "errored": return t("report.errored");
+    case "skipped": return t("report.skipped");
+    default: return outcome;
+  }
+}
+
+function truncate(s: string, n: number): string {
+  return s.length > n ? s.slice(0, n) + "…" : s;
+}

package/src/runner/reporters/index.ts

@@ -0,0 +1,5 @@
+// 内置报告器:Console(默认)/ Artifacts / Json / JUnit。第三方实验跟踪平台也走这条 Reporter 通道。
+
+export { Console } from "./console.ts";
+export { Artifacts } from "./artifacts.ts";
+export { Json, JUnit } from "./json.ts";

package/src/runner/reporters/json.ts

@@ -0,0 +1,52 @@
+// JSON / JUnit 报告器:把运行结果落成机器可读工件,接 CI 或下游 dashboard。
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { Reporter, RunSummary } from "../../types.ts";
+
+export function Json(path: string): Reporter {
+  return {
+    async onRunComplete(summary: RunSummary) {
+      await mkdir(dirname(path), { recursive: true });
+      await writeFile(path, JSON.stringify(summary, null, 2), "utf-8");
+    },
+  };
+}
+
+export function JUnit(path: string): Reporter {
+  return {
+    async onRunComplete(summary: RunSummary) {
+      await mkdir(dirname(path), { recursive: true });
+      const cases = summary.results
+        .map((r) => {
+          const name = xmlAttr(`${r.id} [${r.agent}${r.model ? "/" + r.model : ""}]`);
+          const time = (r.durationMs / 1000).toFixed(3);
+          if (r.outcome === "errored") {
+            return `    <testcase name="${name}" time="${time}"><error message="${xmlAttr(r.error ?? "execution error")}"/></testcase>`;
+          }
+          if (r.outcome === "failed") {
+            const msg = xmlAttr(r.assertions.filter((a) => !a.passed).map((a) => a.name).join("; "));
+            return `    <testcase name="${name}" time="${time}"><failure message="${msg}"/></testcase>`;
+          }
+          if (r.outcome === "skipped") {
+            return `    <testcase name="${name}" time="${time}"><skipped message="${xmlAttr(r.skipReason ?? "")}"/></testcase>`;
+          }
+          return `    <testcase name="${name}" time="${time}"/>`;
+        })
+        .join("\n");
+      const xml =
+        `<?xml version="1.0" encoding="UTF-8"?>\n` +
+        `<testsuite name="niceeval" tests="${summary.results.length}" failures="${summary.failed}" errors="${summary.errored}" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">\n` +
+        `${cases}\n</testsuite>\n`;
+      await writeFile(path, xml, "utf-8");
+    },
+  };
+}
+
+function xmlAttr(s: string): string {
+  return s
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}

package/src/runner/reporters/live.ts

@@ -0,0 +1,178 @@
+// Live terminal reporter:在 TTY 终端里渲染实时状态表,每个 (eval, who) 对占一行。
+// spinner 每 80ms 刷新;attempt 完成后行内显示 ✓/✗/~ 符号。
+// onRunComplete 时清除状态表,打印和网页榜单同口径的表格报告。
+
+import type { Reporter, RunShape, RunSummary } from "../../types.ts";
+import { t } from "../../i18n/index.ts";
+import { renderRunReport } from "./table.ts";
+
+const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+
+const OUTCOME_SYM: Record<string, string> = {
+  passed: "✓",
+  failed: "✗",
+  errored: "!",
+  skipped: "○",
+};
+
+// 进度日志里的基础设施噪声:OTLP 端口、remote-agent 启动提示、trace span 计数。
+// 这些在行尾显示毫无意义,直接丢弃。
+const NOISE_PATTERNS = [
+  /^OTLP /,
+  /^使用 remote agent/,
+  /^using remote agent/,
+  /^驱动 agent/,
+  /^driving agent/,
+  /^trace:\d/,
+  /^agent tracing/,
+  /^agent setup/,
+];
+
+function isNoise(msg: string): boolean {
+  return NOISE_PATTERNS.some((p) => p.test(msg));
+}
+
+export interface LiveRow {
+  evalId: string;
+  who: string;
+  /** 该 (evalId, who) 对的总 attempt 数(含 earlyExit 估算值)。 */
+  total: number;
+}
+
+export interface LiveReporter extends Reporter {
+  /** 被 RunOptions.onProgress 调用,更新行尾的 lastMsg。 */
+  progress(evalId: string, who: string, msg: string): void;
+}
+
+interface RowState extends LiveRow {
+  completed: number;
+  lastMsg: string;
+  dominantOutcome: string | undefined;
+}
+
+export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
+  const stateMap = new Map<string, RowState>();
+  const keyOrder: string[] = [];
+
+  for (const r of rows) {
+    const k = `${r.evalId}|${r.who}`;
+    if (!stateMap.has(k)) {
+      stateMap.set(k, { ...r, completed: 0, lastMsg: "", dominantOutcome: undefined });
+      keyOrder.push(k);
+    } else {
+      // 同一 (evalId, who) 可能在多个 agentRun 里出现(不应发生,但做防御)
+      stateMap.get(k)!.total += r.total;
+    }
+  }
+
+  let spinFrame = 0;
+  let totalCompleted = 0;
+  let drawnLines = 0; // 上次 draw() 写了多少行,用于 \x1B[nA 回跳
+  let intervalId: ReturnType<typeof setInterval> | undefined;
+  let shape: RunShape | undefined;
+
+  const cols = () => process.stderr.columns || 100;
+
+  function renderRow(state: RowState, frame: number): string {
+    const done = state.completed >= state.total;
+    const sym = done
+      ? (OUTCOME_SYM[state.dominantOutcome ?? ""] ?? "?")
+      : SPINNER[frame % SPINNER.length];
+
+    const evalCol = state.evalId.slice(0, 24).padEnd(24);
+    const whoCol = `[${state.who}]`.slice(0, 26).padEnd(26);
+    const cntCol = `${state.completed}/${state.total}`.padEnd(5);
+
+    const prefix = `  ${sym} ${evalCol} ${whoCol} ${cntCol}  `;
+    const budget = Math.max(0, cols() - prefix.length - 1);
+    const msg = done ? "" : state.lastMsg.slice(0, budget);
+
+    return `\x1B[2K${prefix}${msg}`;
+  }
+
+  function renderHeader(): string {
+    const hdr = shape
+      ? t("live.running", {
+          totalRuns: shape.totalRuns,
+          evals: shape.evals,
+          configs: shape.configs,
+          completed: totalCompleted,
+          total: totalAttempts,
+        })
+      : t("live.runningUnknown", { completed: totalCompleted, total: totalAttempts });
+    return `\x1B[2K${hdr}`;
+  }
+
+  function draw(frame: number) {
+    if (!process.stderr.isTTY) return;
+
+    if (drawnLines > 0) {
+      process.stderr.write(`\x1B[${drawnLines}A`);
+    }
+
+    let out = renderHeader() + "\n";
+    for (const k of keyOrder) {
+      out += renderRow(stateMap.get(k)!, frame) + "\n";
+    }
+
+    process.stderr.write(out);
+    drawnLines = 1 + keyOrder.length;
+  }
+
+  function clearDisplay() {
+    if (!process.stderr.isTTY || drawnLines === 0) return;
+    // 回到起点,逐行清空
+    process.stderr.write(`\x1B[${drawnLines}A`);
+    for (let i = 0; i < drawnLines; i++) {
+      process.stderr.write("\x1B[2K\n");
+    }
+    process.stderr.write(`\x1B[${drawnLines}A`);
+    drawnLines = 0;
+  }
+
+  return {
+    progress(evalId, who, msg) {
+      if (isNoise(msg)) return;
+      const state = stateMap.get(`${evalId}|${who}`);
+      if (state) state.lastMsg = msg;
+      // 不在这里 draw();由 interval 驱动,避免每条日志都刷屏
+    },
+
+    onRunStart(_evals, _agent, s) {
+      shape = s;
+      // 初始渲染:让用户看到行表
+      draw(0);
+      intervalId = setInterval(() => {
+        spinFrame = (spinFrame + 1) % SPINNER.length;
+        draw(spinFrame);
+      }, 80);
+    },
+
+    onEvalComplete(result) {
+      const who = result.model ? `${result.agent}/${result.model}` : result.agent;
+      const state = stateMap.get(`${result.id}|${who}`);
+      if (state) {
+        state.completed += 1;
+        totalCompleted += 1;
+        const prev = state.dominantOutcome;
+        if (!prev || (prev === "passed" && result.outcome !== "passed")) {
+          state.dominantOutcome = result.outcome;
+        }
+      } else {
+        totalCompleted += 1;
+      }
+    },
+
+    onRunComplete(summary) {
+      if (intervalId) {
+        clearInterval(intervalId);
+        intervalId = undefined;
+      }
+      // 最后刷一帧(所有行都 done)
+      draw(spinFrame);
+      clearDisplay();
+
+      process.stdout.write(renderRunReport(summary));
+    },
+  };
+}