niceeval 0.1.0

package/src/expect/index.ts

@@ -0,0 +1,243 @@
+// 值级断言匹配器(expect)。每个匹配器产出一个 ValueAssertion:纯函数 score +
+// 可链式改严重级 / 阈值。链式方法返回全新的不可变 ValueAssertion,复用同一个 score。
+
+import type { Severity, ValueAssertion } from "../types.ts";
+import { stripComments } from "../util.ts";
+
+/** includes / excludes 的可选项。stripComments:先剥注释再匹配(只对真实代码生效)。 */
+export interface MatchOptions {
+  stripComments?: boolean;
+}
+
+// ───────────────────────── 内部工厂 ─────────────────────────
+
+/**
+ * 唯一的内部工厂。gate()/atLeast() 都基于它返回新的不可变实例,
+ * 共享同一个 score(只换 severity / threshold)。
+ */
+function createAssertion(
+  name: string,
+  severity: Severity,
+  score: (value: unknown) => number | Promise<number>,
+  threshold?: number,
+): ValueAssertion {
+  const self: ValueAssertion = {
+    name,
+    severity,
+    threshold,
+    score,
+    // 转成硬门槛(失败即整条 eval 不通过)。
+    gate: (t?: number) => createAssertion(name, "gate", score, t),
+    // 软阈值:默认不改变 outcome;--strict 下软阈值失败也会使 outcome=failed。
+    atLeast: (t: number) => createAssertion(name, "soft", score, t),
+  };
+  return Object.freeze(self);
+}
+
+// ───────────────────────── 工具函数 ─────────────────────────
+
+/** 给断言起个可读名时用,JSON.stringify 失败 / 为 undefined 时回退到 String。 */
+function safeLabel(v: unknown): string {
+  try {
+    return JSON.stringify(v) ?? String(v);
+  } catch {
+    return String(v);
+  }
+}
+
+/** 小而全的深比较:处理基本值、NaN、数组、Date、纯对象。 */
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+
+  // NaN === NaN
+  if (typeof a === "number" && typeof b === "number") {
+    return Number.isNaN(a) && Number.isNaN(b);
+  }
+
+  if (a === null || b === null || typeof a !== "object" || typeof b !== "object") {
+    return false;
+  }
+
+  // Date 按时间戳比
+  if (a instanceof Date || b instanceof Date) {
+    return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
+  }
+
+  const aIsArr = Array.isArray(a);
+  const bIsArr = Array.isArray(b);
+  if (aIsArr !== bIsArr) return false;
+
+  if (aIsArr && bIsArr) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false;
+    }
+    return true;
+  }
+
+  const ao = a as Record<string, unknown>;
+  const bo = b as Record<string, unknown>;
+  const ak = Object.keys(ao);
+  const bk = Object.keys(bo);
+  if (ak.length !== bk.length) return false;
+  for (const k of ak) {
+    if (!Object.prototype.hasOwnProperty.call(bo, k)) return false;
+    if (!deepEqual(ao[k], bo[k])) return false;
+  }
+  return true;
+}
+
+/** 经典 DP 编辑距离(滚动两行)。 */
+function levenshtein(a: string, b: string): number {
+  const m = a.length;
+  const n = b.length;
+  if (m === 0) return n;
+  if (n === 0) return m;
+
+  let prev = new Array<number>(n + 1);
+  let curr = new Array<number>(n + 1);
+  for (let j = 0; j <= n; j++) prev[j] = j;
+
+  for (let i = 1; i <= m; i++) {
+    curr[0] = i;
+    for (let j = 1; j <= n; j++) {
+      const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1;
+      curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
+    }
+    const tmp = prev;
+    prev = curr;
+    curr = tmp;
+  }
+  return prev[n];
+}
+
+// ───────────────────────── 匹配器 ─────────────────────────
+
+/** 把 value 转成待匹配字符串;opts.stripComments 时先剥注释。 */
+function toMatchTarget(value: unknown, opts?: MatchOptions): string {
+  const s = String(value);
+  return opts?.stripComments ? stripComments(s) : s;
+}
+
+/** String(value) 含子串 / 命中正则则 1,否则 0。默认硬门槛。opts.stripComments 时只看真实代码。 */
+export function includes(needle: string | RegExp, opts?: MatchOptions): ValueAssertion {
+  const label = needle instanceof RegExp ? needle.toString() : safeLabel(needle);
+  const suffix = opts?.stripComments ? ", stripComments" : "";
+  return createAssertion(`includes(${label}${suffix})`, "gate", (value) => {
+    const s = toMatchTarget(value, opts);
+    if (needle instanceof RegExp) return needle.test(s) ? 1 : 0;
+    return s.includes(needle) ? 1 : 0;
+  });
+}
+
+/** includes 的取反:不含子串 / 不命中正则则 1,否则 0。默认硬门槛。opts.stripComments 时只看真实代码。 */
+export function excludes(needle: string | RegExp, opts?: MatchOptions): ValueAssertion {
+  const label = needle instanceof RegExp ? needle.toString() : safeLabel(needle);
+  const suffix = opts?.stripComments ? ", stripComments" : "";
+  return createAssertion(`excludes(${label}${suffix})`, "gate", (value) => {
+    const s = toMatchTarget(value, opts);
+    if (needle instanceof RegExp) return needle.test(s) ? 0 : 1;
+    return s.includes(needle) ? 0 : 1;
+  });
+}
+
+/** 深相等则 1,否则 0。默认硬门槛。 */
+export function equals(expected: unknown): ValueAssertion {
+  return createAssertion(`equals(${safeLabel(expected)})`, "gate", (value) =>
+    deepEqual(value, expected) ? 1 : 0,
+  );
+}
+
+/**
+ * 用 schema 校验 value。优先 Standard Schema(schema['~standard'].validate),
+ * 否则退化到 zod 风格的 .safeParse / .parse。校验通过 1,否则 0;任何异常 → 0。
+ */
+export function matches(schema: unknown): ValueAssertion {
+  return createAssertion("matches(schema)", "gate", async (value) => {
+    try {
+      const std = (schema as { ["~standard"]?: { validate?: (v: unknown) => unknown } } | null)?.[
+        "~standard"
+      ];
+      if (std && typeof std.validate === "function") {
+        // validate 可能同步也可能返回 Promise;成功结果不带 issues。
+        const result = (await std.validate(value)) as { issues?: unknown } | null | undefined;
+        return result != null && result.issues == null ? 1 : 0;
+      }
+
+      const zodish = schema as {
+        safeParse?: (v: unknown) => { success?: boolean };
+        parse?: (v: unknown) => unknown;
+      } | null;
+
+      if (zodish && typeof zodish.safeParse === "function") {
+        const result = zodish.safeParse(value);
+        return result && result.success ? 1 : 0;
+      }
+      if (zodish && typeof zodish.parse === "function") {
+        zodish.parse(value);
+        return 1;
+      }
+      return 0;
+    } catch {
+      return 0;
+    }
+  });
+}
+
+/** 归一化 Levenshtein 相似度 [0,1]。默认软分,阈值 0.6。 */
+export function similarity(expected: string): ValueAssertion {
+  return createAssertion(
+    `similarity(${safeLabel(expected)})`,
+    "soft",
+    (value) => {
+      const a = String(value);
+      const b = expected;
+      const maxLen = Math.max(a.length, b.length);
+      if (maxLen === 0) return 1; // 两个空串视为完全相同
+      return 1 - levenshtein(a, b) / maxLen;
+    },
+    0.6,
+  );
+}
+
+/** 谓词为真则 1,否则 0。默认硬门槛;name 带上 label 便于报告辨认。 */
+export function satisfies(predicate: (v: unknown) => boolean, label?: string): ValueAssertion {
+  const name = label ? `satisfies(${label})` : "satisfies(predicate)";
+  return createAssertion(name, "gate", (value) => (predicate(value) ? 1 : 0));
+}
+
+/** value 非 null / 非 undefined 则 1,否则 0。省掉 `x !== undefined` + isTrue 的样板。 */
+export function isDefined(label?: string): ValueAssertion {
+  const name = label ? `isDefined(${label})` : "isDefined()";
+  return createAssertion(name, "gate", (value) => (value != null ? 1 : 0));
+}
+
+/** value === true 则 1,否则 0。带 label 的布尔断言(fileExists 等检查用)。 */
+export function isTrue(label?: string): ValueAssertion {
+  const name = label ? `isTrue(${label})` : "isTrue()";
+  return createAssertion(name, "gate", (value) => (value === true ? 1 : 0));
+}
+
+/** CommandResult.exitCode === 0 则通过。 */
+export function commandSucceeded(): ValueAssertion {
+  return createAssertion("commandSucceeded()", "gate", (value) => {
+    if (value === null || typeof value !== "object") return 0;
+    return (value as { exitCode?: unknown }).exitCode === 0 ? 1 : 0;
+  });
+}
+
+/** value === false 则 1,否则 0。带 label 的布尔断言。 */
+export function isFalse(label?: string): ValueAssertion {
+  const name = label ? `isFalse(${label})` : "isFalse()";
+  return createAssertion(name, "gate", (value) => (value === false ? 1 : 0));
+}
+
+/** 自定义断言工厂:直接给名字 / 严重级 / 阈值 / score。severity 省略默认 gate。 */
+export function makeAssertion(spec: {
+  name: string;
+  severity?: Severity;
+  threshold?: number;
+  score: (value: unknown) => number | Promise<number>;
+}): ValueAssertion {
+  return createAssertion(spec.name, spec.severity ?? "gate", spec.score, spec.threshold);
+}

package/src/i18n/en.ts

@@ -0,0 +1,131 @@
+import type { Messages } from "./zh-CN.ts";
+
+export const en = {
+  "agent.installFailed": "Install failed: {{key}}\n{{tail}}",
+  "agent.unknown": "Unknown agent \"{{name}}\". Registered agents: {{known}}.",
+  "bub.diagnose.exitCode": "bub run exited with code {{code}}",
+  "bub.diagnose.lastError": "last error: {{message}}",
+  "bub.diagnose.noTape": "tape was not generated",
+  "bub.diagnose.outputTail": "output tail: {{tail}}",
+  "bub.diagnose.zeroEvents": "tape exists but contains 0 events",
+  "bub.installFailed": "bub install failed after {{attempts}} attempts:\n{{tail}}",
+  "checkpoint.emptyTar": "checkpoint: tar is empty (paths: {{paths}})",
+  "cli.all": "(all)",
+  "cli.browserOpenFailed": "Could not open the browser automatically. Open manually: {{url}}\n",
+  "cli.clean.done": "Deleted .niceeval/ historical run artifacts.\n",
+  "cli.config.missing": "Could not find niceeval.config.ts (run this from the project root).",
+  "cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
+  "cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
+  "cli.dry.noMatches": "(no matches)",
+  "cli.dry.row": "  {{who}}{{experiment}}: {{evals}}  ×{{runs}}\n",
+  "cli.error": "niceeval error: {{error}}\n",
+  "cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
+  "cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
+  "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
+  "cli.exp.agentModelFlagUnsupported": "`--agent` / `--model` cannot override an experiment. Add or copy a config file under experiments/ instead.\n",
+  "cli.experiment.noMatch": "No experiment matched: {{arg}}. Discovered: {{experiments}}\n",
+  "cli.experimentGroup": " group",
+  "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
+  "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
+  "cli.init.done": "Created evals/ and starter niceeval.config.ts.\n",
+  "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
+  "cli.list.header": "Discovered {{count}} evals:\n",
+  "cli.noAgent": "No agent specified (use --agent <name>).\n",
+  "cli.none": "(none)",
+  "cli.pressCtrlC": "Press Ctrl+C to exit.\n",
+  "cli.run.experimentRequired": "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n",
+  "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
+  "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
+  "cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
+  "cli.view.exported": "Exported eval report page: {{out}}\n",
+  "cli.view.url": "niceeval view: {{url}}\n",
+  "context.skipEmpty": "skip() requires a non-empty reason.",
+  "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
+  "context.turnFailedDefault": "This send returned failed (turn status = failed)",
+  "define.agentNameRequired": "defineAgent requires name.",
+  "define.evalIdRejected": "defineEval does not accept id; ids are derived from file paths.",
+  "define.evalTestRequired": "defineEval requires an async test(t) function.",
+  "define.experimentAgentRequired": "defineExperiment requires agent.",
+  "define.experimentIdRejected": "defineExperiment does not accept id; ids are derived from file paths.",
+  "define.sandboxAgentNameRequired": "defineSandboxAgent requires name.",
+  "define.sandboxCreateRequired": "defineSandbox requires a create() function.",
+  "define.sandboxNameRequired": "defineSandbox requires name.",
+  "docker.commandTimeout": "Command timed out after {{timeoutMs}}ms",
+  "docker.containerNotInitialized": "Container not initialized",
+  "docker.imagePullDone": "Docker image ready: {{image}}",
+  "docker.imagePullStart": "Pulling Docker image: {{image}}...",
+  "docker.readFileFailed": "Failed to read file {{path}}: {{stderr}}",
+  "docker.unsupportedRuntime": "Unsupported runtime: {{runtime}}",
+  "judge.apiKeyMissing": "judge is missing an API key (CODEX_API_KEY / OPENAI_API_KEY).",
+  "judge.httpError": "judge HTTP {{status}}: {{body}}",
+  "judge.probeFailed": "judge precheck failed ({{model}}): {{error}}",
+  "judge.probeMissingKey": "judge model {{model}} is missing an API key; configure {{envHint}}",
+  "live.running": "  Running {{totalRuns}} attempts ({{evals}} evals × {{configs}} configs)       {{completed}}/{{total}} done",
+  "live.runningUnknown": "  Running...  {{completed}}/{{total}} done",
+  "report.assertionThreshold": " (got {{score}} < {{threshold}})",
+  "report.error": "error",
+  "report.errored": "errored",
+  "report.failed": "failed",
+  "report.gate": "gate",
+  "report.passed": "passed",
+  "report.result": "\nResult: {{parts}}  ({{duration}} · {{tokens}}{{cost}})\n\n",
+  "report.runStart": "\nRunning {{count}} evals{{extra}}\n\n",
+  "report.runStartExtra": " × {{configs}} configs = {{totalRuns}} runs",
+  "report.skipped": "skipped",
+  "report.soft": "soft",
+  "report.summary.errored": "{{count}} errored",
+  "report.summary.failed": "{{count}} failed",
+  "report.summary.passed": "{{count}} passed",
+  "report.summary.skipped": "{{count}} skipped",
+  "report.table.agent": "Agent",
+  "report.table.avgDuration": "Avg Duration",
+  "report.table.cost": "Cost",
+  "report.table.default": "default",
+  "report.table.duration": "Duration",
+  "report.table.eval": "Eval",
+  "report.table.evalTitle": "Eval Results:",
+  "report.table.experiment": "Experiment",
+  "report.table.experimentsTitle": "Experiments",
+  "report.table.model": "Model",
+  "report.table.reason": "Reason",
+  "report.table.result": "Result",
+  "report.table.runs": "Runs",
+  "report.table.status": "Status",
+  "report.table.successRate": "Success Rate",
+  "report.table.tokens": "Tokens",
+  "runner.diffProgress": "captured diff: {{changed}} changed / {{deleted}} deleted",
+  "runner.driveAgent": "driving agent...",
+  "runner.evalSetup": "eval setup (installing dependencies)...",
+  "runner.interrupted": "  · interrupted: sandbox containers cleaned up; printing partial results completed so far.\n",
+  "runner.judgePrecheck": "  · prechecking judge config...\n",
+  "runner.otlpInSandbox": "OTLP in-sandbox collector -> {{endpoint}}{{proto}}",
+  "runner.otlpOverride": "OTLP receiver (host override) -> {{endpoint}}",
+  "runner.otlpReceiver": "OTLP receiver -> {{endpoint}}{{proto}}",
+  "runner.remoteSandboxUnavailable": "remote agents do not have sandbox.{{method}}; use a sandbox agent or remove workspace assertions.",
+  "runner.reporterDiagnostic": "  · [diagnostic] {{stage}} failed (ignored): {{message}}\n",
+  "runner.scoreJudge": "scoring / judge...",
+  "runner.skip": "skip: {{reason}}",
+  "runner.startAgentSetup": "agent setup (install CLI / write config)...",
+  "runner.startAgentTracing": "agent tracing (write OTEL export config)...",
+  "runner.startSandbox": "starting sandbox...",
+  "runner.timeout": "attempt timed out ({{timeoutMs}}ms)\nRecent progress:\n{{recentLogs}}",
+  "runner.traceSelected": " -> kept {{count}} semantic spans",
+  "runner.resumeCarry": "  · reusing {{carried}} passing results from last run, re-running {{retry}} evals\n",
+  "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
+  "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
+  "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
+  "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
+  "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
+  "sandbox.forceCleanup": "  · [sandbox] force-cleaning {{count}} sandboxes...\n",
+  "sandbox.stopFailed": "  · [sandbox] failed to stop sandbox {{id}} (ignored; backend TTL should clean it up): {{message}}\n",
+  "sandbox.stopTimeout": "stop timed out ({{timeoutMs}}ms)",
+  "scoring.evalError": "evaluation error: {{error}}",
+  "session.fileFallback": "[file]",
+  "session.tools": "{{count}} tools",
+  "session.turn.primary": "turn {{turn}}",
+  "session.turn.secondary": "session {{session}} · turn {{turn}}",
+  "util.requiredEnv": "Missing required environment variable {{name}} (configure it in .env).",
+  "vercel.fileNotFound": "File not found: {{path}}",
+  "vercel.rotateFailed": "[VercelSandbox] session rotate failed ({{seconds}}s): {{error}}",
+  "vercel.rotated": "[VercelSandbox] session rotated after {{seconds}}s -> {{sessionId}}",
+} satisfies Messages;

package/src/i18n/index.ts

@@ -0,0 +1,39 @@
+import { en } from "./en.ts";
+import { zhCN, type MessageKey, type Messages } from "./zh-CN.ts";
+
+export type Locale = "zh-CN" | "en";
+
+type Vars = Record<string, string | number | boolean | undefined>;
+
+const dictionaries: Record<Locale, Messages> = {
+  "zh-CN": zhCN,
+  en,
+};
+
+function normalizeLocale(raw: string | undefined): Locale | undefined {
+  if (!raw) return undefined;
+  const value = raw.trim().toLowerCase().replace("_", "-");
+  if (!value) return undefined;
+  if (value === "c" || value === "posix") return undefined;
+  if (value.startsWith("zh")) return "zh-CN";
+  if (value.startsWith("en")) return "en";
+  return "en";
+}
+
+export function detectLocale(env: NodeJS.ProcessEnv = process.env): Locale {
+  return (
+    normalizeLocale(env.NICEEVAL_LANG) ??
+    normalizeLocale(env.NICEEVAL_LOCALE) ??
+    normalizeLocale(env.LC_ALL) ??
+    normalizeLocale(env.LC_MESSAGES) ??
+    normalizeLocale(env.LANG) ??
+    "zh-CN"
+  );
+}
+
+export function t(key: MessageKey, vars: Vars = {}): string {
+  const message = dictionaries[detectLocale()][key];
+  return message.replace(/\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g, (_match, name: string) =>
+    vars[name] === undefined ? "" : String(vars[name]),
+  );
+}

package/src/i18n/zh-CN.ts

@@ -0,0 +1,132 @@
+export const zhCN = {
+  "agent.installFailed": "安装失败:{{key}}\n{{tail}}",
+  "agent.unknown": "未知 agent \"{{name}}\"。已注册:{{known}}。",
+  "bub.diagnose.exitCode": "bub run 退出码 {{code}}",
+  "bub.diagnose.lastError": "最后错误:{{message}}",
+  "bub.diagnose.noTape": "tape 未生成",
+  "bub.diagnose.outputTail": "输出末尾:{{tail}}",
+  "bub.diagnose.zeroEvents": "tape 存在但 0 事件",
+  "bub.installFailed": "bub 安装失败(重试 {{attempts}} 次):\n{{tail}}",
+  "checkpoint.emptyTar": "checkpoint: tar 为空(paths: {{paths}})",
+  "cli.all": "(全部)",
+  "cli.browserOpenFailed": "无法自动打开浏览器,请手动访问:{{url}}\n",
+  "cli.clean.done": "已删除 .niceeval/ 历史运行工件。\n",
+  "cli.config.missing": "找不到 niceeval.config.ts(请在项目根运行)。",
+  "cli.config.noDefault": "niceeval.config.ts 需要 default export(defineConfig(...))。",
+  "cli.dry.header": "\n[dry] {{evals}} 个 eval × {{configs}} 个运行配置:\n",
+  "cli.dry.noMatches": "(无匹配)",
+  "cli.dry.row": "  {{who}}{{experiment}}: {{evals}}  ×{{runs}}\n",
+  "cli.error": "niceeval 出错:{{error}}\n",
+  "cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
+  "cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
+  "cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
+  "cli.exp.agentModelFlagUnsupported": "`--agent` / `--model` 不能覆盖 experiment。请在 experiments/ 里新增或复制一个配置文件。\n",
+  "cli.experiment.noMatch": "没有匹配的实验:{{arg}}。已发现:{{experiments}}\n",
+  "cli.experimentGroup": "组",
+  "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
+  "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
+  "cli.init.done": "已创建 evals/ 和 niceeval.config.ts 起始文件。\n",
+  "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
+  "cli.list.header": "发现 {{count}} 个 eval:\n",
+  "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
+  "cli.none": "(无)",
+  "cli.pressCtrlC": "按 Ctrl+C 退出。\n",
+  "cli.run.experimentRequired": "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n",
+  "cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
+  "cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
+  "cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
+  "cli.view.exported": "已导出实验查看页:{{out}}\n",
+  "cli.view.url": "niceeval view: {{url}}\n",
+  "context.skipEmpty": "skip() 需要一个非空理由。",
+  "context.turnFailed": "本轮 send 返回 failed(turn status = failed):{{message}}",
+  "context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
+  "define.agentNameRequired": "defineAgent 需要 name。",
+  "define.evalIdRejected": "defineEval 不接受 id —— id 由文件路径推导。",
+  "define.evalTestRequired": "defineEval 需要一个 async test(t) 函数。",
+  "define.experimentAgentRequired": "defineExperiment 需要 agent。",
+  "define.experimentIdRejected": "defineExperiment 不接受 id —— id 由文件路径推导。",
+  "define.sandboxAgentNameRequired": "defineSandboxAgent 需要 name。",
+  "define.sandboxCreateRequired": "defineSandbox 需要一个 create() 函数。",
+  "define.sandboxNameRequired": "defineSandbox 需要 name。",
+  "docker.commandTimeout": "Command timed out after {{timeoutMs}}ms",
+  "docker.containerNotInitialized": "Container not initialized",
+  "docker.imagePullDone": "Docker image ready: {{image}}",
+  "docker.imagePullStart": "Pulling Docker image: {{image}}...",
+  "docker.readFileFailed": "Failed to read file {{path}}: {{stderr}}",
+  "docker.unsupportedRuntime": "Unsupported runtime: {{runtime}}",
+  "judge.apiKeyMissing": "judge 缺少 API key(CODEX_API_KEY / OPENAI_API_KEY)。",
+  "judge.httpError": "judge HTTP {{status}}: {{body}}",
+  "judge.probeFailed": "judge 预检失败({{model}}): {{error}}",
+  "judge.probeMissingKey": "judge 模型 {{model}} 缺少 API key —— 请配置 {{envHint}}",
+  "live.running": "  正在运行 {{totalRuns}} 次 ({{evals}} eval × {{configs}} 配置)       {{completed}}/{{total}} 完成",
+  "live.runningUnknown": "  正在运行…  {{completed}}/{{total}} 完成",
+  "report.assertionThreshold": " (得分 {{score}} < {{threshold}})",
+  "report.error": "错误",
+  "report.errored": "错误",
+  "report.failed": "失败",
+  "report.gate": "gate",
+  "report.passed": "通过",
+  "report.result": "\n结果:{{parts}}  ({{duration}} · {{tokens}}{{cost}})\n\n",
+  "report.runStart": "\n本次运行 {{count}} 个 eval{{extra}}\n\n",
+  "report.runStartExtra": " × {{configs}} 配置 = {{totalRuns}} 次运行",
+  "report.skipped": "跳过",
+  "report.soft": "soft",
+  "report.summary.errored": "{{count}} 错误",
+  "report.summary.failed": "{{count}} 失败",
+  "report.summary.passed": "{{count}} 通过",
+  "report.summary.skipped": "{{count}} 跳过",
+  "report.table.agent": "Agent",
+  "report.table.avgDuration": "平均耗时",
+  "report.table.cost": "预估成本",
+  "report.table.default": "默认",
+  "report.table.duration": "耗时",
+  "report.table.eval": "Eval",
+  "report.table.evalTitle": "各 Eval:",
+  "report.table.experiment": "实验",
+  "report.table.experimentsTitle": "实验",
+  "report.table.model": "模型",
+  "report.table.reason": "原因",
+  "report.table.result": "结果",
+  "report.table.runs": "轮次",
+  "report.table.status": "状态",
+  "report.table.successRate": "成功率",
+  "report.table.tokens": "Tokens",
+  "runner.diffProgress": "采 diff:{{changed}} 改 / {{deleted}} 删",
+  "runner.driveAgent": "驱动 agent…",
+  "runner.evalSetup": "eval setup(装依赖)…",
+  "runner.interrupted": "  · 已中断:沙箱容器已清理,输出本次已完成的部分结果。\n",
+  "runner.judgePrecheck": "  · 预检 judge 配置…\n",
+  "runner.otlpInSandbox": "OTLP in-sandbox collector → {{endpoint}}{{proto}}",
+  "runner.otlpOverride": "OTLP 接收器(覆盖 host) → {{endpoint}}",
+  "runner.otlpReceiver": "OTLP 接收器 → {{endpoint}}{{proto}}",
+  "runner.remoteSandboxUnavailable": "remote agent 没有 sandbox.{{method}};请改用 sandbox agent 或移除 workspace 断言。",
+  "runner.reporterDiagnostic": "  · [diagnostic] {{stage}} 失败(已忽略):{{message}}\n",
+  "runner.scoreJudge": "评分 / judge…",
+  "runner.skip": "skip:{{reason}}",
+  "runner.startAgentSetup": "agent setup(装 CLI / 写配置)…",
+  "runner.startAgentTracing": "agent tracing(写 otel 导出配置)…",
+  "runner.startSandbox": "起沙箱…",
+  "runner.timeout": "attempt 超时({{timeoutMs}}ms)\n最近进度:\n{{recentLogs}}",
+  "runner.traceSelected": " → 留 {{count}}(按语义)",
+  "runner.resumeCarry": "  · 复用上次 {{carried}} 个通过的结果,重跑 {{retry}} 个 eval\n",
+  "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
+  "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
+  "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
+  "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
+  "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
+  "sandbox.forceCleanup": "  · [sandbox] 强制清理 {{count}} 个沙箱…\n",
+  "sandbox.stopFailed": "  · [sandbox] 停沙箱 {{id}} 失败(已忽略,靠后端过期兜底):{{message}}\n",
+  "sandbox.stopTimeout": "stop 超时({{timeoutMs}}ms)",
+  "scoring.evalError": "评估出错: {{error}}",
+  "session.fileFallback": "[file]",
+  "session.tools": "{{count}} 工具",
+  "session.turn.primary": "第{{turn}}轮",
+  "session.turn.secondary": "会话{{session}}·第{{turn}}轮",
+  "util.requiredEnv": "缺少必需的环境变量 {{name}}(请在 .env 里配置)。",
+  "vercel.fileNotFound": "File not found: {{path}}",
+  "vercel.rotateFailed": "[VercelSandbox] session rotate failed ({{seconds}}s): {{error}}",
+  "vercel.rotated": "[VercelSandbox] session rotated after {{seconds}}s → {{sessionId}}",
+} as const;
+
+export type MessageKey = keyof typeof zhCN;
+export type Messages = Record<MessageKey, string>;

package/src/index.ts

@@ -0,0 +1,39 @@
+// niceeval 公开导出(import { … } from "niceeval")。
+// Agent/Adapter 相关见 "niceeval/adapter";Sandbox 相关见 "niceeval/sandbox"。
+
+export { defineEval, defineConfig, defineExperiment } from "./define.ts";
+
+export { requireEnv, getEnv, stripComments } from "./util.ts";
+
+// 类型(eval 作者会用到;跑哪个 agent / 用哪个 sandbox 见对应子路径)
+export type {
+  StreamEvent,
+  ToolName,
+  JsonValue,
+  Usage,
+  Turn,
+  TurnInput,
+  InputFile,
+  TurnHandle,
+  SessionHandle,
+  TestContext,
+  ToolMatch,
+  ValueAssertion,
+  Severity,
+  ResultOutcome,
+  EvalDef,
+  ExperimentDef,
+  Config,
+  LocalizedText,
+  JudgeConfig,
+  Reporter,
+  ReporterEvent,
+  EvalResult,
+  RunSummary,
+  O11ySummary,
+  TraceSpan,
+  SpanKind,
+  DerivedFacts,
+} from "./types.ts";
+
+export type { ParsedTranscript } from "./o11y/parsers/index.ts";

package/src/loaders/index.ts

@@ -0,0 +1,56 @@
+// 数据集加载器:把 YAML / JSON 读进来,配 .map(row => defineEval(...)) 扇出。
+
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+
+export async function loadJson<T = unknown>(path: string): Promise<T> {
+  const raw = await readFile(resolve(process.cwd(), path), "utf-8");
+  return JSON.parse(raw) as T;
+}
+
+export async function loadYaml<T = unknown>(path: string): Promise<T> {
+  const raw = await readFile(resolve(process.cwd(), path), "utf-8");
+  // 尽量用真正的 yaml 解析器(若项目装了);否则退回极简解析。
+  // 用变量 specifier 避免 tsc 静态解析这个可选依赖。
+  const yamlPkg = "yaml";
+  try {
+    const yaml = (await import(yamlPkg)) as { parse(s: string): unknown };
+    return yaml.parse(raw) as T;
+  } catch {
+    return parseSimpleYaml(raw) as T;
+  }
+}
+
+/** 极简 YAML(只够 cases 列表这类扁平结构;复杂结构请 `pnpm add yaml`)。 */
+function parseSimpleYaml(text: string): unknown {
+  const lines = text.split("\n").filter((l) => l.trim() && !l.trim().startsWith("#"));
+  const root: Record<string, unknown> = {};
+  let currentKey: string | undefined;
+  let list: Record<string, unknown>[] | undefined;
+  let item: Record<string, unknown> | undefined;
+  for (const line of lines) {
+    const m = /^(\s*)(- )?([\w.-]+)?:?\s?(.*)$/.exec(line);
+    if (!m) continue;
+    const [, indent, dash, key, value] = m;
+    if (indent === "" && key && !dash) {
+      currentKey = key;
+      list = [];
+      root[currentKey] = list;
+    } else if (dash) {
+      item = {};
+      list?.push(item);
+      if (key) item[key] = coerce(value);
+    } else if (key && item) {
+      item[key] = coerce(value);
+    }
+  }
+  return root;
+}
+
+function coerce(v: string): unknown {
+  const t = v.trim().replace(/^["']|["']$/g, "");
+  if (t === "true") return true;
+  if (t === "false") return false;
+  if (t !== "" && !Number.isNaN(Number(t))) return Number(t);
+  return t;
+}