niceeval 0.9.1 → 0.10.0

package/src/cli.ts

@@ -5,17 +5,20 @@
 //   niceeval clean                   删除 .niceeval/ 历史运行 artifact
 
 import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { join, relative, resolve as resolvePath } from "node:path";
 import { pathToFileURL } from "node:url";
 import { parseArgs as nodeParseArgs } from "node:util";
-import { discoverEvals, discoverExperiments, makeFilter } from "./runner/discover.ts";
+import { discoverEvals, discoverExperiments } from "./runner/discover.ts";
+import { browsableExperimentPaths, matchExperimentSelector } from "./shared/aggregate.ts";
 import { runEvals, type AgentRun } from "./runner/run.ts";
 import { planCarry } from "./runner/fingerprint.ts";
+import { fingerprintEvalsFilter, resolveExperimentEvals, selectedEvalsForRun } from "./runner/eval-selection.ts";
 import { failureDetailFromResult } from "./runner/feedback/failure.ts";
 import { stopAllSandboxes, liveSandboxCount } from "./sandbox/registry.ts";
+import { drainExperimentTeardowns } from "./runner/experiment-cleanup-registry.ts";
+import { CLEANUP_TIMEOUT_MS } from "./runner/cleanup-timeout.ts";
 import { evalLevelStats } from "./shared/verdict.ts";
 import { prepareRunSandboxes, resolvedSandboxRecommendedConcurrency } from "./runner/sandbox-selection.ts";
 import { Json, JUnit } from "./runner/reporters/json.ts";
@@ -47,14 +50,13 @@ import {
 // --report 用的那份(dist/report/**,见 tsconfig.report-build.json)是同一个模块实例——
 // `unique symbol` 品牌与 class 的 instanceof 都按声明处的模块身份判定,raw src 和编译产物
 // 是两份不同源码位置,即使运行时同名同形,TS 类型与 instanceof 都不认。
-import { ReportLoadError } from "../dist/report/load.js";
+import { ReportLoadError } from "../dist/report/runtime/load.js";
 import { runShow } from "./show/index.ts";
 import { t } from "./i18n/index.ts";
 import { formatThrown, upsertManagedBlock } from "./util.ts";
 import type {
   CompletionStatus,
   Config,
-  DiscoveredExperiment,
   ReporterError,
   ReporterRegistration,
   RunCompletion,
@@ -135,9 +137,9 @@ const FLAG_OPTIONS = {
   timeout: { type: "string" },
   /** 整次运行的预算上限(美元)。 */
   budget: { type: "string" },
-  /** `exp` 命令专用:跑完留下 failed/errored attempt 的沙箱现场(= `--keep-sandbox=failed`);`--keep-sandbox=all` 连 passed 也留。事后用 `niceeval sandbox list/enter/stop` 查看与销毁。 */
+  /** `exp` 命令专用:跑完留下 failed/errored attempt 的 Sandbox 现场(= `--keep-sandbox=failed`);`--keep-sandbox=all` 连 passed 也留。事后用 `niceeval sandbox list/enter/stop` 查看与销毁。 */
   "keep-sandbox": { type: "boolean" },
-  /** `sandbox stop` 专用:销毁全部留存沙箱。 */
+  /** `sandbox stop` 专用:销毁全部留存 Sandbox。 */
   all: { type: "boolean" },
   /** `sandbox diff` 专用:只看某个 send 窗口(如 `--window s1/t2`);省略输出全部窗口的串联视图。 */
   window: { type: "string" },
@@ -155,7 +157,7 @@ const FLAG_OPTIONS = {
   out: { type: "string" },
   /** `view` 命令专用:指定本地服务器监听端口。 */
   port: { type: "string" },
-  // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/how-to/viewing-results.mdx)。
+  // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/tutorials/viewing-results.mdx)。
   // 证据切面只认 `@<locator>`(或收窄到单个 eval 的前缀)选出的那一个 attempt——不再有
   // 数字 `--attempt`,选哪个 attempt 由 locator 精确指名,不是「先选 eval 再挑第几次」。
   /** `show` 命令专用:该 attempt 运行时保存的 Eval 源码,gate/soft 断言标回源码行(证据切面)。 */
@@ -170,7 +172,7 @@ const FLAG_OPTIONS = {
   diff: { type: "boolean" },
   /** `show` 命令专用:执行时间轴——对匹配的每个 experiment × eval 分节,逐 attempt 列时间 / verdict / 摘要 / 耗时 / 成本 / locator;与 `--report` 互斥。 */
   history: { type: "boolean" },
-  /** `show` / `view` 命令专用:按路径段前缀收窄 experiment(与 `niceeval exp` 位置参数同一套匹配);组名会选中组内全部配置。`view --out` 时同一收窄决定出站内容。 */
+  /** `show` / `view` 命令专用:按路径段前缀收窄 experiment(与 `niceeval exp` 位置参数同一套匹配);目录路径会选中其下全部配置。`view --out` 时同一收窄决定出站内容。 */
   exp: { type: "string" },
   /** `show` / `view` / `sandbox enter|list|stop` 共用:结果根目录(`.niceeval` 之外的另一个根,如 `copySnapshots` 产出的发布根)。 */
   results: { type: "string" },
@@ -462,32 +464,6 @@ async function openBrowser(url: string): Promise<boolean> {
   });
 }
 
-function evalsFilterFromExperiment(
-  evals: DiscoveredExperiment["evals"],
-  patterns: string[],
-): (id: string) => boolean {
-  const patternFilter = makeFilter(patterns);
-  let expFilter: (id: string) => boolean = () => true;
-  if (Array.isArray(evals)) expFilter = makeFilter(evals);
-  else if (typeof evals === "function") expFilter = evals;
-  return (id) => expFilter(id) && patternFilter(id);
-}
-
-/**
- * evals 过滤器的指纹(进 ExperimentRunInfo.evalFilterFingerprint,供「配置没变」判断):
- * 数组按内容、函数按函数体哈希;CLI 追加的位置参数前缀一并计入。不存过滤器本身——
- * 求值结果在 selectedEvalIds(见 runEvals)。
- */
-function fingerprintEvalsFilter(evals: DiscoveredExperiment["evals"], patterns: string[]): string {
-  const basis =
-    evals === undefined || evals === "*"
-      ? "*"
-      : Array.isArray(evals)
-        ? JSON.stringify([...evals].sort())
-        : evals.toString();
-  return createHash("sha256").update(JSON.stringify({ basis, patterns })).digest("hex").slice(0, 16);
-}
-
 /**
  * run 结束后把 coordinator 累计的诊断折成 `RunCompletion`(见 docs/feature/experiments/cli.md
  * 「运行完成状态不只看 verdict 计数」)。只读已经真实发生过的诊断,不额外发明信号:
@@ -663,7 +639,7 @@ async function main(): Promise<void> {
 
   const agentRuns: AgentRun[] = [];
   let experimentSelection = t("cli.all");
-  let availableExperimentGroups = t("cli.none");
+  let availableExperimentPaths = t("cli.none");
 
   if (command === "exp") {
     if (flags.agent || flags.model) {
@@ -679,16 +655,13 @@ async function main(): Promise<void> {
     const expArg = positionals[0];
     const extraPatterns = positionals.slice(1);
     experimentSelection = positionals.join(" ") || t("cli.all");
-    availableExperimentGroups = [...new Set(experiments.map((experiment) => experiment.group || experiment.id))]
-      .sort()
-      .join(", ") || t("cli.none");
-    const selected = expArg
-      ? experiments.filter((e) => e.group === expArg || e.id === expArg || e.id.startsWith(expArg + "/"))
-      : experiments;
+    availableExperimentPaths = browsableExperimentPaths(experiments.map((e) => e.id)).join(", ") || t("cli.none");
+    const selectedIds = expArg ? new Set(matchExperimentSelector(experiments.map((e) => e.id), expArg)) : undefined;
+    const selected = selectedIds ? experiments.filter((e) => selectedIds.has(e.id)) : experiments;
     if (selected.length === 0) {
       process.stderr.write(t("cli.experiment.noMatch", {
         arg: expArg ?? t("cli.all"),
-        experiments: experiments.map((e) => e.id).join(", ") || t("cli.none"),
+        experiments: availableExperimentPaths,
       }));
       // show / view 是顶层命令。只有同名 experiment 确实不存在时才纠错，不能抢占合法 id。
       if (expArg === "show" || expArg === "view") {
@@ -707,6 +680,15 @@ async function main(): Promise<void> {
     }
     for (const exp of selected) {
       // 一个实验 = 一个配置(单 model)。跨模型对比写多个实验文件,各钉一个 model。
+      // evals 谓词在这里对本次 invocation 的候选 eval 各求值一次;下游(dry-run、sandbox 查表、
+      // fingerprint/carry、attempt 展开)只消费 selectedEvalIds,不重新调用谓词
+      // (见 docs/feature/experiments/library.md「evals」)。
+      const { selectedEvalIds } = resolveExperimentEvals({
+        experimentId: exp.id,
+        selector: exp.evals,
+        cliPatterns: extraPatterns,
+        evals,
+      });
       agentRuns.push({
         agent: exp.agent,
         model: exp.model,
@@ -717,23 +699,25 @@ async function main(): Promise<void> {
         sandbox: exp.sandbox ?? config.sandbox,
         timeoutMs: flags.timeout ?? envNumber("NICEEVAL_TIMEOUT") ?? exp.timeoutMs ?? config.timeoutMs,
         budget: flags.budget ?? envNumber("NICEEVAL_BUDGET") ?? exp.budget,
-        evalFilter: evalsFilterFromExperiment(exp.evals, extraPatterns),
+        selectedEvalIds,
         experimentId: exp.id,
         description: exp.description,
+        labels: exp.labels,
         evalFilterFingerprint: fingerprintEvalsFilter(exp.evals, extraPatterns),
         strict: flags.strict,
         // 实验级并发上限:随 AgentRun 进调度器按实验单独限流(runner 两级信号量),
         // 不再取所有选中实验的最小值钳全局——那会让一个串行实验拖慢整批基线。
         maxConcurrency: exp.maxConcurrency,
+        setup: exp.setup,
       });
     }
   } else {
     // 裸 run / `niceeval <eval>` 不再执行。运行配置必须来自 experiments/,
     // 这样 agent/model/flags/runs/budget 与结果聚合都有可签入的身份。
     const experiments = await discoverExperiments(cwd);
-    const asExp = experiments.filter((e) =>
-      positionals.some((p) => e.group === p || e.id === p || e.id.startsWith(p + "/")),
-    );
+    const ids = experiments.map((e) => e.id);
+    const matchedIds = new Set(positionals.flatMap((p) => matchExperimentSelector(ids, p)));
+    const asExp = experiments.filter((e) => matchedIds.has(e.id));
     process.stderr.write(t("cli.run.experimentRequired"));
     if (asExp.length > 0) {
       process.stderr.write(t("cli.run.experimentRequiredHint", {
@@ -758,14 +742,14 @@ async function main(): Promise<void> {
 
   // matchedByRun[i] 对应 agentRuns[i] 匹配到的 eval 集合;--dry 预览与真正开跑时的
   // RunFeedbackPlan(总量、去重 eval 数)共用同一份计算,不重复过滤一遍。
-  const matchedByRun = agentRuns.map((run) => evals.filter((e) => run.evalFilter(e.id)));
+  const matchedByRun = agentRuns.map((run) => selectedEvalsForRun(evals, run));
   const totalRuns = agentRuns.reduce((sum, run, i) => sum + matchedByRun[i]!.length * run.runs, 0);
   const uniqueEvalIds = new Set(matchedByRun.flat().map((e) => e.id));
 
   if (totalRuns === 0) {
     process.stderr.write(t("cli.experiment.noEvalsSelected", {
       selection: experimentSelection,
-      experiments: availableExperimentGroups,
+      experiments: availableExperimentPaths,
     }));
     process.exit(1);
   }
@@ -873,13 +857,34 @@ async function main(): Promise<void> {
   //   3 次:真不耐烦了,硬退(此时多半已无可清理的)。
   const ctrl = new AbortController();
   let signalCount = 0;
-  // 兜底强清 + 退出:只跑一次,带超时(stopAllSandboxes 内每个 stop 各自有超时)。先停 dashboard
-  // 的 tick/动态区域(coordinator.stopDynamic()),避免硬退时终端卡在半帧 ANSI 状态。
+  // 强清 = 加速收尾,不是绕过收尾(docs/cli.md「中断:三级响应」)。顺序:先强停沙箱(卡在
+  // 沙箱 I/O 上的收尾立刻失败返回),然后事件驱动收口——并发等待「在飞收尾链 settle」与
+  // 「实验级 teardown 注册表排空」(drain 会启动未启动的、等待在飞的同一 memoized promise),
+  // 两者都 settle 即退。兜底上限从单可调用体清理上限推导(docs 声明的不等式:provider stop 8s
+  // < 看门狗 < CLEANUP_TIMEOUT_MS ≤ 本上限),不是第 2 级的语义——settle 才是——只拦
+  // 「收尾可调用体绕过了自己的超时」的失守病态,到点放弃退出(职责同第 3 级硬退)。
+  // 只跑一次;先停 dashboard 的 tick/动态区域(coordinator.stopDynamic()),
+  // 避免硬退时终端卡在半帧 ANSI 状态。
+  const FORCE_SETTLE_CAP_MS = CLEANUP_TIMEOUT_MS * 2;
+  let runInFlight: Promise<unknown> | undefined;
   let forcing = false;
   const forceCleanupAndExit = (code: number) => {
     if (forcing) return;
     forcing = true;
-    void Promise.all([coordinator.stopDynamic(), stopAllSandboxes()]).finally(() => process.exit(code));
+    void (async () => {
+      await Promise.all([coordinator.stopDynamic(), stopAllSandboxes()]);
+      const settled = Promise.allSettled([
+        ...(runInFlight ? [runInFlight] : []),
+        drainExperimentTeardowns(),
+      ]);
+      await Promise.race([
+        settled.then(() => {}),
+        new Promise<void>((r) => {
+          setTimeout(r, FORCE_SETTLE_CAP_MS).unref();
+        }),
+      ]);
+      process.exit(code);
+    })();
   };
   for (const sig of ["SIGINT", "SIGTERM"] as const) {
     process.on(sig, () => {
@@ -887,13 +892,16 @@ async function main(): Promise<void> {
       if (signalCount === 1) {
         reportActivity(t("cli.interruptCleanup").trimEnd());
         ctrl.abort();
-        // 看门狗:graceful 清理 12s 还没让进程自己收口,就强清兜底。
+        // 看门狗:graceful 清理迟迟没让进程自己收口,就强清兜底。取值在 docs/cli.md 声明的
+        // 不等式链里:> provider stop 超时(8s,一次正常停容器超时后才升级,不误伤),
+        // < CLEANUP_TIMEOUT_MS(30s)。
+        const GRACEFUL_WATCHDOG_MS = 12_000;
         setTimeout(() => {
           if (liveSandboxCount() > 0) {
             reportActivity(t("cli.fallbackCleanupTimeout").trimEnd());
             forceCleanupAndExit(130);
           }
-        }, 12_000).unref();
+        }, GRACEFUL_WATCHDOG_MS).unref();
       } else if (signalCount === 2) {
         reportActivity(t("cli.forceCleanupExit").trimEnd());
         forceCleanupAndExit(130);
@@ -921,7 +929,7 @@ async function main(): Promise<void> {
 
   let summary: RunSummary;
   try {
-    summary = await runEvals({
+    const inFlight = runEvals({
       config,
       evals,
       agentRuns,
@@ -933,15 +941,20 @@ async function main(): Promise<void> {
       keepSandbox: flags.keepSandbox,
       niceevalRoot: resolvePath(cwd, ".niceeval"),
     });
+    // 交给强清路径一个可等待的收尾句柄:二次中断/看门狗强清时先有界等它收口,让在飞的
+    // teardown 链跑完,而不是 process.exit 把它们连同进程一起杀掉。
+    runInFlight = inFlight;
+    summary = await inFlight;
   } catch (e) {
     // 真崩溃前先撤下 dashboard,不让半帧 ANSI 状态和下面 main().catch 打印的错误交织。
     await coordinator.stopDynamic();
     throw e;
   }
 
-  // 正常返回(含被中断后走部分汇总)后再兜一刀:Scope finalizer 没停掉的残留沙箱在这里强清。
-  // 跑顺利时登记表已空,是 no-op。
+  // 正常返回(含被中断后走部分汇总)后再兜一刀:Scope finalizer 没停掉的残留沙箱、没被运行
+  // 路径消费的实验级 cleanup 在这里强清。跑顺利时两份登记表都已空,是 no-op。
   await stopAllSandboxes();
+  await drainExperimentTeardowns();
 
   // completion 要先算好,--json/--junit 是否"这次真的写出"才有依据(见下)。
   const completion = assembleRunCompletion(coordinator.state);
@@ -979,7 +992,8 @@ async function main(): Promise<void> {
 
 main().catch(async (e) => {
   process.stderr.write(t("cli.error", { error: formatThrown(e) }));
-  // 真·崩溃路径也别留孤儿:强清还活着的沙箱(带超时),再退。
+  // 真·崩溃路径也别留孤儿:强清还活着的沙箱(带超时)、排空实验级 cleanup 注册表,再退。
   await stopAllSandboxes();
+  await drainExperimentTeardowns();
   process.exit(2);
 });

package/src/context/types.ts

@@ -140,35 +140,35 @@ export interface RespondAnswer {
   readonly text?: string;
 }
 
-/** eval 作者可见的受限沙箱视图:能执行命令 / 文件 IO / 读最终 diff,但不能 stop。 */
+/** 评估用例作者可见的受限 Sandbox 视图:能执行命令 / 文件 IO / 读最终 diff,但不能 stop。 */
 export interface SandboxHandle {
-  /** 沙箱内的工作目录绝对路径。 */
+  /** Sandbox 内的工作目录绝对路径。 */
   readonly workdir: string;
-  /** 在沙箱里执行一条命令(argv 形式,不经 shell)。装系统依赖等需要 root 时传 `{ root: true }`。 */
+  /** 在 Sandbox 里执行一条命令(argv 形式,不经 shell)。装系统依赖等需要 root 时传 `{ root: true }`。 */
   runCommand(cmd: string, args?: string[], opts?: CommandOptions): Promise<CommandResult>;
-  /** 在沙箱里执行一段 shell 脚本(经 shell 解释,支持管道 / 重定向)。 */
+  /** 在 Sandbox 里执行一段 shell 脚本(经 shell 解释,支持管道 / 重定向)。 */
   runShell(script: string, opts?: CommandOptions): Promise<CommandResult>;
-  /** 读沙箱内某文件此刻的文本内容(实时读,不是 diff 快照)。 */
+  /** 读 Sandbox 内某文件此刻的文本内容(实时读,不是 diff 快照)。 */
   readFile(path: string): Promise<string>;
-  /** 沙箱内某路径此刻是否存在。 */
+  /** Sandbox 内某路径此刻是否存在。 */
   fileExists(path: string): Promise<boolean>;
-  /** 按选项批量读回沙箱里的源文件,供比对 / judge 用材料。 */
+  /** 按选项批量读回 Sandbox 里的源文件,供比对 / judge 用材料。 */
   readSourceFiles(opts?: ReadSourceFilesOptions): Promise<SourceFiles>;
-  /** 把一组内容写进沙箱(路径相对 targetDir,默认 workdir)。 */
+  /** 把一组内容写进 Sandbox(路径相对 targetDir,默认 workdir)。 */
   writeFiles(files: Record<string, string>, targetDir?: string): Promise<void>;
-  /** 把一批内存中的文件上传进沙箱。 */
+  /** 把一批内存中的文件上传进 Sandbox。 */
   uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void>;
-  /** 把本地目录整体上传进沙箱;`opts.ignore` 排除指定路径。 */
+  /** 把本地目录整体上传进 Sandbox;`opts.ignore` 排除指定路径。 */
   uploadDirectory(localDir: string, targetDir?: string, opts?: { ignore?: string[] }): Promise<void>;
-  /** 从沙箱下载某文件内容。 */
+  /** 从 Sandbox 下载某文件内容。 */
   downloadFile(path: string): Promise<Buffer>;
-  /** 把一段内容上传成沙箱里的单个文件。 */
+  /** 把一段内容上传成 Sandbox 里的单个文件。 */
   uploadFile(path: string, content: Buffer): Promise<void>;
-  /** 沙箱 provider 分配的实例 id,用于排查 / 关联日志。 */
+  /** Sandbox provider 分配的实例 id,用于排查 / 关联日志。 */
   readonly sandboxId: string;
   /** 相对 git 基线的最终 diff 视图(test() 跑完、finalize 前才落定)。 */
   readonly diff: DiffView;
-  /** 取沙箱内某文件的最终内容,占位延迟到 finalize 才真正读取;只能配合 t.check 使用。 */
+  /** 取 Sandbox 内某文件的最终内容,占位延迟到 finalize 才真正读取;只能配合 t.check 使用。 */
   file(path: string): string;
   /** 断言 diff 里某路径发生了变化(新增或修改)。 */
   fileChanged(path: string): AssertionHandle;
@@ -176,7 +176,7 @@ export interface SandboxHandle {
   fileDeleted(path: string): AssertionHandle;
   /** 断言 diff 的路径与内容都不匹配给定正则(常用来否定式检查不该出现的改动)。 */
   notInDiff(re: RegExp): AssertionHandle;
-  /** 断言沙箱里执行过的命令都没有失败退出。 */
+  /** 断言 Sandbox 里执行过的命令都没有失败退出。 */
   noFailedShellCommands(): AssertionHandle;
 }
 
@@ -285,7 +285,7 @@ export interface TestContext {
   /** 本次 attempt 生效的实验 flags(experiment.flags 的只读视图;实验条件,非命令行开关)。 */
   readonly flags: Readonly<Record<string, unknown>>;
   /**
-   * 作用域反馈:报告 eval 自己执行的长步骤(上传 fixture、跑构建……)。短命状态,scope 固定
+   * 作用域反馈:报告评估用例自己执行的长步骤(上传 Fixture、跑构建……)。短命状态,scope 固定
    * 为 `eval.run`;只报告不断言(见 docs/feature/eval/library/context.md「向运行反馈长步骤」)。
    */
   progress(update: ProgressUpdate): void;
@@ -296,18 +296,18 @@ export interface TestContext {
   diagnostic(input: DiagnosticInput): void;
   /** `progress({ message: msg })` 的别名(调试日志),不出现在最终结果里。 */
   log(msg: string): void;
-  /** 立即中止本 eval 并标记为 skipped(verdict / EvalResult.skipReason),reason 不能为空。 */
+  /** 立即中止本评估用例并标记为 skipped(verdict / EvalResult.skipReason),reason 不能为空。 */
   skip(reason: string): never;
 
   // 值断言
   /**
    * 对任意值跑一个 ValueAssertion,返回可链 `.gate()` / `.atLeast()` 的 AssertionHandle。
-   * 打分延迟到 eval 结束后统一 finalize,调用本身同步、不抛错——不通过只是记一条失败断言,
-   * 不会中止后续代码。要「不满足就立即中止 eval」用 require。
+   * 打分延迟到评估用例结束后统一 finalize,调用本身同步、不抛错——不通过只是记一条失败断言,
+   * 不会中止后续代码。要「不满足就立即中止评估用例」用 require。
    */
   check(value: unknown, assertion: ValueAssertion): AssertionHandle;
   /**
-   * 对任意值跑一个 ValueAssertion,立即(await 时)求值;不满足就抛错中止整个 eval 剩余步骤
+   * 对任意值跑一个 ValueAssertion,立即(await 时)求值;不满足就抛错中止整个评估用例剩余步骤
    * (仍会把这条断言计入报告,不影响已记录的其它断言)。跟 check 的区别:check 只记录、
    * 从不抛错,打分留到最后统一算;require 当场判定、失败即中止,适合「前置条件不满足,
    * 后面写了也没意义」的场景。
@@ -352,7 +352,7 @@ export interface TestContext {
   eventsSatisfy(label: string, predicate: (events: readonly StreamEvent[]) => boolean): AssertionHandle;
 
   // 工作区 / 沙箱
-  /** 受限沙箱视图:能执行命令 / 读写文件 / 看最终 diff,不能 stop 沙箱本身(见 SandboxHandle)。 */
+  /** 受限 Sandbox 视图:能执行命令 / 读写文件 / 看最终 diff,不能 stop Sandbox 本身(见 SandboxHandle)。 */
   readonly sandbox: SandboxHandle;
 
   // 效率 / 成本

package/src/define.ts

@@ -67,6 +67,11 @@ export function defineExperiment(def: ExperimentDef): ExperimentDef {
     throw new Error(t("define.experimentIdRejected"));
   }
   if (!def.agent) throw new Error(t("define.experimentAgentRequired"));
+  // setup 是实验级生命周期钩子(整场一次,宿主机侧,见 runner/types.ts 的 ExperimentDef.setup);
+  // 传成非函数(如误把 sandbox 钩子对象塞进来)在解析时就报,不等到调度才炸。
+  if (def.setup !== undefined && typeof def.setup !== "function") {
+    throw new Error(t("define.experimentSetupNotFunction"));
+  }
   // flags 必须可 JSON 序列化(进结果快照的 ExperimentRunInfo.flags):解析时即校验,
   // 非 JSON 值(函数 / undefined / 循环引用 / bigint)直接报错,不等到落盘才炸。
   if (def.flags !== undefined) {
@@ -76,6 +81,14 @@ export function defineExperiment(def: ExperimentDef): ExperimentDef {
       }
     }
   }
+  // labels 是报告归类坐标(进 ExperimentRunInfo.labels,不透传 ctx/t):值域 string | number,
+  // 解析时即校验,布尔 / 对象 / NaN 直接报错,不等到落盘或报告分组才炸。
+  if (def.labels !== undefined) {
+    for (const [key, value] of Object.entries(def.labels)) {
+      const ok = typeof value === "string" || (typeof value === "number" && Number.isFinite(value));
+      if (!ok) throw new Error(t("define.experimentLabelInvalid", { key }));
+    }
+  }
   return def;
 }
 

package/src/i18n/en.ts

@@ -60,9 +60,16 @@ export const en = {
   "cli.flag.invalidOutput": "Flag --output expects one of auto|human|agent|ci, got \"{{value}}\".\n",
   "runner.budgetUnenforceable":
     "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
+  "runner.experimentTeardownFailed":
+    "teardown for experiment {{experimentId}} failed: {{message}}. Results are unaffected, but host-side resources started by this experiment may not have been released; check manually.\n",
+  "runner.cleanupTimeout": "cleanup timed out after {{timeoutMs}}ms\n",
+  "runner.setupReturnedCleanup":
+    "{{layer}} returned a function. setup does not carry cleanup and the returned value will not be executed — put the cleanup in the paired teardown of the same layer ({{hint}}); see the experiments tutorial on docs-site or docs/runner.md.\n",
+  "runner.experimentTeardownLate":
+    "experiment {{experimentId}}'s teardown was not triggered by the normal countdown path; it has been executed by the end-of-run sweep instead. Results are unaffected; seeing this line means an unlocated intermittent scheduling issue fired — please record this run in the memory ledger.\n",
   "judge.modelMissing":
     "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
-    "  Docs: node_modules/niceeval/docs-site/zh/how-to/scoring-guide.mdx",
+    "  Docs: node_modules/niceeval/docs-site/zh/tutorials/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
   "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
@@ -70,7 +77,7 @@ export const en = {
   "cli.help":
     "niceeval — agent-native evals\n\n" +
     "Usage:\n" +
-    "  niceeval exp [group|experiment] [eval-id-prefix…]   run experiments\n" +
+    "  niceeval exp [path|experiment] [eval-id-prefix…]    run experiments\n" +
     "  niceeval show [eval-id-prefix… | @<locator>]        read results in the terminal\n" +
     "      bare: current verdicts per experiment (composed across runs), each row\n" +
     "        with a compact attempt index (locator + failure reason)\n" +
@@ -121,10 +128,14 @@ export const en = {
   "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
   "cli.exp.agentModelFlagUnsupported": "experiment runs do not support --agent / --model. Add or copy an experiment file and change its model instead.\n",
   "cli.exp.viewerFlagUnsupported": "`{{flag}}` only applies to niceeval {{command}}, not niceeval exp.\n",
-  "cli.experiment.noMatch": "No experiment matched: {{arg}}. Discovered: {{experiments}}\n",
+  "cli.experiment.noMatch":
+    "No experiment matched: {{arg}}. Available paths: {{experiments}}.\n" +
+    "Run `niceeval exp <path> --dry` to preview a plan.\n",
   "cli.experiment.viewerCommandHint": "Did you mean: niceeval {{command}}{{args}}\n",
-  "cli.experiment.noEvalsSelected": "No evals selected: {{selection}} matched 0 evals. Available experiments: {{experiments}}.\n",
-  "cli.experimentGroup": " group",
+  "cli.experiment.noEvalsSelected":
+    "No evals selected: {{selection}} matched 0 evals. Available eval prefixes: {{experiments}}.\n" +
+    "Run `niceeval exp {{selection}} --dry` to see what it covers, or drop the eval filter to run every eval selected by those experiments.\n",
+  "cli.experimentGroup": " path",
   "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
   "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
   "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site/zh).\n",
@@ -135,8 +146,8 @@ export const en = {
   "cli.pressCtrlC": "Press Ctrl+C to exit.\n",
   "cli.resultsPath": "Structured results: {{path}} (snapshot.json + per-attempt result.json / events.json / trace.json / diff.json)\n",
   "cli.run.experimentRequired":
-    "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
-    "  Docs: node_modules/niceeval/docs-site/zh/how-to/write-experiment.mdx\n",
+    "Run evals through an experiment: use `niceeval exp [path|config] [eval id prefix]`.\n" +
+    "  Docs: node_modules/niceeval/docs-site/zh/tutorials/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
   "cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
@@ -147,7 +158,7 @@ export const en = {
   "cli.view.url": "niceeval view: {{url}}\n",
   "context.capabilityMissing":
     "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
-    "  Docs: node_modules/niceeval/docs-site/zh/how-to/sandbox-agent.mdx",
+    "  Docs: node_modules/niceeval/docs-site/zh/tutorials/sandbox-agent.mdx",
   "context.skipEmpty": "skip() requires a non-empty reason.",
   "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
   "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -157,6 +168,8 @@ export const en = {
   "define.evalTestRequired": "defineEval requires an async test(t) function.",
   "define.experimentAgentRequired": "defineExperiment requires agent.",
   "define.experimentFlagNotJson": "experiment.flags.{{key}} is not JSON-serializable (functions / undefined / cycles / bigint are not allowed); flags are persisted verbatim into result snapshots and must be plain JSON.",
+  "define.experimentLabelInvalid": "experiment.labels.{{key}} must be a string or a finite number; labels are report-side grouping coordinates persisted verbatim into result snapshots.",
+  "define.experimentSetupNotFunction": "experiment.setup must be a function ((ctx) => void); use experiment.teardown for cleanup; to prepare the in-sandbox environment per experiment, chain .setup() hooks on the sandbox spec instead.",
   "define.experimentIdRejected": "defineExperiment does not accept id; ids are derived from file paths.",
   "define.sandboxAgentNameRequired": "defineSandboxAgent requires name.",
   "define.sandboxCreateRequired": "defineSandbox requires a create() function.",
@@ -169,7 +182,7 @@ export const en = {
   "docker.unsupportedRuntime": "Unsupported runtime: {{runtime}}",
   "feedback.human.active": "ACTIVE",
   "feedback.human.budgetExhausted": "budget exhausted for {{experimentId}} (spent {{spent}}, unstarted {{unstarted}})",
-  "feedback.human.compare": "Compare: niceeval view {{group}}",
+  "feedback.human.compare": "Compare: niceeval view",
   "feedback.human.counts": "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{completed}} completed",
   "feedback.human.diffHint": "Diff:    niceeval show {{locator}} --diff",
   "feedback.human.evalHint": "Eval:    niceeval show {{locator}} --source",
@@ -194,6 +207,10 @@ export const en = {
   "feedback.phase.evalSetup": "eval setup",
   "feedback.phase.sandboxCreate": "creating sandbox",
   "feedback.phase.sandboxQueue": "queued for sandbox",
+  "feedback.phase.experimentSetup": "experiment setup",
+  "feedback.phase.experimentTeardown": "experiment teardown",
+  "feedback.human.hookDone": "done",
+  "feedback.human.hookFailed": "failed",
   "feedback.phase.sandboxSetup": "sandbox setup",
   "feedback.phase.scoring": "scoring",
   "feedback.phase.teardown": "cleaning up",
@@ -273,13 +290,11 @@ export const en = {
   "runner.startSandboxTeardown": "sandbox teardown (environment provisioning hooks)...",
   "runner.timeout": "attempt timed out ({{timeoutMs}}ms)\nRecent progress:\n{{recentLogs}}",
   "runner.traceSelected": " -> kept {{count}} semantic spans",
-  "runner.resumeCarry": "  · reusing {{carried}} settled results from last run, re-running {{retry}} evals\n",
-  "runner.resumeCarryDetail": "      carried [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
   "sandbox.providerNotImplemented": "{{provider}} sandbox provider is not implemented; use docker, vercel, or e2b",
   "sandbox.missingSpec":
     "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
-    "  Docs: node_modules/niceeval/docs-site/zh/how-to/sandbox-providers.mdx",
+    "  Docs: node_modules/niceeval/docs-site/zh/tutorials/sandbox-providers.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",