niceeval 0.5.3 → 0.6.0

package/src/cli.ts

@@ -2,21 +2,23 @@
 //   niceeval exp [组|配置] [pattern]  跑实验
 //   niceeval show [pattern]          终端读结果:榜单 / 单 eval / 证据切面 / 时间轴 / --report
 //   niceeval list                    只列出发现到的 eval
-//   niceeval clean                   删除 .niceeval/ 历史运行工件
+//   niceeval clean                   删除 .niceeval/ 历史运行 artifact
 
 import { spawn } from "node:child_process";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
-import { join } from "node:path";
+import { join, relative } from "node:path";
 import { pathToFileURL } from "node:url";
 import { parseArgs as nodeParseArgs } from "node:util";
 import { discoverEvals, discoverExperiments, makeFilter } from "./runner/discover.ts";
 import { runEvals, type AgentRun } from "./runner/run.ts";
 import { runWho } from "./runner/types.ts";
+import { planCarry } from "./runner/fingerprint.ts";
 import { stopAllSandboxes, liveSandboxCount } from "./sandbox/registry.ts";
-import { evalLevelStats } from "./shared/outcome.ts";
+import { evalLevelStats } from "./shared/verdict.ts";
 import { sandboxRecommendedConcurrency } from "./sandbox/resolve.ts";
 import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
+import { Quiet as QuietReporter } from "./runner/reporters/quiet.ts";
 import { Json, JUnit } from "./runner/reporters/json.ts";
 import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
 import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
@@ -28,7 +30,11 @@ import {
   IncompatibleResultsError,
   ViewInputError,
 } from "./view/index.ts";
-import { ReportLoadError } from "./report/load.ts";
+// load.ts 本身没有 JSX,但它的 ReportDefinition/ReportLoadError 要和 view 报告槽实际装载
+// --report 用的那份(dist/report/**,见 tsconfig.report-build.json)是同一个模块实例——
+// `unique symbol` 品牌与 class 的 instanceof 都按声明处的模块身份判定,raw src 和编译产物
+// 是两份不同源码位置,即使运行时同名同形,TS 类型与 instanceof 都不认。
+import { ReportLoadError } from "../dist/report/load.js";
 import { runShow } from "./show/index.ts";
 import { t } from "./i18n/index.ts";
 import { formatThrown, upsertManagedBlock } from "./util.ts";
@@ -66,15 +72,14 @@ interface Flags {
   port?: number;
   help: boolean;
   version: boolean;
-  // ── show 专属(位置参数仍是 eval id 前缀;这些 flag 选「怎么看」)──
-  transcript: boolean;
-  trace: boolean;
+  // ── show 专属(位置参数仍是 eval id 前缀 / `@<locator>`;这些 flag 选「怎么看」)──
+  eval: boolean;
+  execution: boolean;
   diff: boolean;
   /** --diff=<路径>(必须 = 连写;空格形式会把路径当 eval id 前缀,按文档如此)。 */
   diffPath?: string;
   history: boolean;
   experiment?: string;
-  attempt?: number;
   run?: string;
   report?: string;
 }
@@ -109,31 +114,31 @@ const FLAG_OPTIONS = {
   /** `view` 命令专用:指定本地服务器监听端口。 */
   port: { type: "string" },
   // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/guides/viewing-results.mdx)。
-  /** `show` 命令专用:渲染单个 eval 的完整对话与工具调用(证据切面)。 */
-  transcript: { type: "boolean" },
-  /** `show` 命令专用:渲染单个 eval 的 trace 瀑布文本版(证据切面)。 */
-  trace: { type: "boolean" },
+  // 证据切面只认 `@<locator>`(或收窄到单个 eval 的前缀)选出的那一个 attempt——不再有
+  // 数字 `--attempt`,选哪个 attempt 由 locator 精确指名,不是「先选 eval 再挑第几次」。
+  /** `show` 命令专用:该 attempt 运行时保存的 Eval 源码,gate/soft 断言标回源码行(证据切面)。 */
+  eval: { type: "boolean" },
+  /** `show` 命令专用:该 attempt 的标准执行事件流(消息、thinking、Skill load、工具调用/结果);有 OTel 时同一节点补时间(证据切面)。 */
+  execution: { type: "boolean" },
   // --diff 是布尔;--diff=<路径> 在 parseArgs 前预扫成 diffPath(路径必须 = 连写,
   // 空格形式的下一个 token 仍是位置参数 = eval id 前缀,与文档一致)。
   /** `show` 命令专用:sandbox 里的文件改动摘要;`--diff=<文件路径>` 看单个文件的完整改动(路径必须 `=` 连写)。 */
   diff: { type: "boolean" },
   /** `show` 命令专用:跨 run 时间轴,只列真实执行;与 `--report` 互斥。 */
   history: { type: "boolean" },
-  /** `show` / `view` 命令专用:选集只留该实验。 */
+  /** `show` / `view` 命令专用:Selection 只留该实验。 */
   experiment: { type: "string" },
-  /** `show` 命令专用:指定详情 / 证据切面看第几次 attempt(与展示一致的 1 计序号)。 */
-  attempt: { type: "string" },
-  /** `show` / `view` 命令专用:钉死看某一个结果目录(历史 run 或 `copySnapshots` 产物)。 */
+  /** `show` / `view` 命令专用:钉死看某一个结果目录(某次快照或 `copySnapshots` 产物)。 */
   run: { type: "string" },
-  /** `show` / `view` 命令专用:把默认榜单整槽换成你的报告文件(默认导出 `defineReport(...)`)。 */
+  /** `show` / `view` 命令专用:渲染你的报告文件(文件默认导出 `defineReport(...)`);show 用它替换 Attempt 索引,view 用它替换默认分析报告。 */
   report: { type: "string" },
   /** 只打印本次会匹配到的 eval × 运行配置,不实际执行。 */
   dry: { type: "boolean" },
-  /** 关闭控制台 / live 进度输出(reporter 仍会写 artifacts)。 */
+  /** 关闭控制台 / live 的逐条结果与末尾汇总(attempt 进度行仍写 stderr);errored / failed 的结果各在 stderr 补一行摘要,passed / skipped 静默;reporter 仍会写 artifacts。 */
   quiet: { type: "boolean" },
   /** 忽略上次运行结果,不跳过已通过的 (experiment, eval) 组合,强制全部重跑。 */
   force: { type: "boolean" },
-  /** CI 中推荐使用:让软阈值(`soft`)失败也计入整条 eval 的 outcome。 */
+  /** CI 中推荐使用:让软阈值(`soft`)失败也计入整条 eval 的 verdict。 */
   strict: { type: "boolean" },
   /** 某个 eval 的一次 attempt 通过后,停止该 eval 剩余的 attempts。 */
   "early-exit": { type: "boolean" },
@@ -211,13 +216,12 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
     open: values["no-open"] === true ? false : values.open === true ? true : undefined,
     help: values.help === true,
     version: values.version === true,
-    transcript: values.transcript === true,
-    trace: values.trace === true,
+    eval: values.eval === true,
+    execution: values.execution === true,
     diff: values.diff === true && diffPath === undefined,
     diffPath,
     history: values.history === true,
     experiment: values.experiment as string | undefined,
-    attempt: numberFlag("attempt", values.attempt as string | undefined),
     run: values.run as string | undefined,
     report: values.report as string | undefined,
   };
@@ -273,13 +277,16 @@ const AGENT_RULES_END = "<!-- END:niceeval-agent-rules -->";
 const AGENT_RULES_CONTENT = [
   "# niceeval is NOT in your training data",
   "",
-  "Its APIs and conventions may differ from anything you have seen. Read the relevant",
-  "guide in `node_modules/niceeval/docs-site/zh/` before writing any eval, experiment,",
-  "adapter, or niceeval config. The bundled docs are Chinese-only — that is the single",
-  "authoritative, always-current version; read it regardless of your working language.",
-  "After a run, drill into failures with `niceeval show <eval id>` (add `--transcript` /",
-  "`--trace` / `--diff` for evidence); the `summary.json` path the CLI prints and the",
-  "artifact files it references are the structured source of truth.",
+  "Its APIs and conventions may differ from anything you have seen. Start with",
+  "`node_modules/niceeval/INDEX.md`, then read the task-specific bundled guides it points",
+  "to before writing any eval, experiment, adapter, or niceeval config. That index and",
+  "the bundled Chinese docs are the authoritative version matching this installation.",
+  "After a run, drill into failures with `niceeval show` — pick an `@<locator>` from the",
+  "compact index it prints, then `niceeval show @<locator>` for a compact overview, or add",
+  "`--eval` / `--execution` / `--diff` for evidence; the snapshot directories the CLI prints",
+  "are the structured source of truth: `snapshot.json` holds the run's metadata and each",
+  "`<evalId>/a<attempt>/result.json` holds that attempt's verdict and assertions, next to",
+  "its artifact files (`events.json` / `trace.json` / `diff.json`).",
 ].join("\n");
 
 // 优先复用已有的 AGENTS.md;项目只有 CLAUDE.md(没有 AGENTS.md)时改写 CLAUDE.md 本身,
@@ -373,7 +380,7 @@ async function main(): Promise<void> {
   }
 
   if (command === "view") {
-    // 位置参数 = eval id 前缀(收窄报告槽选集);存在的文件路径 = 单文件模式;
+    // 位置参数 = eval id 前缀(收窄报告槽 Selection);存在的文件路径 = 单文件模式;
     // 结果目录经 --run 递入;--report 整槽替换报告槽(与 show --report 吃同一个文件)。
     let viewInput: { input?: string; patterns: string[] };
     try {
@@ -404,15 +411,14 @@ async function main(): Promise<void> {
   }
 
   if (command === "show") {
-    // show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --run 指定目录)的落盘结果。
+    // show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --run 指定的某个快照目录)的落盘结果。
     const code = await runShow(cwd, positionals, {
-      transcript: flags.transcript,
-      trace: flags.trace,
+      eval: flags.eval,
+      execution: flags.execution,
       diff: flags.diff,
       diffPath: flags.diffPath,
       history: flags.history,
       experiment: flags.experiment,
-      attempt: flags.attempt,
       run: flags.run,
       report: flags.report,
     });
@@ -522,6 +528,18 @@ async function main(): Promise<void> {
     process.exit(0);
   }
 
+  // 提前算好(而不是等 runEvals 内部算):live 表格要在第一帧就知道哪些行会被携入
+  // (carry),直接渲染成已完成,不然会显示"waiting for a slot"直到进程退出——它们
+  // 永远等不到 eval:start,run.ts 压根不会为携入的 (experimentId, evalId) 派发 attempt。
+  // 两处必须共用同一份 planCarry() 判断,否则各自算一遍,一旦判断不一致,live 表格
+  // 显示的"携入"和 run.ts 实际调度的"携入"就会对不上。
+  const priorResults = flags.force ? undefined : await loadLatestResultsPerEval(join(cwd, ".niceeval"));
+  const carryPlan = priorResults?.length ? await planCarry(evals, agentRuns, priorResults) : undefined;
+  const carriedVerdictByKey = new Map<string, string>();
+  for (const r of carryPlan?.carriedResults ?? []) {
+    if (r.experimentId) carriedVerdictByKey.set(`${r.experimentId}|${r.id}`, r.verdict);
+  }
+
   const reporters: Reporter[] = [];
   let onProgress: ((evalId: string, who: string, msg: string) => void) | undefined;
 
@@ -532,10 +550,13 @@ async function main(): Promise<void> {
       for (const agentRun of agentRuns) {
         // who 必须与 attempt.ts 的进度上报同源(runWho):曾用 agent/model,同 agent 同 model
         // 的实验变体(xxx 与 xxx--agents-md)会被折叠成一行,0/2 看起来像同一 eval 跑两次。
-        const who = runWho(agentRun);
+        const who = runWho({ agentName: agentRun.agent.name, model: agentRun.model, experimentId: agentRun.experimentId });
         const matched = evals.filter((e) => agentRun.evalFilter(e.id));
         for (const evalDef of matched) {
-          liveRows.push({ evalId: evalDef.id, who, total: agentRun.runs });
+          const carriedVerdict = agentRun.experimentId
+            ? carriedVerdictByKey.get(`${agentRun.experimentId}|${evalDef.id}`)
+            : undefined;
+          liveRows.push({ evalId: evalDef.id, who, total: agentRun.runs, carriedVerdict });
         }
       }
       const totalAttempts = liveRows.reduce((s, r) => s + r.total, 0);
@@ -545,9 +566,13 @@ async function main(): Promise<void> {
     } else {
       reporters.push(ConsoleReporter());
     }
+  } else {
+    // --quiet:进度流照旧直写 stderr,结果流换成最小报告器 —— errored / failed 各补一行
+    // stderr,passed / skipped 静默。没有它,attempt 出执行错时控制台会全程无声。
+    reporters.push(QuietReporter());
   }
   const artifacts = ArtifactsReporter();
-  reporters.push(artifacts);
+  reporters.push( artifacts);
   if (flags.junit) reporters.push(JUnit(flags.junit));
   if (flags.json) reporters.push(Json(flags.json));
   reporters.push(...(config.reporters ?? []));
@@ -590,13 +615,11 @@ async function main(): Promise<void> {
     });
   }
 
-  // 无全局默认:并发上限由 sandbox 后端的推荐值决定。
-  // 多个 agentRun 各有 sandbox 时取最小值(最保守的后端决定上限)。
+  // 无全局默认:并发上限由 sandbox provider 的推荐值决定。
+  // 多个 agentRun 各有 sandbox 时取最小值(最保守的 provider 决定上限)。
   const sandboxRecs = agentRuns.map((r) => sandboxRecommendedConcurrency(r.sandbox));
   const sandboxDefaultConcurrency = sandboxRecs.length > 0 ? Math.min(...sandboxRecs) : 10;
 
-  const priorResults = flags.force ? undefined : await loadLatestResultsPerEval(join(cwd, ".niceeval"));
-
   const summary = await runEvals({
     config,
     evals,
@@ -610,22 +633,26 @@ async function main(): Promise<void> {
     signal: ctrl.signal,
     onProgress,
     priorResults,
+    carryPlan,
   });
 
   // 正常返回(含被中断后走部分汇总)后再兜一刀:Scope finalizer 没停掉的残留沙箱在这里强清。
   // 跑顺利时登记表已空,是 no-op。
   await stopAllSandboxes();
 
-  // agent 反馈闭环的入口:跑完直接给出结构化结果路径,agent 读 summary.json 与各
-  // attempt 的工件(events/trace/diff),不必解析人类向的流式输出。--quiet 下也输出。
-  if (artifacts.outputDir()) {
-    process.stdout.write(t("cli.resultsPath", { path: join(artifacts.outputDir(), "summary.json") }));
+  // agent 反馈闭环的入口:跑完直接给出每个已创建快照的目录,agent 读 snapshot.json 与各
+  // attempt 的 result.json / artifact(events/trace/diff),不必解析人类向的流式输出。
+  // --quiet 下也输出。相对 cwd 的路径更友好;结果落在 cwd 外时(relative 路径以 .. 开头)
+  // 原样打印绝对路径。
+  for (const { dir } of artifacts.outputDirs()) {
+    const rel = relative(cwd, dir);
+    process.stdout.write(t("cli.resultsPath", { path: rel && !rel.startsWith("..") ? rel : dir }));
   }
 
-  // 退出码按 eval 级判决,不按 attempt:summary.failed/errored 统计的是每次 attempt,
+  // 退出码按 eval 级判定,不按 attempt:summary.failed/errored 统计的是每次 attempt,
   // 被 runs+earlyExit 重试吸收的失败(先挂一次、后来过了)不该把进程判红——否则
   // 「runs 吸收单次抖动」在 CI 退出码这层永远不成立。折叠口径与报表/view 共用
-  // foldEvalOutcome(任一轮通过 → 该 eval 通过),粒度 experimentId|eval id。
+  // foldEvalVerdict(任一轮通过 → 该 eval 通过),粒度 experimentId|eval id。
   const stats = evalLevelStats(summary.results, (r) => `${r.experimentId ?? ""}|${r.id}`);
   const failedExit = stats.failed > 0 || stats.errored > 0;
   process.exit(failedExit ? 1 : 0);

package/src/context/context.test.ts

@@ -352,11 +352,11 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
     expect(cost.passed).toBe(false);
   });
 
-  it("turn.loadedSkill() checks a load_skill call scoped to this turn", async () => {
+  it("turn.loadedSkill() reads the skill.loaded event scoped to this turn", async () => {
     const agent = scriptedAgent([
       {
         status: "completed",
-        events: [{ type: "action.called", callId: "c1", name: "load_skill", input: { skill: "pdf-export" } }],
+        events: [{ type: "skill.loaded", skill: "pdf-export", callId: "c1" }],
       },
     ]);
     const { context, state } = makeContext(agent);
@@ -366,4 +366,22 @@ describe("TurnHandle scoped assertions (parked/loadedSkill/noFailedActions/maxTo
     const [result] = await state.collector.finalize(baseScoringContext(state));
     expect(result.passed).toBe(true);
   });
+
+  // Skill 加载是一等事件,不是「名字叫 load_skill 的工具调用」——adapter 负责归一
+  // (claude-code parser 就把 Skill tool_use 直接吐成 skill.loaded)。伪装成工具调用的
+  // 加载断言侧看不见,这是 adapter 违约,不是断言该兜的底(见 docs/feature/adapters/architecture/events.md)。
+  it("turn.loadedSkill() does not match a tool call that merely happens to be named load_skill", async () => {
+    const agent = scriptedAgent([
+      {
+        status: "completed",
+        events: [{ type: "action.called", callId: "c1", name: "load_skill", input: { skill: "pdf-export" } }],
+      },
+    ]);
+    const { context, state } = makeContext(agent);
+    const turn = await context.send("export as pdf");
+    turn.loadedSkill("pdf-export");
+
+    const [result] = await state.collector.finalize(baseScoringContext(state));
+    expect(result.passed).toBe(false);
+  });
 });

package/src/context/context.ts

@@ -62,6 +62,8 @@ export interface ContextDeps {
   model?: string;
   reasoningEffort?: string;
   flags: Record<string, unknown>;
+  /** 路径推导出的实验 id(经 send ctx 透给 adapter,见 AgentContext.experimentId)。 */
+  experimentId?: string;
   signal: AbortSignal;
   log(msg: string): void;
   judge: JudgeConfig | undefined;
@@ -93,6 +95,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
     model: deps.model,
     reasoningEffort: deps.reasoningEffort,
     flags: deps.flags,
+    experimentId: deps.experimentId,
     signal: deps.signal,
     log: deps.log,
     telemetry: deps.telemetry,

package/src/context/control-flow.ts

@@ -10,7 +10,7 @@ export class EvalSkipped extends Error {
   }
 }
 
-/** t.require / turn.expectOk 不过:正常的断言失败,中止后续,但已记录的断言决定判决。 */
+/** t.require / turn.expectOk 不过:正常的断言失败,中止后续,但已记录的断言决定判定。 */
 export class EvalRequirementFailed extends Error {
   constructor(public readonly assertionName: string) {
     super(`requirement failed: ${assertionName}`);

package/src/context/session.test.ts

@@ -1,11 +1,102 @@
 import { describe, expect, it } from "vitest";
 
-import { createAgentSession } from "./session.ts";
+import { createAgentSession, SessionManager } from "./session.ts";
+import type { Agent, Sandbox, StreamEvent, Turn, TurnInput } from "../types.ts";
 
 // createAgentSession() 是 ctx.session 的实现——一条会话线的存取器(见
 // docs-site/zh/concepts/adapter.mdx 的 AgentSession 契约)。这里直接测存取器本身;
 // 端到端的「同一条线同一个 ctx.session」由 SessionManager / RunSession 保证。
 
+function fakeSandbox(): Sandbox {
+  return {
+    workdir: "/sandbox/work",
+    runCommand: async () => { throw new Error("not implemented"); },
+    runShell: async () => { throw new Error("not implemented"); },
+    readFile: async () => "",
+    fileExists: async () => false,
+    readSourceFiles: async () => Object.assign([], {
+      text: () => "",
+      code: () => "",
+      fileMatching: () => undefined,
+      fileMatchingAll: () => undefined,
+      hasPath: () => false,
+    }),
+    writeFiles: async () => {},
+    uploadFiles: async () => {},
+    uploadDirectory: async () => {},
+    stop: async () => {},
+    sandboxId: "fake",
+    otlpHost: null,
+    downloadFile: async () => Buffer.from(""),
+    uploadFile: async () => {},
+  };
+}
+
+function agentReturning(turn: Turn): Agent {
+  return {
+    name: "fake-agent",
+    kind: "remote",
+    async send(_input: TurnInput): Promise<Turn> {
+      return turn;
+    },
+  };
+}
+
+function makeManager(turn: Turn) {
+  const lines: string[] = [];
+  const manager = new SessionManager({
+    agent: agentReturning(turn),
+    sandbox: fakeSandbox(),
+    flags: {},
+    signal: new AbortController().signal,
+    log: (msg) => lines.push(msg),
+  });
+  return { manager, lines };
+}
+
+describe("SessionManager.send() 进度行", () => {
+  it("failed 轮:进度行末尾追加最后一条 error 事件的 message", async () => {
+    const events: StreamEvent[] = [
+      { type: "message", role: "assistant", text: "" },
+      { type: "error", message: "402 Insufficient Balance" },
+    ];
+    const { manager, lines } = makeManager({ events, status: "failed" });
+    await manager.send(manager.primary, "hi");
+
+    const progressLine = lines.find((l) => l.includes("← failed"));
+    expect(progressLine).toContain("402 Insufficient Balance");
+  });
+
+  it("failed 轮但没有 error 事件:不追加空后缀,行尾保持原格式", async () => {
+    const { manager, lines } = makeManager({ events: [], status: "failed" });
+    await manager.send(manager.primary, "hi");
+
+    const progressLine = lines.find((l) => l.includes("← failed"));
+    expect(progressLine).toMatch(/\ds$/); // 以 "Ns" 收尾,没有多余的 " · " 后缀
+  });
+
+  it("completed 轮:即使事件里混了 error 事件也不提取原因(只在 failed 时生效)", async () => {
+    const events: StreamEvent[] = [{ type: "error", message: "不该出现在这里" }];
+    const { manager, lines } = makeManager({ events, status: "completed" });
+    await manager.send(manager.primary, "hi");
+
+    const progressLine = lines.find((l) => l.includes("← completed"));
+    expect(progressLine).not.toContain("不该出现在这里");
+  });
+
+  it("原因文本压成单行并截断到 120 字符", async () => {
+    const long = "x".repeat(200);
+    const events: StreamEvent[] = [{ type: "error", message: long }];
+    const { manager, lines } = makeManager({ events, status: "failed" });
+    await manager.send(manager.primary, "hi");
+
+    const progressLine = lines.find((l) => l.includes("← failed"))!;
+    const suffix = progressLine.split(" · ").at(-1)!;
+    expect(suffix.length).toBeLessThanOrEqual(120);
+    expect(suffix.endsWith("…")).toBe(true);
+  });
+});
+
 describe("createAgentSession", () => {
   describe("history()", () => {
     it("新线 get() 是空数组;commit 之后同一条线的 get() 能看见", () => {

package/src/context/session.ts

@@ -91,6 +91,8 @@ export interface SessionDeps {
   flags: Record<string, unknown>;
   signal: AbortSignal;
   log(msg: string): void;
+  /** 路径推导出的实验 id(经 send ctx 透给 adapter,见 AgentContext.experimentId)。 */
+  experimentId?: string;
   /** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
   telemetry?: Telemetry;
   /** 非沙箱 tracing agent 的共享 OTLP 通道(runner 从 run 级池取,经它做逐轮 span 归属)。 */
@@ -139,6 +141,7 @@ export class SessionManager {
       model: this.deps.model,
       reasoningEffort: this.deps.reasoningEffort,
       flags: this.deps.flags,
+      experimentId: this.deps.experimentId,
       sandbox: this.deps.sandbox,
       session,
       telemetry: this.deps.telemetry,
@@ -181,8 +184,9 @@ export class SessionManager {
 
     const tok = (turn.usage?.inputTokens ?? 0) + (turn.usage?.outputTokens ?? 0);
     const tools = turn.events.filter((e) => e.type === "action.called").length;
+    const reason = turn.status === "failed" ? failureReason(turn.events) : undefined;
     this.deps.log(
-      `${turnLabel} ← ${turn.status} · ${t("session.tools", { count: tools })} · ${tok} tok · ${Math.round((Date.now() - t0) / 1000)}s`,
+      `${turnLabel} ← ${turn.status} · ${t("session.tools", { count: tools })} · ${tok} tok · ${Math.round((Date.now() - t0) / 1000)}s${reason ? ` · ${reason}` : ""}`,
     );
     return turn;
   }
@@ -218,6 +222,27 @@ export class SessionManager {
   }
 }
 
+/**
+ * 失败轮的进度行原因摘要:取本轮事件流里最后一个 `type: "error"` 事件的 message
+ * (与 TurnHandle.expectOk() / src/agents/shared.ts 的 diagnoseFailure 同一口径——
+ * 都认「最后一条 error 事件」为本轮失败的权威原因),压成单行并截断,避免 402/超时
+ * 这类关键信息只能事后翻落盘的 result.json 才看得到。提不到时返回 undefined,调用方不补空后缀。
+ */
+function failureReason(events: readonly StreamEvent[]): string | undefined {
+  for (let i = events.length - 1; i >= 0; i--) {
+    const e = events[i];
+    if (e.type === "error") return truncateOneLine(e.message, 120);
+  }
+  return undefined;
+}
+
+/** 单行截断,口径同 src/runner/reporters/table.ts 的 truncateOneLine(折叠空白 + 120 字符上限)。 */
+function truncateOneLine(s: string, width: number): string {
+  const clean = s.replace(/\s+/g, " ").trim();
+  if (clean.length <= width) return clean;
+  return `${clean.slice(0, width - 1)}…`;
+}
+
 export function lastAssistantText(events: readonly StreamEvent[]): string | undefined {
   for (let i = events.length - 1; i >= 0; i--) {
     const e = events[i];

package/src/context/types.ts

@@ -156,7 +156,7 @@ export interface SandboxHandle {
   downloadFile(path: string): Promise<Buffer>;
   /** 把一段内容上传成沙箱里的单个文件。 */
   uploadFile(path: string, content: Buffer): Promise<void>;
-  /** 沙箱后端分配的实例 id,用于排查 / 关联日志。 */
+  /** 沙箱 provider 分配的实例 id,用于排查 / 关联日志。 */
   readonly sandboxId: string;
   /** 相对 git 基线的最终 diff 视图(test() 跑完、finalize 前才落定)。 */
   readonly diff: DiffView;
@@ -268,7 +268,7 @@ export interface TestContext {
   newSession(): SessionHandle;
 
   // 运行上下文
-  /** 本次 attempt 的中止信号;超时 / 早停 / 用户 Ctrl-C 时触发,传给 adapter 的长耗时调用做取消。 */
+  /** 本次 attempt 的中止信号;超时 / 首过即停 / 用户 Ctrl-C 时触发,传给 adapter 的长耗时调用做取消。 */
   readonly signal: AbortSignal;
   /** 本次 attempt 使用的模型名(由 experiment/CLI flag 决定);省略即 agent 原生默认,不代表「无模型」。 */
   readonly model?: string;
@@ -278,10 +278,10 @@ export interface TestContext {
   readonly flags: Readonly<Record<string, unknown>>;
   /** 打一行调试日志;有 live 进度回调时走该回调,否则落到 stderr,不出现在最终结果里。 */
   log(msg: string): void;
-  /** 立即中止本 eval 并标记为 skipped(outcome / EvalResult.skipReason),reason 不能为空。 */
+  /** 立即中止本 eval 并标记为 skipped(verdict / EvalResult.skipReason),reason 不能为空。 */
   skip(reason: string): never;
 
-  // 值级断言
+  // 值断言
   /**
    * 对任意值跑一个 ValueAssertion,返回可链 `.gate()` / `.atLeast()` 的 AssertionHandle。
    * 打分延迟到 eval 结束后统一 finalize,调用本身同步、不抛错——不通过只是记一条失败断言,

package/src/define.test.ts

@@ -0,0 +1,82 @@
+// SandboxSpec 链式钩子(.setup()/.teardown())的构造期契约:不可变、多次追加按顺序累加。
+// 执行顺序(setup 正序 / teardown 逆序 / LIFO cleanup)是 runner 的事,见
+// test/e2e-sandbox-hooks.test.ts;这里只测 dockerSandbox()/vercelSandbox()/e2bSandbox()/
+// defineSandbox() 这四个工厂产出的 spec 对象本身的构造行为。
+import { describe, expect, it } from "vitest";
+import { dockerSandbox, e2bSandbox, vercelSandbox, defineSandbox } from "./define.ts";
+import type { AgentSetup, AgentTeardown } from "./types.ts";
+
+const noopSetup: AgentSetup = () => {};
+const noopTeardown: AgentTeardown = () => {};
+
+describe("sandbox factories: .setup()/.teardown() chain", () => {
+  it("dockerSandbox() starts with empty hook arrays", () => {
+    const spec = dockerSandbox({ image: "custom:latest" });
+    expect(spec.provider).toBe("docker");
+    expect(spec.image).toBe("custom:latest");
+    expect(spec.setupHooks).toEqual([]);
+    expect(spec.teardownHooks).toEqual([]);
+  });
+
+  it(".setup() returns a new spec and does not mutate the original", () => {
+    const base = dockerSandbox();
+    const withSetup = base.setup(noopSetup);
+    expect(withSetup).not.toBe(base);
+    expect(base.setupHooks).toEqual([]);
+    expect(withSetup.setupHooks).toEqual([noopSetup]);
+  });
+
+  it("multiple .setup() calls accumulate in append order", () => {
+    const a: AgentSetup = () => {};
+    const b: AgentSetup = () => {};
+    const spec = dockerSandbox().setup(a).setup(b);
+    expect(spec.setupHooks).toEqual([a, b]);
+  });
+
+  it("multiple .teardown() calls accumulate in append order (execution order is the runner's job)", () => {
+    const x: AgentTeardown = () => {};
+    const y: AgentTeardown = () => {};
+    const spec = dockerSandbox().teardown(x).teardown(y);
+    expect(spec.teardownHooks).toEqual([x, y]);
+  });
+
+  it("chaining preserves provider-specific fields and stacks across calls", () => {
+    const spec = dockerSandbox({ image: "img" }).setup(noopSetup).teardown(noopTeardown).setup(noopSetup);
+    expect(spec.provider).toBe("docker");
+    expect(spec.image).toBe("img");
+    expect(spec.setupHooks).toHaveLength(2);
+    expect(spec.teardownHooks).toHaveLength(1);
+  });
+
+  it("vercelSandbox() / e2bSandbox() chain the same way and keep their own fields", () => {
+    const vercel = vercelSandbox({ snapshotId: "snap-1" }).setup(noopSetup);
+    expect(vercel.provider).toBe("vercel");
+    expect(vercel.snapshotId).toBe("snap-1");
+    expect(vercel.setupHooks).toEqual([noopSetup]);
+
+    const e2b = e2bSandbox({ template: "niceeval-agents" }).teardown(noopTeardown);
+    expect(e2b.provider).toBe("e2b");
+    expect(e2b.template).toBe("niceeval-agents");
+    expect(e2b.teardownHooks).toEqual([noopTeardown]);
+  });
+
+  it("defineSandbox() (custom provider) chains too and keeps create()/name", async () => {
+    const create = async () => {
+      throw new Error("not called in this test");
+    };
+    const spec = defineSandbox({ name: "my-provider", create }).setup(noopSetup).setup(noopSetup);
+    expect(spec.provider).toBe("my-provider");
+    expect(spec.create).toBe(create);
+    expect(spec.setupHooks).toHaveLength(2);
+  });
+
+  it("each chain call produces an independent object (no shared mutable state)", () => {
+    const base = dockerSandbox();
+    const other: AgentSetup = () => {};
+    const branchA = base.setup(noopSetup);
+    const branchB = base.setup(other);
+    expect(branchA.setupHooks).toEqual([noopSetup]);
+    expect(branchB.setupHooks).toEqual([other]);
+    expect(branchA.setupHooks).not.toBe(branchB.setupHooks);
+  });
+});