niceeval 0.4.3 → 0.4.5

package/src/i18n/en.ts

@@ -15,7 +15,12 @@ export const en = {
   "cli.all": "(all)",
   "cli.browserOpenFailed": "Could not open the browser automatically. Open manually: {{url}}\n",
   "cli.clean.done": "Deleted .niceeval/ historical run artifacts.\n",
-  "cli.config.missing": "Could not find niceeval.config.ts (run this from the project root).",
+  "cli.config.missing":
+    "Could not find niceeval.config.ts.\n" +
+    "Ways to fix:\n" +
+    "  - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
+    "  - [cd] Run from the project root that contains niceeval.config.ts\n" +
+    "  Docs: node_modules/niceeval/docs-site/quickstart.mdx",
   "cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
   "cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
   "cli.dry.noMatches": "(no matches)",
@@ -25,7 +30,8 @@ export const en = {
   "runner.budgetUnenforceable":
     "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
   "judge.modelMissing":
-    "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).",
+    "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
   "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
@@ -55,13 +61,16 @@ export const en = {
   "cli.experimentGroup": " group",
   "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
   "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
-  "cli.init.done": "Created evals/ and starter niceeval.config.ts.\n",
+  "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site).\n",
   "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
   "cli.list.header": "Discovered {{count}} evals:\n",
   "cli.noAgent": "No agent specified (use --agent <name>).\n",
   "cli.none": "(none)",
   "cli.pressCtrlC": "Press Ctrl+C to exit.\n",
-  "cli.run.experimentRequired": "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n",
+  "cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
+  "cli.run.experimentRequired":
+    "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
   "cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
@@ -70,7 +79,8 @@ export const en = {
   "cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
   "cli.view.url": "niceeval view: {{url}}\n",
   "context.capabilityMissing":
-    "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.",
+    "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/sandbox-agent.mdx",
   "context.skipEmpty": "skip() requires a non-empty reason.",
   "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
   "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -99,6 +109,7 @@ export const en = {
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge precheck failed ({{model}}): {{error}}",
   "judge.probeMissingKey": "judge model {{model}} is missing an API key; configure {{envHint}}",
+  "live.more": "… {{hidden}} more ({{running}} running · {{waiting}} waiting · {{done}} done)",
   "live.running": "  Running {{totalRuns}} attempts ({{evals}} evals × {{configs}} configs, concurrency {{concurrency}})       {{completed}}/{{total}} done",
   "live.runningUnknown": "  Running...  {{completed}}/{{total}} done",
   "live.waiting": "waiting for a slot...",
@@ -156,9 +167,12 @@ export const en = {
   "runner.timeout": "attempt timed out ({{timeoutMs}}ms)\nRecent progress:\n{{recentLogs}}",
   "runner.traceSelected": " -> kept {{count}} semantic spans",
   "runner.resumeCarry": "  · reusing {{carried}} passing results from last run, re-running {{retry}} evals\n",
+  "runner.resumeCarryDetail": "      carried [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
-  "sandbox.missingSpec": "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").",
+  "sandbox.missingSpec":
+    "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/sandbox-backends.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/i18n/zh-CN.ts

@@ -13,7 +13,12 @@ export const zhCN = {
   "cli.all": "(全部)",
   "cli.browserOpenFailed": "无法自动打开浏览器,请手动访问:{{url}}\n",
   "cli.clean.done": "已删除 .niceeval/ 历史运行工件。\n",
-  "cli.config.missing": "找不到 niceeval.config.ts(请在项目根运行)。",
+  "cli.config.missing":
+    "找不到 niceeval.config.ts。\n" +
+    "修法:\n" +
+    "  - [init] 运行 `npx niceeval init` 生成 niceeval.config.ts 和 evals/\n" +
+    "  - [cd] 切到包含 niceeval.config.ts 的项目根再运行\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/quickstart.mdx",
   "cli.config.noDefault": "niceeval.config.ts 需要 default export(defineConfig(...))。",
   "cli.dry.header": "\n[dry] {{evals}} 个 eval × {{configs}} 个运行配置:\n",
   "cli.dry.noMatches": "(无匹配)",
@@ -23,7 +28,8 @@ export const zhCN = {
   "runner.budgetUnenforceable":
     "{{budgetKey}} 的 budget:连续多个 attempt 完成后都拿不到成本数据(agent 不上报用量且模型不在价格表)——该 agent 的 budget 无法执行,取消护栏继续跑。\n",
   "judge.modelMissing":
-    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。",
+    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") 需要 YAML 解析器:请先 `pnpm add yaml`(或改用 loadJson + JSON 数据集)。",
   "cli.flag.parseError": "{{message}}\n运行 `niceeval --help` 查看用法。\n",
@@ -52,13 +58,16 @@ export const zhCN = {
   "cli.experimentGroup": "组",
   "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
   "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
-  "cli.init.done": "已创建 evals/ 和 niceeval.config.ts 起始文件。\n",
+  "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site)。\n",
   "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
   "cli.list.header": "发现 {{count}} 个 eval:\n",
   "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
   "cli.none": "(无)",
   "cli.pressCtrlC": "按 Ctrl+C 退出。\n",
-  "cli.run.experimentRequired": "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n",
+  "cli.resultsPath": "结构化结果:{{path}}(每条结果的 artifactsDir 下是该次 attempt 的 events.json / trace.json / diff.json)\n",
+  "cli.run.experimentRequired":
+    "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
   "cli.sandboxFlagRemoved": "`--sandbox` 不是 CLI flag。请在 experiment(或 niceeval.config.ts 做全项目兜底)里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n",
@@ -67,7 +76,8 @@ export const zhCN = {
   "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
   "cli.view.url": "niceeval view: {{url}}\n",
   "context.capabilityMissing":
-    "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。",
+    "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-agent.mdx",
   "context.skipEmpty": "skip() 需要一个非空理由。",
   "context.turnFailed": "本轮 send 返回 failed(turn status = failed):{{message}}",
   "context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
@@ -96,6 +106,7 @@ export const zhCN = {
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge 预检失败({{model}}): {{error}}",
   "judge.probeMissingKey": "judge 模型 {{model}} 缺少 API key —— 请配置 {{envHint}}",
+  "live.more": "… 其余 {{hidden}} 项({{running}} 运行中 · {{waiting}} 等待 · {{done}} 已完成)",
   "live.running": "  正在运行 {{totalRuns}} 次 ({{evals}} eval × {{configs}} 配置, 并发 {{concurrency}})       {{completed}}/{{total}} 完成",
   "live.runningUnknown": "  正在运行…  {{completed}}/{{total}} 完成",
   "live.waiting": "排队等待中…",
@@ -153,9 +164,12 @@ export const zhCN = {
   "runner.timeout": "attempt 超时({{timeoutMs}}ms)\n最近进度:\n{{recentLogs}}",
   "runner.traceSelected": " → 留 {{count}}(按语义)",
   "runner.resumeCarry": "  · 复用上次 {{carried}} 个通过的结果,重跑 {{retry}} 个 eval\n",
+  "runner.resumeCarryDetail": "      复用 [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
-  "sandbox.missingSpec": "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。",
+  "sandbox.missingSpec":
+    "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/o11y/parsers/bub.test.ts

@@ -0,0 +1,71 @@
+// bub tape 解析回归锁。
+//
+// 背景(2026-07 实测):模型在同一步里「先说话再调工具」时,bub 的 tape 把伴随文本
+// 记在 tool_call 条目的 payload.content 上(修复前上游直接丢弃,transcript 里
+// 每次 send 后面都看不到 AI 回复 —— 只有最后一步纯文本收尾能出现)。
+// 解析器必须把这段文本还原成 assistant message 事件,且顺序在 action.called 之前。
+
+import { describe, it, expect } from "vitest";
+import { parseBubTranscript } from "./bub.ts";
+
+const line = (obj: object) => JSON.stringify(obj);
+
+describe("parseBubTranscript", () => {
+  it("tool_call 携带 payload.content 时,先 emit assistant message 再 emit action.called", () => {
+    const raw = [
+      line({ kind: "message", payload: { role: "user", content: "migrate the routes" } }),
+      line({
+        kind: "tool_call",
+        payload: {
+          content: "I'll start by listing the pages directory.",
+          calls: [
+            { id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls pages"}' } },
+          ],
+        },
+      }),
+      line({ kind: "tool_result", payload: { results: ["index.js\nabout.js"] } }),
+    ].join("\n");
+
+    const { events } = parseBubTranscript(raw);
+    const types = events.map((e) => e.type);
+    expect(types).toEqual(["message", "message", "action.called", "action.result"]);
+
+    const assistant = events[1];
+    expect(assistant).toMatchObject({
+      type: "message",
+      role: "assistant",
+      text: "I'll start by listing the pages directory.",
+    });
+
+    const call = events[2];
+    expect(call).toMatchObject({ type: "action.called", callId: "call_1", name: "bash", tool: "shell" });
+  });
+
+  it("tool_call 无 content(存量 tape)不产生幻影 assistant message", () => {
+    const raw = [
+      line({ kind: "message", payload: { role: "user", content: "do it" } }),
+      line({
+        kind: "tool_call",
+        payload: { calls: [{ id: "call_1", function: { name: "fs_read", arguments: '{"path":"a.ts"}' } }] },
+      }),
+      line({ kind: "tool_result", payload: { results: ["file body"] } }),
+    ].join("\n");
+
+    const { events } = parseBubTranscript(raw);
+    expect(events.map((e) => e.type)).toEqual(["message", "action.called", "action.result"]);
+  });
+
+  it("tool_call 的 content 为块数组时也能抽出文本", () => {
+    const raw = line({
+      kind: "tool_call",
+      payload: {
+        content: [{ type: "text", text: "Checking the config first." }],
+        calls: [{ id: "call_1", function: { name: "bash", arguments: "{}" } }],
+      },
+    });
+
+    const { events } = parseBubTranscript(raw);
+    expect(events[0]).toMatchObject({ type: "message", role: "assistant", text: "Checking the config first." });
+    expect(events[1]).toMatchObject({ type: "action.called", callId: "call_1" });
+  });
+});

package/src/o11y/parsers/bub.ts

@@ -163,6 +163,11 @@ export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
           const text = extractText(get(payload, "content") ?? get(payload, "text"));
           if (text) events.push({ type: "message", role, text });
         } else if (kind === "tool_call") {
+          // 模型「先说话再调工具」:伴随文本记在 tool_call 的 payload.content 上
+          // (bub 不给它单独的 message 条目)。先还原成 assistant message,再吐 calls,
+          // 否则 transcript 里每次 send 后面都看不到 AI 回复。
+          const accompanying = extractText(get(payload, "content"));
+          if (accompanying) events.push({ type: "message", role: "assistant", text: accompanying });
           const calls = get(payload, "calls");
           const list = Array.isArray(calls) ? calls : [payload];
           lastCallIds = [];

package/src/o11y/types.ts

@@ -3,14 +3,23 @@
 
 import type { JsonValue, SourceLoc } from "../shared/types.ts";
 
-/** 一次运行的 token 用量(沙箱型从 transcript 抠,remote 由 send 返回)。 */
+/** 一次运行的 token 用量(沙箱型从 transcript/OTel span 的 `gen_ai.usage.*` 属性抠,remote 由 send 的 `Turn.usage` 直接返回)。 */
 export interface Usage {
+  /** 输入(prompt)token 数,不含缓存命中部分。 */
   inputTokens: number;
+  /** 输出(completion)token 数。 */
   outputTokens: number;
+  /** 命中 prompt 缓存、按缓存价读取的 token 数(省略表示该 agent 不上报此项)。 */
   cacheReadTokens?: number;
+  /** 写入 prompt 缓存的 token 数(省略表示该 agent 不上报此项)。 */
   cacheWriteTokens?: number;
+  /** 本次运行触发的模型请求次数(多轮/重试可能大于 1)。 */
   requests?: number;
-  /** 网关实测成本(若 agent 带回)——优先于价格表估算。 */
+  /**
+   * 网关/adapter 实测的真实美元成本(只能由 `Turn.usage.costUSD` 显式带回,从不从
+   * token 用量或 OTel span 反推得到)。存在时优先于按价格表(`defineConfig({ pricing })`)
+   * 估算的成本——见 `estimateCost` 的 `usage.costUSD ?? estimateCost(...)` 兜底顺序。
+   */
   costUSD?: number;
 }
 
@@ -28,12 +37,19 @@ export type ToolName =
   | "agent_task"
   | "unknown";
 
+/** HITL 停轮请求的结构化描述,供 `t.requireInputRequest(filter)` / `t.respondAll` 按条件匹配。 */
 export interface InputRequest {
+  /** 请求的唯一标识;多个请求并停时,`InputResponse.requestId` 靠它对位。 */
   readonly id?: string;
+  /** 模型提出的原始问题/文本(用于 `stringMatches` 过滤)。 */
   readonly prompt?: string;
+  /** adapter 自定义的人类可读展示文案,与 `prompt` 二选一或并存,用于 UI/日志展示及过滤匹配。 */
   readonly display?: string;
+  /** 请求关联的动作类型(如某个待批准的工具调用名),供过滤用。 */
   readonly action?: string;
+  /** 请求携带的结构化输入(如待批准命令的参数)。 */
   readonly input?: JsonValue;
+  /** 若请求提供了预设选项(如批准/拒绝),逐项列出;`id` 对应 `InputResponse.optionId`。 */
   readonly options?: readonly { id: string; label?: string }[];
 }
 
@@ -42,19 +58,28 @@ export interface InputRequest {
  * 各 agent 五花八门的原始 transcript 映射成 StreamEvent[];映射完,整套断言免费。
  */
 export type StreamEvent =
+  /** 一条文本消息(assistant 回复或 user 输入);`loc` 是可选的源码位置,用于把消息叠回 eval 源码。 */
   | { type: "message"; role: "assistant" | "user"; text: string; loc?: SourceLoc }
+  /** 发起一次工具/动作调用;`tool` 是归一化后的规范工具名,原始名保留在 DerivedFacts.ToolCall.originalName。 */
   | { type: "action.called"; callId: string; name: string; input: JsonValue; tool?: ToolName }
+  /** 一次工具/动作调用的结果,按 `callId` 与对应的 `action.called` 对位。 */
   | {
       type: "action.result";
       callId: string;
       output?: JsonValue;
       status: "completed" | "failed" | "rejected";
     }
+  /** 发起一次子 agent 调用(如 Task 工具、远程 sub-agent);`remoteUrl` 仅远程子 agent 有。 */
   | { type: "subagent.called"; callId: string; name: string; remoteUrl?: string }
+  /** 一次子 agent 调用的结果,按 `callId` 与对应的 `subagent.called` 对位。 */
   | { type: "subagent.completed"; callId: string; output?: JsonValue; status: "completed" | "failed" }
+  /** 模型停下来向人请求输入(HITL);具体请求内容见 InputRequest。 */
   | { type: "input.requested"; request: InputRequest }
+  /** 模型的思考/推理文本(非最终回复)。 */
   | { type: "thinking"; text: string }
+  /** 上下文被压缩/摘要(如超长会话截断历史);`reason` 是可选的压缩原因说明。 */
   | { type: "compaction"; reason?: string }
+  /** 运行中出现的错误。 */
   | { type: "error"; message: string };
 
 /** core 从事件流折叠出的结构化事实(deriveRunFacts)。 */

package/src/runner/attempt.ts

@@ -36,6 +36,7 @@ import type {
   Telemetry,
   TraceSpan,
 } from "../types.ts";
+import { runWho } from "./types.ts";
 import type { AgentRun, Attempt, RunOptions } from "./types.ts";
 
 export function runAttemptEffect(
@@ -72,7 +73,7 @@ export function runAttemptEffect(
 
   // 流式进度打到宿主 stderr(结果走 stdout,互不干扰)。容器主日志【不】放这些进度标记 ——
   // 那里留给 agent 的原始输出(adapter 给 agent 命令开 { stream: true })。
-  const who = run.model ? `${run.agent.name}/${run.model}` : run.agent.name;
+  const who = runWho(run);
   // 同时保留最近 20 条进度消息,timeout 时嵌入 error 字段方便定位卡在哪一步。
   const recentLogs: string[] = [];
   const log = (m: string) => {

package/src/runner/reporters/artifacts.ts

@@ -20,12 +20,15 @@ function producerVersion(): Promise<string | undefined> {
   return producerVersionPromise;
 }
 
-/** 一个 attempt 的工件子目录(相对 run 根):<evalId>/<agent>/<model>/a<attempt>。 */
+/** 一个 attempt 的工件子目录(相对 run 根):<evalId>/<agent>/<model>[/<experiment>]/a<attempt>。 */
 function attemptDir(r: EvalResult): string {
   const safe = (s: string) => s.replace(/[^\w.@-]/g, "_");
   // evalId 里的 / 保留作目录层级,其余危险字符替换。
   const id = r.id.replace(/[^\w./@-]/g, "_");
-  return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}/a${r.attempt}`;
+  // experiment 段:两个实验可以同 agent 同 model、只差 flags(feature A/B),没有这一段
+  // 它们的工件会互相覆盖。experimentId 里的 / 不作层级(压成 _),整个实验一格。
+  const exp = r.experimentId ? `/${safe(r.experimentId)}` : "";
+  return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}${exp}/a${r.attempt}`;
 }
 
 /** summary.json 用的瘦身结果:去掉大数组(events/trace/o11y/diff/sources),换成目录引用 + 存在标记。 */
@@ -51,7 +54,10 @@ function slimResult(r: EvalResult): EvalResult {
   };
 }
 
-export function Artifacts(root = ".niceeval"): Reporter {
+/** Artifacts 报告器额外暴露输出目录:CLI 在 run 结束时打出 summary.json 路径给 agent 直读。 */
+export type ArtifactsReporter = Reporter & { outputDir(): string };
+
+export function Artifacts(root = ".niceeval"): ArtifactsReporter {
   let outputDir = "";
   const ensureDir = async (): Promise<void> => {
     if (!outputDir) outputDir = join(root, safeTimestamp(new Date()));
@@ -59,6 +65,8 @@ export function Artifacts(root = ".niceeval"): Reporter {
   };
 
   return {
+    outputDir: () => outputDir,
+
     async onRunStart() {
       outputDir = join(root, safeTimestamp(new Date()));
       await mkdir(outputDir, { recursive: true });

package/src/runner/reporters/live.ts

@@ -102,20 +102,57 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
     return `\x1B[2K${hdr}`;
   }
 
-  function draw(frame: number) {
-    if (!process.stderr.isTTY) return;
-
-    if (drawnLines > 0) {
-      process.stderr.write(`\x1B[${drawnLines}A`);
+  // 终端放不下全部行时,光标回跳会被屏幕顶端截断,导致每帧往下追加整表。
+  // 所以按终端高度截断:优先显示运行中的行,放不下的折叠成一行摘要。
+  function frameLines(frame: number): string[] {
+    const lines = [renderHeader()];
+    const termRows = process.stderr.rows || 30;
+    // 预留:1 行表头 + 1 行防止末尾换行触发滚动
+    const budget = Math.max(1, termRows - 2);
+
+    if (keyOrder.length <= budget) {
+      for (const k of keyOrder) lines.push(renderRow(stateMap.get(k)!, frame));
+      return lines;
     }
 
-    let out = renderHeader() + "\n";
+    const running: string[] = [];
+    const waiting: string[] = [];
+    const done: string[] = [];
+    for (const k of keyOrder) {
+      const s = stateMap.get(k)!;
+      if (s.completed >= s.total) done.push(k);
+      else if (s.started) running.push(k);
+      else waiting.push(k);
+    }
+    // 选出要显示的 key(运行中 > 等待 > 已完成),但按原始顺序渲染,避免行来回跳动
+    const shown = new Set([...running, ...waiting, ...done].slice(0, budget - 1));
     for (const k of keyOrder) {
-      out += renderRow(stateMap.get(k)!, frame) + "\n";
+      if (shown.has(k)) lines.push(renderRow(stateMap.get(k)!, frame));
     }
+    lines.push(
+      `\x1B[2K  ${t("live.more", {
+        hidden: keyOrder.length - shown.size,
+        running: running.filter((k) => !shown.has(k)).length,
+        waiting: waiting.filter((k) => !shown.has(k)).length,
+        done: done.filter((k) => !shown.has(k)).length,
+      })}`,
+    );
+    return lines;
+  }
 
+  function draw(frame: number) {
+    if (!process.stderr.isTTY) return;
+
+    const lines = frameLines(frame);
+    let out = drawnLines > 0 ? `\x1B[${drawnLines}A` : "";
+    out += lines.join("\n") + "\n";
+    // 本帧比上帧短(行完成后折叠、终端拉高)时,清掉下方残留的旧行
+    const extra = drawnLines - lines.length;
+    if (extra > 0) {
+      out += "\x1B[2K\n".repeat(extra) + `\x1B[${extra}A`;
+    }
     process.stderr.write(out);
-    drawnLines = 1 + keyOrder.length;
+    drawnLines = lines.length;
   }
 
   function clearDisplay() {

package/src/runner/run.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+import { judgeProbeTargets } from "./run.ts";
+import type { JudgeConfig } from "../types.ts";
+
+// judge 预检的目标收敛:只探测「实际要跑、且源码里出现 judge 字样」的 eval 的生效配置。
+// 这是对 memory/judge-config-precheck-hard-fails-without-key 的修复守护——
+// 全局配了 judge 但选中的 eval 都不用时,不能再因 judge key / 端点问题拦下整次运行。
+describe("judgeProbeTargets", () => {
+  const configJudge: JudgeConfig = { model: "gpt-5.4" };
+
+  it("skips probing when no selected eval mentions judge", () => {
+    const evals = [
+      { source: `t.check(t.reply, includes("2"));`, judge: undefined },
+      { source: `await t.sandbox.exec("pnpm test");`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([]);
+  });
+
+  it("probes config-level judge when a selected eval mentions judge", () => {
+    const evals = [
+      { source: `t.judge.autoevals.closedQA("did it summarize?");`, judge: undefined },
+      { source: `t.check(t.reply, includes("ok"));`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([configJudge]);
+  });
+
+  it("resolves eval-level judge over config-level, like attempt resolution", () => {
+    const evalJudge: JudgeConfig = { model: "deepseek-v4", baseUrl: "http://localhost:8787/v1" };
+    const evals = [{ source: `t.judge.autoevals.factuality("2")`, judge: evalJudge }];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([evalJudge]);
+  });
+
+  it("dedupes identical effective configs across evals", () => {
+    const evals = [
+      { source: `t.judge.autoevals.closedQA("a")`, judge: undefined },
+      { source: `t.judge.autoevals.closedQA("b")`, judge: undefined },
+    ];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([configJudge]);
+  });
+
+  it("returns nothing when judge is used but no config exists (runtime env fallback)", () => {
+    const evals = [{ source: `t.judge.autoevals.closedQA("a")`, judge: undefined }];
+    expect(judgeProbeTargets(evals, undefined)).toEqual([]);
+  });
+
+  it("does not match judge as part of a longer identifier", () => {
+    const evals = [{ source: `const prejudged = true;`, judge: undefined }];
+    expect(judgeProbeTargets(evals, configJudge)).toEqual([]);
+  });
+});