niceeval 0.4.4 → 0.4.6

package/src/i18n/en.ts

@@ -15,7 +15,12 @@ export const en = {
   "cli.all": "(all)",
   "cli.browserOpenFailed": "Could not open the browser automatically. Open manually: {{url}}\n",
   "cli.clean.done": "Deleted .niceeval/ historical run artifacts.\n",
-  "cli.config.missing": "Could not find niceeval.config.ts (run this from the project root).",
+  "cli.config.missing":
+    "Could not find niceeval.config.ts.\n" +
+    "Ways to fix:\n" +
+    "  - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
+    "  - [cd] Run from the project root that contains niceeval.config.ts\n" +
+    "  Docs: node_modules/niceeval/docs-site/quickstart.mdx",
   "cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
   "cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
   "cli.dry.noMatches": "(no matches)",
@@ -25,7 +30,8 @@ export const en = {
   "runner.budgetUnenforceable":
     "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
   "judge.modelMissing":
-    "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).",
+    "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
   "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
@@ -55,13 +61,16 @@ export const en = {
   "cli.experimentGroup": " group",
   "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
   "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
-  "cli.init.done": "Created evals/ and starter niceeval.config.ts.\n",
+  "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site).\n",
   "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
   "cli.list.header": "Discovered {{count}} evals:\n",
   "cli.noAgent": "No agent specified (use --agent <name>).\n",
   "cli.none": "(none)",
   "cli.pressCtrlC": "Press Ctrl+C to exit.\n",
-  "cli.run.experimentRequired": "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n",
+  "cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
+  "cli.run.experimentRequired":
+    "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
   "cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
@@ -70,7 +79,8 @@ export const en = {
   "cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
   "cli.view.url": "niceeval view: {{url}}\n",
   "context.capabilityMissing":
-    "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.",
+    "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/sandbox-agent.mdx",
   "context.skipEmpty": "skip() requires a non-empty reason.",
   "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
   "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -99,6 +109,7 @@ export const en = {
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge precheck failed ({{model}}): {{error}}",
   "judge.probeMissingKey": "judge model {{model}} is missing an API key; configure {{envHint}}",
+  "live.more": "… {{hidden}} more ({{running}} running · {{waiting}} waiting · {{done}} done)",
   "live.running": "  Running {{totalRuns}} attempts ({{evals}} evals × {{configs}} configs, concurrency {{concurrency}})       {{completed}}/{{total}} done",
   "live.runningUnknown": "  Running...  {{completed}}/{{total}} done",
   "live.waiting": "waiting for a slot...",
@@ -159,7 +170,9 @@ export const en = {
   "runner.resumeCarryDetail": "      carried [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "using remote agent (no sandbox created)...",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
-  "sandbox.missingSpec": "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").",
+  "sandbox.missingSpec":
+    "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
+    "  Docs: node_modules/niceeval/docs-site/guides/sandbox-backends.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/i18n/zh-CN.ts

@@ -13,7 +13,12 @@ export const zhCN = {
   "cli.all": "(全部)",
   "cli.browserOpenFailed": "无法自动打开浏览器,请手动访问:{{url}}\n",
   "cli.clean.done": "已删除 .niceeval/ 历史运行工件。\n",
-  "cli.config.missing": "找不到 niceeval.config.ts(请在项目根运行)。",
+  "cli.config.missing":
+    "找不到 niceeval.config.ts。\n" +
+    "修法:\n" +
+    "  - [init] 运行 `npx niceeval init` 生成 niceeval.config.ts 和 evals/\n" +
+    "  - [cd] 切到包含 niceeval.config.ts 的项目根再运行\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/quickstart.mdx",
   "cli.config.noDefault": "niceeval.config.ts 需要 default export(defineConfig(...))。",
   "cli.dry.header": "\n[dry] {{evals}} 个 eval × {{configs}} 个运行配置:\n",
   "cli.dry.noMatches": "(无匹配)",
@@ -23,7 +28,8 @@ export const zhCN = {
   "runner.budgetUnenforceable":
     "{{budgetKey}} 的 budget:连续多个 attempt 完成后都拿不到成本数据(agent 不上报用量且模型不在价格表)——该 agent 的 budget 无法执行,取消护栏继续跑。\n",
   "judge.modelMissing":
-    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。",
+    "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定评判模型(没有内置默认模型)。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
   "loaders.yamlMissing":
     "loadYaml(\"{{path}}\") 需要 YAML 解析器:请先 `pnpm add yaml`(或改用 loadJson + JSON 数据集)。",
   "cli.flag.parseError": "{{message}}\n运行 `niceeval --help` 查看用法。\n",
@@ -52,13 +58,16 @@ export const zhCN = {
   "cli.experimentGroup": "组",
   "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
   "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
-  "cli.init.done": "已创建 evals/ 和 niceeval.config.ts 起始文件。\n",
+  "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site)。\n",
   "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
   "cli.list.header": "发现 {{count}} 个 eval:\n",
   "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
   "cli.none": "(无)",
   "cli.pressCtrlC": "按 Ctrl+C 退出。\n",
-  "cli.run.experimentRequired": "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n",
+  "cli.resultsPath": "结构化结果:{{path}}(每条结果的 artifactsDir 下是该次 attempt 的 events.json / trace.json / diff.json)\n",
+  "cli.run.experimentRequired":
+    "运行 eval 必须通过 experiment:用 `niceeval exp [实验组|配置] [eval id 前缀]`。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
   "cli.run.experimentRequiredHint": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
   "cli.run.experimentRequiredKnown": "已发现实验:{{experiments}}\n",
   "cli.sandboxFlagRemoved": "`--sandbox` 不是 CLI flag。请在 experiment(或 niceeval.config.ts 做全项目兜底)里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n",
@@ -67,7 +76,8 @@ export const zhCN = {
   "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
   "cli.view.url": "niceeval view: {{url}}\n",
   "context.capabilityMissing":
-    "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。",
+    "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-agent.mdx",
   "context.skipEmpty": "skip() 需要一个非空理由。",
   "context.turnFailed": "本轮 send 返回 failed(turn status = failed):{{message}}",
   "context.turnFailedDefault": "本轮 send 返回 failed(turn status = failed)",
@@ -96,6 +106,7 @@ export const zhCN = {
   "judge.httpError": "judge HTTP {{status}}: {{body}}",
   "judge.probeFailed": "judge 预检失败({{model}}): {{error}}",
   "judge.probeMissingKey": "judge 模型 {{model}} 缺少 API key —— 请配置 {{envHint}}",
+  "live.more": "… 其余 {{hidden}} 项({{running}} 运行中 · {{waiting}} 等待 · {{done}} 已完成)",
   "live.running": "  正在运行 {{totalRuns}} 次 ({{evals}} eval × {{configs}} 配置, 并发 {{concurrency}})       {{completed}}/{{total}} 完成",
   "live.runningUnknown": "  正在运行…  {{completed}}/{{total}} 完成",
   "live.waiting": "排队等待中…",
@@ -156,7 +167,9 @@ export const zhCN = {
   "runner.resumeCarryDetail": "      复用 [{{experiment}}] {{evals}}\n",
   "runner.useRemoteAgent": "使用 remote agent(不创建沙箱)…",
   "sandbox.backendNotImplemented": "{{backend}} sandbox backend not implemented; use docker, vercel, or e2b",
-  "sandbox.missingSpec": "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。",
+  "sandbox.missingSpec":
+    "沙箱型 agent 需要一个 sandbox,但没有提供。niceeval 不再自动选默认后端——请在 defineExperiment()/defineConfig() 里把 sandbox 设成 dockerSandbox() / vercelSandbox() / e2bSandbox()(从 \"niceeval/sandbox\" 导入)。\n" +
+    "  文档:node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
   "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
   "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
   "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",

package/src/o11y/parsers/bub.test.ts

@@ -0,0 +1,71 @@
+// bub tape 解析回归锁。
+//
+// 背景(2026-07 实测):模型在同一步里「先说话再调工具」时,bub 的 tape 把伴随文本
+// 记在 tool_call 条目的 payload.content 上(修复前上游直接丢弃,transcript 里
+// 每次 send 后面都看不到 AI 回复 —— 只有最后一步纯文本收尾能出现)。
+// 解析器必须把这段文本还原成 assistant message 事件,且顺序在 action.called 之前。
+
+import { describe, it, expect } from "vitest";
+import { parseBubTranscript } from "./bub.ts";
+
+const line = (obj: object) => JSON.stringify(obj);
+
+describe("parseBubTranscript", () => {
+  it("tool_call 携带 payload.content 时,先 emit assistant message 再 emit action.called", () => {
+    const raw = [
+      line({ kind: "message", payload: { role: "user", content: "migrate the routes" } }),
+      line({
+        kind: "tool_call",
+        payload: {
+          content: "I'll start by listing the pages directory.",
+          calls: [
+            { id: "call_1", type: "function", function: { name: "bash", arguments: '{"cmd":"ls pages"}' } },
+          ],
+        },
+      }),
+      line({ kind: "tool_result", payload: { results: ["index.js\nabout.js"] } }),
+    ].join("\n");
+
+    const { events } = parseBubTranscript(raw);
+    const types = events.map((e) => e.type);
+    expect(types).toEqual(["message", "message", "action.called", "action.result"]);
+
+    const assistant = events[1];
+    expect(assistant).toMatchObject({
+      type: "message",
+      role: "assistant",
+      text: "I'll start by listing the pages directory.",
+    });
+
+    const call = events[2];
+    expect(call).toMatchObject({ type: "action.called", callId: "call_1", name: "bash", tool: "shell" });
+  });
+
+  it("tool_call 无 content(存量 tape)不产生幻影 assistant message", () => {
+    const raw = [
+      line({ kind: "message", payload: { role: "user", content: "do it" } }),
+      line({
+        kind: "tool_call",
+        payload: { calls: [{ id: "call_1", function: { name: "fs_read", arguments: '{"path":"a.ts"}' } }] },
+      }),
+      line({ kind: "tool_result", payload: { results: ["file body"] } }),
+    ].join("\n");
+
+    const { events } = parseBubTranscript(raw);
+    expect(events.map((e) => e.type)).toEqual(["message", "action.called", "action.result"]);
+  });
+
+  it("tool_call 的 content 为块数组时也能抽出文本", () => {
+    const raw = line({
+      kind: "tool_call",
+      payload: {
+        content: [{ type: "text", text: "Checking the config first." }],
+        calls: [{ id: "call_1", function: { name: "bash", arguments: "{}" } }],
+      },
+    });
+
+    const { events } = parseBubTranscript(raw);
+    expect(events[0]).toMatchObject({ type: "message", role: "assistant", text: "Checking the config first." });
+    expect(events[1]).toMatchObject({ type: "action.called", callId: "call_1" });
+  });
+});

package/src/o11y/parsers/bub.ts

@@ -163,6 +163,11 @@ export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
           const text = extractText(get(payload, "content") ?? get(payload, "text"));
           if (text) events.push({ type: "message", role, text });
         } else if (kind === "tool_call") {
+          // 模型「先说话再调工具」:伴随文本记在 tool_call 的 payload.content 上
+          // (bub 不给它单独的 message 条目)。先还原成 assistant message,再吐 calls,
+          // 否则 transcript 里每次 send 后面都看不到 AI 回复。
+          const accompanying = extractText(get(payload, "content"));
+          if (accompanying) events.push({ type: "message", role: "assistant", text: accompanying });
           const calls = get(payload, "calls");
           const list = Array.isArray(calls) ? calls : [payload];
           lastCallIds = [];

package/src/o11y/types.ts

@@ -3,14 +3,23 @@
 
 import type { JsonValue, SourceLoc } from "../shared/types.ts";
 
-/** 一次运行的 token 用量(沙箱型从 transcript 抠,remote 由 send 返回)。 */
+/** 一次运行的 token 用量(沙箱型从 transcript/OTel span 的 `gen_ai.usage.*` 属性抠,remote 由 send 的 `Turn.usage` 直接返回)。 */
 export interface Usage {
+  /** 输入(prompt)token 数,不含缓存命中部分。 */
   inputTokens: number;
+  /** 输出(completion)token 数。 */
   outputTokens: number;
+  /** 命中 prompt 缓存、按缓存价读取的 token 数(省略表示该 agent 不上报此项)。 */
   cacheReadTokens?: number;
+  /** 写入 prompt 缓存的 token 数(省略表示该 agent 不上报此项)。 */
   cacheWriteTokens?: number;
+  /** 本次运行触发的模型请求次数(多轮/重试可能大于 1)。 */
   requests?: number;
-  /** 网关实测成本(若 agent 带回)——优先于价格表估算。 */
+  /**
+   * 网关/adapter 实测的真实美元成本(只能由 `Turn.usage.costUSD` 显式带回,从不从
+   * token 用量或 OTel span 反推得到)。存在时优先于按价格表(`defineConfig({ pricing })`)
+   * 估算的成本——见 `estimateCost` 的 `usage.costUSD ?? estimateCost(...)` 兜底顺序。
+   */
   costUSD?: number;
 }
 
@@ -28,12 +37,19 @@ export type ToolName =
   | "agent_task"
   | "unknown";
 
+/** HITL 停轮请求的结构化描述,供 `t.requireInputRequest(filter)` / `t.respondAll` 按条件匹配。 */
 export interface InputRequest {
+  /** 请求的唯一标识;多个请求并停时,`InputResponse.requestId` 靠它对位。 */
   readonly id?: string;
+  /** 模型提出的原始问题/文本(用于 `stringMatches` 过滤)。 */
   readonly prompt?: string;
+  /** adapter 自定义的人类可读展示文案,与 `prompt` 二选一或并存,用于 UI/日志展示及过滤匹配。 */
   readonly display?: string;
+  /** 请求关联的动作类型(如某个待批准的工具调用名),供过滤用。 */
   readonly action?: string;
+  /** 请求携带的结构化输入(如待批准命令的参数)。 */
   readonly input?: JsonValue;
+  /** 若请求提供了预设选项(如批准/拒绝),逐项列出;`id` 对应 `InputResponse.optionId`。 */
   readonly options?: readonly { id: string; label?: string }[];
 }
 
@@ -42,19 +58,28 @@ export interface InputRequest {
  * 各 agent 五花八门的原始 transcript 映射成 StreamEvent[];映射完,整套断言免费。
  */
 export type StreamEvent =
+  /** 一条文本消息(assistant 回复或 user 输入);`loc` 是可选的源码位置,用于把消息叠回 eval 源码。 */
   | { type: "message"; role: "assistant" | "user"; text: string; loc?: SourceLoc }
+  /** 发起一次工具/动作调用;`tool` 是归一化后的规范工具名,原始名保留在 DerivedFacts.ToolCall.originalName。 */
   | { type: "action.called"; callId: string; name: string; input: JsonValue; tool?: ToolName }
+  /** 一次工具/动作调用的结果,按 `callId` 与对应的 `action.called` 对位。 */
   | {
       type: "action.result";
       callId: string;
       output?: JsonValue;
       status: "completed" | "failed" | "rejected";
     }
+  /** 发起一次子 agent 调用(如 Task 工具、远程 sub-agent);`remoteUrl` 仅远程子 agent 有。 */
   | { type: "subagent.called"; callId: string; name: string; remoteUrl?: string }
+  /** 一次子 agent 调用的结果,按 `callId` 与对应的 `subagent.called` 对位。 */
   | { type: "subagent.completed"; callId: string; output?: JsonValue; status: "completed" | "failed" }
+  /** 模型停下来向人请求输入(HITL);具体请求内容见 InputRequest。 */
   | { type: "input.requested"; request: InputRequest }
+  /** 模型的思考/推理文本(非最终回复)。 */
   | { type: "thinking"; text: string }
+  /** 上下文被压缩/摘要(如超长会话截断历史);`reason` 是可选的压缩原因说明。 */
   | { type: "compaction"; reason?: string }
+  /** 运行中出现的错误。 */
   | { type: "error"; message: string };
 
 /** core 从事件流折叠出的结构化事实(deriveRunFacts)。 */

package/src/runner/reporters/artifacts.ts

@@ -20,12 +20,15 @@ function producerVersion(): Promise<string | undefined> {
   return producerVersionPromise;
 }
 
-/** 一个 attempt 的工件子目录(相对 run 根):<evalId>/<agent>/<model>/a<attempt>。 */
+/** 一个 attempt 的工件子目录(相对 run 根):<evalId>/<agent>/<model>[/<experiment>]/a<attempt>。 */
 function attemptDir(r: EvalResult): string {
   const safe = (s: string) => s.replace(/[^\w.@-]/g, "_");
   // evalId 里的 / 保留作目录层级,其余危险字符替换。
   const id = r.id.replace(/[^\w./@-]/g, "_");
-  return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}/a${r.attempt}`;
+  // experiment 段:两个实验可以同 agent 同 model、只差 flags(feature A/B),没有这一段
+  // 它们的工件会互相覆盖。experimentId 里的 / 不作层级(压成 _),整个实验一格。
+  const exp = r.experimentId ? `/${safe(r.experimentId)}` : "";
+  return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}${exp}/a${r.attempt}`;
 }
 
 /** summary.json 用的瘦身结果:去掉大数组(events/trace/o11y/diff/sources),换成目录引用 + 存在标记。 */
@@ -51,7 +54,10 @@ function slimResult(r: EvalResult): EvalResult {
   };
 }
 
-export function Artifacts(root = ".niceeval"): Reporter {
+/** Artifacts 报告器额外暴露输出目录:CLI 在 run 结束时打出 summary.json 路径给 agent 直读。 */
+export type ArtifactsReporter = Reporter & { outputDir(): string };
+
+export function Artifacts(root = ".niceeval"): ArtifactsReporter {
   let outputDir = "";
   const ensureDir = async (): Promise<void> => {
     if (!outputDir) outputDir = join(root, safeTimestamp(new Date()));
@@ -59,6 +65,8 @@ export function Artifacts(root = ".niceeval"): Reporter {
   };
 
   return {
+    outputDir: () => outputDir,
+
     async onRunStart() {
       outputDir = join(root, safeTimestamp(new Date()));
       await mkdir(outputDir, { recursive: true });

package/src/runner/reporters/live.ts

@@ -102,20 +102,57 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
     return `\x1B[2K${hdr}`;
   }
 
-  function draw(frame: number) {
-    if (!process.stderr.isTTY) return;
-
-    if (drawnLines > 0) {
-      process.stderr.write(`\x1B[${drawnLines}A`);
+  // 终端放不下全部行时,光标回跳会被屏幕顶端截断,导致每帧往下追加整表。
+  // 所以按终端高度截断:优先显示运行中的行,放不下的折叠成一行摘要。
+  function frameLines(frame: number): string[] {
+    const lines = [renderHeader()];
+    const termRows = process.stderr.rows || 30;
+    // 预留:1 行表头 + 1 行防止末尾换行触发滚动
+    const budget = Math.max(1, termRows - 2);
+
+    if (keyOrder.length <= budget) {
+      for (const k of keyOrder) lines.push(renderRow(stateMap.get(k)!, frame));
+      return lines;
     }
 
-    let out = renderHeader() + "\n";
+    const running: string[] = [];
+    const waiting: string[] = [];
+    const done: string[] = [];
+    for (const k of keyOrder) {
+      const s = stateMap.get(k)!;
+      if (s.completed >= s.total) done.push(k);
+      else if (s.started) running.push(k);
+      else waiting.push(k);
+    }
+    // 选出要显示的 key(运行中 > 等待 > 已完成),但按原始顺序渲染,避免行来回跳动
+    const shown = new Set([...running, ...waiting, ...done].slice(0, budget - 1));
     for (const k of keyOrder) {
-      out += renderRow(stateMap.get(k)!, frame) + "\n";
+      if (shown.has(k)) lines.push(renderRow(stateMap.get(k)!, frame));
     }
+    lines.push(
+      `\x1B[2K  ${t("live.more", {
+        hidden: keyOrder.length - shown.size,
+        running: running.filter((k) => !shown.has(k)).length,
+        waiting: waiting.filter((k) => !shown.has(k)).length,
+        done: done.filter((k) => !shown.has(k)).length,
+      })}`,
+    );
+    return lines;
+  }
 
+  function draw(frame: number) {
+    if (!process.stderr.isTTY) return;
+
+    const lines = frameLines(frame);
+    let out = drawnLines > 0 ? `\x1B[${drawnLines}A` : "";
+    out += lines.join("\n") + "\n";
+    // 本帧比上帧短(行完成后折叠、终端拉高)时,清掉下方残留的旧行
+    const extra = drawnLines - lines.length;
+    if (extra > 0) {
+      out += "\x1B[2K\n".repeat(extra) + `\x1B[${extra}A`;
+    }
     process.stderr.write(out);
-    drawnLines = 1 + keyOrder.length;
+    drawnLines = lines.length;
   }
 
   function clearDisplay() {

package/src/runner/run.ts

@@ -9,9 +9,9 @@ import { t } from "../i18n/index.ts";
 import { cacheKey, computeFingerprint } from "./fingerprint.ts";
 import { OtelReceiverPool } from "../o11y/otlp/turn-otel.ts";
 import { runAttemptEffect } from "./attempt.ts";
-import { runReporter, emitReporterEvent, summarize } from "./report.ts";
-import type { Agent, EvalResult, JudgeConfig, RunShape, RunSummary } from "../types.ts";
-import type { Attempt, RunOptions } from "./types.ts";
+import { runReporter, emitReporterEvent, scopeReporter, summarize } from "./report.ts";
+import type { Agent, EvalResult, JudgeConfig, Reporter, RunShape, RunSummary } from "../types.ts";
+import type { AgentRun, Attempt, RunOptions } from "./types.ts";
 
 export type { AgentRun, RunOptions } from "./types.ts";
 
@@ -102,7 +102,11 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     for (let i = 0; i < run.runs; i++) {
       for (const evalDef of evals) {
         if (run.experimentId && priorRunKeys.has(`${run.experimentId}|${evalDef.id}`)) continue;
-        const key = `${run.agent.name}|${run.model ?? ""}|${evalDef.id}`;
+        // key 标识「同一个运行配置下的同一条 eval」,earlyExit 的跳过/abort 只应作用于
+        // 同 key 的重试轮。experimentId 必须进 key:两个实验可以同 agent 同 model、只差
+        // flags(feature A/B 正是这种形状),漏掉它会让先过的实验把其它实验的同名 eval
+        // 整个跳掉——花了钱还丢结果。
+        const key = `${run.experimentId ?? ""}|${run.agent.name}|${run.model ?? ""}|${evalDef.id}`;
         attempts.push({
           evalDef,
           run,
@@ -159,11 +163,37 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     totalRuns: attempts.length,
     maxConcurrency: opts.maxConcurrency,
   };
-  for (const r of opts.reporters) {
+  // eval 级 reporters:实例只观测引用它的 eval(经 scopeReporter 过滤转发)。
+  // 已经挂在全局 reporters 里的同一实例不重复挂;同一实例被多个 eval 引用时合并观测集
+  // (共享一个目的地,如同一个 Braintrust 实验)。本次没有任何被观测 eval 要跑时整个跳过。
+  const scopedSets = new Map<Reporter, Set<string>>();
+  for (const e of opts.evals) {
+    for (const r of e.reporters ?? []) {
+      if (opts.reporters.includes(r)) continue;
+      let ids = scopedSets.get(r);
+      if (!ids) scopedSets.set(r, (ids = new Set()));
+      ids.add(e.id);
+    }
+  }
+  const reporters: Reporter[] = [...opts.reporters];
+  for (const [r, ids] of scopedSets) {
+    const scopedRuns = attempts.filter((a) => ids.has(a.evalDef.id)).length;
+    if (scopedRuns === 0) continue;
+    reporters.push(
+      scopeReporter(r, ids, {
+        evals: [...ids].filter((id) => runningIds.has(id)).length,
+        configs: opts.agentRuns.length,
+        totalRuns: scopedRuns,
+        maxConcurrency: opts.maxConcurrency,
+      }),
+    );
+  }
+
+  for (const r of reporters) {
     // reporter 只是结果消费方:单个 reporter 抛错记 diagnostic,不能让整次调度崩(P2)。
     await runReporter("onRunStart", () => r.onRunStart?.(runningEvals, firstAgent as Agent, shape));
   }
-  await emitReporterEvent(opts.reporters, {
+  await emitReporterEvent(reporters, {
     type: "run:start",
     evals: runningEvals,
     agent: firstAgent as Agent,
@@ -209,6 +239,18 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   // 未显式指定时跟 maxConcurrency 走——各 backend 的推荐值已在 cli 层写进 maxConcurrency 默认值。
   const sandboxSem = Effect.runSync(Effect.makeSemaphore(opts.maxConcurrency));
 
+  // 两级并发闸:全局(opts.maxConcurrency)+ 实验级(AgentRun.maxConcurrency,可选)。
+  // 实验级信号量让「有共享状态、必须串行」的实验(如跨 eval 累积记忆,maxConcurrency: 1)
+  // 只在自己内部排队,同批其它实验照常并发——旧行为是 CLI 取所有选中实验的最小值钳全局,
+  // 一个串行实验会把整批基线拖成串行。等于全局上限的实验级值不建闸(与全局闸重复)。
+  const globalSem = Effect.runSync(Effect.makeSemaphore(opts.maxConcurrency));
+  const runSems = new Map<AgentRun, Effect.Semaphore>();
+  for (const run of opts.agentRuns) {
+    if (run.maxConcurrency !== undefined && run.maxConcurrency < opts.maxConcurrency) {
+      runSems.set(run, Effect.runSync(Effect.makeSemaphore(Math.max(1, run.maxConcurrency))));
+    }
+  }
+
   // 非沙箱 tracing agent 的共享 OTLP 接收池:被测应用是长驻进程,端点不能随
   // attempt 换 —— receiver 粒度跟被测进程走(每 agent 一个,整个 run 复用),run 结束回收。
   if (!opts.otelPool) {
@@ -224,8 +266,11 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
     }
   }
 
-  // 有界并发调度:Effect.forEach({ concurrency }) 取代手写 queue / inFlight / Promise.race。
-  // 每个 attempt 跑在自己的 fiber;runAttemptEffect 只把「执行错误」收进 EvalResult.error(不 fail),
+  // 有界并发调度:forEach 本身 unbounded(每个 attempt 立刻有自己的 fiber),真正的
+  // 并发上限由上面两级信号量把守——执行体先过实验级闸(若有)再占全局 permit 才开跑。
+  // 获取定序恒为 runSem → globalSem,无环等待;实验级闸的持有者在等全局 permit 时
+  // 不占别的实验的并发位(并发位就是 globalSem 的 permit,不再是 forEach 的 fiber 槽)。
+  // runAttemptEffect 只把「执行错误」收进 EvalResult.error(不 fail),
   // 但中断(Ctrl+C / kill)照常向上传播 —— 所以一条挂掉不会中断其它 attempt,而中断能停掉全部。
   //
   // signal:把 opts.signal 喂给 run → abort 触发根 fiber 中断 → forEach 中断所有子 fiber
@@ -240,14 +285,15 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   const exit = await Effect.runPromiseExit(
     Effect.forEach(
       attempts,
-      (a) =>
-        Effect.gen(function* () {
+      (a) => {
+        const body = Effect.gen(function* () {
             // 早停:同 key 已通过,或已 errored(重跑只会重复同一个框架错误)且开了 earlyExit
-            // → 跳过未启动的 attempt。
+            // → 跳过未启动的 attempt。检查必须在拿到 permit 之后(fiber 是 unbounded 一次性
+            // 全建的,建时结果还没出来);跳过路径短暂占一个 permit,可忽略。
             if (a.run.earlyExit && (passedKeys.has(a.key) || erroredKeys.has(a.key))) {
               yield* reportMutex.withPermits(1)(
                 Effect.promise(() =>
-                  emitReporterEvent(opts.reporters, {
+                  emitReporterEvent(reporters, {
                     type: "run:earlyExit",
                     evalId: a.evalDef.id,
                     experimentId: a.run.experimentId,
@@ -270,7 +316,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
                     budgetReported.add(budgetKey);
                     yield* reportMutex.withPermits(1)(
                       Effect.promise(() =>
-                        emitReporterEvent(opts.reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
+                        emitReporterEvent(reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
                       ),
                     );
                   }
@@ -305,7 +351,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
 
             yield* reportMutex.withPermits(1)(
               Effect.promise(() =>
-                emitReporterEvent(opts.reporters, {
+                emitReporterEvent(reporters, {
                   type: "eval:start",
                   eval: { id: a.evalDef.id },
                   agent: a.run.agent,
@@ -354,7 +400,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
               // 停掉后续 attempt(P2)。
               Effect.promise(() =>
                 Promise.all(
-                  opts.reporters.map((r) =>
+                  reporters.map((r) =>
                     runReporter("onEvalComplete", () => r.onEvalComplete?.(result)),
                   ),
                 ),
@@ -364,10 +410,14 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
             // attempt 各自触发的 eval:complete 会绕开 permit=1 直接并发跑,和文档承诺的
             // 「报告回调串行化」不一致。
             yield* reportMutex.withPermits(1)(
-              Effect.promise(() => emitReporterEvent(opts.reporters, { type: "eval:complete", result })),
+              Effect.promise(() => emitReporterEvent(reporters, { type: "eval:complete", result })),
             );
-          }),
-      { concurrency: opts.maxConcurrency, discard: true },
+          });
+        const gated = globalSem.withPermits(1)(body);
+        const runSem = runSems.get(a.run);
+        return runSem ? runSem.withPermits(1)(gated) : gated;
+      },
+      { concurrency: "unbounded", discard: true },
     ).pipe(
       // 中断(用户 Ctrl+C):finalizer 已在中断过程中跑完(容器已停),这里只是把它咽下,
       // 好让流程走到 summarize / onRunComplete,用已完成的 results 出一份部分汇总,而不是抛栈。
@@ -403,10 +453,10 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
   );
 
   const summary = summarize(allResults, firstAgent?.name ?? "", startedAt, Date.now() - t0, opts.config.name);
-  await emitReporterEvent(opts.reporters, { type: "run:summary", summary });
-  for (const r of opts.reporters) {
+  await emitReporterEvent(reporters, { type: "run:summary", summary });
+  for (const r of reporters) {
     await runReporter("onRunComplete", () => r.onRunComplete?.(summary));
   }
-  await emitReporterEvent(opts.reporters, { type: "run:saved", summary });
+  await emitReporterEvent(reporters, { type: "run:saved", summary });
   return summary;
 }