niceeval 0.10.3-canary.7 → 0.11.0

package/src/i18n/en.ts

@@ -74,6 +74,14 @@ export const en = {
     "experiment {{experimentId}}'s teardown was not triggered by the normal countdown path; it has been executed by the end-of-run sweep instead. Results are unaffected; seeing this line means an unlocated intermittent scheduling issue fired — please record this run in the memory ledger.\n",
   "runner.teardownRegistrationWriteFailed":
     "writing the crash-recovery teardown registration for experiment {{experimentId}} failed: {{message}}. The run continues normally, but a SIGKILL during this run cannot be recovered via `niceeval exp --teardown` or the startup self-heal — check disk space/permissions under .niceeval/teardowns/.\n",
+  "runner.lockTakenOver":
+    "took over an expired case lock for {{experimentId}}/{{evalId}} (previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns dispatching this case.\n",
+  "runner.gateLeaseTakenOver":
+    "took over an expired concurrency-slot lease for experiment {{experimentId}} (slot {{slot}}, previously held by pid {{pid}} on {{host}}; its heartbeat went stale) — that run likely died without releasing it; this run now owns the slot.\n",
+  "runner.gateLeaseWaiting":
+    "waiting on another run for experiment {{experimentId}}'s concurrency slots: all {{effectiveN}} in use ({{holders}}). Concurrent runs share this experiment's slots, and the smallest maxConcurrency in play wins — this run declared {{declaredN}}. Nothing dispatches until a slot frees up; the other run's slots release when its attempts finish, or 30s after it dies.\n",
+  "runner.dispatchHaltedExperiment": "experiment halted (dispatch-halted): {{message}}\n",
+  "runner.dispatchHaltedEval": "eval halted: {{message}}\n",
   "judge.modelMissing":
     "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
     "  Docs: node_modules/niceeval/docs-site/zh/tutorials/scoring-guide.mdx",
@@ -236,6 +244,7 @@ export const en = {
   "define.experimentFlagNotJson": "experiment.flags.{{key}} is not JSON-serializable (functions / undefined / cycles / bigint are not allowed); flags are persisted verbatim into result snapshots and must be plain JSON.",
   "define.experimentLabelInvalid": "experiment.labels.{{key}} must be a string or a finite number; labels are report-side grouping coordinates persisted verbatim into result snapshots.",
   "define.experimentSetupNotFunction": "experiment.setup must be a function ((ctx) => void); use experiment.teardown for cleanup; to prepare the in-sandbox environment per experiment, chain .setup() hooks on the sandbox spec instead.",
+  "define.experimentClassifyFailureNotFunction": "experiment.classifyFailure must be a function ((failure) => FailureClass | undefined); it classifies failures that surface as third-party errors and must return undefined for anything it does not recognize.",
   "define.experimentIdRejected": "defineExperiment does not accept id; ids are derived from file paths.",
   "define.sandboxAgentNameRequired": "defineSandboxAgent requires name.",
   "define.sandboxCreateRequired": "defineSandbox requires a create() function.",
@@ -249,7 +258,8 @@ export const en = {
   "feedback.human.active": "ACTIVE",
   "feedback.human.budgetExhausted": "budget exhausted for {{experimentId}} (spent {{spent}}, unstarted {{unstarted}})",
   "feedback.human.compare": "Compare: niceeval view",
-  "feedback.human.counts": "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{completed}} completed",
+  "feedback.human.counts":
+    "{{total}} total · {{reused}} reused · {{running}} running · {{queued}} queued · {{passed}} passed · {{failed}} failed · {{errored}} errored · {{skipped}} skipped",
   "feedback.human.diffHint": "Diff:    niceeval show {{locator}} --diff",
   "feedback.human.evalHint": "Eval:    niceeval show {{locator}} --source",
   "feedback.human.failuresHeader": "FAILURES",
@@ -282,6 +292,15 @@ export const en = {
   "feedback.human.hookFailed": "failed",
   "feedback.human.precheckJudge": "prechecking judge config",
   "feedback.human.precheckJudgeDone": "judge config ok",
+  "feedback.human.countsWithElsewhere":
+    "{{total}} total · {{reused}} reused · {{running}} running · {{elsewhere}} elsewhere · {{queued}} queued · {{passed}} passed · {{failed}} failed · {{errored}} errored · {{skipped}} skipped",
+  "feedback.human.waitingOnAnotherRun": "waiting on another run",
+  "feedback.human.lockWaitDetail": "{{count}} evals · pid {{pid}}",
+  "feedback.human.lockWaitStarted": "waiting on another run · {{experimentId}} ({{count}} evals, pid {{pid}})",
+  "feedback.human.lockWaitResolved": "lock wait resolved · {{experimentId}} ({{summary}}, {{elapsed}})",
+  "feedback.human.lockWaitCarried": "{{count}} carried",
+  "feedback.human.lockWaitDispatched": "{{count}} to run",
+  "feedback.human.lockedRowSuffix": "locked",
   "feedback.phase.sandboxSetup": "sandbox setup",
   "feedback.phase.scoring": "scoring",
   "feedback.phase.teardown": "cleaning up",

package/src/i18n/zh-CN.ts

@@ -71,6 +71,14 @@ export const zhCN = {
     "实验 {{experimentId}} 的 teardown 未被正常计数路径触发,已在运行收尾兜底执行。结果不受影响;这行出现说明命中了一个未定位的调度间歇问题,请把本次运行信息记入 memory 台账。\n",
   "runner.teardownRegistrationWriteFailed":
     "实验 {{experimentId}} 的强杀恢复收尾登记写入失败:{{message}}。本次运行照常继续,但这次运行期间若被 SIGKILL,`niceeval exp --teardown` 与启动自愈都无法找到它——检查 .niceeval/teardowns/ 下的磁盘空间或权限。\n",
+  "runner.lockTakenOver":
+    "接管了 {{experimentId}}/{{evalId}} 的过期用例锁(原持有者 pid {{pid}}@{{host}},心跳已过期)——那次运行大概率没能正常释放它;本次运行现在接手派发这条用例。\n",
+  "runner.gateLeaseTakenOver":
+    "接管了实验 {{experimentId}} 的过期并发名额租约(槽位 {{slot}},原持有者 pid {{pid}}@{{host}},心跳已过期)——那次运行大概率没能正常释放它;本次运行现在接手这个名额。\n",
+  "runner.gateLeaseWaiting":
+    "在等别的运行让出实验 {{experimentId}} 的并发名额:生效的 {{effectiveN}} 个位子全被占着({{holders}})。并行运行共用同一实验的名额,生效值取在场声明里最小的那个——本次运行声明的是 {{declaredN}}。名额腾出来之前不会派发任何 attempt;对方的名额会在它的 attempt 跑完时释放,它若已死则 30s 后过期被接管。\n",
+  "runner.dispatchHaltedExperiment": "实验已止损(dispatch-halted):{{message}}\n",
+  "runner.dispatchHaltedEval": "eval 已止损:{{message}}\n",
   "judge.modelMissing":
     "judge 未配置模型:在 defineConfig({ judge: { model: \"...\" } })、eval 的 judge 配置或环境变量 NICEEVAL_JUDGE_MODEL 里指定裁判模型(没有内置默认模型)。\n" +
     "  文档:node_modules/niceeval/docs-site/zh/tutorials/scoring-guide.mdx",
@@ -229,6 +237,7 @@ export const zhCN = {
   "define.experimentFlagNotJson": "experiment.flags.{{key}} 不是可 JSON 序列化的值(函数 / undefined / 循环引用 / bigint 不允许);flags 会原样进入结果快照,必须是纯 JSON。",
   "define.experimentLabelInvalid": "experiment.labels.{{key}} 必须是字符串或有限数字;labels 是报告侧的归类坐标,会原样进入结果快照。",
   "define.experimentSetupNotFunction": "experiment.setup 必须是函数((ctx) => void);要清理请挂 experiment.teardown;要按实验准备沙箱内环境请挂 sandbox spec 的 .setup() 钩子链。",
+  "define.experimentClassifyFailureNotFunction": "experiment.classifyFailure 必须是函数((failure) => FailureClass | undefined):它识别以第三方错误形态浮出的失败,认不出的一律返回 undefined 交给后续链路。",
   "define.experimentIdRejected": "defineExperiment 不接受 id —— id 由文件路径推导。",
   "define.sandboxAgentNameRequired": "defineSandboxAgent 需要 name。",
   "define.sandboxCreateRequired": "defineSandbox 需要一个 create() 函数。",
@@ -242,7 +251,8 @@ export const zhCN = {
   "feedback.human.active": "ACTIVE",
   "feedback.human.budgetExhausted": "{{experimentId}} 预算已耗尽(已花 {{spent}},未跑 {{unstarted}})",
   "feedback.human.compare": "Compare: niceeval view",
-  "feedback.human.counts": "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 已完成 {{completed}}",
+  "feedback.human.counts":
+    "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}",
   "feedback.human.diffHint": "Diff:    niceeval show {{locator}} --diff",
   "feedback.human.evalHint": "Eval:    niceeval show {{locator}} --source",
   "feedback.human.failuresHeader": "FAILURES",
@@ -275,6 +285,15 @@ export const zhCN = {
   "feedback.human.hookFailed": "失败",
   "feedback.human.precheckJudge": "预检 judge 配置",
   "feedback.human.precheckJudgeDone": "judge 配置就绪",
+  "feedback.human.countsWithElsewhere":
+    "共 {{total}} · 复用 {{reused}} · 运行中 {{running}} · 等待中 {{elsewhere}} · 排队 {{queued}} · 通过 {{passed}} · 失败 {{failed}} · 出错 {{errored}} · 跳过 {{skipped}}",
+  "feedback.human.waitingOnAnotherRun": "等待另一个并行 run",
+  "feedback.human.lockWaitDetail": "{{count}} 条用例 · pid {{pid}}",
+  "feedback.human.lockWaitStarted": "等待另一个并行 run · {{experimentId}}({{count}} 条用例,pid {{pid}})",
+  "feedback.human.lockWaitResolved": "等待结束 · {{experimentId}}({{summary}},{{elapsed}})",
+  "feedback.human.lockWaitCarried": "{{count}} 条携入",
+  "feedback.human.lockWaitDispatched": "{{count}} 条自跑",
+  "feedback.human.lockedRowSuffix": "locked",
   "feedback.phase.sandboxSetup": "沙箱预置",
   "feedback.phase.scoring": "评分",
   "feedback.phase.teardown": "清理中",

package/src/index.ts

@@ -5,6 +5,10 @@ export { defineEval, defineScoreEval, defineConfig, defineExperiment } from "./d
 
 export { requireEnv, getEnv, stripComments } from "./util.ts";
 
+// 执行失败分类:抛出点糖衣类(声明死因波及多远)+ 结构守卫。判据、分类链与止损语义见
+// docs/feature/error-classification/README.md;`niceeval/adapter` 复导出同一份词表类型。
+export { ExperimentFatalError, EvalFatalError, failureClassOf } from "./shared/failure-class.ts";
+
 // 类型(eval 作者会用到;跑哪个 agent / 用哪个 sandbox 见对应子路径)
 export type {
   StreamEvent,
@@ -59,6 +63,10 @@ export type {
   TraceSpan,
   SpanKind,
   DerivedFacts,
+  FailureClass,
+  FailureScope,
+  AttemptFailureInfo,
+  AttemptFailureClassifier,
 } from "./types.ts";
 
 export type { ParsedTranscript } from "./o11y/parsers/index.ts";

package/src/o11y/cost.ts

@@ -56,8 +56,10 @@ function lookupBuiltin(model: string): Price | undefined {
 }
 
 /**
- * 按 token 桶 × 单价估算一次运行的美元成本。cache 桶缺专门单价时退回 input 价
- * (cache token 本质也是 input)。无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
+ * 按 token 桶 × 单价估算一次运行的美元成本。逐桶相加成立的前提是 Usage 桶恒互斥
+ * (inputTokens 不含 cache 命中,归一义务在 adapter,见 docs/feature/results/architecture.md#usage);
+ * cache 桶缺专门单价时退回 input 价——宁可高估,不静默低估。
+ * 无 model / 查不到价(用户表 + 内置快照都没有)→ undefined。
  */
 export function estimateCost(
   model: string | undefined,

package/src/o11y/derive.test.ts

@@ -58,6 +58,46 @@ describe("deriveRunFacts:pending 折叠", () => {
   });
 });
 
+describe("deriveRunFacts:callId 跨轮复用不覆盖", () => {
+  // 回归:adapter 常按轮各自编号(OpenAI 兼容 / transcript 归一复用 c1/c2…)。同一 callId 在其
+  // result 之后再次出现是新的一次调用——旧折叠按 callId 存 Map 会被后一轮覆盖,跨轮聚合断言
+  // (t.calledTool)于是「只扫最后一轮」,前几轮的工具调用被抹掉(定稿见 events.md 不变量 2)。
+  it("同一 callId 完成后在后一轮再次 called,折叠成两条独立调用而非覆盖成一条", () => {
+    const events: StreamEvent[] = [
+      // 第一轮:读 init、读 INDEX(都用 c1/c2 编号)
+      { type: "action.called", callId: "c1", name: "read", input: { path: "init.md" } },
+      { type: "action.result", callId: "c1", output: "init contents", status: "completed" },
+      { type: "action.called", callId: "c2", name: "read", input: { path: "INDEX.md" } },
+      { type: "action.result", callId: "c2", output: "index contents", status: "completed" },
+      { type: "message", role: "assistant", text: "第一轮做完" },
+      // 第二轮:续轮,adapter 从 c1 重新编号
+      { type: "action.called", callId: "c1", name: "write", input: { path: "note.md" } },
+      { type: "action.result", callId: "c1", output: "ok", status: "completed" },
+      { type: "message", role: "assistant", text: "答复" },
+    ];
+    const facts = deriveRunFacts(events);
+    expect(facts.toolCalls).toHaveLength(3); // 两轮共 3 次调用,后一轮不覆盖前一轮
+    expect(facts.toolCalls.map((tc) => tc.originalName)).toEqual(["read", "read", "write"]);
+    // 第一轮的两次 read 都在,且各配对到自己的 result(不是都取最后一条)
+    expect(facts.toolCalls[0]!.input).toEqual({ path: "init.md" });
+    expect(facts.toolCalls[0]!.output).toBe("init contents");
+    expect(facts.toolCalls[1]!.input).toEqual({ path: "INDEX.md" });
+    expect(facts.toolCalls[1]!.output).toBe("index contents");
+  });
+
+  it("subagent 折叠同理:同一 callId 完成后再次 called 起新记录", () => {
+    const events: StreamEvent[] = [
+      { type: "subagent.called", callId: "s1", name: "researcher" },
+      { type: "subagent.completed", callId: "s1", output: "round1", status: "completed" },
+      { type: "subagent.called", callId: "s1", name: "writer" },
+      { type: "subagent.completed", callId: "s1", output: "round2", status: "completed" },
+    ];
+    const facts = deriveRunFacts(events);
+    expect(facts.subagentCalls).toHaveLength(2);
+    expect(facts.subagentCalls.map((sc) => sc.name)).toEqual(["researcher", "writer"]);
+  });
+});
+
 describe("deriveRunFacts:contextInjections 计数", () => {
   it("统计事件流里 context.injected 事件的次数,不与 messageCount 混计", () => {
     const events: StreamEvent[] = [

package/src/o11y/derive.ts

@@ -61,10 +61,16 @@ function pickExitCode(output: JsonValue | undefined): number | undefined {
 // ───────────────────────── deriveRunFacts ─────────────────────────
 
 export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
-  const toolCallMap = new Map<string, ToolCall>();
-  const toolCallOrder: string[] = [];
-  const subagentMap = new Map<string, SubagentCall>();
-  const subagentOrder: string[] = [];
+  // 折叠是逐条按发生顺序进行的:called 追加一条新调用,result 回填「当前还没配上 result 的
+  // 同 callId 调用」。callId 只在一个 called→result 配对内保证稳定,不保证跨轮唯一——adapter
+  // 常按轮各自编号(OpenAI 兼容协议、transcript 归一都会复用 c1/c2…)。所以一个 callId 在其
+  // result 之后再次以 called 出现,是新的一次调用,起一条新记录,不覆盖前一轮那条(否则跨轮
+  // 聚合会把前面几轮的工具调用抹成「只剩最后一轮」)。用 open*ByCallId 只跟踪各 callId 当前
+  // 敞口的那条,配上 result 即关闭。
+  const toolCalls: ToolCall[] = [];
+  const openToolByCallId = new Map<string, number>();
+  const subagentCalls: SubagentCall[] = [];
+  const openSubagentByCallId = new Map<string, number>();
   const inputRequests: InputRequest[] = [];
   let messageCount = 0;
   let compactions = 0;
@@ -81,8 +87,8 @@ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
         break;
 
       case "action.called": {
-        if (!toolCallMap.has(ev.callId)) toolCallOrder.push(ev.callId);
-        toolCallMap.set(ev.callId, {
+        openToolByCallId.set(ev.callId, toolCalls.length);
+        toolCalls.push({
           callId: ev.callId,
           name: ev.tool ?? "unknown",
           originalName: ev.name,
@@ -94,14 +100,14 @@ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
       }
 
       case "action.result": {
-        const existing = toolCallMap.get(ev.callId);
-        if (existing) {
-          existing.output = ev.output;
-          existing.status = ev.status;
+        const idx = openToolByCallId.get(ev.callId);
+        if (idx !== undefined) {
+          toolCalls[idx].output = ev.output;
+          toolCalls[idx].status = ev.status;
+          openToolByCallId.delete(ev.callId);
         } else {
           // 只有结果、没配上调用:补一条占位 ToolCall。
-          toolCallOrder.push(ev.callId);
-          toolCallMap.set(ev.callId, {
+          toolCalls.push({
             callId: ev.callId,
             name: "unknown",
             input: null,
@@ -113,8 +119,8 @@ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
       }
 
       case "subagent.called": {
-        if (!subagentMap.has(ev.callId)) subagentOrder.push(ev.callId);
-        subagentMap.set(ev.callId, {
+        openSubagentByCallId.set(ev.callId, subagentCalls.length);
+        subagentCalls.push({
           callId: ev.callId,
           name: ev.name,
           remoteUrl: ev.remoteUrl,
@@ -125,13 +131,13 @@ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
       }
 
       case "subagent.completed": {
-        const existing = subagentMap.get(ev.callId);
-        if (existing) {
-          existing.output = ev.output;
-          existing.status = ev.status;
+        const idx = openSubagentByCallId.get(ev.callId);
+        if (idx !== undefined) {
+          subagentCalls[idx].output = ev.output;
+          subagentCalls[idx].status = ev.status;
+          openSubagentByCallId.delete(ev.callId);
         } else {
-          subagentOrder.push(ev.callId);
-          subagentMap.set(ev.callId, {
+          subagentCalls.push({
             callId: ev.callId,
             name: "unknown",
             output: ev.output,
@@ -164,8 +170,8 @@ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
   }
 
   return {
-    toolCalls: toolCallOrder.map((id) => toolCallMap.get(id)!),
-    subagentCalls: subagentOrder.map((id) => subagentMap.get(id)!),
+    toolCalls,
+    subagentCalls,
     inputRequests,
     parked,
     messageCount,

package/src/o11y/otlp/sandbox-receiver.test.ts

@@ -0,0 +1,201 @@
+// cases: docs/engineering/testing/unit/experiments-runner.md
+// bug: memory/insandbox-otlp-port-wait-3s-no-retry.md
+// 沙箱内 OTLP 采集器的启动韧性:这条路径外面没有任何重试兜底(命令执行不进 IO 重试、
+// provision 重试只覆盖 create、runner 没有 attempt 级重试),起不来就是一条 errored attempt。
+// 两层 fixture:
+//   · 脚本化 fake sandbox —— 重试轮次、每轮换路径、重试前杀上一轮,确定性断言;
+//   · 真实 /bin/sh 执行 —— 生成的 shell 脚本本身能跑(语法、算术、退出边),以及采集器进程
+//     一起来就死时不空等满预算(观察实测耗时,不看脚本字节)。
+
+import { mkdtemp, rm, writeFile, chmod, readdir } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Effect, Exit, Cause } from "effect";
+import { afterEach, describe, expect, it } from "vitest";
+import type { CommandResult, Sandbox } from "../../types.ts";
+import { createInSandboxTraceReceiver } from "./sandbox-receiver.ts";
+
+const notUsed = () => {
+  throw new Error("not used by the in-sandbox receiver");
+};
+
+function baseSandbox(overrides: Partial<Sandbox>): Sandbox {
+  return {
+    workdir: "/work",
+    sandboxId: "fake",
+    otlpHost: null,
+    runCommand: notUsed,
+    runShell: notUsed,
+    readFile: notUsed,
+    fileExists: notUsed,
+    writeFiles: async () => {},
+    uploadFiles: notUsed,
+    uploadDirectory: notUsed,
+    downloadDirectory: notUsed,
+    downloadFile: notUsed,
+    uploadFile: notUsed,
+    stop: async () => {},
+    ...overrides,
+  } as Sandbox;
+}
+
+const isStart = (script: string) => script.includes("niceeval-otlp-collector") && script.includes("&");
+const isKill = (script: string) => script.trimStart().startsWith("kill ");
+
+/**
+ * 启动脚本按轮次回放一条预置 stdout(`PID\n端口`,端口空表示这一轮没等到),日志读取回放
+ * 固定内容,kill 一律成功。按脚本形态派发而不是按调用序号——被测实现每轮发几条命令是它自己
+ * 的事,fixture 不该把这个数字焊死。
+ */
+function scriptedSandbox(startOutputs: string[], log = "") {
+  const shells: string[] = [];
+  const written: string[] = [];
+  const sandbox = baseSandbox({
+    writeFiles: async (files: Record<string, string>) => {
+      written.push(...Object.keys(files));
+    },
+    runShell: async (script: string): Promise<CommandResult> => {
+      shells.push(script);
+      const stdout = isStart(script) ? (startOutputs.shift() ?? "") : script.startsWith("cat ") ? log : "";
+      return { stdout, stderr: "", exitCode: 0 };
+    },
+  });
+  return { sandbox, shells, written };
+}
+
+/** 在 Scope 内取端点,并把「此刻为止发过的命令」快照出来——release 阶段的收尾 kill 不混进来。 */
+async function receiverExit(sandbox: Sandbox, shells: string[] = []) {
+  return Effect.runPromiseExit(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const receiver = yield* createInSandboxTraceReceiver(sandbox);
+        return { endpoint: receiver.endpoint(""), shells: [...shells] };
+      }),
+    ),
+  );
+}
+
+describe("沙箱内 OTLP 采集器启动:脚本化 fixture", () => {
+  it("第一轮拿不到端口时重试,换一套路径并先杀掉上一轮的进程", async () => {
+    // 第一轮:拿到 PID 但端口行为空(采集器慢/死);第二轮:端口写回来了。
+    const { sandbox, shells, written } = scriptedSandbox(["4242\n", "4243\n61000"]);
+
+    const exit = await receiverExit(sandbox, shells);
+
+    expect(Exit.isSuccess(exit) && exit.value.endpoint).toBe("http://127.0.0.1:61000/v1/traces");
+    const sent = Exit.isSuccess(exit) ? exit.value.shells : [];
+    expect(sent.filter(isStart)).toHaveLength(2);
+    // 上一轮那个「慢但还活着」的采集器必须被杀掉:留着它会占内存,还会在重试之后才把端口
+    // 写进它自己那份文件。
+    expect(sent.filter(isKill)).toEqual([expect.stringContaining("kill 4242")]);
+    // 每轮换一套带随机后缀的脚本路径:上一轮迟到的采集器写不进新一轮的端口 / spans 文件。
+    expect(written).toHaveLength(2);
+    expect(written[0]).not.toBe(written[1]);
+  });
+
+  it("首轮就拿到端口时只起一次,不杀任何进程", async () => {
+    const { sandbox, shells } = scriptedSandbox(["4242\n61001"]);
+
+    const exit = await receiverExit(sandbox, shells);
+
+    expect(Exit.isSuccess(exit) && exit.value.endpoint).toBe("http://127.0.0.1:61001/v1/traces");
+    const sent = Exit.isSuccess(exit) ? exit.value.shells : [];
+    expect(sent.filter(isStart)).toHaveLength(1);
+    expect(sent.filter(isKill)).toEqual([]);
+  });
+
+  it("重试用尽后抛错,带上预算、轮次与采集器自己的日志", async () => {
+    const { sandbox, shells } = scriptedSandbox(["4242\n", "4243\n"], "node: not found");
+
+    const exit = await receiverExit(sandbox, shells);
+
+    expect(Exit.isFailure(exit)).toBe(true);
+    const message = Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined;
+    expect(String(message)).toContain("within 20s");
+    expect(String(message)).toContain("2 attempts");
+    expect(String(message)).toContain("node: not found");
+    expect(shells.filter(isStart)).toHaveLength(2);
+  });
+});
+
+describe("沙箱内 OTLP 采集器启动:真实 /bin/sh 执行", () => {
+  const dirs: string[] = [];
+  const pids: number[] = [];
+
+  afterEach(async () => {
+    for (const pid of pids.splice(0)) {
+      try {
+        process.kill(pid);
+      } catch {
+        // 已经退出
+      }
+    }
+    for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true });
+    // 本用例真的会往 /tmp 写采集器脚本 / 端口文件,跑完清掉自己那批。
+    for (const name of await readdir(tmpdir())) {
+      if (name.startsWith(".niceeval-otlp-")) await rm(join(tmpdir(), name), { force: true }).catch(() => {});
+    }
+  });
+
+  /** runShell 真的交给 /bin/sh 跑,writeFiles 真的落盘——生成的脚本语法错误在这里现形。 */
+  function shellSandbox(pathPrefix?: string) {
+    return baseSandbox({
+      writeFiles: async (files: Record<string, string>) => {
+        for (const [path, content] of Object.entries(files)) await writeFile(path, content);
+      },
+      runShell: (script: string) =>
+        new Promise<CommandResult>((resolve) => {
+          const child = spawn("/bin/sh", ["-c", script], {
+            env: { ...process.env, ...(pathPrefix ? { PATH: `${pathPrefix}:${process.env.PATH}` } : {}) },
+          });
+          let stdout = "";
+          let stderr = "";
+          child.stdout.on("data", (c) => (stdout += c));
+          child.stderr.on("data", (c) => (stderr += c));
+          child.on("close", (code) => resolve({ stdout, stderr, exitCode: code ?? 0 }));
+        }),
+    });
+  }
+
+  it("真实 shell 下起得来:端口写回宿主,采集器确实在监听", async () => {
+    const sandbox = shellSandbox();
+
+    const result = await Effect.runPromise(
+      Effect.scoped(
+        Effect.gen(function* () {
+          const receiver = yield* createInSandboxTraceReceiver(sandbox);
+          const endpoint = receiver.endpoint("");
+          const status = yield* Effect.promise(async () => {
+            const res = await fetch(endpoint, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            });
+            return res.status;
+          });
+          return { endpoint, status };
+        }),
+      ),
+    );
+
+    expect(result.endpoint).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1\/traces$/);
+    expect(result.status).toBe(200);
+  });
+
+  it("采集器一起来就死时不空等满预算,错误里带着它的日志", async () => {
+    const shimDir = await mkdtemp(join(tmpdir(), "niceeval-node-shim-"));
+    dirs.push(shimDir);
+    await writeFile(join(shimDir, "node"), '#!/bin/sh\necho "node: simulated crash" >&2\nexit 1\n');
+    await chmod(join(shimDir, "node"), 0o755);
+
+    const startedAt = Date.now();
+    const exit = await receiverExit(shellSandbox(shimDir));
+    const elapsed = Date.now() - startedAt;
+
+    expect(Exit.isFailure(exit)).toBe(true);
+    expect(String(Exit.isFailure(exit) ? Cause.squash(exit.cause) : "")).toContain("node: simulated crash");
+    // 进程已死时循环立刻 break,两轮加起来也远小于一轮的等待预算(20s)。
+    expect(elapsed).toBeLessThan(5_000);
+  });
+});

package/src/o11y/otlp/sandbox-receiver.ts

@@ -11,7 +11,8 @@
 //   6. close() 尝试 kill PID(沙箱本身也会停,所以 best-effort)
 //
 // 文件路径带随机后缀:同一沙箱跨 eval 复用时,每个 receiver 实例的脚本 / spans /
-// 端口文件互不串扰,collector 也不会读到上一个 eval 的 span。
+// 端口文件互不串扰,collector 也不会读到上一个 eval 的 span。启动重试同样每轮换一套后缀
+// (见 startCollector)。
 
 import { randomUUID } from "node:crypto";
 import { Effect } from "effect";
@@ -56,6 +57,16 @@ server.listen(0, '127.0.0.1', () => {
 `;
 }
 
+// 端口等待预算。这条路径外面没有任何一层重试兜着——`runShell` 不进 `withSandboxIoRetry`
+// (命令执行有不可重复副作用),`withProvisionRetry` 只覆盖沙箱 create,runner 也没有
+// attempt 级重试:等不到端口就是一条 errored attempt,agent 一次都没跑
+// (memory/insandbox-otlp-port-wait-3s-no-retry.md)。所以预算按冷沙箱首次起 node 的最坏
+// 情况给,而不是按热沙箱的常见值给。
+const PORT_WAIT_MS = 20_000;
+// 启动整体的重试次数。慢不需要重试(一次等满预算就够),这里兜的是 collector 起来就死
+// (镜像里没有 node、脚本被 OOM kill 等)——那种情况下沙箱侧循环会立刻 break,重试很便宜。
+const START_ATTEMPTS = 2;
+
 export function createInSandboxTraceReceiver(sandbox: Sandbox) {
   return Effect.acquireRelease(
     Effect.promise(() => makeInSandboxReceiver(sandbox)),
@@ -63,35 +74,70 @@ export function createInSandboxTraceReceiver(sandbox: Sandbox) {
   );
 }
 
-async function makeInSandboxReceiver(sandbox: Sandbox): Promise<TraceReceiver> {
-  let cached: TraceSpan[] = [];
+interface StartedCollector {
+  pid: number;
+  port: number;
+  spansPath: string;
+}
 
-  const tag = randomUUID().slice(0, 8);
-  const collectorPath = `/tmp/.niceeval-otlp-collector-${tag}.cjs`;
-  const spansPath = `/tmp/.niceeval-otlp-spans-${tag}.jsonl`;
-  const portPath = `/tmp/.niceeval-otlp-port-${tag}`;
-  const logPath = `/tmp/.niceeval-otlp-collector-${tag}.log`;
-
-  // 上传 collector 脚本
-  await sandbox.writeFiles({ [collectorPath]: collectorScript(spansPath, portPath) });
-
-  // 后台启动 + 等端口文件,折进一次 shell 往返(远程沙箱一次 exec 要 100-500ms,
-  // host 侧逐次轮询会把几秒的启动等待放大成 N 个 API round-trip)。
-  // 输出两行:PID、端口(等不到则空)。
-  const startResult = await sandbox.runShell(
-    `node ${collectorPath} >${logPath} 2>&1 & echo $!; ` +
-      `i=0; while [ $i -lt 30 ] && [ ! -s ${portPath} ]; do sleep 0.1; i=$((i+1)); done; ` +
-      `cat ${portPath} 2>/dev/null || true`,
-  );
-  const [pidLine, portLine] = startResult.stdout.trim().split("\n");
-  const pid = parseInt((pidLine ?? "").trim(), 10);
-  const port = parseInt((portLine ?? "").trim(), 10) || 0;
-  if (!port) {
-    const log = await sandbox.runShell(`cat ${logPath} 2>/dev/null || true`).catch(() => undefined);
-    throw new Error(
-      `in-sandbox OTLP collector failed to report its port within 3s. Collector log:\n${log?.stdout.trim() || "(empty)"}`,
+/**
+ * 上传脚本 + 后台起 collector + 等端口文件,失败重试。每一轮换一套带随机后缀的路径:
+ * 上一轮那个「慢但还活着」的 collector 如果在重试之后才起来,会写回它自己那份端口 / spans
+ * 文件,不会污染新一轮。
+ */
+async function startCollector(sandbox: Sandbox): Promise<StartedCollector> {
+  let lastLog = "";
+  for (let attempt = 1; ; attempt++) {
+    const tag = randomUUID().slice(0, 8);
+    const collectorPath = `/tmp/.niceeval-otlp-collector-${tag}.cjs`;
+    const spansPath = `/tmp/.niceeval-otlp-spans-${tag}.jsonl`;
+    const portPath = `/tmp/.niceeval-otlp-port-${tag}`;
+    const logPath = `/tmp/.niceeval-otlp-collector-${tag}.log`;
+
+    await sandbox.writeFiles({ [collectorPath]: collectorScript(spansPath, portPath) });
+
+    // 后台启动 + 等端口文件,折进一次 shell 往返(远程沙箱一次 exec 要 100-500ms,
+    // host 侧逐次轮询会把几秒的启动等待放大成 N 个 API round-trip)。循环两条退出边:
+    //   · `kill -0` 失败 → collector 已经死了,别再空等满预算,立刻回 host 重试;
+    //   · 到 deadline → 真的太慢。
+    // 预算按 `date +%s` 的墙钟算而不是数 tick:`sleep` 的小数秒支持因镜像而异,数 tick 会让
+    // 不支持小数的镜像瞬间跑完循环、伪装成「等了 20s」。
+    // 输出两行:PID、端口(等不到则空)。
+    const startResult = await sandbox.runShell(
+      `node ${collectorPath} >${logPath} 2>&1 & pid=$!; echo $pid; ` +
+        `end=$(( $(date +%s) + ${Math.ceil(PORT_WAIT_MS / 1000)} )); ` +
+        `while [ ! -s ${portPath} ]; do ` +
+        `kill -0 $pid 2>/dev/null || break; ` +
+        `[ "$(date +%s)" -lt "$end" ] || break; ` +
+        `sleep 0.1 2>/dev/null || sleep 1; ` +
+        `done; ` +
+        `cat ${portPath} 2>/dev/null || true`,
     );
+    const [pidLine, portLine] = startResult.stdout.trim().split("\n");
+    const pid = parseInt((pidLine ?? "").trim(), 10);
+    const port = parseInt((portLine ?? "").trim(), 10) || 0;
+    if (port) return { pid, port, spansPath };
+
+    const log = await sandbox.runShell(`cat ${logPath} 2>/dev/null || true`).catch(() => undefined);
+    lastLog = log?.stdout.trim() ?? "";
+    // 这一轮可能只是慢、进程还活着:重试前先杀掉,不留孤儿 collector 占内存
+    //(沙箱复用 / --keep-sandbox 下它会一直在)。
+    if (Number.isFinite(pid) && pid > 0) {
+      await sandbox.runShell(`kill ${pid} 2>/dev/null || true`).catch(() => {});
+    }
+    if (attempt >= START_ATTEMPTS) {
+      throw new Error(
+        `in-sandbox OTLP collector failed to report its port within ${Math.round(PORT_WAIT_MS / 1000)}s ` +
+          `(${attempt} attempts). Collector log:\n${lastLog || "(empty)"}`,
+      );
+    }
   }
+}
+
+async function makeInSandboxReceiver(sandbox: Sandbox): Promise<TraceReceiver> {
+  let cached: TraceSpan[] = [];
+
+  const { pid, port, spansPath } = await startCollector(sandbox);
 
   return {
     endpoint: (_host) => `http://127.0.0.1:${port}/v1/traces`,

package/src/o11y/parsers/bub.test.ts

@@ -0,0 +1,30 @@
+// cases: docs/engineering/testing/unit/results.md
+// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:bub tape 的 usage 是 Chat Completions
+// 形状,prompt_tokens_details.cached_tokens 是 prompt_tokens 子集,聚合前扣减;
+// 同对象的 cost 是实测计费,照常累进 costUSD。
+// bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
+
+import { describe, expect, it } from "vitest";
+
+import { parseBubTranscript } from "./bub.ts";
+
+describe("parseBubTranscript usage 归一(OpenAI 口径)", () => {
+  it("cached_tokens 从 prompt_tokens 里扣出,cost 累进实测 costUSD", () => {
+    const line = JSON.stringify({
+      kind: "event",
+      payload: {
+        name: "run",
+        data: {
+          usage: {
+            prompt_tokens: 1000,
+            completion_tokens: 20,
+            prompt_tokens_details: { cached_tokens: 900 },
+            cost: 0.05,
+          },
+        },
+      },
+    });
+    const parsed = parseBubTranscript(line);
+    expect(parsed.usage).toMatchObject({ inputTokens: 100, cacheReadTokens: 900, outputTokens: 20, costUSD: 0.05 });
+  });
+});