niceeval 0.10.3-canary.6 → 0.11.0

package/src/runner/feedback/coordinator.ts

@@ -28,6 +28,7 @@ import {
   type ExperimentHookInput,
   type ExperimentProgressInput,
   type PrecheckInput,
+  type LockWaitInput,
   type FailureInput,
   type FeedbackSink,
   type KeptInput,
@@ -198,6 +199,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       type: "diagnostic",
       at: io.clock.now(),
       key: input.key,
+      ...(input.code !== undefined ? { code: input.code } : {}),
       severity: input.severity,
       message: input.message,
       identity: input.identity,
@@ -272,6 +274,22 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     });
   }
 
+  function lockWait(input: LockWaitInput): void {
+    emit({
+      type: "lock-wait",
+      at: io.clock.now(),
+      experimentId: input.experimentId,
+      evalId: input.evalId,
+      status: input.status,
+      ...(input.holderPid !== undefined ? { holderPid: input.holderPid } : {}),
+      ...(input.holderHost !== undefined ? { holderHost: input.holderHost } : {}),
+      ...(input.attempts !== undefined ? { attempts: input.attempts } : {}),
+      ...(input.carried !== undefined ? { carried: input.carried } : {}),
+      ...(input.dispatched !== undefined ? { dispatched: input.dispatched } : {}),
+      ...(input.waitedMs !== undefined ? { waitedMs: input.waitedMs } : {}),
+    });
+  }
+
   function reporterError(input: { reporter: string; required: boolean; message: string }): void {
     emit({
       type: "reporter-error",
@@ -313,6 +331,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       experimentHook,
       experimentProgress,
       precheck,
+      lockWait,
       lifecycle,
     });
     emit({ type: "plan", at: startedAtMs, plan });
@@ -378,6 +397,7 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     experimentHook,
     experimentProgress,
     precheck,
+    lockWait,
     lifecycle,
     stopDynamic,
     finish,
@@ -404,8 +424,10 @@ function fallbackTextFor(event: DurableFeedbackEvent): string | undefined {
     case "saved":
     case "experiment-hook":
     case "precheck":
-      // 钩子/预检起止不是失败证据:setup 失败的每条 attempt 另有 "failure" 事件兜底,
-      // 预检失败以既有错误路径中止运行,renderer 崩溃丢一行起止不丢数据。
+    case "lock-wait":
+      // 钩子/预检/锁等待起止都不是失败证据:setup 失败的每条 attempt 另有 "failure" 事件
+      // 兜底,预检失败以既有错误路径中止运行,锁等待解决与否不改变任何 verdict——
+      // renderer 崩溃丢一行起止不丢数据。
       return undefined;
   }
 }

package/src/runner/feedback/eval-conclusions.ts

@@ -40,12 +40,15 @@ export interface EvalConclusionRow {
   rate?: number;
 }
 
-/** `diagnostics` 里 `fail-fast:` 记录按 (experiment, eval) 求和的未派发次数;没有 `identity`
- *  的记录(理论上不应出现——fail-fast 诊断恒带 identity,见 run.ts)不计入任何 key。 */
+/** `diagnostics` 里 fail-fast 记录按 (experiment, eval) 求和的未派发次数;没有 `identity`
+ *  的记录(理论上不应出现——fail-fast 诊断恒带 identity,见 run.ts)不计入任何 key。
+ *  归类按稳定词法 `code`(省略时回落到 key 首段,缺省 key 恒是 `${code}:${identity}`),不按
+ *  `key` 前缀:key 里编着折叠身份,拿它匹配会在身份编码变动时静默失配,减不掉的那部分
+ *  fail-fast 份额会当成真实的 earlyExit 省略数报出去。 */
 function failFastByEval(diagnostics: readonly DiagnosticNotice[]): ReadonlyMap<string, number> {
   const result = new Map<string, number>();
   for (const d of diagnostics) {
-    if (!d.key.startsWith("fail-fast:") || !d.identity) continue;
+    if ((d.code ?? d.key.split(":", 1)[0]) !== "fail-fast" || !d.identity) continue;
     const key = evalConclusionKey(d.identity);
     result.set(key, (result.get(key) ?? 0) + d.count);
   }

package/src/runner/feedback/human.test.ts

@@ -5,10 +5,10 @@
 // boxed 能力下产生可识别的框线字符与正确的面板顺序/分隔,plain/非 TTY 下不产生任何框字符。
 
 import { afterEach, describe, expect, it } from "vitest";
-import { createHumanRenderer, renderDurableLines } from "./human.ts";
+import { createHumanRenderer, renderDurableLines, renderHumanDryPlan } from "./human.ts";
 import { createFakeFeedbackIO } from "./testing.ts";
-import { createInitialRunFeedbackState } from "./reducer.ts";
-import { encodeAttemptKey } from "../types.ts";
+import { createInitialRunFeedbackState, reduceRunFeedback } from "./reducer.ts";
+import { encodeAttemptKey, HALT_DIAGNOSTIC_CODE } from "../types.ts";
 import { stringWidth } from "../../report/model/text-layout.ts";
 import type { DurableFeedbackEvent, InvocationCompletion, InvocationSummary, RunFeedbackPlan, RunFeedbackState } from "../types.ts";
 import type { AttemptLocator } from "../../results/locator.ts";
@@ -182,7 +182,8 @@ describe("live dashboard — 接线到 panel.ts", () => {
       reused: 6,
       running: 19,
       queued: 12,
-      completed: 8,
+      passed: 6,
+      failed: 2,
       elapsedMs: 134_000,
       estimatedCostUSD: 0.84,
       active: new Map([[key, { identity, who: "compare/bub-e2b", phase: "eval.run", phaseStartedAt: 0 }]]),
@@ -260,7 +261,8 @@ describe("live dashboard — 宽终端下 ACTIVE 行与身份列分配", () => {
       reused: 6,
       running: 19,
       queued: 12,
-      completed: 8,
+      passed: 6,
+      failed: 2,
       elapsedMs: 134_000,
       active: new Map([
         [key, { identity, who: "compare/bub-e2b", phase: "eval.run", phaseStartedAt: 0, detail: longDetail }],
@@ -358,7 +360,7 @@ describe("live dashboard — 宽终端下 ACTIVE 行与身份列分配", () => {
       ...createInitialRunFeedbackState(),
       total: 2,
       running: 1,
-      completed: 1,
+      passed: 1,
       active: new Map([[shortKey, { identity: shortIdentity, who, phase: "eval.run", phaseStartedAt: 1 }]]),
     };
     renderer.onLifecycle?.(
@@ -418,3 +420,295 @@ describe("live dashboard — 宽终端下 ACTIVE 行与身份列分配", () => {
     for (const l of framedLines) expect(stringWidth(l)).toBe(100);
   });
 });
+
+// cases: docs/engineering/testing/unit/experiments-runner.md「用例锁与并发 Invocation」——
+// 字节级精确渲染归 E2E · CLI「反馈输出格式」;这里只做与 precheck/experiment-hook 同等级别
+// 的最小 smoke 断言(行是否出现、关键子串是否存在),不断言列宽算术。
+describe("用例锁等待(elsewhere)的显示", () => {
+  it("TTY:等待期间显示运行级行,面板首行的 elsewhere 计数非零", () => {
+    const { io, stderr } = createFakeFeedbackIO({ stderr: { isTTY: true, columns: 100, rows: 30 } });
+    const renderer = createHumanRenderer({ io, command: "niceeval exp compare/codex" });
+    const state: RunFeedbackState = {
+      ...createInitialRunFeedbackState(),
+      total: 3,
+      elsewhere: 2,
+      queued: 1,
+      lockWaits: new Map([
+        [
+          "compare/codex",
+          {
+            experimentId: "compare/codex",
+            waiting: new Map([
+              ["memory/a", { startedAt: 0, holderPid: 41267, holderHost: "mba.local" }],
+              ["memory/b", { startedAt: 5, holderPid: 41267, holderHost: "mba.local" }],
+            ]),
+            resolvedCarried: 0,
+            resolvedDispatched: 0,
+          },
+        ],
+      ]),
+    };
+    renderer.redrawDynamic?.(state);
+
+    const plain = stripAnsi(stderr.writes.join(""));
+    expect(plain).toMatch(/├─ ACTIVE ─+┤/);
+    expect(plain).toContain("waiting on another run");
+    expect(plain).toContain("compare/codex");
+    expect(plain).toContain("2 evals");
+    expect(plain).toContain("pid 41267");
+    expect(plain).toContain("2 elsewhere");
+  });
+
+  it("TTY appendDurable 对 lock-wait 直接返回,不写 scrollback 永久行(运行级行由 state.lockWaits 驱动)", () => {
+    const { io, stdout, stderr } = createFakeFeedbackIO({ stderr: { isTTY: true, columns: 100, rows: 30 } });
+    const renderer = createHumanRenderer({ io, command: "niceeval exp compare/codex" });
+    const state: RunFeedbackState = { ...createInitialRunFeedbackState(), total: 1, elsewhere: 1 };
+    renderer.appendDurable(
+      { type: "lock-wait", at: 0, experimentId: "compare/codex", evalId: "memory/a", status: "started", attempts: 1, holderPid: 1, holderHost: "h" },
+      state,
+    );
+    expect(stdout.writes.join("") + stderr.writes.join("")).toBe("");
+  });
+
+  it("非 TTY:started 只在窗口第一次打开(唯一等待用例)时追加一行,中途加入的用例不逐条刷屏", () => {
+    const { io, stdout } = createFakeFeedbackIO({ stderr: { isTTY: false } });
+    const renderer = createHumanRenderer({ io, command: "niceeval exp compare/codex" });
+    const firstState: RunFeedbackState = {
+      ...createInitialRunFeedbackState(),
+      total: 2,
+      elsewhere: 1,
+      lockWaits: new Map([
+        [
+          "compare/codex",
+          {
+            experimentId: "compare/codex",
+            waiting: new Map([["memory/a", { startedAt: 0, holderPid: 41267 }]]),
+            resolvedCarried: 0,
+            resolvedDispatched: 0,
+          },
+        ],
+      ]),
+    };
+    renderer.appendDurable(
+      { type: "lock-wait", at: 0, experimentId: "compare/codex", evalId: "memory/a", status: "started", attempts: 1, holderPid: 41267, holderHost: "h" },
+      firstState,
+    );
+    const secondState: RunFeedbackState = {
+      ...firstState,
+      elsewhere: 2,
+      lockWaits: new Map([
+        [
+          "compare/codex",
+          {
+            ...firstState.lockWaits.get("compare/codex")!,
+            waiting: new Map([
+              ["memory/a", { startedAt: 0, holderPid: 41267 }],
+              ["memory/b", { startedAt: 1, holderPid: 41267 }],
+            ]),
+          },
+        ],
+      ]),
+    };
+    renderer.appendDurable(
+      { type: "lock-wait", at: 1, experimentId: "compare/codex", evalId: "memory/b", status: "started", attempts: 1, holderPid: 41267, holderHost: "h" },
+      secondState,
+    );
+
+    const out = stdout.writes.join("");
+    expect(out).toContain("waiting on another run · compare/codex");
+    // 只出现一次:第二条(memory/b 加入)是同一窗口内的非首条,静默不刷屏。
+    expect(out.split("waiting on another run").length - 1).toBe(1);
+  });
+
+  it("非 TTY:resolved 只在窗口最后一次关闭(全部等待用例都已解决)时追加聚合收尾行", () => {
+    const { io, stdout } = createFakeFeedbackIO({ stderr: { isTTY: false } });
+    const renderer = createHumanRenderer({ io, command: "niceeval exp compare/codex" });
+    // 还剩一个用例没解决:窗口未关闭,静默。
+    const stillWaitingState: RunFeedbackState = {
+      ...createInitialRunFeedbackState(),
+      lockWaits: new Map([
+        [
+          "compare/codex",
+          {
+            experimentId: "compare/codex",
+            waiting: new Map([["memory/b", { startedAt: 1, holderPid: 1 }]]),
+            resolvedCarried: 2,
+            resolvedDispatched: 0,
+          },
+        ],
+      ]),
+    };
+    renderer.appendDurable(
+      { type: "lock-wait", at: 5, experimentId: "compare/codex", evalId: "memory/a", status: "resolved", carried: 2, dispatched: 0, waitedMs: 5_000 },
+      stillWaitingState,
+    );
+    expect(stdout.writes.join("")).toBe("");
+
+    // 最后一个也解决了:窗口关闭,打印聚合收尾行(carried + dispatched 混合的措辞两面都要覆盖)。
+    const closedState: RunFeedbackState = {
+      ...createInitialRunFeedbackState(),
+      lockWaits: new Map([
+        [
+          "compare/codex",
+          {
+            experimentId: "compare/codex",
+            waiting: new Map(),
+            resolvedCarried: 2,
+            resolvedDispatched: 1,
+          },
+        ],
+      ]),
+    };
+    renderer.appendDurable(
+      { type: "lock-wait", at: 94_000, experimentId: "compare/codex", evalId: "memory/b", status: "resolved", carried: 0, dispatched: 1, waitedMs: 94_000 },
+      closedState,
+    );
+    const out = stdout.writes.join("");
+    expect(out).toContain("lock wait resolved · compare/codex");
+    expect(out).toContain("2 carried");
+    expect(out).toContain("1 to run");
+  });
+});
+
+describe("诊断行:标题是「阶段标签 · code」,止损闸落闸是一行 error 级通知", () => {
+  /** 把同一条诊断喂 N 次(emitter 刷新 data.unstarted 时就是这个形状),返回每一次的渲染行。 */
+  function replayDiagnostic(event: DurableFeedbackEvent & { type: "diagnostic" }, times: number): string[][] {
+    let state = createInitialRunFeedbackState();
+    const out: string[][] = [];
+    for (let i = 0; i < times; i++) {
+      state = reduceRunFeedback(state, event);
+      out.push(renderDurableLines(event, state, { mode: "plain", width: 100 }));
+    }
+    return out;
+  }
+
+  it("普通诊断的标题用 code,不把编了身份的去重 key 甩进人读的一行", () => {
+    const [lines] = replayDiagnostic(
+      {
+        type: "diagnostic",
+        at: 0,
+        key: "lock-taken-over:compare/codex|memory/retention",
+        code: "lock-taken-over",
+        severity: "warning",
+        message: "took over a stale lock from pid 41267",
+      },
+      1,
+    );
+    expect(lines![0]).toBe("! lock-taken-over");
+    expect(lines![1]).toContain("took over a stale lock");
+  });
+
+  it("同一 key 再次出现时标题带 ×N 折叠计数(非止损闸诊断保持既有形态)", () => {
+    const rounds = replayDiagnostic(
+      { type: "diagnostic", at: 0, key: "memory-warmup-degraded", severity: "warning", message: "cold index" },
+      3,
+    );
+    expect(rounds[0]![0]).toBe("! memory-warmup-degraded");
+    expect(rounds[2]![0]).toBe("! memory-warmup-degraded (3 attempts)");
+  });
+
+  it("attempt 级诊断的标题是「阶段标签 · code」,阶段标签与失败行同一投影", () => {
+    // 失败行已有的 ` · <阶段标签>` 段是同一个投影的既有出口:拿它当参照,断言诊断行没有
+    // 另写一份阶段词表(不硬编码语言,en/zh-CN 两种 locale 下都成立)。
+    const failureLines = renderDurableLines(
+      {
+        type: "failure",
+        at: 0,
+        locator: locator("@1bwcxxiy"),
+        identity: { experimentId: "compare/codex", evalId: "memory/x", attempt: 1 },
+        who: "codex",
+        verdict: "errored",
+        reason: "boom",
+        phase: "sandbox.setup",
+      },
+      createInitialRunFeedbackState(),
+      { mode: "plain", width: 100 },
+    );
+    const label = failureLines[0]!.split("\n")[0]!.split(" · ")[1]!;
+
+    const rounds = replayDiagnostic(
+      {
+        type: "diagnostic",
+        at: 0,
+        // 作者没传 dedupeKey 时 attempt.ts 折出来的 key —— 编了身份,不能出现在人读标题里。
+        key: "memory-warmup-degraded:compare/codex|memory/x|1",
+        code: "memory-warmup-degraded",
+        severity: "warning",
+        message: "Memory warmup failed; continuing with a cold index",
+        identity: { experimentId: "compare/codex", evalId: "memory/x", attempt: 1 },
+        data: { phase: "sandbox.setup" },
+      },
+      12,
+    );
+    expect(rounds[0]![0]).toBe(`! ${label} · memory-warmup-degraded`);
+    expect(rounds[11]![0]).toBe(`! ${label} · memory-warmup-degraded (12 attempts)`);
+    expect(rounds[11]![1]).toBe("  Memory warmup failed; continuing with a cold index");
+  });
+
+  it("运行级诊断没有 phase:标题只有 code,不留空的 · 分隔符", () => {
+    const [lines] = replayDiagnostic(
+      {
+        type: "diagnostic",
+        at: 0,
+        key: "budget-unenforceable:compare/codex",
+        code: "budget-unenforceable",
+        severity: "warning",
+        message: "no cost data; budget cannot be enforced",
+        data: { experimentId: "compare/codex" },
+      },
+      1,
+    );
+    expect(lines![0]).toBe("! budget-unenforceable");
+  });
+
+  it("实验闸落闸:一行 error 级通知,文案就是契约字面,不再多一行标题", () => {
+    const [lines] = replayDiagnostic(
+      {
+        type: "diagnostic",
+        at: 0,
+        key: "dispatch-halted:experiment:compare/codex",
+        code: HALT_DIAGNOSTIC_CODE,
+        severity: "error",
+        message: "experiment halted (dispatch-halted): shared service is down; restart the tunnel",
+        data: { experimentId: "compare/codex", scope: "experiment", phase: "eval.run", unstarted: 0 },
+      },
+      1,
+    );
+    expect(lines).toEqual(["✗ experiment halted (dispatch-halted): shared service is down; restart the tunnel"]);
+  });
+
+  it("每条未派发 attempt 刷一次的后续声明零输出:被中止的等待集不逐条刷屏,数量归完成状态的 unstarted", () => {
+    const rounds = replayDiagnostic(
+      {
+        type: "diagnostic",
+        at: 0,
+        key: "dispatch-halted:eval:compare/codex|memory/retention",
+        code: HALT_DIAGNOSTIC_CODE,
+        severity: "error",
+        message: "eval halted: fixture db is empty; run scripts/seed.ts",
+        data: { experimentId: "compare/codex", scope: "eval", evalId: "memory/retention", phase: "eval.run", unstarted: 4 },
+      },
+      5,
+    );
+    expect(rounds[0]).toEqual(["✗ eval halted: fixture db is empty; run scripts/seed.ts"]);
+    expect(rounds.slice(1).flat()).toEqual([]);
+  });
+});
+
+describe("renderHumanDryPlan: locked 标注", () => {
+  it("locked 为 true 的行尾标注 locked;false/省略的行不受影响", () => {
+    const text = renderHumanDryPlan({
+      totalAttempts: 2,
+      evals: 2,
+      configs: 1,
+      runs: 1,
+      rows: [
+        { experimentId: "compare/codex", evalId: "memory/a", locked: true },
+        { experimentId: "compare/codex", evalId: "memory/b" },
+      ],
+    });
+    const lines = text.trim().split("\n");
+    expect(lines.find((l) => l.includes("memory/a"))).toContain("locked");
+    expect(lines.find((l) => l.includes("memory/b"))).not.toContain("locked");
+  });
+});