niceeval 0.9.0 → 0.10.0

package/src/runner/feedback/human.test.ts

@@ -12,6 +12,7 @@ import { createFeedbackCoordinator, type FeedbackCoordinator } from "./coordinat
 import { createFakeFeedbackIO, type FakeFeedbackIO } from "./testing.ts";
 import { activeFeedbackSinkCount } from "./sink.ts";
 import { createHumanRenderer, formatElapsed, formatTokenCount, renderDurableLines } from "./human.ts";
+import { t } from "../../i18n/index.ts";
 import { createInitialRunFeedbackState } from "./reducer.ts";
 import type { AttemptLocator } from "../../results/locator.ts";
 import type { AttemptRef, RunCompletion, RunFeedbackPlan, RunSummary } from "../types.ts";
@@ -356,7 +357,7 @@ describe("完成页:失败优先摘要 + locator + show/view 下一步 + 快照
     expect(text).toContain("Eval:    niceeval show @17m2k9p --source");
     expect(text).toContain("Trace:   niceeval show @17m2k9p --execution");
     expect(text).toContain("Diff:    niceeval show @17m2k9p --diff");
-    expect(text).toContain("Compare: niceeval view compare");
+    expect(text).toContain("Compare: niceeval view");
     expect(text).toContain(".niceeval/compare/bub-e2b/2026-07-13T000000");
     // 不再是 renderRunReport() 的大表(表格标志性的竖线分隔表头在这里不应出现)。
     expect(text).not.toMatch(/-\+-/);
@@ -614,3 +615,109 @@ describe("renderDurableLines: 纯函数,不需要经过 coordinator 也能验证
     expect(lines.join("\n")).toContain("interrupted");
   });
 });
+
+// ───────────────────────── 实验级钩子:运行级行 / 起止行(cases.md「实验级生命周期」) ─────────────────────────
+
+describe("TTY dashboard: 实验级钩子的运行级行", () => {
+  it("setup started 后 ACTIVE 区出现运行级行,progress 只更新该行 detail,done 后行消失且不写 scrollback", async () => {
+    const { fake, coordinator } = setupTty();
+    coordinator.start(plan({ totalRuns: 2, evals: 2 }));
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "started" });
+    fake.advance(1000);
+    await flush();
+    const rowLabel = `${t("feedback.phase.experimentSetup")} · compare/bub-e2b`;
+    expect(fake.stderr.writes.join("")).toContain(rowLabel);
+
+    coordinator.experimentProgress({ experimentId: "compare/bub-e2b", detail: "starting tunnel (2/5)" });
+    fake.advance(1000);
+    await flush();
+    expect(fake.stderr.writes.join("")).toContain("starting tunnel (2/5)");
+
+    const before = fake.stderr.writes.length;
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "done", durationMs: 42_000 });
+    fake.advance(1000);
+    await flush();
+    // done 之后的帧不再含运行级行;成功钩子也不追加任何 scrollback 永久行。
+    expect(fake.stderr.writes.slice(before).join("")).not.toContain(t("feedback.phase.experimentSetup"));
+    expect(fake.stderr.writes.join("")).not.toContain(t("feedback.human.hookDone"));
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+
+  it("运行级行排在 attempt 行前面", async () => {
+    const { fake, coordinator } = setupTty();
+    coordinator.start(plan({ totalRuns: 2, evals: 2 }));
+    coordinator.emit({ type: "attempt:start", at: 0, identity: ref("memory/a"), who: "bub-e2b", phase: "eval.run" });
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "started" });
+    fake.advance(1000);
+    await flush();
+    const lastFrame = fake.stderr.writes.filter((w) => w.includes(ESC)).at(-1)!;
+    const hookIdx = lastFrame.indexOf(t("feedback.phase.experimentSetup"));
+    const attemptIdx = lastFrame.indexOf("memory/a");
+    expect(hookIdx).toBeGreaterThan(-1);
+    expect(attemptIdx).toBeGreaterThan(-1);
+    expect(hookIdx).toBeLessThan(attemptIdx);
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+});
+
+describe("非 TTY human: 实验级钩子起止各追加一行", () => {
+  it("started / done 各一行 human 文案,done 带耗时;failed 行同形;全程无 ANSI", async () => {
+    const { fake, coordinator } = setupPlain();
+    coordinator.start(plan({ totalRuns: 2, evals: 2 }));
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "started" });
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "done", durationMs: 42_000 });
+    coordinator.experimentHook({ experimentId: "compare/codex", hook: "setup", status: "started" });
+    coordinator.experimentHook({ experimentId: "compare/codex", hook: "setup", status: "failed", durationMs: 3_000 });
+    await flush();
+    const all = fake.stderr.writes.join("");
+    const label = t("feedback.phase.experimentSetup");
+    expect(all).toContain(`${label} · compare/bub-e2b`);
+    expect(all).toContain(`${label} ${t("feedback.human.hookDone")} · compare/bub-e2b (42s)`);
+    expect(all).toContain(`${label} ${t("feedback.human.hookFailed")} · compare/codex (3s)`);
+    expect(all).not.toContain(ESC);
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+
+  it("实验级 progress 在非 TTY human 下零输出(短命状态没有可覆盖的区域)", async () => {
+    const { fake, coordinator } = setupPlain();
+    coordinator.start(plan());
+    await flush(); // 先让 plan 行落盘,再取零输出基线
+    const before = fake.stderr.writes.length;
+    coordinator.experimentProgress({ experimentId: "compare/bub-e2b", detail: "starting tunnel" });
+    await flush();
+    expect(fake.stderr.writes.slice(before).join("")).toBe("");
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+});
+
+// ───────────────────────── 运行级瞬时通知(activity)(cases.md「实验级生命周期」) ─────────────────────────
+
+describe("运行级瞬时通知(activity): human 追加一行", () => {
+  it("TTY:activity 落进 scrollback,dashboard 撤下后重建(通知行不被下一帧覆盖)", async () => {
+    const { fake, coordinator } = setupTty();
+    coordinator.start(plan());
+    coordinator.emit({ type: "attempt:start", at: 0, identity: ref("memory/a"), who: "bub-e2b", phase: "eval.run" });
+    fake.advance(1000);
+    await flush();
+    coordinator.activity("Braintrust experiment: https://example.test/exp/1");
+    fake.advance(1000);
+    await flush();
+    const all = fake.stderr.writes.join("");
+    expect(all).toContain("Braintrust experiment: https://example.test/exp/1");
+    // 通知之后 dashboard 还在:activity 之后仍有 ANSI 重画帧。
+    const activityIdx = fake.stderr.writes.findIndex((w) => w.includes("Braintrust"));
+    expect(fake.stderr.writes.slice(activityIdx + 1).some((w) => w.includes(ESC))).toBe(true);
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+
+  it("非 TTY:activity 追加一行,无 ANSI", async () => {
+    const { fake, coordinator } = setupPlain();
+    coordinator.start(plan());
+    coordinator.activity("prechecking judge config...");
+    await flush();
+    const all = fake.stderr.writes.join("");
+    expect(all).toContain("prechecking judge config...");
+    expect(all).not.toContain(ESC);
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+});

package/src/runner/feedback/human.ts

@@ -23,7 +23,9 @@ import { assertionSummaryLines } from "../../scoring/display.ts";
 import { encodeAttemptKey } from "../types.ts";
 import type {
   ActiveAttempt,
+  ActiveExperimentHook,
   AttemptKey,
+  ExperimentHookName,
   LifecyclePhase,
   DurableFeedbackEvent,
   RunFeedbackPlan,
@@ -98,6 +100,16 @@ export function renderDurableLines(event: DurableFeedbackEvent, state: RunFeedba
       // 留存授予单条不即时打印;run 摘要后由 buildSummaryLines 汇总成 Kept sandboxes 块
       // (见 docs/feature/sandbox/cli.md「run 收尾输出」)。
       return [];
+    case "experiment-hook": {
+      // 只服务非 TTY 退化流(TTY dashboard 的 appendDurable 对这个事件直接返回,运行级行
+      // 由 state.experimentHooks 驱动,成功钩子不进 scrollback,见 cli.md「实验级钩子的显示」)。
+      const label = experimentHookLabel(event.hook);
+      const duration = event.durationMs !== undefined ? ` (${formatElapsed(event.durationMs)})` : "";
+      if (event.status === "started") return [`${label} · ${event.experimentId}`];
+      const statusWord =
+        event.status === "done" ? t("feedback.human.hookDone") : t("feedback.human.hookFailed");
+      return [`${label} ${statusWord} · ${event.experimentId}${duration}`];
+    }
     case "summary":
       return buildSummaryLines(event, state);
     case "saved":
@@ -217,9 +229,9 @@ function buildSummaryLines(event: DurableFeedbackEvent & { type: "summary" }, st
 
 function buildSavedLines(event: DurableFeedbackEvent & { type: "saved" }): string[] {
   const paths = event.paths;
-  const lines: string[] = [];
-  const group = deriveResultGroup(paths);
-  if (group) lines.push(t("feedback.human.compare", { group }));
+  // 比较命令直接是 `niceeval view`——它读整个结果根,不需要(也不该被)目录路径收窄成
+  // 一个 eval 位置参数(那是选择语义,不是报告分组语义);见 docs/feature/experiments/cli.md。
+  const lines: string[] = [t("feedback.human.compare")];
   if (paths.length === 0) return lines;
   if (paths.length === 1) {
     lines.push(`${t("feedback.human.resultsHeader")} ${paths[0]}`);
@@ -233,18 +245,6 @@ function buildSavedLines(event: DurableFeedbackEvent & { type: "saved" }): strin
   return lines;
 }
 
-/** 快照路径的共同「组」段(如 `.niceeval/compare/bub-e2b/<snapshot>` → `compare`)。
- *  多个 experiment 组混在同一次 invocation 里(不带 group 前缀的 `niceeval exp` 全量运行)时
- *  组名不唯一,不猜一个可能就 wrong 的 view 目标,直接省略 Compare 行 —— Results 路径仍然完整。 */
-function deriveResultGroup(paths: readonly string[]): string | undefined {
-  const groups = new Set<string>();
-  for (const p of paths) {
-    const seg = p.split("/")[1];
-    if (seg) groups.add(seg);
-  }
-  return groups.size === 1 ? [...groups][0] : undefined;
-}
-
 function formatSummaryDetail(durationMs: number, state: RunFeedbackState): string {
   const parts = [formatElapsed(durationMs)];
   const fullReuse = state.total > 0 && state.total === state.reused;
@@ -281,6 +281,12 @@ export function formatTokenCount(n: number): string {
  *  机器面(agent/ci 的 `phase=` 与落盘)保留精确的点分名,收尾段在 Human 侧合并显示为一档。 */
 function phaseLabel(phase: LifecyclePhase): string {
   switch (phase) {
+    // 实验级两员不会作为 ActiveAttempt.phase 出现(钩子跑的时候没有活跃 attempt),
+    // 这里只服务 failure 行的 phase 标注(experiment.setup 失败的合成 errored 结果)。
+    case "experiment.setup":
+      return t("feedback.phase.experimentSetup");
+    case "experiment.teardown":
+      return t("feedback.phase.teardown");
     case "sandbox.queue":
       return t("feedback.phase.sandboxQueue");
     case "sandbox.create":
@@ -317,6 +323,13 @@ function phaseLabel(phase: LifecyclePhase): string {
   }
 }
 
+/** 实验级钩子的运行级行标签。`phaseLabel` 把 `experiment.teardown` 与其它收尾段合并成
+ *  「cleaning up」一档(那是 attempt failure 行的语境);运行级行没有 attempt 语境,必须
+ *  自报家门是哪个实验级钩子,所以这里用独立的两个词。 */
+function experimentHookLabel(hook: ExperimentHookName): string {
+  return hook === "setup" ? t("feedback.phase.experimentSetup") : t("feedback.phase.experimentTeardown");
+}
+
 function formatCounts(state: RunFeedbackState): string {
   const counts = t("feedback.human.counts", {
     total: state.total,
@@ -365,18 +378,24 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
     if (state.total > 0 && state.total === state.reused) return [];
     const columns = io.stderr.columns;
     const lines: string[] = [formatCommandLine(command, state.elapsedMs, columns), formatCounts(state)];
-    if (activeOrder.length === 0) return lines.map((l) => truncateNoWrap(l, columns));
+    // 实验级钩子的运行级行排在 attempt 行前面(见 cli.md「实验级钩子的显示」):它解释了
+    // 为什么后面的 attempt 还停在 queued。Map 按插入序迭代,天然满足稳定 slot。
+    const hookRows = [...state.experimentHooks.values()];
+    if (activeOrder.length === 0 && hookRows.length === 0) return lines.map((l) => truncateNoWrap(l, columns));
 
     lines.push("", t("feedback.human.active"));
     const rowBudget = Math.max(0, io.stderr.rows - lines.length - DASHBOARD_ROW_RESERVE);
-    const total = activeOrder.length;
+    const total = hookRows.length + activeOrder.length;
     // 窄/矮终端先减 active slots(减少行数),而不是先压缩单行内容 ——
     // 单行内容的截断在 formatActiveRow 里按 columns 单独处理。
     const showCount = total <= rowBudget ? total : Math.max(0, rowBudget - 1);
-    for (let i = 0; i < showCount; i++) {
-      const active = state.active.get(activeOrder[i]!);
-      if (active) lines.push(formatActiveRow(active, io));
+    const rows: string[] = hookRows.map((hookRow) => formatExperimentHookRow(hookRow, io));
+    for (const key of activeOrder) {
+      if (rows.length >= showCount) break;
+      const active = state.active.get(key);
+      if (active) rows.push(formatActiveRow(active, io));
     }
+    lines.push(...rows.slice(0, showCount));
     if (total > showCount) {
       lines.push(t("feedback.human.moreActive", { count: total - showCount }));
     }
@@ -412,8 +431,17 @@ function createDashboardRenderer(io: FeedbackIO, command: string): FeedbackRende
 
   return {
     appendDurable(event, state) {
+      // 实验级钩子起止在 TTY 下只驱动运行级 active 行(state.experimentHooks 已由 reducer
+      // 更新,coordinator 紧接着的 redrawDynamic 会画出来);成功钩子不写 scrollback 永久行
+      // (见 cli.md「实验级钩子的显示」)。非 TTY 退化流才逐行追加(见 renderDurableLines)。
+      if (event.type === "experiment-hook") return;
       writeDurable(io, event, state);
     },
+    activity(text) {
+      // 运行级瞬时通知(judge 预检、provider 一次性通知……):coordinator 已按
+      // clearDynamic → activity → redrawDynamic 包好顺序,这里只管把这一行落进 scrollback。
+      io.stderr.write(text.endsWith("\n") ? text : `${text}\n`);
+    },
     clearDynamic() {
       if (linesDrawn === 0) return; // 幂等:coordinator 收尾时会无条件再调一次
       let out = `\x1B[${linesDrawn}A`;
@@ -469,6 +497,22 @@ function formatActiveRow(active: ActiveAttempt, io: FeedbackIO): string {
   return prefix + detail.slice(0, budget);
 }
 
+/** 实验级钩子的运行级行:与 attempt 行同一套列宽算法,label 跨过 evalId+who 两列的宽度,
+ *  elapsed 列因此对齐;detail 来自实验级 `ctx.progress`,没有就只留标签行。 */
+function formatExperimentHookRow(hook: ActiveExperimentHook, io: FeedbackIO): string {
+  const columns = io.stderr.columns;
+  const elapsed = formatElapsed(io.clock.now() - hook.startedAt).padStart(6);
+  const sym = "● ";
+  const fixedWidth = sym.length + elapsed.length + 6;
+  const detailReserve = Math.min(80, Math.max(0, Math.floor(columns * 0.35)));
+  const remaining = Math.max(0, columns - fixedWidth - detailReserve);
+  // attempt 行的身份区是 evalCol + 两格间隔 + whoCol(合计 remaining + 2),这里用同一跨度。
+  const label = padTrunc(`${experimentHookLabel(hook.hook)} · ${hook.experimentId}`, remaining + 2);
+  const prefix = `${sym}${label}  ${elapsed}  `;
+  const budget = Math.max(0, columns - prefix.length);
+  return prefix + (hook.detail ?? "").slice(0, budget);
+}
+
 // ───────────────────────── 非 TTY:human 文案的纯追加流 ─────────────────────────
 
 function createPlainRenderer(io: FeedbackIO): FeedbackRenderer {
@@ -480,6 +524,12 @@ function createPlainRenderer(io: FeedbackIO): FeedbackRenderer {
       lastDurableAtMs = event.at;
       writeDurable(io, event, state);
     },
+    activity(text) {
+      // 运行级瞬时通知按永久行追加(非 TTY 没有可覆盖的动态区域),并重置 heartbeat 计时 ——
+      // 刚有输出就不需要紧跟一条「还活着」。
+      lastDurableAtMs = io.clock.now();
+      io.stderr.write(text.endsWith("\n") ? text : `${text}\n`);
+    },
     onTick(event, state) {
       if (event.at - lastDurableAtMs < NON_TTY_HEARTBEAT_IDLE_MS) return;
       lastDurableAtMs = event.at;
@@ -487,7 +537,7 @@ function createPlainRenderer(io: FeedbackIO): FeedbackRenderer {
         `${t("feedback.human.heartbeat", { elapsed: formatElapsed(state.elapsedMs), counts: formatCounts(state) })}\n`,
       );
     },
-    // 没有 clearDynamic/redrawDynamic/activity/onLifecycle:非 TTY 退化流不维护动态区域,
+    // 没有 clearDynamic/redrawDynamic/onLifecycle:非 TTY 退化流不维护动态区域,
     // 不展示 active attempt 的逐次阶段变化,也不逐次输出 provisioning retry/backoff ——
     // 这些行为由「不实现对应可选钩子」天然满足,不需要在这里写 profile 分支。
   };

package/src/runner/feedback/reducer.test.ts

@@ -393,3 +393,67 @@ describe("reduceRunFeedback: 守恒公式", () => {
     expect(activeAfterPhase?.detail).toBeUndefined();
   });
 });
+
+describe("reduceRunFeedback: 实验级钩子", () => {
+  const plan: RunFeedbackEvent = {
+    type: "plan",
+    at: 0,
+    plan: { shape: { evals: 2, configs: 1, totalRuns: 2, maxConcurrency: 2 }, reused: 0, reusedFailures: [] },
+  };
+
+  it("started 添加运行级行,done 移除;等待 setup 的 attempt 保持 queued(计数不变量不受钩子影响)", () => {
+    const afterStart = replay([
+      plan,
+      { type: "experiment-hook", at: 1, experimentId: "compare/bub-e2b", hook: "setup", status: "started" },
+    ]);
+    expect(afterStart).toMatchObject({ total: 2, running: 0, queued: 2, completed: 0 });
+    expect(afterStart.experimentHooks.get("compare/bub-e2b")).toMatchObject({
+      experimentId: "compare/bub-e2b",
+      hook: "setup",
+      startedAt: 1,
+    });
+
+    const afterDone = replay([
+      plan,
+      { type: "experiment-hook", at: 1, experimentId: "compare/bub-e2b", hook: "setup", status: "started" },
+      { type: "experiment-hook", at: 5, experimentId: "compare/bub-e2b", hook: "setup", status: "done", durationMs: 4 },
+    ]);
+    expect(afterDone.experimentHooks.size).toBe(0);
+  });
+
+  it("failed 同样移除运行级行(setup 抛错的证据走每条 attempt 的 failure 事件,不留孤儿行)", () => {
+    const state = replay([
+      plan,
+      { type: "experiment-hook", at: 1, experimentId: "compare/codex", hook: "setup", status: "started" },
+      { type: "experiment-hook", at: 3, experimentId: "compare/codex", hook: "setup", status: "failed", durationMs: 2 },
+    ]);
+    expect(state.experimentHooks.size).toBe(0);
+  });
+
+  it("experiment:progress 只覆盖对应行的 detail;没有对应行时静默忽略", () => {
+    const state = replay([
+      plan,
+      { type: "experiment-hook", at: 1, experimentId: "compare/bub-e2b", hook: "setup", status: "started" },
+      { type: "experiment:progress", at: 2, experimentId: "compare/bub-e2b", detail: "starting tunnel (1/3)" },
+      { type: "experiment:progress", at: 3, experimentId: "compare/bub-e2b", detail: "starting tunnel (2/3)" },
+      { type: "experiment:progress", at: 3, experimentId: "compare/unknown", detail: "ignored" },
+    ]);
+    expect(state.experimentHooks.get("compare/bub-e2b")?.detail).toBe("starting tunnel (2/3)");
+    expect(state.experimentHooks.has("compare/unknown")).toBe(false);
+  });
+
+  it("plan 事件清空残留的运行级行(reducer 复用于多次 run 时不带上一次的钩子状态)", () => {
+    let state = createInitialRunFeedbackState();
+    state = reduceRunFeedback(state, plan);
+    state = reduceRunFeedback(state, {
+      type: "experiment-hook",
+      at: 1,
+      experimentId: "compare/bub-e2b",
+      hook: "teardown",
+      status: "started",
+    });
+    expect(state.experimentHooks.size).toBe(1);
+    state = reduceRunFeedback(state, plan);
+    expect(state.experimentHooks.size).toBe(0);
+  });
+});

package/src/runner/feedback/reducer.ts

@@ -23,6 +23,7 @@ export function createInitialRunFeedbackState(): RunFeedbackState {
     earlyExitSkipped: 0,
     elapsedMs: 0,
     active: new Map(),
+    experimentHooks: new Map(),
     failures: [],
     freshFailureCount: 0,
     diagnostics: [],
@@ -47,6 +48,7 @@ export function reduceRunFeedback(state: RunFeedbackState, event: RunFeedbackEve
         running: 0,
         completed: 0,
         active: new Map(),
+        experimentHooks: new Map(),
         failures: (event.plan.reusedFailures ?? []).map((failure) => ({ ...failure, at: event.at })),
         freshFailureCount: 0,
         diagnostics: [],
@@ -120,6 +122,30 @@ export function reduceRunFeedback(state: RunFeedbackState, event: RunFeedbackEve
       };
     }
 
+    case "experiment-hook": {
+      // 运行级行的增删:started 添加,done/failed 移除(见 cli.md「实验级钩子的显示」)。
+      // 不动 running/queued 计数——等待 setup 的 attempt 保持 queued,计数不变量不受钩子影响。
+      const experimentHooks = new Map(state.experimentHooks);
+      if (event.status === "started") {
+        experimentHooks.set(event.experimentId, {
+          experimentId: event.experimentId,
+          hook: event.hook,
+          startedAt: event.at,
+        });
+      } else {
+        experimentHooks.delete(event.experimentId);
+      }
+      return { ...state, experimentHooks };
+    }
+
+    case "experiment:progress": {
+      const existing = state.experimentHooks.get(event.experimentId);
+      if (!existing) return state; // 防御:没有对应运行级行时静默忽略,不让 renderer 崩
+      const experimentHooks = new Map(state.experimentHooks);
+      experimentHooks.set(event.experimentId, { ...existing, detail: event.detail });
+      return { ...state, experimentHooks };
+    }
+
     case "attempt:early-exit":
       // 首过即停下已知 verdict 的省略次数:折进 completed(结论已经确定,不再需要派发),
       // 不产生 failures/diagnostics —— 这不是一次失败或异常,只是省下的重复验证

package/src/runner/feedback/sink.ts

@@ -16,7 +16,7 @@
 
 import { writeStderrLine } from "../../tty-line.ts";
 import { t } from "../../i18n/index.ts";
-import type { AttemptLifecycleEvent, AttemptRef, FailureDetail } from "../types.ts";
+import type { AttemptLifecycleEvent, AttemptRef, ExperimentHookName, FailureDetail } from "../types.ts";
 import type { Verdict } from "../../scoring/types.ts";
 import type { JsonValue } from "../../shared/types.ts";
 import type { AttemptLocator } from "../../results/locator.ts";
@@ -51,6 +51,22 @@ export interface BudgetExhaustedInput {
   unstarted: number;
 }
 
+/** `sink.experimentHook()` 的输入 —— 与 `DurableFeedbackEvent` 的 "experiment-hook" 变体字段
+ *  一致,省略 `type`/`at`(由 coordinator 补上)。调用方(run.ts)在钩子真正开始/结束时各调
+ *  一次;`durationMs` 只在 done/failed 上给。 */
+export interface ExperimentHookInput {
+  experimentId: string;
+  hook: ExperimentHookName;
+  status: "started" | "done" | "failed";
+  durationMs?: number;
+}
+
+/** `sink.experimentProgress()` 的输入 —— 实验级 `ctx.progress` 压好的单行文本。 */
+export interface ExperimentProgressInput {
+  experimentId: string;
+  detail: string;
+}
+
 /** `sink.kept()` 的输入 —— 与 `DurableFeedbackEvent` 的 "kept" 变体字段一致,省略 type/at。 */
 export interface KeptInput {
   locator: AttemptLocator;
@@ -76,6 +92,11 @@ export interface FeedbackSink {
   budgetExhausted(input: BudgetExhaustedInput): void;
   /** 一次留存授予(--keep-sandbox);见 `KeptInput` 与 docs/feature/sandbox/cli.md。 */
   kept(input: KeptInput): void;
+  /** 实验级钩子(`ExperimentDef.setup` / teardown)的起止(见 `ExperimentHookInput`)。 */
+  experimentHook(input: ExperimentHookInput): void;
+  /** 实验级 `ctx.progress` 的短命投影:只更新运行级行的 detail。与 `lifecycle` 同级别的
+   *  「只服务正在画着的 dashboard」信号,没有活跃 coordinator 时静默丢弃是安全的。 */
+  experimentProgress(input: ExperimentProgressInput): void;
   /** attempt 生命周期事件(queued/start/phase/progress/complete/early-exit),见
    *  `AttemptLifecycleEvent`。只驱动 human dashboard 的 active slot,不落 RunSummary/结果文件,
    *  所以没有活跃 coordinator 时(见 `reportAttemptLifecycle`)静默丢弃是安全的 —— 这类信息
@@ -155,6 +176,24 @@ export function reportKept(input: KeptInput): void {
   writeStderrLine(`kept sandbox ${input.sandboxId} (${input.provider}) — ${input.identity.evalId} #${input.identity.attempt} ${input.verdict}\n`);
 }
 
+/** 实验级钩子的起止(见 `ExperimentHookInput`)。没有活跃 coordinator 时退回一行 stderr ——
+ *  长 setup 的可见性正是这条通道存在的理由,不能像 lifecycle 那样静默丢弃。 */
+export function reportExperimentHook(input: ExperimentHookInput): void {
+  const sink = current();
+  if (sink) {
+    sink.experimentHook(input);
+    return;
+  }
+  const duration = input.durationMs !== undefined ? ` (${Math.round(input.durationMs / 1000)}s)` : "";
+  writeStderrLine(`experiment ${input.hook} ${input.status} · ${input.experimentId}${duration}\n`);
+}
+
+/** 实验级 `ctx.progress` 的短命投影。与 `reportAttemptLifecycle` 同理:只服务正在画着的
+ *  dashboard,没有活跃 coordinator 时静默 no-op。 */
+export function reportExperimentProgress(input: ExperimentProgressInput): void {
+  current()?.experimentProgress(input);
+}
+
 export function reportBudgetExhausted(input: BudgetExhaustedInput): void {
   const sink = current();
   if (sink) {

package/src/runner/fingerprint.ts

@@ -7,6 +7,7 @@ import { sandboxRunInfo } from "../sandbox/resolve.ts";
 import type { DiscoveredEval, EvalResult, SandboxOption } from "../types.ts";
 import type { AgentRun } from "./types.ts";
 import { prepareRunSandboxes, sandboxForEval } from "./sandbox-selection.ts";
+import { selectedEvalsForRun } from "./eval-selection.ts";
 
 export function cacheKey(run: AgentRun, evalId: string): string {
   return `${run.experimentId ?? ""}|${evalId}`;
@@ -76,7 +77,7 @@ export async function planCarry(
   const plannedFingerprints = new Map<string, string>();
   const jobs: Promise<void>[] = [];
   for (const run of agentRuns) {
-    for (const evalDef of evals.filter((e) => run.evalFilter(e.id))) {
+    for (const evalDef of selectedEvalsForRun(evals, run)) {
       jobs.push(
         computeFingerprint(evalDef, run, sourceCache, configSandbox).then((fp) => {
           plannedFingerprints.set(cacheKey(run, evalDef.id), fp);

package/src/runner/report.test.ts

@@ -45,6 +45,18 @@ describe("filterSummary", () => {
     expect(sub.estimatedCostUSD).toBe(0.1);
     expect(sub.completedAt).toBe("2026-07-07T00:01:00.000Z");
   });
+
+  it("保留原 summary 的 model 字段(docs/engineering/unit-tests/experiments-runner/cases.md「汇总判定与退出码」)", () => {
+    const s = { ...summary([result("a/1")]), model: "deepseek-chat" };
+    const sub = filterSummary(s, new Set(["a/1"]));
+    expect(sub.model).toBe("deepseek-chat");
+  });
+
+  it("未声明 model 时该字段省略,不伪造成空字符串或 undefined 字面量", () => {
+    const s = summary([result("a/1")]);
+    const sub = filterSummary(s, new Set(["a/1"]));
+    expect(sub.model).toBeUndefined();
+  });
 });
 
 describe("scopeReporter", () => {
@@ -136,6 +148,8 @@ describe("runReporter / emitReporterEvent · required/best-effort 原样转发,
       failure() {},
       budgetExhausted() {},
       kept() {},
+      experimentHook() {},
+      experimentProgress() {},
       lifecycle() {},
     });
     return fn(calls).finally(deactivate);

package/src/runner/report.ts

@@ -46,7 +46,7 @@ export async function emitReporterEvent(
 /** 按 eval id 过滤 RunSummary 并重新计数 —— eval 级 reporter 只看它观测的那部分。 */
 export function filterSummary(summary: RunSummary, ids: ReadonlySet<string>): RunSummary {
   const results = summary.results.filter((r) => ids.has(r.id));
-  const sub = summarize(results, summary.agent, summary.startedAt, summary.durationMs, summary.name);
+  const sub = summarize(results, summary.agent, summary.startedAt, summary.durationMs, summary.name, summary.model);
   // completedAt 用原值(summarize 会重新取 now);name 等其余字段原样保留。
   return { ...summary, ...sub, completedAt: summary.completedAt };
 }
@@ -101,6 +101,7 @@ export function summarize(
   startedAt: string,
   durationMs: number,
   name?: LocalizedText,
+  model?: string,
 ): RunSummary {
   const counts = { passed: 0, failed: 0, skipped: 0, errored: 0 };
   let inTok = 0;
@@ -115,6 +116,7 @@ export function summarize(
   return {
     name,
     agent,
+    model,
     startedAt,
     completedAt: new Date().toISOString(),
     passed: counts.passed,