niceeval 0.9.0 → 0.10.0

package/src/runner/eval-selection.ts

@@ -0,0 +1,109 @@
+// 纯选题边界:发现结果 → EvalDescriptor 投影 → 用户谓词求值 → selectedEvalIds。
+// CLI 在构造每个 AgentRun 前对候选 eval 各调用一次谓词并把结果落进这里;下游(dry-run、
+// sandbox 查表、fingerprint/carry、attempt 展开、hook ctx、快照)只消费返回的
+// selectedEvalIds,不重新调用用户谓词(见 docs/feature/experiments/library.md「evals」)。
+
+import { createHash } from "node:crypto";
+import { evalPrefixPredicate } from "../shared/aggregate.ts";
+import type { AgentRun, DiscoveredEval, EvalDescriptor, ExperimentDef } from "./types.ts";
+
+/** `DiscoveredEval` → 用户谓词可见的显式白名单投影;不透传内部路径/执行字段。 */
+export function evalDescriptorOf(evalDef: DiscoveredEval): EvalDescriptor {
+  return Object.freeze({
+    id: evalDef.id,
+    ...(evalDef.description !== undefined ? { description: evalDef.description } : {}),
+    tags: Object.freeze([...(evalDef.tags ?? [])]),
+    ...(evalDef.environment !== undefined ? { environment: evalDef.environment } : {}),
+    ...(evalDef.metadata !== undefined ? { metadata: Object.freeze({ ...evalDef.metadata }) } : {}),
+  }) as EvalDescriptor;
+}
+
+export interface ResolveExperimentEvalsInput {
+  experimentId: string;
+  selector: ExperimentDef["evals"];
+  cliPatterns: readonly string[];
+  evals: readonly DiscoveredEval[];
+}
+
+export interface ResolveExperimentEvalsResult {
+  selectedEvals: readonly DiscoveredEval[];
+  selectedEvalIds: readonly string[];
+}
+
+/**
+ * 对某个 experiment 的候选 eval 集合各求值谓词一次,与 CLI 追加的位置参数前缀取交集。
+ * 返回顺序 = discovery 稳定顺序,id 去重——这是 dry-run、attempt 派发顺序与落盘
+ * `selectedEvalIds` 的共同来源(见 docs/feature/eval/README.md「路径即身份」)。
+ */
+export function resolveExperimentEvals(input: ResolveExperimentEvalsInput): ResolveExperimentEvalsResult {
+  const { experimentId, selector, cliPatterns, evals } = input;
+  const patternFilter = evalPrefixPredicate(cliPatterns.length > 0 ? [...cliPatterns] : undefined);
+
+  let selectorFilter: (evalDef: DiscoveredEval) => boolean;
+  if (selector === undefined || selector === "*") {
+    selectorFilter = () => true;
+  } else if (typeof selector === "function") {
+    const predicate = selector;
+    selectorFilter = (evalDef) => {
+      let result: unknown;
+      try {
+        result = predicate(evalDescriptorOf(evalDef));
+      } catch (e) {
+        throw new Error(
+          `experiment "${experimentId}" evals predicate threw for eval "${evalDef.id}": ` +
+            `${e instanceof Error ? e.message : String(e)}`,
+          { cause: e },
+        );
+      }
+      if (result instanceof Promise) {
+        throw new Error(
+          `experiment "${experimentId}" evals predicate returned a Promise for eval "${evalDef.id}"; ` +
+            "the predicate must be synchronous — do not await inside evals().",
+        );
+      }
+      if (typeof result !== "boolean") {
+        throw new Error(
+          `experiment "${experimentId}" evals predicate returned ${JSON.stringify(result)} (not a boolean) ` +
+            `for eval "${evalDef.id}".`,
+        );
+      }
+      return result;
+    };
+  } else {
+    const arrayFilter = evalPrefixPredicate([...selector]);
+    selectorFilter = (evalDef) => arrayFilter(evalDef.id);
+  }
+
+  const seen = new Set<string>();
+  const selectedEvals: DiscoveredEval[] = [];
+  for (const evalDef of evals) {
+    if (seen.has(evalDef.id)) continue;
+    if (!selectorFilter(evalDef) || !patternFilter(evalDef.id)) continue;
+    seen.add(evalDef.id);
+    selectedEvals.push(evalDef);
+  }
+  return { selectedEvals, selectedEvalIds: selectedEvals.map((e) => e.id) };
+}
+
+/**
+ * `evals` 选择器的审计指纹(数组内容 / 函数体哈希),进 `ExperimentRunInfo.evalFilterFingerprint`,
+ * 供「配置没变」判断;不存选择器本身、不参与报告选题(选题权威是 `selectedEvalIds`)。
+ */
+export function fingerprintEvalsFilter(evals: ExperimentDef["evals"], patterns: readonly string[]): string {
+  const basis =
+    evals === undefined || evals === "*"
+      ? "*"
+      : Array.isArray(evals)
+        ? JSON.stringify([...evals].sort())
+        : evals.toString();
+  return createHash("sha256").update(JSON.stringify({ basis, patterns })).digest("hex").slice(0, 16);
+}
+
+/** 所有消费者按已解析的 `selectedEvalIds` 取 eval;不持有、不调用用户谓词。 */
+export function selectedEvalsForRun(
+  all: readonly DiscoveredEval[],
+  run: Pick<AgentRun, "selectedEvalIds">,
+): DiscoveredEval[] {
+  const ids = new Set(run.selectedEvalIds);
+  return all.filter((e) => ids.has(e.id));
+}

package/src/runner/experiment-cleanup-registry.test.ts

@@ -0,0 +1,89 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// bug: memory/force-exit-skips-experiment-teardown.md
+import { describe, expect, it } from "vitest";
+import {
+  drainExperimentTeardowns,
+  pendingExperimentTeardownCount,
+  registerExperimentTeardown,
+  unregisterExperimentTeardown,
+} from "./experiment-cleanup-registry.ts";
+
+/** 模拟 runner 登记的执行体形态(见 run.ts 的 runExperimentTeardown):memoized 一次性
+ *  promise,settle 后自行注销——注册表契约的一半在登记方,测试按同一形态构造。 */
+function registerMemoizedEntry(id: string, body: () => Promise<void>): () => Promise<void> {
+  let p: Promise<void> | undefined;
+  const run = () =>
+    (p ??= body()
+      .catch(() => {})
+      .finally(() => unregisterExperimentTeardown(id)));
+  registerExperimentTeardown(id, run);
+  return run;
+}
+
+describe("experiment-cleanup-registry", () => {
+  it("drain 启动所有登记条目各一次并等到 settle;条目自注销后再次 drain 无动作", async () => {
+    const calls: string[] = [];
+    registerMemoizedEntry("exp-a", async () => {
+      calls.push("a");
+    });
+    registerMemoizedEntry("exp-b", async () => {
+      calls.push("b");
+    });
+    expect(pendingExperimentTeardownCount()).toBe(2);
+
+    expect(await drainExperimentTeardowns()).toBe(2);
+    expect(calls.sort()).toEqual(["a", "b"]);
+    expect(pendingExperimentTeardownCount()).toBe(0);
+
+    expect(await drainExperimentTeardowns()).toBe(0);
+    expect(calls).toHaveLength(2);
+  });
+
+  it("drain 与正常路径并发到达同一入口:执行恰好一次,双方都等到 settle", async () => {
+    let ran = 0;
+    let release!: () => void;
+    const gate = new Promise<void>((r) => {
+      release = r;
+    });
+    const run = registerMemoizedEntry("exp-concurrent", async () => {
+      ran += 1;
+      await gate;
+    });
+
+    let normalSettled = false;
+    let drainSettled = false;
+    const normal = run().then(() => {
+      normalSettled = true;
+    });
+    const drain = drainExperimentTeardowns().then(() => {
+      drainSettled = true;
+    });
+    // 双方都在等同一个在飞 promise:gate 未放行前谁都不 settle。
+    await Promise.resolve();
+    expect(ran).toBe(1);
+    expect(normalSettled).toBe(false);
+    expect(drainSettled).toBe(false);
+
+    release();
+    await Promise.all([normal, drain]);
+    expect(ran).toBe(1);
+    expect(pendingExperimentTeardownCount()).toBe(0);
+  });
+
+  it("settle 已注销(正常路径收尾完成)后 drain 无动作;执行体抛错不让 drain 抛", async () => {
+    let ran = 0;
+    const run = registerMemoizedEntry("exp-consumed", async () => {
+      ran += 1;
+    });
+    await run(); // 正常路径先收尾:settle 后自注销
+    expect(pendingExperimentTeardownCount()).toBe(0);
+
+    registerMemoizedEntry("exp-throws", async () => {
+      throw new Error("boom");
+    });
+
+    expect(await drainExperimentTeardowns()).toBe(1);
+    expect(ran).toBe(1);
+    expect(pendingExperimentTeardownCount()).toBe(0);
+  });
+});

package/src/runner/experiment-cleanup-registry.ts

@@ -0,0 +1,39 @@
+// 实验级 teardown 的宿主机侧登记表 + 强清兜底,与 sandbox/registry.ts 同一模式。
+//
+// 正常路径(per-attempt 计数归零 / run 收尾扫尾)在 runner 的 fiber 里执行 teardown;强清退出
+// (二次 Ctrl+C / 看门狗 / main() 崩溃路径)时 fiber 可能来不及走到那一步,cli 需要一个独立于
+// Effect 的入口把「还没 settle 的实验级 teardown」收口,否则隧道/容器这类宿主机资源随
+// process.exit 变成孤儿(契约见 docs/cli.md「中断:三级响应」)。
+//
+// 登记的闭包由 runner 构造,执行体是 memoized 的一次性 promise(见 run.ts 的
+// runExperimentTeardown):正常路径、drain、崩溃路径谁先到都启动同一个 promise,后到者等到
+// 同一个结果——不双跑、也不空转。条目在 settle 后由闭包自己注销,所以 drain 的完整语义就是
+// 「启动全部未启动 + 等待全部未 settle」;在飞中的 teardown 对 drain 同样可等待,这正是
+// 强清「事件驱动收口」的数据基础。
+
+const pending = new Map<string, () => Promise<void>>();
+
+/** 登记一个实验的 teardown 入口(实验生命周期触发时点调用;setup 尚未完成也已可达)。 */
+export function registerExperimentTeardown(experimentId: string, run: () => Promise<void>): void {
+  pending.set(experimentId, run);
+}
+
+/** teardown settle 后注销;不存在时是 no-op。 */
+export function unregisterExperimentTeardown(experimentId: string): void {
+  pending.delete(experimentId);
+}
+
+export function pendingExperimentTeardownCount(): number {
+  return pending.size;
+}
+
+/**
+ * 强清兜底:启动所有还登记着的实验级 teardown 并等待它们 settle(各执行体自己有界、自己兜错,
+ * 绝不抛;已在飞的返回同一个 memoized promise,等待而非重跑)。返回本次等待的条目数。
+ * 与 stopAllSandboxes 同语义:重复调用安全,表空时是 no-op。
+ */
+export async function drainExperimentTeardowns(): Promise<number> {
+  const entries = [...pending.values()];
+  await Promise.allSettled(entries.map((run) => run()));
+  return entries.length;
+}

package/src/runner/experiment-labels.test.ts

@@ -0,0 +1,71 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
+// 登记行:ExperimentDef.labels 值域 string | number(解析时校验),原样投影进快照
+// ExperimentRunInfo.labels;不透传 ctx / t,不参与可比性配置。
+
+import { describe, expect, it } from "vitest";
+
+import { defineExperiment } from "../define.ts";
+import { experimentRunInfo } from "./attempt.ts";
+import { comparabilityConfigOf, deepEqualJson } from "../results/select.ts";
+import type { Snapshot } from "../results/types.ts";
+import type { Agent } from "../agents/types.ts";
+import type { AgentRun } from "./types.ts";
+
+const fakeAgent = { name: "fake" } as unknown as Agent;
+
+function runWith(labels?: Record<string, string | number>): AgentRun {
+  return {
+    agent: fakeAgent,
+    flags: {},
+    runs: 1,
+    earlyExit: true,
+    selectedEvalIds: [],
+    ...(labels !== undefined ? { labels } : {}),
+  };
+}
+
+describe("ExperimentDef.labels", () => {
+  it("值域 string | number:合法值通过,布尔 / 对象在解析时报错", () => {
+    expect(() =>
+      defineExperiment({ agent: fakeAgent, labels: { line: "codex", contextK: 32 } }),
+    ).not.toThrow();
+    expect(() =>
+      defineExperiment({ agent: fakeAgent, labels: { on: true as unknown as string } }),
+    ).toThrow(/labels\.on/);
+    expect(() =>
+      defineExperiment({ agent: fakeAgent, labels: { nested: { a: 1 } as unknown as string } }),
+    ).toThrow(/labels\.nested/);
+    expect(() =>
+      defineExperiment({ agent: fakeAgent, labels: { nan: Number.NaN } }),
+    ).toThrow(/labels\.nan/);
+  });
+
+  it("原样投影进 ExperimentRunInfo.labels;未声明时字段缺省", () => {
+    const labels = { line: "codex", memory: "mempal" };
+    expect(experimentRunInfo(runWith(labels))?.labels).toEqual(labels);
+    expect(experimentRunInfo(runWith())?.labels).toBeUndefined();
+    // 空对象不落盘(与 flags 同一态度:不写空壳字段)
+    expect(experimentRunInfo(runWith({}))?.labels).toBeUndefined();
+  });
+
+  it("不参与可比性配置:仅 labels 不同的两快照仍互相可比", () => {
+    const snapshotWith = (labels?: Record<string, string | number>): Snapshot =>
+      ({
+        experimentId: "mem/codex",
+        agent: "codex",
+        model: "gpt-5.4",
+        experiment: {
+          runs: 1,
+          earlyExit: true,
+          selectedEvalIds: [],
+          flags: { web: true },
+          ...(labels !== undefined ? { labels } : {}),
+        },
+      }) as unknown as Snapshot;
+    const a = comparabilityConfigOf(snapshotWith({ line: "codex" }));
+    const b = comparabilityConfigOf(snapshotWith({ line: "renamed", memory: "mempal" }));
+    const c = comparabilityConfigOf(snapshotWith());
+    expect(deepEqualJson(a, b)).toBe(true);
+    expect(deepEqualJson(a, c)).toBe(true);
+  });
+});

package/src/runner/feedback/agent.test.ts

@@ -406,7 +406,7 @@ describe("最终 stdout handoff:status/summary/快照/最多 5 个失败/show
     expect(text).toContain("failures: 12 total, showing 5");
     for (let i = 0; i < 5; i++) expect(text).toContain(`memory/eval-${i}`);
     expect(text).not.toContain("memory/eval-5 ");
-    expect(text).toContain("… 7 more; inspect the JSON result or run `niceeval view compare`");
+    expect(text).toContain("… 7 more; inspect the JSON result or run `niceeval view`");
   });
 
   it("快照路径超过上限时折叠成 '… N more'", async () => {
@@ -502,3 +502,35 @@ describe("`--dry --output agent`:不经 coordinator,直接渲染 PLAN envelope",
     expect(text.split("\n")).toHaveLength(3);
   });
 });
+
+// ───────────────────────── 实验级钩子起止行(cases.md「实验级生命周期」) ─────────────────────────
+
+describe("agent: 实验级钩子起止各追加一行", () => {
+  it("experiment_setup / experiment_teardown 各含 experiment 与 status 字段,done/failed 带 duration", async () => {
+    const { fake, coordinator } = setup();
+    coordinator.start(plan({ totalRuns: 2, evals: 2 }));
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "started" });
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "setup", status: "done", durationMs: 42_000 });
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "teardown", status: "started" });
+    coordinator.experimentHook({ experimentId: "compare/bub-e2b", hook: "teardown", status: "failed", durationMs: 3_000 });
+    await flush();
+    const lines = fake.stderr.writes.join("").split("\n");
+    expect(lines).toContain("NICEEVAL experiment_setup experiment=compare/bub-e2b status=started");
+    expect(lines).toContain("NICEEVAL experiment_setup experiment=compare/bub-e2b status=done duration=42s");
+    expect(lines).toContain("NICEEVAL experiment_teardown experiment=compare/bub-e2b status=started");
+    expect(lines).toContain("NICEEVAL experiment_teardown experiment=compare/bub-e2b status=failed duration=3s");
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+
+  it("实验级 progress 与 activity 在 agent 下零输出(短命状态不进 checkpoint 流)", async () => {
+    const { fake, coordinator } = setup();
+    coordinator.start(plan());
+    await flush(); // 先排空 start 事件的队列任务,再取零输出基线
+    const before = fake.stderr.writes.length;
+    coordinator.experimentProgress({ experimentId: "compare/bub-e2b", detail: "starting tunnel" });
+    coordinator.activity("prechecking judge config...");
+    await flush();
+    expect(fake.stderr.writes.slice(before).join("")).toBe("");
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+});

package/src/runner/feedback/agent.ts

@@ -126,6 +126,17 @@ export function createAgentRenderer(options: AgentRendererOptions): FeedbackRend
           return;
         }
 
+        case "experiment-hook": {
+          // 实验级钩子起止各一行(见 cli.md「实验级钩子的显示」):agent 没有动态区域,
+          // 长 setup 期间只有 heartbeat 的日志无法区分「钩子在跑」和「挂死」。
+          noteCheckpoint(event.at);
+          const word = event.hook === "setup" ? "experiment_setup" : "experiment_teardown";
+          const parts = [`NICEEVAL ${word}`, kv("experiment", event.experimentId), kv("status", event.status)];
+          if (event.durationMs !== undefined) parts.push(kv("duration", formatElapsedSeconds(event.durationMs)));
+          io.stderr.write(parts.join(" ") + "\n");
+          return;
+        }
+
         case "interrupted": {
           noteCheckpoint(event.at);
           if (!isFirstOccurrence(state, "interrupted")) return;
@@ -287,19 +298,6 @@ function summaryLine(summary: RunSummary, completion: RunCompletion, reused: num
   return `summary: ${summary.passed} passed, ${summary.failed} failed, ${summary.errored} errored (${paren.join(", ")})`;
 }
 
-/** 快照路径的共同「组」段(如 `.niceeval/compare/bub-e2b/<snapshot>` → `compare`),用于
- *  over-limit 提示里的 `niceeval view <group>`。多个 experiment 组混在一次 invocation 里时
- *  组名不唯一,不猜一个可能错的 view 目标。与 human.ts 里同名逻辑刻意分开实现(见文件顶部
- *  注释),不是共享同一个函数。 */
-function deriveResultGroup(paths: readonly string[]): string | undefined {
-  const groups = new Set<string>();
-  for (const p of paths) {
-    const seg = p.split("/")[1];
-    if (seg) groups.add(seg);
-  }
-  return groups.size === 1 ? [...groups][0] : undefined;
-}
-
 function writeHandoff(
   io: FeedbackIO,
   pending: { summary: RunSummary; completion: RunCompletion; reused: number } | undefined,
@@ -323,11 +321,9 @@ function writeHandoff(
   if (failures.length > 0) {
     const shown = failures.slice(0, AGENT_FAILURE_CAP);
     if (failures.length > AGENT_FAILURE_CAP) {
-      const group = deriveResultGroup(paths);
-      const hint = group ? `run \`niceeval view ${group}\`` : "check the results directory";
       lines.push(`failures: ${failures.length} total, showing ${AGENT_FAILURE_CAP}`);
       for (const f of shown) lines.push(...handoffFailureLines(f));
-      lines.push(`  … ${failures.length - AGENT_FAILURE_CAP} more; inspect the JSON result or ${hint}`);
+      lines.push(`  … ${failures.length - AGENT_FAILURE_CAP} more; inspect the JSON result or run \`niceeval view\``);
     } else {
       lines.push("failures:");
       for (const f of shown) lines.push(...handoffFailureLines(f));

package/src/runner/feedback/ci.test.ts

@@ -197,7 +197,7 @@ describe("start 立即追加;仅连续 60 秒无永久事件才 heartbeat;failur
       "niceeval: result=failed passed=23 failed=1 errored=0 reused=18 duration=128s",
       "niceeval: json=.niceeval/ci-summary.json",
       "niceeval: junit=.niceeval/junit.xml",
-      "niceeval: snapshots=.niceeval/ci/<3 snapshots>",
+      "niceeval: snapshots=<3 snapshots>",
     ]);
     expect(fake.stderr.writes.join("")).toBe("");
   });
@@ -560,3 +560,37 @@ describe("集成测试覆盖 cli.md「CI 常见 case」的三条命令", () => {
     expect(computeCiExitCode(summary({ passed: 36, failed: 0, errored: 0 }), runCompletion)).toBe(1);
   });
 });
+
+// ───────────────────────── 实验级钩子起止行(cases.md「实验级生命周期」) ─────────────────────────
+
+describe("ci: 实验级钩子起止各追加一行", () => {
+  it("experiment_setup / experiment_teardown 各含 experiment 与 status 字段,done/failed 带 duration", async () => {
+    const { fake, coordinator } = setup();
+    coordinator.start(plan({ totalRuns: 2, evals: 2 }));
+    coordinator.experimentHook({ experimentId: "ci/claude", hook: "setup", status: "started" });
+    coordinator.experimentHook({ experimentId: "ci/claude", hook: "setup", status: "done", durationMs: 42_000 });
+    coordinator.experimentHook({ experimentId: "ci/claude", hook: "teardown", status: "started" });
+    coordinator.experimentHook({ experimentId: "ci/claude", hook: "teardown", status: "failed", durationMs: 3_000 });
+    await flush();
+    const lines = fake.stdout.writes.join("").split("\n");
+    expect(lines).toContain("niceeval: experiment_setup experiment=ci/claude status=started");
+    expect(lines).toContain("niceeval: experiment_setup experiment=ci/claude status=done duration=42s");
+    expect(lines).toContain("niceeval: experiment_teardown experiment=ci/claude status=started");
+    expect(lines).toContain("niceeval: experiment_teardown experiment=ci/claude status=failed duration=3s");
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+
+  it("实验级 progress 与 activity 在 ci 下零输出(短命状态不进事件流)", async () => {
+    const { fake, coordinator } = setup();
+    coordinator.start(plan());
+    await flush(); // 先让 start 行落盘,再取零输出基线
+    const beforeOut = fake.stdout.writes.length;
+    const beforeErr = fake.stderr.writes.length;
+    coordinator.experimentProgress({ experimentId: "ci/claude", detail: "starting tunnel" });
+    coordinator.activity("prechecking judge config...");
+    await flush();
+    expect(fake.stdout.writes.slice(beforeOut).join("")).toBe("");
+    expect(fake.stderr.writes.slice(beforeErr).join("")).toBe("");
+    await coordinator.finish({ summary: summary(), completion: completion(), paths: [] });
+  });
+});

package/src/runner/feedback/ci.ts

@@ -137,6 +137,17 @@ export function createCiRenderer(options: CiRendererOptions): FeedbackRenderer {
           return;
         }
 
+        case "experiment-hook": {
+          // 实验级钩子起止各一行(见 cli.md「实验级钩子的显示」):CI 日志没有动态区域,
+          // 长 setup 期间只有 heartbeat 无法区分「钩子在跑」和「挂死」。
+          noteCheckpoint(event.at);
+          const word = event.hook === "setup" ? "experiment_setup" : "experiment_teardown";
+          const parts = [`niceeval: ${word}`, kv("experiment", event.experimentId), kv("status", event.status)];
+          if (event.durationMs !== undefined) parts.push(kv("duration", formatCiSeconds(event.durationMs)));
+          io.stdout.write(parts.join(" ") + "\n");
+          return;
+        }
+
         case "interrupted": {
           noteCheckpoint(event.at);
           if (!isFirstOccurrence(state, "interrupted")) return;
@@ -299,26 +310,13 @@ function resultLine(summary: RunSummary, completion: RunCompletion, reused: numb
   return parts.join(" ");
 }
 
-/** 快照路径的共同「组」段(如 `.niceeval/ci/bub/<snapshot>` → `ci`),用于折叠成
- *  `.niceeval/<group>/<N snapshots>`。多个 experiment 组混在一次 invocation 里时组名不唯一,
- *  不猜一个可能错的前缀,退化成不带路径前缀的 `<N snapshots>`——与 human.ts/agent.ts 同名逻辑
- *  刻意分开实现(见文件顶部注释),不是共享同一个函数。 */
-function deriveResultGroup(paths: readonly string[]): string | undefined {
-  const groups = new Set<string>();
-  for (const p of paths) {
-    const seg = p.split("/")[1];
-    if (seg) groups.add(seg);
-  }
-  return groups.size === 1 ? [...groups][0] : undefined;
-}
-
-/** 单条路径直接原样打印;多条折叠成 `<N snapshots>`(见 cli.md「CI 怎么用」的字面例子:
- *  `.niceeval/ci/<3 snapshots>`)——CI「同一种事件一行」的整体约束(checklist/docs 都强调
- *  一行一事件,方便日志搜索)决定了这里不能像 human/agent 那样逐条缩进列出。 */
+/** 单条路径直接原样打印;多条折叠成不带路径前缀的 `<N snapshots>`(见 cli.md「CI 怎么用」的
+ *  字面例子)——不再从路径推导「共同目录段」去拼前缀,那是运行选择的展示,不是报告分组,
+ *  也不该由这里悄悄猜一个可能跨实验误配的前缀。CI「同一种事件一行」的整体约束(checklist/docs
+ *  都强调一行一事件,方便日志搜索)决定了这里不能像 human/agent 那样逐条缩进列出。 */
 function snapshotsValue(paths: readonly string[]): string {
   if (paths.length === 1) return paths[0]!;
-  const group = deriveResultGroup(paths);
-  return group ? `.niceeval/${group}/<${paths.length} snapshots>` : `<${paths.length} snapshots>`;
+  return `<${paths.length} snapshots>`;
 }
 
 function writeResultBlock(

package/src/runner/feedback/coordinator.ts

@@ -25,6 +25,8 @@ import {
   activateFeedbackSink,
   type BudgetExhaustedInput,
   type DiagnosticInput,
+  type ExperimentHookInput,
+  type ExperimentProgressInput,
   type FailureInput,
   type FeedbackSink,
   type KeptInput,
@@ -144,6 +146,10 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       case "attempt:early-exit":
         queue.push(() => renderer.onLifecycle?.(event, snapshot));
         return;
+      case "experiment:progress":
+        // 短命状态,只进 reducer(上面已更新 state.experimentHooks 的 detail);下一次 tick 的
+        // 重画会读到新值,不为每条 progress 单独排一次渲染任务。
+        return;
       case "tick":
         queue.push(() => renderer.onTick?.(event, snapshot));
         return;
@@ -242,6 +248,21 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     });
   }
 
+  function experimentHook(input: ExperimentHookInput): void {
+    emit({
+      type: "experiment-hook",
+      at: io.clock.now(),
+      experimentId: input.experimentId,
+      hook: input.hook,
+      status: input.status,
+      ...(input.durationMs !== undefined ? { durationMs: input.durationMs } : {}),
+    });
+  }
+
+  function experimentProgress(input: ExperimentProgressInput): void {
+    emit({ type: "experiment:progress", at: io.clock.now(), experimentId: input.experimentId, detail: input.detail });
+  }
+
   function reporterError(input: { reporter: string; required: boolean; message: string }): void {
     emit({
       type: "reporter-error",
@@ -280,6 +301,8 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
       failure,
       budgetExhausted,
       kept,
+      experimentHook,
+      experimentProgress,
       lifecycle,
     });
     emit({ type: "plan", at: startedAtMs, plan });
@@ -343,6 +366,8 @@ export function createFeedbackCoordinator(options: FeedbackCoordinatorOptions):
     failure,
     budgetExhausted,
     kept,
+    experimentHook,
+    experimentProgress,
     lifecycle,
     stopDynamic,
     finish,
@@ -367,6 +392,9 @@ function fallbackTextFor(event: DurableFeedbackEvent): string | undefined {
     case "plan":
     case "summary":
     case "saved":
+    case "experiment-hook":
+      // 钩子起止不是失败证据:setup 失败的每条 attempt 另有 "failure" 事件兜底,
+      // renderer 崩溃丢一行起止不丢数据。
       return undefined;
   }
 }