niceeval 0.9.0 → 0.10.0

package/src/runner/run.test.ts

@@ -12,7 +12,13 @@ import { encodeAttemptLocator } from "../results/locator.ts";
 import { equals } from "../expect/index.ts";
 import { createFeedbackCoordinator, type FeedbackCoordinator } from "./feedback/coordinator.ts";
 import { createFakeFeedbackIO } from "./feedback/testing.ts";
-import { activeFeedbackSinkCount } from "./feedback/sink.ts";
+import {
+  activateFeedbackSink,
+  activeFeedbackSinkCount,
+  type ExperimentHookInput,
+  type ExperimentProgressInput,
+} from "./feedback/sink.ts";
+import { drainExperimentTeardowns, pendingExperimentTeardownCount } from "./experiment-cleanup-registry.ts";
 import type { CapturedEvalSource } from "./eval-source.ts";
 import type { CarryPlan } from "./fingerprint.ts";
 import type { AgentRun, RunFeedbackPlan, RunOptions } from "./types.ts";
@@ -251,7 +257,7 @@ describe("runEvals · fresh EvalResult.locator 在 reporter 观察到之前已
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["ok", "boom"],
       experimentId,
     };
 
@@ -281,7 +287,7 @@ describe("runEvals · fresh EvalResult.locator 在 reporter 观察到之前已
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["algebra/q1"],
       experimentId,
     });
 
@@ -310,7 +316,7 @@ describe("runEvals · fresh EvalResult.locator 在 reporter 观察到之前已
       earlyExit: false, // 两次都要真的跑,不能被首过即停吞掉其中一次
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["flaky"],
       experimentId,
     };
 
@@ -358,7 +364,7 @@ describe("runEvals · fresh EvalResult.locator 在 reporter 观察到之前已
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: [evalId],
       experimentId,
     };
 
@@ -410,7 +416,7 @@ describe("runEvals · fresh EvalResult.locator 在 reporter 观察到之前已
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["c-slow", "c-mid", "c-fast"],
       experimentId,
     };
     const completionOrder: string[] = [];
@@ -487,7 +493,7 @@ describe("runEvals · failure 永久事件在真实失败/errored attempt 上被
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["ok", "boom", "gate-fail"],
       experimentId,
     };
     const plan: RunFeedbackPlan = {
@@ -555,7 +561,7 @@ describe("runEvals · budget-exhausted 永久事件按每个被跳过的 attempt
       earlyExit: false,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["a", "b", "c"],
       experimentId,
       budget: 0, // 花费从 0 起算,>= budget 恒成立——每个 attempt 在 preflight 就被跳过。
     };
@@ -604,7 +610,7 @@ describe("runEvals · budget-unenforceable 只统计真正发起过 agent turn
       earlyExit: false,
       sandbox: missingTemplate,
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["a", "b", "c"],
       experimentId,
       budget: 10,
     };
@@ -637,7 +643,7 @@ describe("runEvals · budget-unenforceable 只统计真正发起过 agent turn
       earlyExit: false,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: ["a", "b", "c"],
       experimentId,
       budget: 10,
     };
@@ -703,7 +709,7 @@ describe("runEvals · 携入数量少于本次请求的 runs 时,差额必须真
       earlyExit: true,
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: [evalId],
       experimentId,
     };
     const plan: RunFeedbackPlan = {
@@ -767,7 +773,7 @@ describe("runEvals · 携入数量少于本次请求的 runs 时,差额必须真
       earlyExit: true, // failed 不触发 earlyExit(只有 passed/errored 会),回填的两次应该真的跑
       sandbox: fakeSandboxSpec(),
       timeoutMs: 5_000,
-      evalFilter: () => true,
+      selectedEvalIds: [evalId],
       experimentId,
     };
     const plan: RunFeedbackPlan = {
@@ -805,3 +811,375 @@ describe("runEvals · 携入数量少于本次请求的 runs 时,差额必须真
     });
   });
 });
+
+// ───────────────────────── 实验级生命周期(ExperimentDef.setup / .teardown) ─────────────────────────
+// 契约见 docs/feature/experiments/architecture.md「实验级生命周期」与 docs/runner.md「环境预置不进
+// 运行器,但按顺序调它」:成对 setup/teardown,setup 不返回值——teardown 是独立字段,当且仅当
+// 同层 setup 时点走到过才触发(setup 抛错不豁免、未声明 setup 不影响触发、时点没走到则跳过);
+// setup 抛错 → 本实验所有 attempt 逐条合成 errored;teardown 抛错只作运行级诊断。
+
+describe("runEvals · 实验级 setup/teardown", () => {
+  function runWithHooks(
+    experimentId: string,
+    setup: AgentRun["setup"],
+    teardown: AgentRun["teardown"],
+    overrides: Partial<AgentRun> = {},
+  ): AgentRun {
+    return {
+      agent: makeAgent(`agent-${experimentId}`),
+      flags: {},
+      runs: 1,
+      earlyExit: true,
+      sandbox: fakeSandboxSpec(),
+      timeoutMs: 5_000,
+      selectedEvalIds: [],
+      experimentId,
+      setup,
+      teardown,
+      ...overrides,
+    };
+  }
+
+  it("setup 整场恰好一次:并发 attempt 共享 memoized 结果,teardown 在全部 attempt 收尾后恰好一次", async () => {
+    let setupCalls = 0;
+    let teardownCalls = 0;
+    let completedAtTeardown = -1;
+    let completed = 0;
+    const evals = ["a", "b", "c"].map((id) =>
+      makeEval(id, () => {
+        completed += 1;
+      }),
+    );
+    const agentRun = runWithHooks(
+      "lifecycle-exp",
+      async () => {
+        setupCalls += 1;
+        // 给并发的其它 attempt 一个真实的等待窗口,验证它们不各自重跑 setup
+        await new Promise((r) => setTimeout(r, 20));
+      },
+      () => {
+        teardownCalls += 1;
+        completedAtTeardown = completed;
+      },
+      { runs: 2, selectedEvalIds: ["a", "b", "c"] },
+    );
+
+    const { summary } = await run(evals, [agentRun], { maxConcurrency: 4 });
+
+    expect(setupCalls).toBe(1);
+    expect(teardownCalls).toBe(1);
+    expect(summary.results).toHaveLength(6);
+    expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
+    // teardown 必须晚于本实验全部 attempt 的执行(runs:2 但 earlyExit 会省略第二轮,
+    // 至少 3 条 eval 各完成一次)
+    expect(completedAtTeardown).toBeGreaterThanOrEqual(3);
+  });
+
+  it("两个实验各自的 setup/teardown 各跑一次,互不共享", async () => {
+    const calls: string[] = [];
+    const evals = [makeEval("shared", () => {})];
+    const mk = (id: string) =>
+      runWithHooks(
+        id,
+        () => {
+          calls.push(`setup:${id}`);
+        },
+        () => {
+          calls.push(`teardown:${id}`);
+        },
+        { selectedEvalIds: ["shared"] },
+      );
+
+    await run(evals, [mk("exp-a"), mk("exp-b")]);
+
+    expect(calls.filter((c) => c === "setup:exp-a")).toHaveLength(1);
+    expect(calls.filter((c) => c === "setup:exp-b")).toHaveLength(1);
+    expect(calls.filter((c) => c === "teardown:exp-a")).toHaveLength(1);
+    expect(calls.filter((c) => c === "teardown:exp-b")).toHaveLength(1);
+  });
+
+  it("全部结果被 carry 携入、无 attempt 派发时 setup 与 teardown 都不执行", async () => {
+    let setupCalls = 0;
+    let teardownCalls = 0;
+    const experimentId = "carry-exp";
+    const evalDef = makeEval("done", () => {});
+    const agentRun = runWithHooks(
+      experimentId,
+      () => {
+        setupCalls += 1;
+      },
+      () => {
+        teardownCalls += 1;
+      },
+      { selectedEvalIds: ["done"] },
+    );
+    const carried: EvalResult = {
+      id: "done",
+      experimentId,
+      agent: agentRun.agent.name,
+      verdict: "passed",
+      attempt: 0,
+      durationMs: 1,
+      assertions: [],
+    };
+
+    const { summary } = await run([evalDef], [agentRun], {
+      carryPlan: {
+        plannedFingerprints: new Map(),
+        priorRunKeys: new Set([`${experimentId}|done`]),
+        carriedResults: [carried],
+      },
+    });
+
+    expect(setupCalls).toBe(0);
+    expect(teardownCalls).toBe(0);
+    expect(summary.results).toHaveLength(1);
+    expect(summary.results[0]!.verdict).toBe("passed");
+  });
+
+  it("setup 抛错:本实验所有 attempt 合成 errored(code/phase 结构化),同批其它实验不受影响", async () => {
+    const evals = [makeEval("m1", () => {}), makeEval("m2", () => {})];
+    const broken = runWithHooks(
+      "broken-exp",
+      () => {
+        throw new Error("tunnel refused to start");
+      },
+      undefined,
+      { runs: 2, earlyExit: false, selectedEvalIds: ["m1", "m2"] },
+    );
+    const healthy = runWithHooks("healthy-exp", () => {}, undefined, { selectedEvalIds: ["m1", "m2"] });
+
+    const { summary } = await run(evals, [broken, healthy], { maxConcurrency: 4 });
+
+    const brokenResults = summary.results.filter((r) => r.experimentId === "broken-exp");
+    // 2 eval × runs 2 全部逐条 errored——setup 失败不派发 agent、零成本,不被 fail-fast 截短
+    expect(brokenResults).toHaveLength(4);
+    for (const r of brokenResults) {
+      expect(r.verdict).toBe("errored");
+      expect(r.error).toMatchObject({ code: "experiment-setup-failed", phase: "experiment.setup" });
+      expect(r.error?.message).toContain("tunnel refused to start");
+      expect(r.locator).toBeDefined();
+    }
+    const healthyResults = summary.results.filter((r) => r.experimentId === "healthy-exp");
+    expect(healthyResults).toHaveLength(2);
+    expect(healthyResults.every((r) => r.verdict === "passed")).toBe(true);
+  });
+
+  it("setup 抛错后 teardown 仍执行:半初始化现场同样要扫尾,setup 抛错不豁免", async () => {
+    let teardownCalls = 0;
+    const evals = [makeEval("m1", () => {}), makeEval("m2", () => {})];
+    const broken = runWithHooks(
+      "broken-with-teardown-exp",
+      () => {
+        throw new Error("tunnel refused to start");
+      },
+      () => {
+        teardownCalls += 1;
+      },
+      { runs: 2, earlyExit: false, selectedEvalIds: ["m1", "m2"] },
+    );
+
+    const { summary } = await run(evals, [broken], { maxConcurrency: 4 });
+
+    expect(teardownCalls).toBe(1);
+    expect(summary.results).toHaveLength(4);
+    expect(summary.results.every((r) => r.verdict === "errored")).toBe(true);
+  });
+
+  it("运行被中断(signal abort)时 teardown 仍执行", async () => {
+    let teardownCalls = 0;
+    const controller = new AbortController();
+    const evalDef = makeEval("abort-me", async () => {
+      controller.abort();
+      await new Promise((r) => setTimeout(r, 100));
+    });
+    const agentRun = runWithHooks(
+      "interrupted-exp",
+      () => {},
+      () => {
+        teardownCalls += 1;
+      },
+      { selectedEvalIds: ["abort-me"] },
+    );
+
+    await run([evalDef], [agentRun], { signal: controller.signal });
+
+    expect(teardownCalls).toBe(1);
+  });
+
+  it("ctx 携带 experimentId / selectedEvalIds / signal;未声明 teardown 时无收尾动作、也不产生诊断", async () => {
+    let seen: { experimentId: string; selectedEvalIds: readonly string[]; hasSignal: boolean } | undefined;
+    const controller = new AbortController();
+    const evals = [makeEval("ctx-a", () => {}), makeEval("ctx-b", () => {})];
+    const experimentId = "ctx-exp";
+    const agentRun = runWithHooks(
+      experimentId,
+      (ctx) => {
+        seen = {
+          experimentId: ctx.experimentId,
+          selectedEvalIds: ctx.selectedEvalIds,
+          hasSignal: ctx.signal !== undefined,
+        };
+        ctx.progress({ message: "warming" });
+      },
+      undefined,
+      { selectedEvalIds: ["ctx-a", "ctx-b"] },
+    );
+    const plan: RunFeedbackPlan = {
+      shape: { evals: 2, configs: 1, totalRuns: 2, maxConcurrency: 3 },
+      reused: 0,
+      reusedFailures: [],
+    };
+
+    await withCoordinator(plan, async (coordinator) => {
+      const { summary } = await run(evals, [agentRun], { signal: controller.signal });
+
+      expect(seen).toEqual({ experimentId: "ctx-exp", selectedEvalIds: ["ctx-a", "ctx-b"], hasSignal: true });
+      expect(summary.results.every((r) => r.verdict === "passed")).toBe(true);
+      expect(coordinator.state.diagnostics.find((d) => d.key === `experiment-teardown-failed:${experimentId}`)).toBeUndefined();
+    });
+  });
+
+  it("钩子起止经 feedback sink 发布:成功 started+done、抛错 started+failed(done/failed 带耗时),progress 压成运行级 detail", async () => {
+    const hookEvents: ExperimentHookInput[] = [];
+    const progressEvents: ExperimentProgressInput[] = [];
+    const deactivate = activateFeedbackSink({
+      activity() {},
+      diagnostic() {},
+      interrupted() {},
+      reporterError() {},
+      failure() {},
+      budgetExhausted() {},
+      kept() {},
+      experimentHook(input) {
+        hookEvents.push(input);
+      },
+      experimentProgress(input) {
+        progressEvents.push(input);
+      },
+      lifecycle() {},
+    });
+    try {
+      const evals = [makeEval("ok", () => {})];
+      const good = runWithHooks(
+        "good-exp",
+        (ctx) => {
+          ctx.progress({ message: "starting tunnel", current: 2, total: 5 });
+        },
+        () => {},
+        { selectedEvalIds: ["ok"] },
+      );
+      const bad = runWithHooks(
+        "bad-exp",
+        () => {
+          throw new Error("boom");
+        },
+        undefined,
+        { selectedEvalIds: ["ok"] },
+      );
+      await run(evals, [good, bad], { maxConcurrency: 2 });
+    } finally {
+      deactivate();
+    }
+
+    const good = hookEvents.filter((e) => e.experimentId === "good-exp");
+    expect(good.map((e) => `${e.hook}:${e.status}`)).toEqual([
+      "setup:started",
+      "setup:done",
+      "teardown:started",
+      "teardown:done",
+    ]);
+    expect(good[0]!.durationMs).toBeUndefined();
+    expect(good[1]!.durationMs).toBeGreaterThanOrEqual(0);
+    const bad = hookEvents.filter((e) => e.experimentId === "bad-exp").map((e) => `${e.hook}:${e.status}`);
+    expect(bad).toEqual(["setup:started", "setup:failed"]);
+    expect(progressEvents).toContainEqual({ experimentId: "good-exp", detail: "starting tunnel (2/5)" });
+  });
+});
+
+describe("runEvals · 实验级 teardown 失败只作运行级诊断", () => {
+  afterEach(() => {
+    expect(activeFeedbackSinkCount()).toBe(0);
+  });
+
+  // bug: memory/force-exit-skips-experiment-teardown.md
+  it("正常完整跑完后强清兜底注册表为空:teardown 已被运行路径消费恰好一次,drain 无动作", async () => {
+    let teardownCalls = 0;
+    const evalDef = makeEval("tidy", () => {});
+    const agentRun: AgentRun = {
+      agent: makeAgent("agent-registry"),
+      flags: {},
+      runs: 1,
+      earlyExit: true,
+      sandbox: fakeSandboxSpec(),
+      timeoutMs: 5_000,
+      selectedEvalIds: ["tidy"],
+      experimentId: "registry-exp",
+      setup: () => {},
+      teardown: () => {
+        teardownCalls += 1;
+      },
+    };
+
+    await run([evalDef], [agentRun]);
+
+    expect(teardownCalls).toBe(1);
+    expect(pendingExperimentTeardownCount()).toBe(0);
+    expect(await drainExperimentTeardowns()).toBe(0);
+    expect(teardownCalls).toBe(1);
+  });
+
+  it("teardown 抛错:verdict 不变,产生 experiment-teardown-failed 诊断", async () => {
+    const experimentId = "leaky-exp";
+    const evalDef = makeEval("ok", () => {});
+    const agentRun: AgentRun = {
+      agent: makeAgent("agent-leaky"),
+      flags: {},
+      runs: 1,
+      earlyExit: true,
+      sandbox: fakeSandboxSpec(),
+      timeoutMs: 5_000,
+      selectedEvalIds: ["ok"],
+      experimentId,
+      setup: () => {},
+      teardown: () => {
+        throw new Error("port already released");
+      },
+    };
+    const plan: RunFeedbackPlan = {
+      shape: { evals: 1, configs: 1, totalRuns: 1, maxConcurrency: 3 },
+      reused: 0,
+      reusedFailures: [],
+    };
+
+    await withCoordinator(plan, async (coordinator) => {
+      const { summary } = await run([evalDef], [agentRun]);
+
+      expect(summary.results).toHaveLength(1);
+      expect(summary.results[0]!.verdict).toBe("passed");
+      const diag = coordinator.state.diagnostics.find((d) => d.key === `experiment-teardown-failed:${experimentId}`);
+      expect(diag).toBeDefined();
+      expect(diag!.severity).toBe("warning");
+      expect(diag!.message).toContain("port already released");
+    });
+  });
+});
+
+describe("computeFingerprint · 实验级钩子不进 fingerprint", () => {
+  it("只改 setup / teardown 函数体不改变 fingerprint(改钩子要重跑用 --force,与 sandbox 钩子同规则)", async () => {
+    const evalDef = makeEval("fp", () => {});
+    const base: AgentRun = {
+      agent: makeAgent("agent-fp"),
+      flags: {},
+      runs: 1,
+      earlyExit: true,
+      timeoutMs: 5_000,
+      selectedEvalIds: ["fp"],
+      experimentId: "fp-exp",
+    };
+    const withHook: AgentRun = { ...base, setup: () => {}, teardown: () => {} };
+
+    const { computeFingerprint } = await import("./fingerprint.ts");
+    expect(await computeFingerprint(evalDef, withHook)).toBe(await computeFingerprint(evalDef, base));
+  });
+});