niceeval 0.6.1 → 0.7.0

package/src/runner/attempt.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
 // 路径提升单测:agent.setup 写进沙箱 `__niceeval__/agent-setup.json` 的安装 manifest,
 // runAttemptEffect 在 setup 之后把它读出来、原样挂到 EvalResult.agentSetup(见
 // docs/feature/results/architecture.md「agent-setup.json」、src/agents/manifest.ts 的注释)。
@@ -11,7 +12,7 @@ import { runAttemptEffect } from "./attempt.ts";
 import { defineSandboxAgent, defineSandbox } from "../define.ts";
 import { writeAgentSetupManifest, AGENT_SETUP_MANIFEST_PATH } from "../agents/manifest.ts";
 import type { CapturedEvalSource } from "./eval-source.ts";
-import type { Attempt, AgentRun, RunOptions } from "./types.ts";
+import type { Attempt, AgentRun, LifecyclePhase, RunOptions } from "./types.ts";
 import type {
   AgentSetupManifest,
   Agent,
@@ -29,6 +30,8 @@ class FakeSandbox implements Partial<Sandbox> {
   readonly otlpHost = null;
   readonly files = new Map<string, string>();
 
+  constructor(private readonly stopDelayMs = 0) {}
+
   async runShell(): Promise<CommandResult> {
     return { stdout: "", stderr: "", exitCode: 0 };
   }
@@ -63,21 +66,31 @@ class FakeSandbox implements Partial<Sandbox> {
   async readSourceFiles(): Promise<never> {
     throw new Error("not implemented");
   }
-  async stop(): Promise<void> {}
+  async stop(): Promise<void> {
+    if (this.stopDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, this.stopDelayMs));
+  }
 }
 
 const asSandbox = (box: FakeSandbox): Sandbox => box as unknown as Sandbox;
 
 const source: CapturedEvalSource = { path: "fake.eval.ts", content: "", sha256: "0".repeat(64) };
 
-/** 跑一次 attempt:给定 agent,返回 EvalResult。沙箱用内存 fake,不起容器/不联网。 */
-async function runOnce(agent: Agent, box: FakeSandbox): Promise<import("../types.ts").EvalResult> {
+/** 跑一次 attempt:给定 agent,返回 EvalResult。沙箱用内存 fake,不起容器/不联网。
+ *  可选 `evalDefOverrides` 覆盖默认 evalDef 的字段(如挂一个 `setup`);可选 `onPhase` 透传给
+ *  `runAttemptEffect` 的第五个参数,原样转发 attempt.ts 的 enterPhase 边界(见下方
+ *  onPhase 回调专用的 describe 块)。 */
+async function runOnce(
+  agent: Agent,
+  box: FakeSandbox,
+  opts: { evalDefOverrides?: Partial<DiscoveredEval>; onPhase?: (phase: LifecyclePhase) => void } = {},
+): Promise<import("../types.ts").EvalResult> {
   const evalDef: DiscoveredEval = {
     id: "fake/eval",
     baseDir: "/project",
     sourcePath: "/project/fake.eval.ts",
     source,
     test: () => {},
+    ...opts.evalDefOverrides,
   };
   const run: AgentRun = {
     agent,
@@ -91,16 +104,15 @@ async function runOnce(agent: Agent, box: FakeSandbox): Promise<import("../types
   };
   const attempt: Attempt = { evalDef, run, attempt: 0, key: "fake/eval", fingerprint: "" };
   const config: Config = {};
-  const opts: RunOptions = {
+  const runOpts: RunOptions = {
     config,
     evals: [evalDef],
     agentRuns: [run],
     reporters: [],
     maxConcurrency: 1,
-    onProgress: () => {}, // 静音进度日志,不污染测试输出
   };
   const sandboxSem = Effect.runSync(Effect.makeSemaphore(1));
-  return Effect.runPromise(runAttemptEffect(attempt, opts, sandboxSem));
+  return Effect.runPromise(runAttemptEffect(attempt, runOpts, sandboxSem, undefined, opts.onPhase));
 }
 
 describe("runAttemptEffect · agent-setup 路径提升(沙箱 __niceeval__/agent-setup.json → EvalResult.agentSetup)", () => {
@@ -168,3 +180,122 @@ describe("runAttemptEffect · agent-setup 路径提升(沙箱 __niceeval__/agent
     expect(result.agentSetup).toBeUndefined();
   });
 });
+
+// run.ts 的 reportFailure() 需要「失败发生时所在的阶段」,但 attempt:complete 一发出 coordinator
+// 就把 active map 里对应条目删了,run.ts 没法事后反查——只能在 attempt.ts 每次真正跨入一个新
+// phase 边界时同步拿到通知。这里直接单测 runAttemptEffect 的第五个参数(onPhase)是否真的随
+// enterPhase 同步触发、顺序是否符合「没有对应 hook/配置的步骤直接跳过」的契约
+//(docs/feature/experiments/cli.md「Attempt 阶段」),而不是只在 run.ts 集成测试里间接验证。
+describe("runAttemptEffect · onPhase 回调随 enterPhase 同步触发", () => {
+  it("挂了 eval.setup 与 agent.setup 时,phase 序列包含两者且不产生空阶段", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-with-setup",
+      setup: async () => {},
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: { setup: async () => {} },
+      onPhase: (phase) => phases.push(phase),
+    });
+
+    expect(result.error).toBeUndefined();
+    // sandbox-setup(没有 SandboxSpec.setup 钩子)与 telemetry-setup(没有 tracing)都该跳过——
+    // 不产生空阶段,序列只含实际执行到的边界,严格按生命周期顺序出现一次。
+    expect(phases).toEqual([
+      "sandbox.queue",
+      "sandbox.create",
+      "workspace.baseline",
+      "eval.setup",
+      "agent.setup",
+      "eval.run",
+      "workspace.diff",
+      "scoring.evaluate",
+    ]);
+  });
+
+  it("没有 eval.setup / agent.setup 时,对应阶段整个不出现(不是出现后立刻跳过的空事件)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-no-setup",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    await runOnce(agent, box, { onPhase: (phase) => phases.push(phase) });
+
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "eval.run", "workspace.diff", "scoring.evaluate"]);
+  });
+
+  it("test() 抛出的普通执行错误不设置 skipReason,diff/scoring 仍照常进入", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-throws",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: {
+        test: () => {
+          throw new Error("boom-from-eval");
+        },
+      },
+      onPhase: (phase) => phases.push(phase),
+    });
+
+    expect(result.error?.message).toContain("boom-from-eval");
+    expect(result.error?.phase).toBe("eval.run");
+    // test() 里的普通异常被 runAttemptBody 内层 try/catch 收作 result.error,不设置
+    // skipReason——所以 diff/scoring 的跳过条件(`!skipReason`)不成立,两个阶段仍会进入,
+    // 最后落 teardown。这是「running 阶段失败」的真实序列。
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "eval.run", "workspace.diff", "scoring.evaluate"]);
+  });
+
+  it("agent.setup 中途抛错时,phase 序列停在 agent-setup 就跳进 teardown(不会假装跑到了 running)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-setup-throws",
+      setup: async () => {
+        throw new Error("boom-from-setup");
+      },
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, { onPhase: (phase) => phases.push(phase) });
+
+    expect(result.error?.message).toContain("boom-from-setup");
+    expect(result.error?.phase).toBe("agent.setup");
+    // 失败发生在 agent-setup:之后不再出现 running/diff/scoring —— run.ts 的 reportFailure()
+    // 靠的正是这个真实的「最后已知阶段」,不是硬编码成 running(见 run.ts 的 lastPhase 注释)。
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "agent.setup"]);
+  });
+});
+
+describe("runAttemptEffect · 主链与 Scope 收尾的计时边界", () => {
+  it("sandbox.stop 只计入收尾,主链 phase 合计不超过 durationMs", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-timed-stop",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const result = await runOnce(agent, new FakeSandbox(40));
+    const phases = result.phases ?? [];
+    const closing = new Set<LifecyclePhase>([
+      "eval.teardown",
+      "agent.teardown",
+      "sandbox.teardown",
+      "sandbox.suspend",
+      "sandbox.stop",
+    ]);
+    const mainDurationMs = phases
+      .filter((phase) => !closing.has(phase.name))
+      .reduce((sum, phase) => sum + phase.durationMs, 0);
+    const stop = phases.find((phase) => phase.name === "sandbox.stop");
+
+    expect(stop?.durationMs).toBeGreaterThanOrEqual(30);
+    expect(mainDurationMs).toBeLessThanOrEqual(result.durationMs);
+  });
+});