niceeval 0.6.1 → 0.6.2

package/src/results/writer.ts

@@ -9,16 +9,28 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import type { AgentSetupManifest, EvalResult, ExperimentRunInfo, LocalizedText } from "../types.ts";
-import type { DiffData, O11ySummary, SourceArtifact, StreamEvent, TraceSpan } from "../types.ts";
+import type { DiffArtifact, O11ySummary, SourceArtifact, StreamEvent, TraceSpan } from "../types.ts";
 import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../types.ts";
 import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, attemptDirOf, experimentDirOf } from "./format.ts";
 import { encodeAttemptLocator } from "./locator.ts";
 import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
+import { truncateEvents, truncateSpans } from "./truncate.ts";
 import type { Producer, SnapshotMeta } from "./types.ts";
 
 export interface ResultsWriterOptions {
   /** 谁在写这份结果:niceeval 自己,或第三方 harness(name 如实写,别冒充 "niceeval")。 */
   producer: Producer;
+  /**
+   * 本次 invocation 的快照身份锚点(ISO 时间戳,即 runner 的 `RunShape.snapshotStartedAt`)。
+   * `writeAttemptFor()` 的隐式 snapshot 声明统一用它做 `startedAt`,不再按「该 experiment
+   * 第一条落盘 result 的 attempt startedAt」猜 —— 那个锚点依赖并发完成顺序,不确定;
+   * 同一 writer 处理多个 experiment 时也共享这同一个值(locator 身份还含 experimentId,
+   * 不会碰撞)。省略时退回旧行为:每次隐式声明按 `result.startedAt ?? now()` 各自取锚,
+   * 供未提供该值的直调场景使用(测试、结果转换脚本、第三方 harness 直接调
+   * `createResultsWriter()` 而不经过 niceeval 自己的 runner)。显式调用 `writer.snapshot()`
+   * 声明快照的调用方不受这个选项影响,必须自己传 `SnapshotDeclaration.startedAt`。
+   */
+  snapshotStartedAt?: string;
 }
 
 /** 快照级声明:一个 experiment 声明一次,这些字段不塞进每条 attempt。 */
@@ -70,7 +82,7 @@ export interface AttemptArtifacts {
   o11y?: O11ySummary;
   /** agent setup 的安装清单(沙箱型 coding agent 装了 Skill / plugin / MCP 才有)。 */
   agentSetup?: AgentSetupManifest;
-  diff?: DiffData;
+  diff?: DiffArtifact;
   sources?: SourceArtifact[];
 }
 
@@ -118,7 +130,7 @@ export function createResultsWriter(root: string, opts: ResultsWriterOptions): R
       producer: opts.producer,
       experimentId: decl.experimentId,
       // 运行配置不带 id:身份的家是顶层 experimentId,重复一份只会引出「以谁为准」。
-      ...(decl.experiment !== undefined ? { experiment: stripInfoId(decl.experiment) } : {}),
+      ...(decl.experiment !== undefined ? { experiment: (decl.experiment) } : {}),
       agent: decl.agent,
       ...(decl.model !== undefined ? { model: decl.model } : {}),
       startedAt: decl.startedAt,
@@ -172,8 +184,11 @@ export function createResultsWriter(root: string, opts: ResultsWriterOptions): R
       experimentId: result.experimentId,
       agent: result.agent,
       model: result.model,
-      // 快照 startedAt 以该实验首条落盘结果的 attempt 时刻为锚(首条 ≈ 实验开跑)。
-      startedAt: result.startedAt ?? new Date().toISOString(),
+      // 快照 startedAt 优先用 writer 级的 invocation 锚点(见 ResultsWriterOptions.snapshotStartedAt)——
+      // niceeval 自己的 runner 恒会提供,多个 experiment 共享同一个值。省略时(第三方直调
+      // createResultsWriter() 未传该选项)退回旧行为:以本次调用这个 result 自己的 attempt
+      // startedAt 为锚,首条落盘的 result 决定了这个 experiment 快照的身份锚点。
+      startedAt: opts.snapshotStartedAt ?? result.startedAt ?? new Date().toISOString(),
       experiment: result.experiment,
     });
 
@@ -288,9 +303,14 @@ async function writeAttemptFiles(
   const hasTrace = !!(artifacts?.trace && artifacts.trace.length);
 
   const writes: Promise<unknown>[] = [];
-  if (hasEvents) writes.push(writeFile(join(attemptDir, "events.json"), JSON.stringify(artifacts!.events), "utf-8"));
+  // 大值截断只发生在这里(序列化的那一刻):events 的事件字段与 trace 的 span 属性里的任意
+  // 字符串值按 ARTIFACT_VALUE_MAX_BYTES 截断并留结构化 truncated 标记;运行时(断言 / o11y
+  // 派生)看到的始终是完整值。sources 与 diff 不截断(见 docs/feature/results/architecture.md)。
+  if (hasEvents)
+    writes.push(writeFile(join(attemptDir, "events.json"), JSON.stringify(truncateEvents(artifacts!.events!)), "utf-8"));
   if (hasSources) writes.push(writeSourcesRef(snapDir, attemptDir, artifacts!.sources!, sourceStore));
-  if (hasTrace) writes.push(writeFile(join(attemptDir, "trace.json"), JSON.stringify(artifacts!.trace), "utf-8"));
+  if (hasTrace)
+    writes.push(writeFile(join(attemptDir, "trace.json"), JSON.stringify(truncateSpans(artifacts!.trace!)), "utf-8"));
   if (artifacts?.o11y) writes.push(writeFile(join(attemptDir, "o11y.json"), JSON.stringify(artifacts.o11y), "utf-8"));
   if (artifacts?.agentSetup) {
     writes.push(
@@ -303,6 +323,9 @@ async function writeAttemptFiles(
   // locator:caller(如第三方 harness 直接调 SnapshotWriter.writeAttempt)已经带了就尊重,
   // 否则按当前身份元组算一份 —— 这条路径只服务「非携带」的新写入,携带条目走
   // writeAttemptForImpl 的 artifactBase 分支,原样透传 result.locator,从不落到这里重算。
+  // niceeval 自己的 runner(src/runner/run.ts)在 fresh attempt 完成时已经把 locator 写进
+  // entry.locator(用的正是同一个 snapshot.startedAt),所以这条 encodeAttemptLocator 兜底
+  // 对 niceeval 自身运行永不触发,只在第三方 harness 未预先算好 locator 时才会走到。
   const locator =
     entry.locator ??
     encodeAttemptLocator({
@@ -365,12 +388,7 @@ async function createSnapshotDir(root: string, experimentId: string): Promise<st
   throw new Error(`Could not create a unique snapshot directory under "${parent}" after 5 attempts (${String(lastError)}).`);
 }
 
-/** 运行配置落盘前剥掉 id:experimentId 的家在 snapshot.json 顶层。 */
-function stripInfoId(info: ExperimentRunInfo): ExperimentRunInfo {
-  const { id, ...rest } = info;
-  void id;
-  return rest;
-}
+
 
 /** 快照目录名的时间戳段:Date#toISOString 把 : 与 . 换成 -(与 docs/feature/results/architecture.md 一致)。 */
 function safeTimestamp(d: Date): string {

package/src/runner/attempt.test.ts

@@ -1,3 +1,4 @@
+// cases: docs/engineering/unit-tests/experiments-runner/cases.md
 // 路径提升单测:agent.setup 写进沙箱 `__niceeval__/agent-setup.json` 的安装 manifest,
 // runAttemptEffect 在 setup 之后把它读出来、原样挂到 EvalResult.agentSetup(见
 // docs/feature/results/architecture.md「agent-setup.json」、src/agents/manifest.ts 的注释)。
@@ -11,7 +12,7 @@ import { runAttemptEffect } from "./attempt.ts";
 import { defineSandboxAgent, defineSandbox } from "../define.ts";
 import { writeAgentSetupManifest, AGENT_SETUP_MANIFEST_PATH } from "../agents/manifest.ts";
 import type { CapturedEvalSource } from "./eval-source.ts";
-import type { Attempt, AgentRun, RunOptions } from "./types.ts";
+import type { Attempt, AgentRun, LifecyclePhase, RunOptions } from "./types.ts";
 import type {
   AgentSetupManifest,
   Agent,
@@ -29,6 +30,8 @@ class FakeSandbox implements Partial<Sandbox> {
   readonly otlpHost = null;
   readonly files = new Map<string, string>();
 
+  constructor(private readonly stopDelayMs = 0) {}
+
   async runShell(): Promise<CommandResult> {
     return { stdout: "", stderr: "", exitCode: 0 };
   }
@@ -63,21 +66,31 @@ class FakeSandbox implements Partial<Sandbox> {
   async readSourceFiles(): Promise<never> {
     throw new Error("not implemented");
   }
-  async stop(): Promise<void> {}
+  async stop(): Promise<void> {
+    if (this.stopDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, this.stopDelayMs));
+  }
 }
 
 const asSandbox = (box: FakeSandbox): Sandbox => box as unknown as Sandbox;
 
 const source: CapturedEvalSource = { path: "fake.eval.ts", content: "", sha256: "0".repeat(64) };
 
-/** 跑一次 attempt:给定 agent,返回 EvalResult。沙箱用内存 fake,不起容器/不联网。 */
-async function runOnce(agent: Agent, box: FakeSandbox): Promise<import("../types.ts").EvalResult> {
+/** 跑一次 attempt:给定 agent,返回 EvalResult。沙箱用内存 fake,不起容器/不联网。
+ *  可选 `evalDefOverrides` 覆盖默认 evalDef 的字段(如挂一个 `setup`);可选 `onPhase` 透传给
+ *  `runAttemptEffect` 的第五个参数,原样转发 attempt.ts 的 enterPhase 边界(见下方
+ *  onPhase 回调专用的 describe 块)。 */
+async function runOnce(
+  agent: Agent,
+  box: FakeSandbox,
+  opts: { evalDefOverrides?: Partial<DiscoveredEval>; onPhase?: (phase: LifecyclePhase) => void } = {},
+): Promise<import("../types.ts").EvalResult> {
   const evalDef: DiscoveredEval = {
     id: "fake/eval",
     baseDir: "/project",
     sourcePath: "/project/fake.eval.ts",
     source,
     test: () => {},
+    ...opts.evalDefOverrides,
   };
   const run: AgentRun = {
     agent,
@@ -91,16 +104,15 @@ async function runOnce(agent: Agent, box: FakeSandbox): Promise<import("../types
   };
   const attempt: Attempt = { evalDef, run, attempt: 0, key: "fake/eval", fingerprint: "" };
   const config: Config = {};
-  const opts: RunOptions = {
+  const runOpts: RunOptions = {
     config,
     evals: [evalDef],
     agentRuns: [run],
     reporters: [],
     maxConcurrency: 1,
-    onProgress: () => {}, // 静音进度日志,不污染测试输出
   };
   const sandboxSem = Effect.runSync(Effect.makeSemaphore(1));
-  return Effect.runPromise(runAttemptEffect(attempt, opts, sandboxSem));
+  return Effect.runPromise(runAttemptEffect(attempt, runOpts, sandboxSem, undefined, opts.onPhase));
 }
 
 describe("runAttemptEffect · agent-setup 路径提升(沙箱 __niceeval__/agent-setup.json → EvalResult.agentSetup)", () => {
@@ -168,3 +180,122 @@ describe("runAttemptEffect · agent-setup 路径提升(沙箱 __niceeval__/agent
     expect(result.agentSetup).toBeUndefined();
   });
 });
+
+// run.ts 的 reportFailure() 需要「失败发生时所在的阶段」,但 attempt:complete 一发出 coordinator
+// 就把 active map 里对应条目删了,run.ts 没法事后反查——只能在 attempt.ts 每次真正跨入一个新
+// phase 边界时同步拿到通知。这里直接单测 runAttemptEffect 的第五个参数(onPhase)是否真的随
+// enterPhase 同步触发、顺序是否符合「没有对应 hook/配置的步骤直接跳过」的契约
+//(docs/feature/experiments/cli.md「Attempt 阶段」),而不是只在 run.ts 集成测试里间接验证。
+describe("runAttemptEffect · onPhase 回调随 enterPhase 同步触发", () => {
+  it("挂了 eval.setup 与 agent.setup 时,phase 序列包含两者且不产生空阶段", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-with-setup",
+      setup: async () => {},
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: { setup: async () => {} },
+      onPhase: (phase) => phases.push(phase),
+    });
+
+    expect(result.error).toBeUndefined();
+    // sandbox-setup(没有 SandboxSpec.setup 钩子)与 telemetry-setup(没有 tracing)都该跳过——
+    // 不产生空阶段,序列只含实际执行到的边界,严格按生命周期顺序出现一次。
+    expect(phases).toEqual([
+      "sandbox.queue",
+      "sandbox.create",
+      "workspace.baseline",
+      "eval.setup",
+      "agent.setup",
+      "eval.run",
+      "workspace.diff",
+      "scoring.evaluate",
+    ]);
+  });
+
+  it("没有 eval.setup / agent.setup 时,对应阶段整个不出现(不是出现后立刻跳过的空事件)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-no-setup",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    await runOnce(agent, box, { onPhase: (phase) => phases.push(phase) });
+
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "eval.run", "workspace.diff", "scoring.evaluate"]);
+  });
+
+  it("test() 抛出的普通执行错误不设置 skipReason,diff/scoring 仍照常进入", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-throws",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, {
+      evalDefOverrides: {
+        test: () => {
+          throw new Error("boom-from-eval");
+        },
+      },
+      onPhase: (phase) => phases.push(phase),
+    });
+
+    expect(result.error?.message).toContain("boom-from-eval");
+    expect(result.error?.phase).toBe("eval.run");
+    // test() 里的普通异常被 runAttemptBody 内层 try/catch 收作 result.error,不设置
+    // skipReason——所以 diff/scoring 的跳过条件(`!skipReason`)不成立,两个阶段仍会进入,
+    // 最后落 teardown。这是「running 阶段失败」的真实序列。
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "eval.run", "workspace.diff", "scoring.evaluate"]);
+  });
+
+  it("agent.setup 中途抛错时,phase 序列停在 agent-setup 就跳进 teardown(不会假装跑到了 running)", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-setup-throws",
+      setup: async () => {
+        throw new Error("boom-from-setup");
+      },
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const phases: LifecyclePhase[] = [];
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box, { onPhase: (phase) => phases.push(phase) });
+
+    expect(result.error?.message).toContain("boom-from-setup");
+    expect(result.error?.phase).toBe("agent.setup");
+    // 失败发生在 agent-setup:之后不再出现 running/diff/scoring —— run.ts 的 reportFailure()
+    // 靠的正是这个真实的「最后已知阶段」,不是硬编码成 running(见 run.ts 的 lastPhase 注释)。
+    expect(phases).toEqual(["sandbox.queue", "sandbox.create", "workspace.baseline", "agent.setup"]);
+  });
+});
+
+describe("runAttemptEffect · 主链与 Scope 收尾的计时边界", () => {
+  it("sandbox.stop 只计入收尾,主链 phase 合计不超过 durationMs", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-timed-stop",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+    const result = await runOnce(agent, new FakeSandbox(40));
+    const phases = result.phases ?? [];
+    const closing = new Set<LifecyclePhase>([
+      "eval.teardown",
+      "agent.teardown",
+      "sandbox.teardown",
+      "sandbox.suspend",
+      "sandbox.stop",
+    ]);
+    const mainDurationMs = phases
+      .filter((phase) => !closing.has(phase.name))
+      .reduce((sum, phase) => sum + phase.durationMs, 0);
+    const stop = phases.find((phase) => phase.name === "sandbox.stop");
+
+    expect(stop?.durationMs).toBeGreaterThanOrEqual(30);
+    expect(mainDurationMs).toBeLessThanOrEqual(result.durationMs);
+  });
+});