niceeval 0.5.3 → 0.6.0

package/src/runner/attempt.test.ts

@@ -0,0 +1,170 @@
+// 路径提升单测:agent.setup 写进沙箱 `__niceeval__/agent-setup.json` 的安装 manifest,
+// runAttemptEffect 在 setup 之后把它读出来、原样挂到 EvalResult.agentSetup(见
+// docs/feature/results/architecture.md「agent-setup.json」、src/agents/manifest.ts 的注释)。
+// 沙箱是内存 fake(记文件,不起容器)——这里要验的是运行器自己「何时读、读到什么、读不到
+// 怎么办」这段编排逻辑,不是 adapter 侧的 manifest 构造规则(那部分已在 agents/skills.test.ts
+// 覆盖)。
+
+import { describe, expect, it } from "vitest";
+import { Effect } from "effect";
+import { runAttemptEffect } from "./attempt.ts";
+import { defineSandboxAgent, defineSandbox } from "../define.ts";
+import { writeAgentSetupManifest, AGENT_SETUP_MANIFEST_PATH } from "../agents/manifest.ts";
+import type { CapturedEvalSource } from "./eval-source.ts";
+import type { Attempt, AgentRun, RunOptions } from "./types.ts";
+import type {
+  AgentSetupManifest,
+  Agent,
+  CommandResult,
+  Config,
+  DiscoveredEval,
+  Sandbox,
+  SandboxFile,
+} from "../types.ts";
+
+/** 内存沙箱:writeFiles/readFile 记文件,runShell 恒成功(供 initGitAndCommit / diff 采集用)。 */
+class FakeSandbox implements Partial<Sandbox> {
+  readonly workdir = "/workspace";
+  readonly sandboxId = "fake";
+  readonly otlpHost = null;
+  readonly files = new Map<string, string>();
+
+  async runShell(): Promise<CommandResult> {
+    return { stdout: "", stderr: "", exitCode: 0 };
+  }
+  async runCommand(): Promise<CommandResult> {
+    return { stdout: "", stderr: "", exitCode: 0 };
+  }
+  async writeFiles(files: Record<string, string>, targetDir?: string): Promise<void> {
+    for (const [path, content] of Object.entries(files)) {
+      this.files.set(targetDir ? `${targetDir}/${path}` : path, content);
+    }
+  }
+  async uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void> {
+    for (const f of files) {
+      this.files.set(targetDir ? `${targetDir}/${f.path}` : f.path, f.content.toString());
+    }
+  }
+  async uploadFile(path: string, content: Buffer): Promise<void> {
+    this.files.set(path, content.toString());
+  }
+  async uploadDirectory(): Promise<void> {}
+  async downloadFile(path: string): Promise<Buffer> {
+    return Buffer.from(this.files.get(path) ?? "");
+  }
+  async fileExists(path: string): Promise<boolean> {
+    return this.files.has(path);
+  }
+  async readFile(path: string): Promise<string> {
+    const hit = this.files.get(path);
+    if (hit === undefined) throw new Error(`no such file: ${path}`);
+    return hit;
+  }
+  async readSourceFiles(): Promise<never> {
+    throw new Error("not implemented");
+  }
+  async stop(): Promise<void> {}
+}
+
+const asSandbox = (box: FakeSandbox): Sandbox => box as unknown as Sandbox;
+
+const source: CapturedEvalSource = { path: "fake.eval.ts", content: "", sha256: "0".repeat(64) };
+
+/** 跑一次 attempt:给定 agent,返回 EvalResult。沙箱用内存 fake,不起容器/不联网。 */
+async function runOnce(agent: Agent, box: FakeSandbox): Promise<import("../types.ts").EvalResult> {
+  const evalDef: DiscoveredEval = {
+    id: "fake/eval",
+    baseDir: "/project",
+    sourcePath: "/project/fake.eval.ts",
+    source,
+    test: () => {},
+  };
+  const run: AgentRun = {
+    agent,
+    flags: {},
+    runs: 1,
+    earlyExit: true,
+    // 自定义 provider:create() 直接返回内存 fake,绕开真实沙箱 provider。
+    sandbox: defineSandbox({ name: "fake-provider", create: async () => asSandbox(box) }),
+    timeoutMs: 5_000,
+    evalFilter: () => true,
+  };
+  const attempt: Attempt = { evalDef, run, attempt: 0, key: "fake/eval", fingerprint: "" };
+  const config: Config = {};
+  const opts: RunOptions = {
+    config,
+    evals: [evalDef],
+    agentRuns: [run],
+    reporters: [],
+    maxConcurrency: 1,
+    onProgress: () => {}, // 静音进度日志,不污染测试输出
+  };
+  const sandboxSem = Effect.runSync(Effect.makeSemaphore(1));
+  return Effect.runPromise(runAttemptEffect(attempt, opts, sandboxSem));
+}
+
+describe("runAttemptEffect · agent-setup 路径提升(沙箱 __niceeval__/agent-setup.json → EvalResult.agentSetup)", () => {
+  it("沙箱内有 manifest 时,原样读出挂到 EvalResult.agentSetup(不做任何转换/裁剪)", async () => {
+    const manifest: AgentSetupManifest = {
+      skills: [
+        { kind: "local", name: "effect-ts", path: "skills/effect-ts", sha256: "a".repeat(64) },
+        { kind: "repo", source: "anthropics/skills", ref: "9d2f1ae187231d8199c64b5b762e1bdf2244733d", skills: ["pdf", "docx"] },
+      ],
+      nativePlugins: [
+        {
+          agent: "claude-code",
+          marketplace: { name: "duyet", source: "duyet/codex-claude-plugins", ref: "82de4021a311034a9596e891baf3a8266fb33bf7" },
+          name: "example-plugin",
+          resolvedVersion: "1.2.3",
+        },
+      ],
+      mcpServers: [{ name: "fs", command: "npx", args: ["-y", "@modelcontextprotocol/server-filesystem"] }],
+    };
+
+    const agent = defineSandboxAgent({
+      name: "fake-agent",
+      setup: async (sandbox) => {
+        await writeAgentSetupManifest(sandbox, manifest);
+      },
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box);
+
+    expect(result.error).toBeUndefined();
+    // 沙箱内确实落了这个文件(否则下面的断言测不出"提升"这一步真的发生了)。
+    expect(box.files.has(`${box.workdir}/${AGENT_SETUP_MANIFEST_PATH}`)).toBe(true);
+    expect(result.agentSetup).toEqual(manifest); // 深相等:内容原样保留,没有裁剪或改形。
+  });
+
+  it("沙箱内没有 manifest 时(没装任何 Skill/plugin/MCP 的基线场景),不生成空/伪造的 artifact", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-no-install",
+      // agent.setup 跑了(比如只装了 CLI 本体),但没有任何 skill/plugin/mcp 可写,
+      // 所以从不调用 writeAgentSetupManifest —— 这是「基线场景」的真实形状。
+      setup: async () => {},
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box);
+
+    expect(result.error).toBeUndefined();
+    expect(box.files.has(`${box.workdir}/${AGENT_SETUP_MANIFEST_PATH}`)).toBe(false);
+    expect(result.agentSetup).toBeUndefined();
+  });
+
+  it("agent 根本没有 setup 钩子时(非 coding agent adapter),同样不生成 agentSetup", async () => {
+    const agent = defineSandboxAgent({
+      name: "fake-agent-no-setup",
+      send: async () => ({ events: [], status: "completed" }),
+    });
+
+    const box = new FakeSandbox();
+    const result = await runOnce(agent, box);
+
+    expect(result.error).toBeUndefined();
+    expect(result.agentSetup).toBeUndefined();
+  });
+});

package/src/runner/attempt.ts

@@ -1,6 +1,6 @@
 // 单个 attempt 的完整生命周期:资源(沙箱 / OTLP 接收器)经 Effect.Scope 的
 // acquireRelease 接管,无论 body 成功 / 抛错 / 被中断,stop() / close() 都保证执行。
-// 沙箱编排的固定段在 runAttemptBody(基线→setup→驱动 test→采 diff→评分→判决→收 trace),
+// 沙箱编排的固定段在 runAttemptBody(基线→setup→驱动 test→采 diff→评分→判定→收 trace),
 // adapter 只填「把 agent 跑起来」一段。
 
 import { resolve as resolvePath } from "node:path";
@@ -14,16 +14,21 @@ import { selectTraceSpans, enrichTraceWithIO } from "../o11y/otlp/select.ts";
 import { mapGenericSpans } from "../o11y/otlp/mappers/index.ts";
 import { createEvalContext } from "../context/context.ts";
 import { createAgentSession } from "../context/session.ts";
+import { readAgentSetupManifest } from "../agents/manifest.ts";
 import { EvalRequirementFailed, EvalSkipped, TurnFailed } from "../context/control-flow.ts";
-import { computeOutcome } from "../scoring/verdict.ts";
+import { computeVerdict } from "../scoring/verdict.ts";
 import { deriveRunFacts, buildO11ySummary } from "../o11y/derive.ts";
 import { estimateCost } from "../o11y/cost.ts";
 import { t } from "../i18n/index.ts";
 import { formatThrown } from "../util.ts";
 import { captureGeneratedFiles, initGitAndCommit } from "./sandbox-prep.ts";
 import { createRemoteSandbox, withEvalLocalPaths } from "./remote-sandbox.ts";
+import type { CapturedEvalSource } from "./eval-source.ts";
 import type {
   AgentContext,
+  AgentSetup,
+  AgentSetupManifest,
+  AgentTeardown,
   Cleanup,
   Config,
   EvalResult,
@@ -56,7 +61,7 @@ export function runAttemptEffect(
     experiment: experimentRunInfo(run),
     agent: run.agent.name,
     model: run.model,
-    outcome: "errored",
+    verdict: "errored",
     fingerprint: a.fingerprint,
     attempt,
     startedAt: new Date(t0).toISOString(),
@@ -73,7 +78,7 @@ export function runAttemptEffect(
 
   // 流式进度打到宿主 stderr(结果走 stdout,互不干扰)。容器主日志【不】放这些进度标记 ——
   // 那里留给 agent 的原始输出(adapter 给 agent 命令开 { stream: true })。
-  const who = runWho(run);
+  const who = runWho({ agentName: run.agent.name, model: run.model, experimentId: run.experimentId });
   // 同时保留最近 20 条进度消息,timeout 时嵌入 error 字段方便定位卡在哪一步。
   const recentLogs: string[] = [];
   const log = (m: string) => {
@@ -88,6 +93,15 @@ export function runAttemptEffect(
 
   return Effect.scoped(
     Effect.gen(function* () {
+      // run.sandbox ?? config.sandbox 是同一个 SandboxSpec 对象,既用来起沙箱 provider,
+      // 也是 sandbox.setup / sandbox.teardown 钩子(SandboxSpec.setup()/.teardown() 链式挂的)的来源。
+      const sandboxSpec = run.sandbox ?? config.sandbox;
+      // 退避重试(resolve.ts → retry.ts)期间临时归还这个名额:被限流的 provider 只是在
+      // setTimeout 里睡觉,不该攥着 sandboxSem 的槽位陪跑,不然一批 429 能把整体并发拖成个位数。
+      const provisionSlot = {
+        release: () => Effect.runPromise(sandboxSem.release(1)).then(() => {}),
+        reacquire: () => Effect.runPromise(sandboxSem.take(1)).then(() => {}),
+      };
       const sandbox =
         run.agent.kind === "sandbox"
           ? yield* sandboxSem.withPermits(1)(
@@ -96,7 +110,8 @@ export function runAttemptEffect(
                 // sandboxSem 只覆盖「容器创建」阶段;容器起好后立即释放,后续 npm install / agent 不占位。
                 log(t("runner.startSandbox"));
                 return yield* createSandbox({
-                  sandbox: run.sandbox ?? config.sandbox,
+                  sandbox: sandboxSpec,
+                  provisionSlot,
                   timeout: timeoutMs,
                   runtime: "node24",
                 });
@@ -161,6 +176,8 @@ export function runAttemptEffect(
       return yield* Effect.promise((interruptSignal) =>
         runAttemptBody(a, config, t0, base, {
           sandbox,
+          sandboxSetupHooks: sandboxSpec?.setupHooks ?? [],
+          sandboxTeardownHooks: sandboxSpec?.teardownHooks ?? [],
           receiver,
           telemetry,
           otel: otelChannel,
@@ -203,6 +220,10 @@ function causeToError(cause: Cause.Cause<never>): string {
 
 interface AttemptResources {
   sandbox: Sandbox;
+  /** SandboxSpec.setup() 链式挂的钩子,按追加顺序;非沙箱 agent 传空数组(usesSandbox 挡住不会跑)。 */
+  sandboxSetupHooks: readonly AgentSetup[];
+  /** SandboxSpec.teardown() 链式挂的钩子,按追加顺序保存,执行时逆序。 */
+  sandboxTeardownHooks: readonly AgentTeardown[];
   receiver?: TraceReceiver;
   telemetry?: Telemetry;
   /** 非沙箱 tracing agent 的共享 OTLP 通道(run 级池持有,不随 attempt 关)。 */
@@ -211,7 +232,7 @@ interface AttemptResources {
   log: (m: string) => void;
 }
 
-// attempt 的固定段(上传→基线→setup→驱动 agent→采 diff→脚本→评分→判决)。
+// attempt 的固定段(上传→基线→setup→驱动 agent→采 diff→脚本→评分→判定)。
 // 资源已由 runAttemptEffect 的 Scope 持有;这里只在 finally 跑 agent 自己的 cleanup/teardown。
 async function runAttemptBody(
   a: Attempt,
@@ -221,7 +242,7 @@ async function runAttemptBody(
   res: AttemptResources,
 ): Promise<EvalResult> {
   const { evalDef, run, attempt } = a;
-  const { sandbox, receiver, telemetry, otel, signal, log } = res;
+  const { sandbox, sandboxSetupHooks, sandboxTeardownHooks, receiver, telemetry, otel, signal, log } = res;
   const usesSandbox = run.agent.kind === "sandbox";
   // 整个 attempt 共用一份 agent ctx(sandbox 钩子 / agent setup / tracing configure / teardown 都用它)。
   const attemptCtx: AgentContext = {
@@ -229,6 +250,7 @@ async function runAttemptBody(
     model: run.model,
     reasoningEffort: run.reasoningEffort,
     flags: run.flags,
+    experimentId: run.experimentId,
     sandbox,
     session: createAgentSession(),
     telemetry,
@@ -236,15 +258,31 @@ async function runAttemptBody(
   };
   let agentCleanup: Cleanup | void = undefined;
   let agentDidSetup = false;
+  /** agent.setup 写进沙箱的安装清单(装了 Skill / plugin / MCP 的沙箱型 adapter 才有)。 */
+  let agentSetup: AgentSetupManifest | undefined;
+  // EvalDef.setup() 返回的 cleanup 闭包;finally 里按 LIFO 跑(见下)。
+  let evalCleanup: Cleanup | void = undefined;
+  // SandboxSpec.setup() 返回的 cleanup 闭包,按调用顺序收集;finally 里 LIFO 跑(见下)。
+  const sandboxCleanups: Cleanup[] = [];
   try {
     if (usesSandbox) {
+      // 沙箱级生命周期钩子(SandboxSpec.setup):环境预置层,先于 workspace 上传 / git 基线 /
+      // eval.setup 跑——改动进 git 基线,不会被误算进 agent 产出的 diff。按追加顺序依次执行;
+      // 单个抛错走下面的执行错误路径(与 eval.setup / agent.setup 同一条),已跑过的 cleanup /
+      // sandbox.teardown 钩子仍在 finally 里跑(见 catch/finally)。
+      if (sandboxSetupHooks.length > 0) log(t("runner.startSandboxSetup"));
+      for (const hook of sandboxSetupHooks) {
+        const cleanup = await hook(sandbox, attemptCtx);
+        if (typeof cleanup === "function") sandboxCleanups.push(cleanup);
+      }
+
       await initGitAndCommit(sandbox);
 
       // eval 级 setup(starter prep:npm install / 装系统依赖等)。命令默认非 root;
       // setup 里需要 root 的(apt/pip)自己传 { root: true }。
       if (evalDef.setup) {
         log(t("runner.evalSetup"));
-        await evalDef.setup(withEvalLocalPaths(sandbox, evalDef.baseDir));
+        evalCleanup = await evalDef.setup(withEvalLocalPaths(sandbox, evalDef.baseDir));
       }
     }
 
@@ -255,6 +293,11 @@ async function runAttemptBody(
       agentCleanup = await run.agent.setup(sandbox, attemptCtx);
     }
 
+    // 安装 manifest:adapter 在 setup 收尾写进沙箱的固定路径,核心只把它抬成 attempt artifact
+    // (不解释内容、不按 agent 名字分支)。什么都没装的 adapter 不写这个文件 → undefined,
+    // 不生成空 artifact。读在 setup 之后:test 阶段抛错也留得住这份「这次装了什么」的证据。
+    if (usesSandbox && agentDidSetup) agentSetup = await readAgentSetupManifest(sandbox);
+
     // OTLP 导出配置(file-based,如 codex 的 config.toml [otel] 块):与 setup 分开,
     // 在主配置写完后追加。仅当 tracing 开 + 有 endpoint 时调一次(env-based 的不实现 configure)。
     if (telemetry && run.agent.tracing?.configure) {
@@ -271,6 +314,7 @@ async function runAttemptBody(
       model: run.model,
       reasoningEffort: run.reasoningEffort,
       flags: run.flags,
+      experimentId: run.experimentId,
       signal,
       log,
       judge,
@@ -335,7 +379,7 @@ async function runAttemptBody(
     };
     if (!skipReason) log(t("runner.scoreJudge"));
     const assertions = skipReason ? [] : await state.collector.finalize(scoringContext);
-    const outcome = computeOutcome({ error, assertions, skipReason, strict: run.strict });
+    const verdict = computeVerdict({ error, assertions, skipReason, strict: run.strict });
 
     // 收 OTLP trace:给最后一批导出留点落地时间,再 collect(空则不挂)。
     // codex 的 OTLP 把内部 Rust tracing 全导出来(handle_responses / append_items … 上万条);
@@ -373,7 +417,7 @@ async function runAttemptBody(
     if (cost !== undefined) o11y.estimatedCostUSD = cost;
 
     // 收 test 引用到的 eval 源码(按 send / 断言的 loc 去重),供 view 渲染代码视图。
-    const sources = await collectSources(events, assertions);
+    const sources = await collectSources(events, assertions, evalDef.source);
 
     return {
       id: evalDef.id,
@@ -382,7 +426,7 @@ async function runAttemptBody(
       experiment: experimentRunInfo(run),
       agent: run.agent.name,
       model: run.model,
-      outcome,
+      verdict,
       fingerprint: a.fingerprint,
       attempt,
       startedAt: new Date(t0).toISOString(),
@@ -396,6 +440,7 @@ async function runAttemptBody(
       sources,
       o11y,
       trace,
+      agentSetup,
       diff,
     };
   } catch (e) {
@@ -403,10 +448,12 @@ async function runAttemptBody(
       ...base,
       durationMs: Date.now() - t0,
       error: formatThrown(e),
+      ...(agentSetup !== undefined ? { agentSetup } : {}),
     };
   } finally {
-    // teardown / cleanup 一律在 finally 跑(失败也跑),不改判决,各自兜错(diagnostic)。
-    // LIFO:先 agent(setup 最晚),再沙箱 Scope。
+    // teardown / cleanup 一律在 finally 跑(失败也跑),不改判定,各自兜错(diagnostic)。
+    // LIFO:agent 级(setup 最晚 → cleanup / teardown 先跑)在前,sandbox 级(最早就绪)收尾在后
+    //(即 sandbox.setup 返回的 cleanup、sandbox.teardown 钩子最后跑,沙箱销毁前)。
     // 沙箱 stop / 接收器 close 不在这里 —— 由 runAttemptEffect 的 Scope 在本函数返回后回收。
     try {
       if (typeof agentCleanup === "function") await agentCleanup();
@@ -414,22 +461,57 @@ async function runAttemptBody(
     } catch {
       // teardown 失败只是 diagnostic,不影响已出的结果
     }
+    try {
+      // eval.setup 返回的 cleanup:排在 agent 级之后、sandbox 级之前(LIFO,与 setup 顺序对称)。
+      if (typeof evalCleanup === "function") await evalCleanup();
+    } catch {
+      // 同上,只作 diagnostic
+    }
+    if (usesSandbox) {
+      // sandbox.setup 返回的 cleanup:LIFO(后 setup 先 cleanup),与 agent 级同构。
+      for (let i = sandboxCleanups.length - 1; i >= 0; i--) {
+        try {
+          await sandboxCleanups[i]();
+        } catch {
+          // 同上,只作 diagnostic
+        }
+      }
+      // sandbox.teardown 钩子:按追加的逆序执行,沙箱销毁前最后一步。
+      if (sandboxTeardownHooks.length > 0) {
+        log(t("runner.startSandboxTeardown"));
+        for (let i = sandboxTeardownHooks.length - 1; i >= 0; i--) {
+          try {
+            await sandboxTeardownHooks[i](sandbox, attemptCtx);
+          } catch {
+            // 同上,只作 diagnostic
+          }
+        }
+      }
+    }
   }
 }
 
 /**
- * 收集 test 引用到的 eval 源码:从 send(user message)与断言的 loc 去重出文件集,逐个读回。
- * loc.file 相对项目根(= 进程 cwd,CLI 从那儿发现 / 跑 eval),所以按 cwd 解析。读不到就跳过。
+ * 收集 test 引用到的 eval 源码:从 send(user message)与断言的 loc 去重出文件集。
+ * 命中 eval 自己的定义文件(绝大多数情况——send / 断言几乎总在 eval 主体里直接调用)时,
+ * 直接用 discovery 时已经读好、归一化、算过哈希的 `evalSource`,不重新读盘;loc 指向
+ * 其它文件(共享 helper 里包装 t.send / 断言的少见情形)才现读现取,读不到就跳过
+ * (路径在沙箱内 / 已删 / 权限),view 用 loc 也能降级显示行号。
  */
 async function collectSources(
   events: readonly StreamEvent[],
   assertions: readonly EvalResult["assertions"][number][],
+  evalSource: CapturedEvalSource,
 ): Promise<SourceArtifact[]> {
   const paths = new Set<string>();
   for (const e of events) if (e.type === "message" && e.loc) paths.add(e.loc.file);
   for (const a of assertions) if (a.loc) paths.add(a.loc.file);
   const out: SourceArtifact[] = [];
   for (const path of paths) {
+    if (path === evalSource.path) {
+      out.push({ path, content: evalSource.content });
+      continue;
+    }
     try {
       out.push({ path, content: await readSourceFile(resolvePath(process.cwd(), path), "utf-8") });
     } catch {

package/src/runner/discover.test.ts

@@ -0,0 +1,65 @@
+// discoverEvals 的 eval 源码捕获:同一文件(数组默认导出)共享一份 CapturedEvalSource,
+// 内容/路径/哈希与 captureEvalSource() 直接调出来的一致(定稿见 docs/concepts.md「标注 Eval 源码」)。
+
+import { describe, expect, it, afterEach } from "vitest";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { discoverEvals } from "./discover.ts";
+import { captureEvalSource } from "./eval-source.ts";
+
+const roots: string[] = [];
+async function makeRoot(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-discover-"));
+  roots.push(root);
+  return root;
+}
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
+});
+
+describe("discoverEvals · 源码捕获", () => {
+  it("单个默认导出:source 与 captureEvalSource() 直接调出来的一致(路径/内容/哈希)", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const file = join(root, "evals", "hello.eval.ts");
+    await writeFile(file, 'export default {\n  test() {},\n};\n', "utf-8");
+
+    const evals = await discoverEvals(root);
+    expect(evals).toHaveLength(1);
+    const expected = await captureEvalSource(file, { root });
+    expect(evals[0]!.source).toEqual(expected);
+    expect(evals[0]!.source.path).toBe("evals/hello.eval.ts");
+  });
+
+  it("数组默认导出:多个 eval 共享同一份 CapturedEvalSource 引用(同哈希,同一个文件只读一次)", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const file = join(root, "evals", "batch.eval.ts");
+    await writeFile(
+      file,
+      "export default [\n  { test() {} },\n  { test() {} },\n];\n",
+      "utf-8",
+    );
+
+    const evals = await discoverEvals(root);
+    expect(evals.map((e) => e.id)).toEqual(["batch/0000", "batch/0001"]);
+    expect(evals[0]!.source).toBe(evals[1]!.source); // 同一份引用,不是内容相等的两份拷贝
+    expect(evals[0]!.source.sha256).toHaveLength(64);
+  });
+
+  it("CRLF 源码归一化后哈希与 LF 版本一致(discovery 侧与 collectSources/annotated-source 共用归一化)", async () => {
+    const rootLf = await makeRoot();
+    const rootCrlf = await makeRoot();
+    await mkdir(join(rootLf, "evals"), { recursive: true });
+    await mkdir(join(rootCrlf, "evals"), { recursive: true });
+    const body = "export default {\n  test() {},\n};\n";
+    await writeFile(join(rootLf, "evals", "a.eval.ts"), body, "utf-8");
+    await writeFile(join(rootCrlf, "evals", "a.eval.ts"), body.replace(/\n/g, "\r\n"), "utf-8");
+
+    const [lf] = await discoverEvals(rootLf);
+    const [crlf] = await discoverEvals(rootCrlf);
+    expect(lf!.source.sha256).toBe(crlf!.source.sha256);
+    expect(crlf!.source.content).toBe(body); // 归一化后不含 \r
+  });
+});

package/src/runner/discover.ts

@@ -5,6 +5,7 @@ import { readdir } from "node:fs/promises";
 import { dirname, join, relative, sep } from "node:path";
 import { pathToFileURL } from "node:url";
 import { pad4 } from "../util.ts";
+import { captureEvalSource } from "./eval-source.ts";
 import type { DiscoveredEval, DiscoveredExperiment, EvalDef, ExperimentDef } from "../types.ts";
 
 const SKIP_DIRS = new Set(["node_modules", ".git", ".niceeval", "dist", ".next"]);
@@ -42,10 +43,13 @@ export async function discoverEvals(root: string): Promise<DiscoveredEval[]> {
     if (!def) continue;
     const baseId = relative(dir, file).replace(/\.eval\.tsx?$/, "").split(sep).join("/");
     const baseDir = dirname(file);
+    // discovery 时读一次、归一化、算 SHA-256:同一文件(数组默认导出多个 eval)只读一次盘,
+    // 全部共享同一份 CapturedEvalSource 引用——写入面按哈希去重靠的就是这份内容天然相同。
+    const source = await captureEvalSource(file, { root });
     if (Array.isArray(def)) {
-      def.forEach((d, i) => out.push({ ...d, id: `${baseId}/${pad4(i)}`, baseDir, sourcePath: file }));
+      def.forEach((d, i) => out.push({ ...d, id: `${baseId}/${pad4(i)}`, baseDir, sourcePath: file, source }));
     } else {
-      out.push({ ...def, id: baseId, baseDir, sourcePath: file });
+      out.push({ ...def, id: baseId, baseDir, sourcePath: file, source });
     }
   }
   return out;

package/src/runner/eval-source.test.ts

@@ -0,0 +1,121 @@
+// captureEvalSource 的单测(定稿见 docs/feature/results/architecture.md「sources.json」)。
+// 覆盖:哈希确定性、path 相对 root 计算、CRLF/BOM 归一化行为、与 results/source-hash.ts
+// 算法保持一致(两处哈希必须逐字节相同,见 eval-source.ts 顶部注释)。
+
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { hashEvalSource, normalizeEvalSource } from "../results/source-hash.ts";
+import { captureEvalSource } from "./eval-source.ts";
+
+const roots: string[] = [];
+async function makeRoot(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-eval-source-"));
+  roots.push(root);
+  return root;
+}
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
+});
+
+describe("captureEvalSource", () => {
+  it("computes a deterministic sha256 for the same file content", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const file = join(root, "evals", "weather.eval.ts");
+    await writeFile(file, "export default { id: 'weather' };\n", "utf-8");
+
+    const first = await captureEvalSource(file, { root });
+    const second = await captureEvalSource(file, { root });
+
+    expect(first.sha256).toBe(second.sha256);
+    expect(first.sha256).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it("returns a path relative to the given root, using forward slashes", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals", "nested"), { recursive: true });
+    const file = join(root, "evals", "nested", "weather.eval.ts");
+    await writeFile(file, "x\n", "utf-8");
+
+    const captured = await captureEvalSource(file, { root });
+
+    expect(captured.path).toBe("evals/nested/weather.eval.ts");
+  });
+
+  it("defaults root to process.cwd() when omitted", async () => {
+    const root = await makeRoot();
+    const cwdBefore = process.cwd();
+    process.chdir(root);
+    try {
+      // 用 chdir 之后的 process.cwd() 拼绝对路径:mkdtemp 返回的 root 与 chdir 后的
+      // process.cwd() 在 macOS 上可能因 /var → /private/var 的符号链接解析而不是同一字符串,
+      // 直接比较 root 会因这条无关的平台差异误报,不是本函数要覆盖的行为。
+      const file = join(process.cwd(), "weather.eval.ts");
+      await writeFile(file, "x\n", "utf-8");
+      const captured = await captureEvalSource(file);
+      expect(captured.path).toBe("weather.eval.ts");
+    } finally {
+      process.chdir(cwdBefore);
+    }
+  });
+
+  it("normalizes CRLF to LF so cross-platform checkouts hash identically", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const lfFile = join(root, "evals", "lf.eval.ts");
+    const crlfFile = join(root, "evals", "crlf.eval.ts");
+    const body = "line one\nline two\nline three\n";
+    await writeFile(lfFile, body, "utf-8");
+    await writeFile(crlfFile, body.replace(/\n/g, "\r\n"), "utf-8");
+
+    const lf = await captureEvalSource(lfFile, { root });
+    const crlf = await captureEvalSource(crlfFile, { root });
+
+    expect(crlf.content).toBe(lf.content);
+    expect(crlf.sha256).toBe(lf.sha256);
+  });
+
+  it("strips a UTF-8 BOM from the captured content", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const withBom = join(root, "evals", "bom.eval.ts");
+    const withoutBom = join(root, "evals", "no-bom.eval.ts");
+    await writeFile(withBom, "export default {};\n", "utf-8");
+    await writeFile(withoutBom, "export default {};\n", "utf-8");
+
+    const a = await captureEvalSource(withBom, { root });
+    const b = await captureEvalSource(withoutBom, { root });
+
+    expect(a.content).toBe(b.content);
+    expect(a.content.startsWith("")).toBe(false);
+    expect(a.sha256).toBe(b.sha256);
+  });
+
+  it("matches results/source-hash.ts's normalize+hash algorithm exactly", async () => {
+    const root = await makeRoot();
+    const file = join(root, "weather.eval.ts");
+    const raw = "export default {};\r\n// comment\r\n";
+    await writeFile(file, raw, "utf-8");
+
+    const captured = await captureEvalSource(file, { root });
+
+    expect(captured.content).toBe(normalizeEvalSource(raw));
+    expect(captured.sha256).toBe(hashEvalSource(normalizeEvalSource(raw)));
+  });
+
+  it("rejects two files with different content by producing different hashes", async () => {
+    const root = await makeRoot();
+    await mkdir(join(root, "evals"), { recursive: true });
+    const fileA = join(root, "evals", "a.eval.ts");
+    const fileB = join(root, "evals", "b.eval.ts");
+    await writeFile(fileA, "content a\n", "utf-8");
+    await writeFile(fileB, "content b\n", "utf-8");
+
+    const a = await captureEvalSource(fileA, { root });
+    const b = await captureEvalSource(fileB, { root });
+
+    expect(a.sha256).not.toBe(b.sha256);
+  });
+});