niceeval 0.3.0 → 0.4.0

package/src/results/open.ts

@@ -0,0 +1,207 @@
+// openResults:扫描结果目录,返回类型化句柄(设计见 docs/results-lib.md「读:openResults」)。
+//
+// 三条铁律:
+// - runs 忠实磁盘,不合并不去重;合并/聚合永远发生在消费方。
+// - 读不了的 run 进 skipped,不静默丢,也不抛错(单个坏 run 不拖垮整次扫描)。
+// - 重工件全部懒加载:缺失返回 null(存在性判断被方法语义吸收),同一 handle 内记忆化。
+
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import type { EvalResult } from "../runner/types.ts";
+import type { O11ySummary, StreamEvent, TraceSpan } from "../o11y/types.ts";
+import type { DiffData } from "../scoring/types.ts";
+import type { SourceArtifact } from "../shared/types.ts";
+import { artifactFileOf, classifySummary, experimentKeyOf } from "./format.ts";
+import type { ArtifactKind, AttemptHandle, ResultsCollection, RunHandle, SkippedResultsRun, SnapshotHandle } from "./types.ts";
+
+/**
+ * 打开 `.niceeval/` 根目录、单个 run 目录,或直接指向某个 summary.json 的路径。
+ * 目录不存在返回空集合(还没跑过 eval 不是错误);任何读不了的 run 进 skipped,不抛错。
+ */
+export async function openResults(dir: string): Promise<ResultsCollection> {
+  const target = resolve(dir);
+  const collection: ResultsCollection = { runs: [], snapshots: [], skipped: [], warnings: [] };
+
+  let targetStat;
+  try {
+    targetStat = await stat(target);
+  } catch {
+    return collection;
+  }
+  const summaryPaths = targetStat.isFile() ? [target] : await findSummaryFiles(target);
+
+  for (const path of summaryPaths) {
+    const outcome = await readRun(path);
+    if (outcome.kind === "run") collection.runs.push(outcome.run);
+    else if (outcome.kind === "skipped") collection.skipped.push(outcome.entry);
+    // not-a-report:无关 JSON,静默忽略。
+  }
+
+  // 最新在前;run 目录名是时间戳,startedAt 同刻时按目录名降序兜底。
+  collection.runs.sort((a, b) => b.summary.startedAt.localeCompare(a.summary.startedAt) || b.dir.localeCompare(a.dir));
+  collection.skipped.sort((a, b) => b.dir.localeCompare(a.dir));
+
+  for (const run of collection.runs) {
+    collection.snapshots.push(...sliceSnapshots(run, collection.warnings));
+  }
+  return collection;
+}
+
+type ReadRunOutcome =
+  | { kind: "run"; run: RunHandle }
+  | { kind: "skipped"; entry: SkippedResultsRun }
+  | { kind: "not-a-report" };
+
+async function readRun(path: string): Promise<ReadRunOutcome> {
+  const runDir = dirname(path);
+  const skipped = (entry: Omit<SkippedResultsRun, "dir" | "path">): ReadRunOutcome => ({
+    kind: "skipped",
+    entry: { dir: runDir, path, ...entry },
+  });
+
+  let text: string;
+  try {
+    text = await readFile(path, "utf-8");
+  } catch (e) {
+    return skipped({ reason: "malformed", detail: `cannot read file (${e instanceof Error ? e.message : String(e)})` });
+  }
+  let raw: unknown;
+  try {
+    raw = JSON.parse(text);
+  } catch {
+    return skipped({ reason: "malformed", detail: "invalid JSON" });
+  }
+
+  const classified = classifySummary(raw);
+  switch (classified.kind) {
+    case "not-a-report":
+      return { kind: "not-a-report" };
+    case "malformed":
+      return skipped({ reason: "malformed", detail: classified.detail });
+    case "incompatible":
+      return skipped({
+        reason: "incompatible-version",
+        schemaVersion: classified.schemaVersion,
+        producerVersion: classified.producerVersion,
+      });
+    case "ok": {
+      const run: RunHandle = { dir: runDir, summary: classified.summary, attempts: [] };
+      run.attempts = classified.summary.results.map((r, i) => makeAttempt(run, r, i));
+      return { kind: "run", run };
+    }
+  }
+}
+
+// ───────────────────────── attempt 懒加载 ─────────────────────────
+
+function makeAttempt(run: RunHandle, result: EvalResult, index: number): AttemptHandle {
+  // 候选工件目录:本 run 下的 artifactsDir 为主;--resume / 跨实验携带合入的条目,其工件
+  // 留在旧 run 目录里,summary 里的 artifactBase(相对结果根目录)指向那里,作为回退。
+  const candidates: string[] = [];
+  if (result.artifactsDir) candidates.push(join(run.dir, result.artifactsDir));
+  if (result.artifactBase) candidates.push(join(dirname(run.dir), result.artifactBase));
+
+  return {
+    run,
+    ref: { run: basename(run.dir), result: index },
+    result,
+    events: lazyArtifact<StreamEvent[]>(candidates, "events", result.events),
+    trace: lazyArtifact<TraceSpan[]>(candidates, "trace", result.trace),
+    o11y: lazyArtifact<O11ySummary>(candidates, "o11y", result.o11y),
+    diff: lazyArtifact<DiffData>(candidates, "diff", result.diff),
+    sources: lazyArtifact<SourceArtifact[]>(candidates, "sources", result.sources),
+  };
+}
+
+/**
+ * 单个工件的懒加载器:缺失返回 null(不抛错);同一 handle 内记忆化,diff 这类可达百 MB 的
+ * 文件绝不读两遍。summary 里内联了该字段时直接用(Json reporter 全量输出/外部工具转换的场景)。
+ * 文件存在但 JSON 损坏是真错误,抛英文错误而不是伪装成缺失;失败不缓存,允许重试。
+ */
+function lazyArtifact<T>(candidateDirs: string[], kind: ArtifactKind, inline: T | undefined): () => Promise<T | null> {
+  let memo: Promise<T | null> | undefined;
+  const load = async (): Promise<T | null> => {
+    if (inline !== undefined) return inline;
+    for (const dir of candidateDirs) {
+      const file = join(dir, artifactFileOf(kind));
+      let text: string;
+      try {
+        text = await readFile(file, "utf-8");
+      } catch (e) {
+        if (isMissingFile(e)) continue;
+        throw new Error(`Cannot read artifact ${file} (${e instanceof Error ? e.message : String(e)}).`);
+      }
+      try {
+        return JSON.parse(text) as T;
+      } catch {
+        throw new Error(`Artifact ${file} is not valid JSON. The file may be corrupted; re-run the eval or delete this attempt directory.`);
+      }
+    }
+    return null;
+  };
+  return () => {
+    memo ??= load().catch((e: unknown) => {
+      memo = undefined;
+      throw e;
+    });
+    return memo;
+  };
+}
+
+function isMissingFile(e: unknown): boolean {
+  const code = (e as NodeJS.ErrnoException | null)?.code;
+  return code === "ENOENT" || code === "ENOTDIR";
+}
+
+// ───────────────────────── 快照切片 ─────────────────────────
+
+/**
+ * 把一个 run 按 experiment 身份切成快照;只切片、不合并、不去重。
+ * experimentId 缺失时以 "<agent>/<model>" 合成键,并出集合级 warning(每 run 每键一条)。
+ */
+function sliceSnapshots(run: RunHandle, warnings: string[]): SnapshotHandle[] {
+  const byExperiment = new Map<string, SnapshotHandle>();
+  const evalIdSeen = new Map<string, Set<string>>();
+  for (const attempt of run.attempts) {
+    const key = experimentKeyOf(attempt.result);
+    let snapshot = byExperiment.get(key.id);
+    if (!snapshot) {
+      if (key.synthesized) {
+        warnings.push(
+          `warning: run "${run.dir}" has results without experimentId; grouped as "${key.id}" for snapshot identity.`,
+        );
+      }
+      snapshot = {
+        experimentId: key.id,
+        run,
+        startedAt: run.summary.startedAt,
+        agent: attempt.result.agent,
+        model: attempt.result.model,
+        attempts: [],
+        evalIds: [],
+        ...(key.synthesized ? { synthetic: true } : {}),
+      };
+      byExperiment.set(key.id, snapshot);
+      evalIdSeen.set(key.id, new Set());
+    }
+    snapshot.attempts.push(attempt);
+    const seen = evalIdSeen.get(key.id)!;
+    if (!seen.has(attempt.result.id)) {
+      seen.add(attempt.result.id);
+      snapshot.evalIds.push(attempt.result.id);
+    }
+  }
+  return [...byExperiment.values()];
+}
+
+// ───────────────────────── 目录扫描 ─────────────────────────
+
+/** 递归找出全部 summary.json(与 view 的扫描同一姿势;run 目录嵌套深度不做假设)。 */
+async function findSummaryFiles(dir: string): Promise<string[]> {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const direct = entries.filter((e) => e.isFile() && e.name === "summary.json").map((e) => join(dir, e.name));
+  const nested = await Promise.all(
+    entries.filter((e) => e.isDirectory()).map((e) => findSummaryFiles(join(dir, e.name))),
+  );
+  return [...direct, ...nested.flat()];
+}

package/src/results/results.test.ts

@@ -0,0 +1,359 @@
+// niceeval/results 读取面的单测:临时目录里手工构造最小 summary.json / 工件 fixture,
+// 覆盖快照切片、latest-per-experiment、残缺警告、resume 去重、版本 skipped、懒加载。
+// fixture 的目录名/工件路径全部手写(不 import 库的路径函数),让测试独立于实现充当格式基准。
+
+import { afterEach, describe, expect, it } from "vitest";
+import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  RESULTS_FORMAT,
+  RESULTS_SCHEMA_VERSION,
+  copyRun,
+  dedupeAttempts,
+  latestPerExperiment,
+  openResults,
+  type EvalResult,
+  type RunSummary,
+} from "./index.ts";
+
+// ───────────────────────── fixture 工具 ─────────────────────────
+
+const roots: string[] = [];
+async function makeRoot(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "niceeval-results-"));
+  roots.push(root);
+  return root;
+}
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
+});
+
+function res(over: Partial<EvalResult> & Pick<EvalResult, "id" | "agent">): EvalResult {
+  return { outcome: "passed", attempt: 1, durationMs: 1000, assertions: [], ...over };
+}
+
+function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
+  const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
+  return {
+    format: RESULTS_FORMAT,
+    schemaVersion: RESULTS_SCHEMA_VERSION,
+    producer: { name: "niceeval", version: "0.3.0" },
+    agent: results[0]?.agent ?? "agent",
+    startedAt: "2026-07-01T08:00:00.000Z",
+    completedAt: "2026-07-01T08:10:00.000Z",
+    passed: count("passed"),
+    failed: count("failed"),
+    skipped: count("skipped"),
+    errored: count("errored"),
+    durationMs: 60_000,
+    results,
+    ...over,
+  };
+}
+
+async function writeRun(root: string, dirName: string, summary: unknown): Promise<string> {
+  const dir = join(root, dirName);
+  await mkdir(dir, { recursive: true });
+  await writeFile(join(dir, "summary.json"), typeof summary === "string" ? summary : JSON.stringify(summary, null, 2), "utf-8");
+  return dir;
+}
+
+async function writeArtifact(runDir: string, relDir: string, file: string, data: unknown): Promise<string> {
+  const dir = join(runDir, relDir);
+  await mkdir(dir, { recursive: true });
+  const path = join(dir, file);
+  await writeFile(path, JSON.stringify(data), "utf-8");
+  return path;
+}
+
+async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ───────────────────────── 测试 ─────────────────────────
+
+describe("openResults · 快照切片", () => {
+  it("一个 run 装两个 experiment:按 experimentId 切成两个快照,agent/model/evalIds 取自切片自身", async () => {
+    const root = await makeRoot();
+    await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "algebra/q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", startedAt: "2026-07-01T08:01:00.000Z" }),
+      res({ id: "algebra/q2", agent: "bub", model: "gpt-5", experimentId: "compare/bub", outcome: "failed" }),
+      res({ id: "algebra/q1", agent: "codex", model: "o3", experimentId: "compare/codex" }),
+    ]));
+
+    const results = await openResults(root);
+    expect(results.runs).toHaveLength(1);
+    expect(results.skipped).toHaveLength(0);
+    expect(results.warnings).toHaveLength(0);
+    expect(results.snapshots).toHaveLength(2);
+
+    const bub = results.snapshots.find((s) => s.experimentId === "compare/bub")!;
+    expect(bub.agent).toBe("bub");
+    expect(bub.model).toBe("gpt-5");
+    expect(bub.attempts).toHaveLength(2);
+    expect(bub.evalIds).toEqual(["algebra/q1", "algebra/q2"]);
+    expect(bub.startedAt).toBe(results.runs[0].summary.startedAt);
+    expect(bub.run).toBe(results.runs[0]);
+
+    const codex = results.snapshots.find((s) => s.experimentId === "compare/codex")!;
+    expect(codex.agent).toBe("codex");
+    expect(codex.model).toBe("o3");
+    expect(codex.attempts).toHaveLength(1);
+  });
+
+  it("experimentId 缺失时以 <agent>/<model> 合成快照键,并出英文 warning(无 model 用 default)", async () => {
+    const root = await makeRoot();
+    await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "codex" }),
+      res({ id: "q2", agent: "codex" }),
+    ]));
+
+    const results = await openResults(root);
+    expect(results.snapshots).toHaveLength(1);
+    expect(results.snapshots[0].experimentId).toBe("codex/default");
+    expect(results.warnings).toHaveLength(1);
+    expect(results.warnings[0]).toMatch(/without experimentId/);
+    expect(results.warnings[0]).toMatch(/"codex\/default"/);
+  });
+});
+
+describe("latestPerExperiment", () => {
+  it("跨两个 run 各取所属:每个 experiment 拿自己最新的快照", async () => {
+    const root = await makeRoot();
+    const monday = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
+      res({ id: "q2", agent: "bub", experimentId: "mid/a" }),
+      res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
+      res({ id: "q2", agent: "codex", experimentId: "mid/b" }),
+    ], { startedAt: "2026-07-01T08:00:00.000Z" }));
+    const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
+      res({ id: "q2", agent: "codex", experimentId: "mid/b" }),
+    ], { startedAt: "2026-07-02T08:00:00.000Z" }));
+
+    const all = await openResults(root);
+    const { snapshots, warnings } = latestPerExperiment(all.snapshots);
+    expect(warnings).toHaveLength(0);
+    expect(snapshots.map((s) => s.experimentId)).toEqual(["mid/a", "mid/b"]);
+    expect(snapshots.find((s) => s.experimentId === "mid/a")!.run.dir).toBe(monday);
+    expect(snapshots.find((s) => s.experimentId === "mid/b")!.run.dir).toBe(tuesday);
+
+    // experiments 前缀过滤,同 CLI 语义;尾斜杠写法等价。
+    expect(latestPerExperiment(all.snapshots, { experiments: "mid/a" }).snapshots).toHaveLength(1);
+    expect(latestPerExperiment(all.snapshots, { experiments: "mid/" }).snapshots).toHaveLength(2);
+    expect(latestPerExperiment(all.snapshots, { experiments: "other" }).snapshots).toHaveLength(0);
+  });
+
+  it("最新快照残缺(只重跑了一道题)时生成 covers x of y 警告,仍返回该快照", async () => {
+    const root = await makeRoot();
+    await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "algebra/q1", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
+      res({ id: "algebra/q2", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
+      res({ id: "algebra/q3", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
+    ], { startedAt: "2026-07-01T08:00:00.000Z" }));
+    await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
+      res({ id: "algebra/q1", agent: "bub", experimentId: "midterm/bub-gpt-5.4" }),
+    ], { startedAt: "2026-07-05T08:00:00.000Z" }));
+
+    const all = await openResults(root);
+    const { snapshots, warnings } = latestPerExperiment(all.snapshots);
+    expect(snapshots).toHaveLength(1);
+    expect(snapshots[0].startedAt).toBe("2026-07-05T08:00:00.000Z");
+    expect(warnings).toHaveLength(1);
+    expect(warnings[0]).toMatch(/^warning: snapshot "midterm\/bub-gpt-5\.4" @ 2026-07-05T08:00:00\.000Z covers 1 of 3 evals seen in history\./);
+    expect(warnings[0]).toMatch(/Re-run `niceeval exp midterm\/bub-gpt-5\.4` for a full snapshot/);
+  });
+});
+
+describe("dedupeAttempts", () => {
+  it("resume 原样合入的重复 attempt 按身份键去重,保留最新 run 的那份;缺 startedAt 不去重并出 warning", async () => {
+    const root = await makeRoot();
+    // 周一:q1 通过、q2 失败;q3 没有 startedAt(异常数据)。
+    const monday = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", experimentId: "e", startedAt: "2026-07-01T08:01:00.000Z" }),
+      res({ id: "q2", agent: "bub", experimentId: "e", outcome: "failed", startedAt: "2026-07-01T08:02:00.000Z" }),
+      res({ id: "q3", agent: "bub", experimentId: "e" }),
+    ], { startedAt: "2026-07-01T08:00:00.000Z" }));
+    // 周二 --resume:q1 原样合入(身份键完全相同),q2 重跑出新 startedAt,q3 依旧没有 startedAt。
+    const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", experimentId: "e", startedAt: "2026-07-01T08:01:00.000Z" }),
+      res({ id: "q2", agent: "bub", experimentId: "e", startedAt: "2026-07-02T08:02:00.000Z" }),
+      res({ id: "q3", agent: "bub", experimentId: "e" }),
+    ], { startedAt: "2026-07-02T08:00:00.000Z" }));
+
+    const all = await openResults(root);
+    const attempts = all.snapshots.flatMap((s) => s.attempts);
+    expect(attempts).toHaveLength(6);
+
+    const { attempts: deduped, warnings } = dedupeAttempts(attempts);
+    // q1 合并成一条;q2 两条(startedAt 不同,是两次真实运行);q3 两条(缺 startedAt,不敢去重)。
+    expect(deduped).toHaveLength(5);
+    const q1 = deduped.filter((a) => a.result.id === "q1");
+    expect(q1).toHaveLength(1);
+    expect(q1[0].run.dir).toBe(tuesday);
+    expect(deduped.filter((a) => a.result.id === "q2")).toHaveLength(2);
+    expect(deduped.filter((a) => a.result.id === "q3")).toHaveLength(2);
+    expect(warnings).toHaveLength(2);
+    expect(warnings[0]).toMatch(/"q3".*has no startedAt/);
+    void monday;
+  });
+});
+
+describe("openResults · 版本与坏文件", () => {
+  it("schemaVersion 不匹配进 skipped 带 producerVersion;坏 JSON 记 malformed;无关 JSON 静默忽略;legacy 无信封照读", async () => {
+    const root = await makeRoot();
+    const incompatible = await writeRun(root, "2026-07-03T08-00-00-000Z", {
+      format: RESULTS_FORMAT,
+      schemaVersion: RESULTS_SCHEMA_VERSION + 1,
+      producer: { name: "niceeval", version: "9.9.9" },
+      startedAt: "2026-07-03T08:00:00.000Z",
+      results: [],
+    });
+    await writeRun(root, "2026-07-04T08-00-00-000Z", "not json {");
+    await writeRun(root, "unrelated", { hello: 1 });
+    // 只有 results 键、没有 startedAt:不满足 legacy 启发式,按无关 JSON 静默忽略(不进 skipped)
+    await writeRun(root, "alien-results", { results: [] });
+    await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([res({ id: "q1", agent: "bub", experimentId: "e" })]));
+    // legacy:引入版本号之前的存量报告,没有 format 信封,按 schemaVersion 1 读。
+    const legacy = summaryOf([res({ id: "q1", agent: "bub", experimentId: "old" })], { startedAt: "2026-06-01T08:00:00.000Z" });
+    delete legacy.format;
+    delete legacy.schemaVersion;
+    delete legacy.producer;
+    await writeRun(root, "2026-06-01T08-00-00-000Z", legacy);
+
+    const results = await openResults(root);
+    expect(results.runs).toHaveLength(2);
+    expect(results.skipped).toHaveLength(2);
+
+    const versionSkip = results.skipped.find((s) => s.reason === "incompatible-version")!;
+    expect(versionSkip.dir).toBe(incompatible);
+    expect(versionSkip.schemaVersion).toBe(RESULTS_SCHEMA_VERSION + 1);
+    expect(versionSkip.producerVersion).toBe("9.9.9");
+
+    const malformed = results.skipped.find((s) => s.reason === "malformed")!;
+    expect(malformed.detail).toBe("invalid JSON");
+  });
+});
+
+describe("AttemptHandle · 懒加载", () => {
+  it("缺文件返回 null 不抛错;读过一次即缓存(文件删了也还在);没有 artifactsDir 时全部为 null", async () => {
+    const root = await makeRoot();
+    const runDir = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/a1", hasEvents: true }),
+      res({ id: "q2", agent: "bub", model: "gpt-5", experimentId: "e" }),
+    ]));
+    const eventsPath = await writeArtifact(runDir, "q1/bub/gpt-5/a1", "events.json", [
+      { type: "message", role: "assistant", text: "hi" },
+    ]);
+
+    const results = await openResults(root);
+    const [q1, q2] = results.runs[0].attempts;
+
+    // ref 契约:run 目录名 + summary.results 下标,Reports 的 refs 与 view 深链直接可用
+    expect(q1.ref).toEqual({ run: "2026-07-01T08-00-00-000Z", result: 0 });
+    expect(q2.ref).toEqual({ run: "2026-07-01T08-00-00-000Z", result: 1 });
+
+    const events = await q1.events();
+    expect(events).toHaveLength(1);
+    // summary 只有 hasEvents/hasTrace/hasSources 三个标记,o11y/diff 没有标记 —— 全靠方法语义吸收。
+    expect(await q1.trace()).toBeNull();
+    expect(await q1.o11y()).toBeNull();
+    expect(await q1.diff()).toBeNull();
+    expect(await q1.sources()).toBeNull();
+
+    // 记忆化:同一 handle 读一次缓存,文件删掉后再读仍返回同一份数据。
+    await rm(eventsPath);
+    expect(await q1.events()).toBe(events);
+
+    // 条目没有 artifactsDir(异常/极简数据):不猜路径,全部 null。
+    expect(await q2.events()).toBeNull();
+    expect(await q2.diff()).toBeNull();
+  });
+});
+
+describe("copyRun", () => {
+  it("按指定工件种类复制选中快照,重建的 summary 只含选中条目并保留版本元数据;产物可被 openResults 回读", async () => {
+    const root = await makeRoot();
+    const monday = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5/a1", hasEvents: true, hasTrace: true, startedAt: "2026-07-01T08:01:00.000Z", estimatedCostUSD: 0.5, usage: { inputTokens: 10, outputTokens: 5 } }),
+      res({ id: "q1", agent: "codex", model: "o3", experimentId: "compare/codex", artifactsDir: "q1/codex/o3/a1", hasEvents: true, outcome: "failed", estimatedCostUSD: 0.25, usage: { inputTokens: 20, outputTokens: 2 } }),
+    ], { startedAt: "2026-07-01T08:00:00.000Z" }));
+    await writeArtifact(monday, "q1/bub/gpt-5/a1", "events.json", [{ type: "message" }]);
+    await writeArtifact(monday, "q1/bub/gpt-5/a1", "trace.json", [{ name: "turn" }]);
+    await writeArtifact(monday, "q1/bub/gpt-5/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
+    await writeArtifact(monday, "q1/codex/o3/a1", "events.json", [{ type: "message" }]);
+    const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5/a1", hasEvents: true, startedAt: "2026-07-02T08:01:00.000Z" }),
+    ], { startedAt: "2026-07-02T08:00:00.000Z" }));
+    await writeArtifact(tuesday, "q1/bub/gpt-5/a1", "events.json", [{ type: "message" }, { type: "message" }]);
+
+    const all = await openResults(root);
+    const picked = latestPerExperiment(all.snapshots, { experiments: "compare/" });
+    const dest = join(await makeRoot(), "site/data/run");
+    const copied = await copyRun(picked.snapshots, dest, { artifacts: ["events", "sources"] });
+
+    expect(copied.warnings).toHaveLength(0);
+    expect(copied.summary.format).toBe(RESULTS_FORMAT);
+    expect(copied.summary.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
+    expect(copied.summary.producer?.version).toBe("0.3.0");
+    expect(copied.summary.results).toHaveLength(2);
+    expect(copied.summary.passed).toBe(1);
+    expect(copied.summary.failed).toBe(1);
+    // bub 取周二那份(最新),codex 取周一那份。
+    const bubEntry = copied.summary.results.find((r) => r.experimentId === "compare/bub")!;
+    expect(bubEntry.startedAt).toBe("2026-07-02T08:01:00.000Z");
+    expect(bubEntry.hasEvents).toBe(true);
+    expect(bubEntry.hasTrace).toBe(false); // trace 没被选中,存在标记按目标目录重算
+
+    // 磁盘:只有选中的工件种类被复制。
+    // 目标目录带 experiment 段(与 writer 的 attemptDir 同规则),源 fixture 的旧式路径靠 artifactsDir 定位
+    expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/events.json"))).toBe(true);
+    expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/trace.json"))).toBe(false);
+    expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/diff.json"))).toBe(false);
+    expect(await exists(join(dest, "q1/codex/o3/compare_codex/a1/events.json"))).toBe(true);
+
+    // 回读:产物是合法 run 目录,懒加载语义不变。
+    const reopened = await openResults(dest);
+    expect(reopened.runs).toHaveLength(1);
+    expect(reopened.snapshots.map((s) => s.experimentId).sort()).toEqual(["compare/bub", "compare/codex"]);
+    const bubAttempt = reopened.snapshots.find((s) => s.experimentId === "compare/bub")!.attempts[0];
+    expect(await bubAttempt.events()).toHaveLength(2);
+    expect(await bubAttempt.trace()).toBeNull();
+
+    // 同一 experiment 的两个快照(未 dedupe)落到同一工件目录:保留最新并出 warning。
+    const bubSnapshots = all.snapshots.filter((s) => s.experimentId === "compare/bub");
+    expect(bubSnapshots).toHaveLength(2);
+    const dest2 = join(await makeRoot(), "run2");
+    const collided = await copyRun(bubSnapshots, dest2);
+    expect(collided.warnings).toHaveLength(1);
+    expect(collided.warnings[0]).toMatch(/multiple attempts map to "q1\/bub\/gpt-5\/compare_bub\/a1"/);
+    expect(collided.summary.results).toHaveLength(1);
+    expect(collided.summary.results[0].startedAt).toBe("2026-07-02T08:01:00.000Z");
+  });
+
+  it("同 agent 同 model 只差 experiment 的两个实验互不覆盖(目录带 experiment 段)", async () => {
+    // memory-evals 的典型配对:bub-gpt-5.4 vs bub-gpt-5.4--agents-md,只差 flags。
+    // attemptDirOf 少了 experiment 段时,这两份工件会在拷贝时碰撞、其中一份被静默丢弃。
+    const root = await makeRoot();
+    const run = await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
+      res({ id: "q1", agent: "bub", model: "gpt-5.4", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5.4/compare_bub/a1", hasEvents: true }),
+      res({ id: "q1", agent: "bub", model: "gpt-5.4", experimentId: "compare/bub--agents-md", artifactsDir: "q1/bub/gpt-5.4/compare_bub--agents-md/a1", hasEvents: true }),
+    ]));
+    await writeArtifact(run, "q1/bub/gpt-5.4/compare_bub/a1", "events.json", [{ n: 1 }]);
+    await writeArtifact(run, "q1/bub/gpt-5.4/compare_bub--agents-md/a1", "events.json", [{ n: 2 }]);
+
+    const all = await openResults(root);
+    const dest = join(await makeRoot(), "pair");
+    const copied = await copyRun(all.snapshots, dest);
+    expect(copied.warnings).toHaveLength(0);
+    expect(copied.summary.results).toHaveLength(2);
+    expect(await exists(join(dest, "q1/bub/gpt-5.4/compare_bub/a1/events.json"))).toBe(true);
+    expect(await exists(join(dest, "q1/bub/gpt-5.4/compare_bub--agents-md/a1/events.json"))).toBe(true);
+  });
+});

package/src/results/select.ts

@@ -0,0 +1,101 @@
+// 快照选择器与 attempt 去重(设计见 docs/results-lib.md「选择快照」「身份键与去重」)。
+//
+// 选择器不是 DSL,只是最常用的那次筛选;更细的口径消费方自己 .filter()。
+// 选择器必须诚实:残缺(最新快照覆盖的 eval 少于历史并集)永远被算出来,以 warnings 返回,
+// 渲染与否在消费方,但缺口不静默。
+
+import { experimentKeyOf } from "./format.ts";
+import type { AttemptHandle, RunHandle, SnapshotHandle } from "./types.ts";
+
+/**
+ * 每个 experiment 取最新一次快照(按所属 run 的 startedAt,同刻按 run 目录名兜底)。
+ * `experiments` 是 experiment id 前缀过滤,同 CLI 位置参数语义("compare" 匹配自身与
+ * "compare/..." 子级;允许写成 "compare/",等价)。
+ */
+export function latestPerExperiment(
+  snapshots: SnapshotHandle[],
+  opts?: { experiments?: string | string[] },
+): { snapshots: SnapshotHandle[]; warnings: string[] } {
+  const filtered = filterByExperiments(snapshots, opts?.experiments);
+
+  const byExperiment = new Map<string, SnapshotHandle[]>();
+  for (const snapshot of filtered) {
+    const group = byExperiment.get(snapshot.experimentId);
+    if (group) group.push(snapshot);
+    else byExperiment.set(snapshot.experimentId, [snapshot]);
+  }
+
+  const picked: SnapshotHandle[] = [];
+  const warnings: string[] = [];
+  for (const [experimentId, group] of byExperiment) {
+    let latest = group[0];
+    for (const candidate of group) {
+      if (isNewerRun(candidate.run, latest.run)) latest = candidate;
+    }
+    // 残缺检测:与该 experiment 全部历史快照的 evalId 并集对比。
+    // 位置参数允许只重跑一道题,产出的「最新快照」可能只有一道题 —— 不能安静吞下。
+    const historyIds = new Set<string>();
+    for (const snapshot of group) for (const id of snapshot.evalIds) historyIds.add(id);
+    if (latest.evalIds.length < historyIds.size) {
+      // 合成键不是真实 experiment id,拼不出可执行的 `niceeval exp` 命令,提示退化成中性说法。
+      const hint = latest.synthetic
+        ? "  Re-run the experiment for a full snapshot, or pick another via .filter()."
+        : `  Re-run \`niceeval exp ${experimentId}\` for a full snapshot, or pick another via .filter().`;
+      warnings.push(
+        `warning: snapshot "${experimentId}" @ ${latest.startedAt} covers ${latest.evalIds.length} of ${historyIds.size} evals seen in history.\n` +
+          hint,
+      );
+    }
+    picked.push(latest);
+  }
+
+  picked.sort((a, b) => a.experimentId.localeCompare(b.experimentId));
+  return { snapshots: picked, warnings };
+}
+
+/**
+ * 跨快照聚合前的身份键去重:(experimentId, evalId, attempt, startedAt)。
+ * --resume 会把上一轮已通过的结果原样合入新 run 的 summary,同一 attempt 因此存在于多份落盘;
+ * 重复时保留最新 run 里的那份(位置取首次出现处,顺序稳定)。
+ * startedAt 缺失时宁可不去重也不误删,并记入 warnings。
+ */
+export function dedupeAttempts(attempts: AttemptHandle[]): { attempts: AttemptHandle[]; warnings: string[] } {
+  const deduped: AttemptHandle[] = [];
+  const indexByKey = new Map<string, number>();
+  const warnings: string[] = [];
+
+  for (const attempt of attempts) {
+    const r = attempt.result;
+    if (!r.startedAt) {
+      warnings.push(
+        `warning: attempt "${r.id}" (attempt ${r.attempt}) in run "${attempt.run.dir}" has no startedAt; kept as-is without dedupe.`,
+      );
+      deduped.push(attempt);
+      continue;
+    }
+    const key = JSON.stringify([experimentKeyOf(r).id, r.id, r.attempt, r.startedAt]);
+    const existing = indexByKey.get(key);
+    if (existing === undefined) {
+      indexByKey.set(key, deduped.length);
+      deduped.push(attempt);
+    } else if (isNewerRun(attempt.run, deduped[existing].run)) {
+      deduped[existing] = attempt;
+    }
+  }
+  return { attempts: deduped, warnings };
+}
+
+/** run 新旧比较:startedAt 优先,同刻按目录名(时间戳目录,字典序即时序)。 */
+export function isNewerRun(a: RunHandle, b: RunHandle): boolean {
+  const byStart = a.summary.startedAt.localeCompare(b.summary.startedAt);
+  if (byStart !== 0) return byStart > 0;
+  return a.dir.localeCompare(b.dir) > 0;
+}
+
+function filterByExperiments(snapshots: SnapshotHandle[], experiments?: string | string[]): SnapshotHandle[] {
+  if (experiments === undefined) return snapshots;
+  const prefixes = (Array.isArray(experiments) ? experiments : [experiments])
+    // 允许 "compare/" 这种带尾斜杠的写法,与 "compare" 等价。
+    .map((p) => p.replace(/\/+$/, ""));
+  return snapshots.filter((s) => prefixes.some((p) => s.experimentId === p || s.experimentId.startsWith(p + "/")));
+}