niceeval 0.4.6 → 0.5.0

package/src/results/format.ts

@@ -1,12 +1,11 @@
-// Results Format 的布局与版本知识(读取面),规则见 docs/results-format.md。
+// Results Format 的布局与版本知识,规则见 docs/results-format.md。
 //
-// 版本判定与 src/view/loader.ts 的 normalizeSummary 同一口径(view 抛错驱动 CLI 提示,
-// 这里返回分类值驱动 skipped 列表);attempt 目录规则与 src/runner/reporters/artifacts.ts
-// 的 attemptDir 同一规则。按 results-lib 设计,这份知识最终应只住在本库,
-// view / reporter 改吃这里是后续收编步骤 —— 本实验不动它们的代码。
+// 按 docs/results-lib.md,这份知识只住在本库:写入面(writer.ts / copy.ts)与
+// 读取面(open.ts)共用这里的目录规则与版本判定;src/runner/reporters/artifacts.ts
+// 是 writer 的薄壳,view(src/view/data.ts)经 openResults 消费,不自带布局知识。
 
-import type { EvalResult, RunSummary } from "../runner/types.ts";
-import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../runner/types.ts";
+import type { EvalResult, RunSummary } from "../types.ts";
+import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../types.ts";
 import type { ArtifactKind } from "./types.ts";
 
 /** attempt 工件子目录(相对 run 根):<evalId>/<agent>/<model>[/<experimentId>]/a<attempt>。 */
@@ -14,8 +13,8 @@ export function attemptDirOf(r: Pick<EvalResult, "id" | "agent" | "model" | "att
   const safe = (s: string) => s.replace(/[^\w.@-]/g, "_");
   // evalId 里的 / 保留作目录层级,其余危险字符替换。
   const id = r.id.replace(/[^\w./@-]/g, "_");
-  // experiment 段与 writer(artifacts.ts 的 attemptDir)同一规则:两个实验可以同 agent 同 model、
-  // 只差 flags,少这一段它们的工件会互相覆盖;experimentId 里的 / 不作层级(整段压成 _)。
+  // experiment 段:两个实验可以同 agent 同 model、只差实验参数,少这一段它们的工件会互相
+  // 覆盖;experimentId 里的 / 不作层级(整段压成 _),整个实验一格。
   const exp = r.experimentId ? `/${safe(r.experimentId)}` : "";
   return `${id}/${safe(r.agent)}/${safe(r.model ?? "default")}${exp}/a${r.attempt}`;
 }
@@ -28,7 +27,7 @@ export function artifactFileOf(kind: ArtifactKind): string {
 /** summary.json 的版本判定结果;openResults 按它分流 ok / skipped / 静默忽略。 */
 export type SummaryClassification =
   | { kind: "ok"; summary: RunSummary }
-  | { kind: "incompatible"; schemaVersion: number; producerVersion?: string }
+  | { kind: "incompatible"; schemaVersion: number; producer?: RunSummary["producer"] }
   | { kind: "malformed"; detail: string }
   | { kind: "not-a-report" };
 
@@ -49,7 +48,9 @@ export function classifySummary(raw: unknown): SummaryClassification {
     const version = data.schemaVersion ?? 1;
     if (typeof version !== "number") return { kind: "malformed", detail: "schemaVersion is not a number" };
     if (version !== RESULTS_SCHEMA_VERSION) {
-      return { kind: "incompatible", schemaVersion: version, producerVersion: data.producer?.version };
+      // skipped 必须带完整 producer(name + version):npx 提示只对 name === "niceeval" 成立,
+      // 第三方 harness 的落盘只给裸版本号的话,消费方连做对这个分支的信息都没有。
+      return { kind: "incompatible", schemaVersion: version, producer: data.producer };
     }
     if (!Array.isArray(data.results) || typeof data.startedAt !== "string") {
       return { kind: "malformed", detail: "missing results[] or startedAt" };

package/src/results/index.ts

@@ -1,27 +1,44 @@
-// niceeval/results —— 实验结果数据的读写库(设计见 docs/results-lib.md)。
+// niceeval/results —— 实验结果数据的读写库(定稿见 docs/results-lib.md)。
 //
-// 本实验先落读取面:openResults / latestPerExperiment / dedupeAttempts / copyRun。
-// TODO(results-lib): 写入面 createRunWriter 落地后,src/runner/reporters/artifacts.ts
-// 变薄壳、src/view/loader.ts 改吃这里的 reader —— 布局知识全宇宙只剩这一份实现。
+// 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、runDirs、latest() 选集);
+// 写:createRunWriter(快照级声明 + attempt 增量落盘 + summary 收尾);
+// 发布:copySnapshots(格式感知复制 + knownEvalIds 补记);
+// 身份:dedupeAttempts(跨快照聚合前按 (experimentId, evalId, attempt, startedAt) 去重)。
+// 布局知识(路径、清洗、拆分、版本)全宇宙只有这一份实现;
+// src/runner/reporters/artifacts.ts 是写入面的薄壳,view 的读取收编是下一波。
 
 export { openResults } from "./open.ts";
-export { dedupeAttempts, latestPerExperiment } from "./select.ts";
-export { copyRun, type CopiedRun, type CopyRunOptions } from "./copy.ts";
+export { dedupeAttempts } from "./select.ts";
+export { copySnapshots, type CopySnapshotsOptions, type CopySnapshotsResult } from "./copy.ts";
+export {
+  createRunWriter,
+  type AttemptArtifacts,
+  type AttemptEntry,
+  type FinishOverrides,
+  type RunWriter,
+  type RunWriterOptions,
+  type SnapshotDeclaration,
+  type SnapshotWriter,
+} from "./writer.ts";
 export {
   ARTIFACT_KINDS,
   type ArtifactKind,
   type AttemptHandle,
   type AttemptRef,
-  type ResultsCollection,
-  type RunHandle,
-  type SkippedResultsRun,
-  type SnapshotHandle,
+  type DedupeWarning,
+  type Eval,
+  type Experiment,
+  type Results,
+  type RunDir,
+  type Selection,
+  type SelectionWarning,
+  type SkippedRun,
+  type Snapshot,
 } from "./types.ts";
 
-// 结果数据类型的家还没搬(类型迁移是设计里单独的一步);先从这里 re-export,
-// 让消费方从一个入口拿全「句柄 + 数据类型 + 格式常量」。
-export { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../runner/types.ts";
-export type { EvalResult, ExperimentRunInfo, RunSummary } from "../runner/types.ts";
-export type { O11ySummary, StreamEvent, TraceSpan, Usage } from "../o11y/types.ts";
-export type { DiffData } from "../scoring/types.ts";
-export type { SourceArtifact } from "../shared/types.ts";
+// 结果数据类型的家还没搬(facade 迁移是下一波);先从这里 re-export,
+// 让消费方从一个入口拿全「分层句柄 + 数据类型 + 格式常量」。
+export { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION } from "../types.ts";
+export type { EvalResult, ExperimentRunInfo, RunSummary } from "../types.ts";
+export type { O11ySummary, StreamEvent, TraceSpan, Usage } from "../types.ts";
+export type { DiffData, SourceArtifact } from "../types.ts";

package/src/results/open.ts

@@ -1,75 +1,132 @@
-// openResults:扫描结果目录,返回类型化句柄(设计见 docs/results-lib.md「读:openResults」)。
+// openResults:扫描结果目录,返回「实验 → 快照 → eval → attempt」的类型化层次
+// (定稿见 docs/results-lib.md「读:openResults」)。
 //
 // 三条铁律:
-// - runs 忠实磁盘,不合并不去重;合并/聚合永远发生在消费方。
-// - 读不了的 run 进 skipped,不静默丢,也不抛错(单个坏 run 不拖垮整次扫描)。
+// - 忠实磁盘:快照与实验归组只切片,不合并、不聚合、不去重;合并/聚合永远发生在消费方。
+// - 读不了的落盘进 skipped(三种原因),不静默丢,也不抛错(单个坏 run 不拖垮整次扫描)。
 // - 重工件全部懒加载:缺失返回 null(存在性判断被方法语义吸收),同一 handle 内记忆化。
 
 import { readFile, readdir, stat } from "node:fs/promises";
 import { basename, dirname, join, resolve } from "node:path";
-import type { EvalResult } from "../runner/types.ts";
-import type { O11ySummary, StreamEvent, TraceSpan } from "../o11y/types.ts";
-import type { DiffData } from "../scoring/types.ts";
-import type { SourceArtifact } from "../shared/types.ts";
+import type { EvalResult, RunSummary } from "../types.ts";
+import type { O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
+import type { DiffData, SourceArtifact } from "../types.ts";
 import { artifactFileOf, classifySummary, experimentKeyOf } from "./format.ts";
-import type { ArtifactKind, AttemptHandle, ResultsCollection, RunHandle, SkippedResultsRun, SnapshotHandle } from "./types.ts";
+import { isNewerRunDir, selectLatest } from "./select.ts";
+import type {
+  ArtifactKind,
+  AttemptHandle,
+  Eval,
+  Experiment,
+  Results,
+  RunDir,
+  Selection,
+  SkippedRun,
+  Snapshot,
+} from "./types.ts";
+import { ARTIFACT_KINDS } from "./types.ts";
+
+// copySnapshots 补记 knownEvalIds 需要「复制时刻该实验的 evalIds」,而 Snapshot 上按定稿
+// 不挂 Experiment 反向指针 —— 用模块级 WeakMap 记归属,只供库内部(copy.ts)取用。
+const experimentBySnapshot = new WeakMap<Snapshot, Experiment>();
+
+/** 库内部:快照所属的 Experiment(仅对 openResults 产出的快照存在)。 */
+export function experimentOfSnapshot(snapshot: Snapshot): Experiment | undefined {
+  return experimentBySnapshot.get(snapshot);
+}
 
 /**
  * 打开 `.niceeval/` 根目录、单个 run 目录,或直接指向某个 summary.json 的路径。
- * 目录不存在返回空集合(还没跑过 eval 不是错误);任何读不了的 run 进 skipped,不抛错。
+ * 目录不存在返回空集合(还没跑过 eval 不是错误);任何读不了的落盘进 skipped,不抛错。
  */
-export async function openResults(dir: string): Promise<ResultsCollection> {
+export async function openResults(dir: string): Promise<Results> {
   const target = resolve(dir);
-  const collection: ResultsCollection = { runs: [], snapshots: [], skipped: [], warnings: [] };
+  const runDirs: RunDir[] = [];
+  const skipped: SkippedRun[] = [];
 
   let targetStat;
   try {
     targetStat = await stat(target);
   } catch {
-    return collection;
+    return makeResults([], skipped, runDirs);
+  }
+
+  const summaryPaths: string[] = [];
+  if (targetStat.isFile()) {
+    summaryPaths.push(target);
+  } else {
+    const direct = join(target, "summary.json");
+    if (await fileExists(direct)) {
+      // target 本身是 run 目录:它的子目录是 attempt 工件目录,不做 incomplete 探测。
+      summaryPaths.push(direct, ...(await findSummaryFiles(target, direct)));
+    } else {
+      // target 是结果根:逐个 immediate child 判定 —— 有 summary 的收 run;
+      // 没有 summary 但有 attempt 工件的 = crash 没收尾,进 skipped("incomplete")。
+      const entries = await readdir(target, { withFileTypes: true });
+      for (const entry of entries.filter((e) => e.isDirectory())) {
+        const childDir = join(target, entry.name);
+        const found = await findSummaryFiles(childDir);
+        if (found.length > 0) {
+          summaryPaths.push(...found);
+        } else if (await hasArtifactFiles(childDir)) {
+          skipped.push({ dir: childDir, reason: "incomplete" });
+        }
+      }
+    }
   }
-  const summaryPaths = targetStat.isFile() ? [target] : await findSummaryFiles(target);
 
   for (const path of summaryPaths) {
     const outcome = await readRun(path);
-    if (outcome.kind === "run") collection.runs.push(outcome.run);
-    else if (outcome.kind === "skipped") collection.skipped.push(outcome.entry);
+    if (outcome.kind === "run") runDirs.push(outcome.run);
+    else if (outcome.kind === "skipped") skipped.push(outcome.entry);
     // not-a-report:无关 JSON,静默忽略。
   }
 
   // 最新在前;run 目录名是时间戳,startedAt 同刻时按目录名降序兜底。
-  collection.runs.sort((a, b) => b.summary.startedAt.localeCompare(a.summary.startedAt) || b.dir.localeCompare(a.dir));
-  collection.skipped.sort((a, b) => b.dir.localeCompare(a.dir));
+  runDirs.sort((a, b) => (isNewerRunDir(a, b) ? -1 : 1));
+  skipped.sort((a, b) => b.dir.localeCompare(a.dir));
 
-  for (const run of collection.runs) {
-    collection.snapshots.push(...sliceSnapshots(run, collection.warnings));
-  }
-  return collection;
+  const snapshots: Snapshot[] = [];
+  for (const run of runDirs) snapshots.push(...sliceSnapshots(run));
+  return makeResults(buildExperiments(snapshots), skipped, runDirs);
+}
+
+function makeResults(experiments: Experiment[], skipped: SkippedRun[], runDirs: RunDir[]): Results {
+  return {
+    experiments,
+    skipped,
+    runDirs,
+    latest(opts?: { experiments?: string | string[] }): Selection {
+      return selectLatest(experiments, opts);
+    },
+  };
 }
 
+// ───────────────────────── 单个 run 的读取 ─────────────────────────
+
 type ReadRunOutcome =
-  | { kind: "run"; run: RunHandle }
-  | { kind: "skipped"; entry: SkippedResultsRun }
+  | { kind: "run"; run: RunDir }
+  | { kind: "skipped"; entry: SkippedRun }
   | { kind: "not-a-report" };
 
 async function readRun(path: string): Promise<ReadRunOutcome> {
   const runDir = dirname(path);
-  const skipped = (entry: Omit<SkippedResultsRun, "dir" | "path">): ReadRunOutcome => ({
+  const skippedEntry = (entry: Omit<SkippedRun, "dir">): ReadRunOutcome => ({
     kind: "skipped",
-    entry: { dir: runDir, path, ...entry },
+    entry: { dir: runDir, ...entry },
   });
 
   let text: string;
   try {
     text = await readFile(path, "utf-8");
   } catch (e) {
-    return skipped({ reason: "malformed", detail: `cannot read file (${e instanceof Error ? e.message : String(e)})` });
+    return skippedEntry({ reason: "malformed", detail: `cannot read file (${e instanceof Error ? e.message : String(e)})` });
   }
   let raw: unknown;
   try {
     raw = JSON.parse(text);
   } catch {
-    return skipped({ reason: "malformed", detail: "invalid JSON" });
+    return skippedEntry({ reason: "malformed", detail: "invalid JSON" });
   }
 
   const classified = classifySummary(raw);
@@ -77,34 +134,114 @@ async function readRun(path: string): Promise<ReadRunOutcome> {
     case "not-a-report":
       return { kind: "not-a-report" };
     case "malformed":
-      return skipped({ reason: "malformed", detail: classified.detail });
+      return skippedEntry({ reason: "malformed", detail: classified.detail });
     case "incompatible":
-      return skipped({
+      return skippedEntry({
         reason: "incompatible-version",
         schemaVersion: classified.schemaVersion,
-        producerVersion: classified.producerVersion,
+        ...(classified.producer ? { producer: classified.producer } : {}),
       });
     case "ok": {
-      const run: RunHandle = { dir: runDir, summary: classified.summary, attempts: [] };
-      run.attempts = classified.summary.results.map((r, i) => makeAttempt(run, r, i));
+      const run: RunDir = { dir: runDir, summary: classified.summary, attempts: [] };
       return { kind: "run", run };
     }
   }
 }
 
+// ───────────────────────── 快照切片与实验归组 ─────────────────────────
+
+/**
+ * 把一个 run 按 experiment 身份切成快照,同时填充 run.attempts(按 results[] 下标顺序)。
+ * 只切片、不合并、不去重;experimentId 缺失时以 "<agent>/<model>" 合成键,synthetic: true
+ * (对应警告由 latest() 生成,归属 Selection)。
+ */
+function sliceSnapshots(run: RunDir): Snapshot[] {
+  const summary = run.summary;
+  const byExperiment = new Map<string, Snapshot>();
+  const evalsByExperiment = new Map<string, Map<string, Eval>>();
+
+  summary.results.forEach((result, index) => {
+    const key = experimentKeyOf(result);
+    let snapshot = byExperiment.get(key.id);
+    if (!snapshot) {
+      const meta = summary.snapshots?.[key.id];
+      snapshot = {
+        experimentId: key.id,
+        startedAt: meta?.startedAt ?? summary.startedAt,
+        agent: result.agent,
+        ...(result.model !== undefined ? { model: result.model } : {}),
+        ...(summary.producer ? { producer: summary.producer } : {}),
+        schemaVersion: summary.schemaVersion ?? 1,
+        evals: [],
+        attempts: [],
+        runDir: run,
+        ...(key.synthesized ? { synthetic: true } : {}),
+        ...(meta?.knownEvalIds ? { knownEvalIds: [...meta.knownEvalIds] } : {}),
+      };
+      byExperiment.set(key.id, snapshot);
+      evalsByExperiment.set(key.id, new Map());
+    }
+    const attempt = makeAttempt(run, key.id, result, index);
+    run.attempts.push(attempt);
+    const evals = evalsByExperiment.get(key.id)!;
+    let ev = evals.get(result.id);
+    if (!ev) {
+      ev = { id: result.id, attempts: [] };
+      evals.set(result.id, ev);
+      snapshot.evals.push(ev);
+    }
+    ev.attempts.push(attempt);
+  });
+
+  for (const snapshot of byExperiment.values()) {
+    // attempts 平铺 = evals 逐题展开(同题的重试相邻)。
+    snapshot.attempts = snapshot.evals.flatMap((ev) => ev.attempts);
+  }
+  return [...byExperiment.values()];
+}
+
+/** 同一 experiment id 的历次快照归在一起;实验按 id 字典序,快照最新在前。 */
+function buildExperiments(snapshots: Snapshot[]): Experiment[] {
+  const byId = new Map<string, Snapshot[]>();
+  for (const snapshot of snapshots) {
+    const group = byId.get(snapshot.experimentId);
+    if (group) group.push(snapshot);
+    else byId.set(snapshot.experimentId, [snapshot]);
+  }
+
+  const experiments: Experiment[] = [];
+  for (const [id, group] of byId) {
+    group.sort((a, b) => b.startedAt.localeCompare(a.startedAt) || b.runDir.dir.localeCompare(a.runDir.dir));
+    // 已知并集 = 本地历史(各快照覆盖的题)∪ 各快照携带的 knownEvalIds ——
+    // 不是「优先字段」:把快照复制进已有历史的目录时,本地并集可能更大,优先字段会让分母缩水。
+    const ids = new Set<string>();
+    for (const snapshot of group) {
+      for (const ev of snapshot.evals) ids.add(ev.id);
+      for (const known of snapshot.knownEvalIds ?? []) ids.add(known);
+    }
+    const experiment: Experiment = { id, snapshots: group, latest: group[0], evalIds: [...ids].sort() };
+    for (const snapshot of group) experimentBySnapshot.set(snapshot, experiment);
+    experiments.push(experiment);
+  }
+  experiments.sort((a, b) => a.id.localeCompare(b.id));
+  return experiments;
+}
+
 // ───────────────────────── attempt 懒加载 ─────────────────────────
 
-function makeAttempt(run: RunHandle, result: EvalResult, index: number): AttemptHandle {
+function makeAttempt(run: RunDir, experimentId: string, result: EvalResult, index: number): AttemptHandle {
   // 候选工件目录:本 run 下的 artifactsDir 为主;--resume / 跨实验携带合入的条目,其工件
-  // 留在旧 run 目录里,summary 里的 artifactBase(相对结果根目录)指向那里,作为回退。
+  // 留在原 run 目录里,summary 里的 artifactBase(相对结果根目录)指向那里,作为回退。
   const candidates: string[] = [];
   if (result.artifactsDir) candidates.push(join(run.dir, result.artifactsDir));
   if (result.artifactBase) candidates.push(join(dirname(run.dir), result.artifactBase));
 
   return {
-    run,
-    ref: { run: basename(run.dir), result: index },
+    evalId: result.id,
+    experimentId,
     result,
+    ref: { run: basename(run.dir), result: index },
+    runDir: run,
     events: lazyArtifact<StreamEvent[]>(candidates, "events", result.events),
     trace: lazyArtifact<TraceSpan[]>(candidates, "trace", result.trace),
     o11y: lazyArtifact<O11ySummary>(candidates, "o11y", result.o11y),
@@ -153,55 +290,38 @@ function isMissingFile(e: unknown): boolean {
   return code === "ENOENT" || code === "ENOTDIR";
 }
 
-// ───────────────────────── 快照切片 ─────────────────────────
-
-/**
- * 把一个 run 按 experiment 身份切成快照;只切片、不合并、不去重。
- * experimentId 缺失时以 "<agent>/<model>" 合成键,并出集合级 warning(每 run 每键一条)。
- */
-function sliceSnapshots(run: RunHandle, warnings: string[]): SnapshotHandle[] {
-  const byExperiment = new Map<string, SnapshotHandle>();
-  const evalIdSeen = new Map<string, Set<string>>();
-  for (const attempt of run.attempts) {
-    const key = experimentKeyOf(attempt.result);
-    let snapshot = byExperiment.get(key.id);
-    if (!snapshot) {
-      if (key.synthesized) {
-        warnings.push(
-          `warning: run "${run.dir}" has results without experimentId; grouped as "${key.id}" for snapshot identity.`,
-        );
-      }
-      snapshot = {
-        experimentId: key.id,
-        run,
-        startedAt: run.summary.startedAt,
-        agent: attempt.result.agent,
-        model: attempt.result.model,
-        attempts: [],
-        evalIds: [],
-        ...(key.synthesized ? { synthetic: true } : {}),
-      };
-      byExperiment.set(key.id, snapshot);
-      evalIdSeen.set(key.id, new Set());
-    }
-    snapshot.attempts.push(attempt);
-    const seen = evalIdSeen.get(key.id)!;
-    if (!seen.has(attempt.result.id)) {
-      seen.add(attempt.result.id);
-      snapshot.evalIds.push(attempt.result.id);
-    }
-  }
-  return [...byExperiment.values()];
-}
-
 // ───────────────────────── 目录扫描 ─────────────────────────
 
-/** 递归找出全部 summary.json(与 view 的扫描同一姿势;run 目录嵌套深度不做假设)。 */
-async function findSummaryFiles(dir: string): Promise<string[]> {
+/** 递归找出全部 summary.json(run 目录嵌套深度不做假设);skip 用于排除已收录的直达文件。 */
+async function findSummaryFiles(dir: string, skip?: string): Promise<string[]> {
   const entries = await readdir(dir, { withFileTypes: true });
-  const direct = entries.filter((e) => e.isFile() && e.name === "summary.json").map((e) => join(dir, e.name));
+  const direct = entries
+    .filter((e) => e.isFile() && e.name === "summary.json")
+    .map((e) => join(dir, e.name))
+    .filter((p) => p !== skip);
   const nested = await Promise.all(
-    entries.filter((e) => e.isDirectory()).map((e) => findSummaryFiles(join(dir, e.name))),
+    entries.filter((e) => e.isDirectory()).map((e) => findSummaryFiles(join(dir, e.name), skip)),
   );
   return [...direct, ...nested.flat()];
 }
+
+/** 目录下(递归)是否存在任何 attempt 工件文件 —— incomplete 判定的依据。 */
+async function hasArtifactFiles(dir: string): Promise<boolean> {
+  const artifactNames = new Set<string>(ARTIFACT_KINDS.map((kind) => artifactFileOf(kind)));
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.isFile() && artifactNames.has(entry.name)) return true;
+  }
+  for (const entry of entries) {
+    if (entry.isDirectory() && (await hasArtifactFiles(join(dir, entry.name)))) return true;
+  }
+  return false;
+}
+
+async function fileExists(path: string): Promise<boolean> {
+  try {
+    return (await stat(path)).isFile();
+  } catch {
+    return false;
+  }
+}