niceeval 0.2.1 → 0.4.0

package/src/report/aggregate.ts

@@ -0,0 +1,215 @@
+// 两级聚合引擎:去重 → 按维度分组 → 组内按 (eval × 快照) 折叠(perEval)→ 跨题折叠(across)→ MetricCell。
+//
+// 为什么是两级:earlyExit 默认开,失败的题天然比通过的题样本多;平铺求均值会把分数
+// 和重试策略纠缠在一起(eval A=[1]、eval B=[0,0,0] 平铺 = 0.25,两级宏平均 = 0.5)。
+// 自定义维度把同一道题的 attempt 分进不同组时,第一级折叠发生在各组内部。
+
+import type { EvalResult } from "../types.ts";
+import type { AttemptHandle, SnapshotHandle } from "../results/types.ts";
+import type {
+  Aggregator,
+  AttemptRef,
+  Dimension,
+  Metric,
+  MetricCell,
+  MetricColumn,
+} from "./types.ts";
+import { formatMetricValue } from "./format.ts";
+
+// 复合键分隔符:NUL 不会出现在 eval id / experimentId / ISO 时间里,拼接键不会串味
+const KEY_SEP = "\u0000";
+
+/** 展平后的一条样本:attempt + 它所属的快照(维度解析与题级折叠都需要快照身份)。 */
+export interface Item {
+  snapshot: SnapshotHandle;
+  attempt: AttemptHandle;
+}
+
+export function experimentIdOf(item: Item): string {
+  return item.attempt.result.experimentId ?? item.snapshot.experimentId;
+}
+
+/** 快照身份:与 view Compare 同口径的 (experimentId, startedAt)。 */
+function snapshotIdentity(snapshot: SnapshotHandle): string {
+  return `${snapshot.experimentId}${KEY_SEP}${snapshot.startedAt}`;
+}
+
+/** 身份键 (experimentId, evalId, attempt, startedAt);startedAt 缺失 → null(宁可不去重也不误删)。 */
+function identityKey(snapshot: SnapshotHandle, result: EvalResult): string | null {
+  if (!result.startedAt) return null;
+  return [
+    result.experimentId ?? snapshot.experimentId,
+    result.id,
+    result.attempt,
+    result.startedAt,
+  ].join(KEY_SEP);
+}
+
+/**
+ * 跨快照聚合前按身份键去重:--resume 会把上一轮已通过的结果原样合入新 run 的 summary,
+ * 同一 attempt 因此存在于多份落盘。重复时保留最新 run(按 run.summary.startedAt)里的那份。
+ *
+ * TODO(results-lib):这是「去重是消费方义务」的本地实现(docs/results-lib.md「身份键与去重」);
+ * writer 给合入结果打标之后,按 niceeval/results 的规则收编,本函数随 handles.ts 一起换掉。
+ */
+export function dedupeAttempts(snapshots: SnapshotHandle[]): Item[] {
+  // 第一遍:按身份键选赢家(run.startedAt 是 ISO 字符串,字典序即时间序;并列保留后出现的)。
+  const winner = new Map<string, { attempt: AttemptHandle; runStartedAt: string }>();
+  for (const snapshot of snapshots) {
+    for (const attempt of snapshot.attempts) {
+      const key = identityKey(snapshot, attempt.result);
+      if (key === null) continue;
+      const runStartedAt = snapshot.run.summary.startedAt ?? "";
+      const prev = winner.get(key);
+      if (!prev || runStartedAt >= prev.runStartedAt) winner.set(key, { attempt, runStartedAt });
+    }
+  }
+  // 第二遍:按原始遍历顺序输出,只放行赢家;无 startedAt 的一律放行(不去重)。
+  const items: Item[] = [];
+  for (const snapshot of snapshots) {
+    for (const attempt of snapshot.attempts) {
+      const key = identityKey(snapshot, attempt.result);
+      if (key === null || winner.get(key)?.attempt === attempt) items.push({ snapshot, attempt });
+    }
+  }
+  return items;
+}
+
+/**
+ * eval id 前缀过滤,同 CLI 位置参数的分段语义(src/runner/discover.ts):
+ * "algebra" 匹配自身与 "algebra/..." 子级,不误配 "algebra2";允许 "algebra/" 尾斜杠写法,等价。
+ */
+export function evalPrefixPredicate(evals?: string | string[]): (id: string) => boolean {
+  if (evals === undefined) return () => true;
+  const prefixes = (Array.isArray(evals) ? evals : [evals]).map((p) => p.replace(/\/+$/, ""));
+  return (id) => prefixes.some((prefix) => id === prefix || id.startsWith(prefix + "/"));
+}
+
+export function filterItems(items: Item[], evals?: string | string[]): Item[] {
+  if (evals === undefined) return items;
+  const match = evalPrefixPredicate(evals);
+  return items.filter((item) => match(item.attempt.result.id));
+}
+
+// ───────────────────────── 维度 ─────────────────────────
+
+export function dimensionName(dimension: Dimension): string {
+  return typeof dimension === "string" ? dimension : dimension.name;
+}
+
+/** eval id 的第一段:"algebra/quadratic" → "algebra";没有 "/" 时就是 id 本身。 */
+export function evalGroupOf(id: string): string {
+  const slash = id.indexOf("/");
+  return slash === -1 ? id : id.slice(0, slash);
+}
+
+export function dimensionKey(dimension: Dimension, item: Item): string {
+  if (typeof dimension !== "string") return dimension.of(item.attempt);
+  const result = item.attempt.result;
+  switch (dimension) {
+    case "agent":
+      return result.agent;
+    case "model":
+      return result.model ?? item.snapshot.model ?? "(none)";
+    case "experiment":
+      return experimentIdOf(item);
+    case "eval":
+      return result.id;
+    case "evalGroup":
+      return evalGroupOf(result.id);
+    case "snapshot":
+      return `${item.snapshot.experimentId} @ ${item.snapshot.startedAt}`;
+    default: {
+      // 穷尽检查:新增内置维度而漏改这里时编译期报错
+      const exhausted: never = dimension;
+      throw new Error(`Unknown dimension: ${String(exhausted)}`);
+    }
+  }
+}
+
+/** 按维度分组,保持首次出现顺序(无 sort 时表格行序即此序)。 */
+export function groupItems(items: Item[], dimension: Dimension): Map<string, Item[]> {
+  const groups = new Map<string, Item[]>();
+  for (const item of items) {
+    const key = dimensionKey(dimension, item);
+    const list = groups.get(key);
+    if (list) list.push(item);
+    else groups.set(key, [item]);
+  }
+  return groups;
+}
+
+// ───────────────────────── 聚合 ─────────────────────────
+
+export function applyAggregator(aggregator: Aggregator, values: number[]): number {
+  if (typeof aggregator === "function") return aggregator(values);
+  switch (aggregator) {
+    case "mean":
+      return values.reduce((a, b) => a + b, 0) / values.length;
+    case "sum":
+      return values.reduce((a, b) => a + b, 0);
+    case "min":
+      return Math.min(...values);
+    case "max":
+      return Math.max(...values);
+  }
+}
+
+/** where 不满足 → null,语义等价于 value 开头 return null。 */
+export async function evaluateMetric(metric: Metric, attempt: AttemptHandle): Promise<number | null> {
+  if (metric.where && !metric.where(attempt)) return null;
+  return metric.value(attempt);
+}
+
+export function displayValue(metric: Metric, value: number | null): string {
+  // null 的纯文本兜底;组件把 null 渲染成「缺数据」,绝不画 0
+  if (value === null) return "—";
+  return metric.display ? metric.display(value) : formatMetricValue(value, metric.unit);
+}
+
+/**
+ * 一个格子:组内 attempt → 两级聚合 → 终值。
+ * null 值不进聚合但计入 total(覆盖率经 samples/total 如实暴露);全 null → value null。
+ */
+export async function computeCell(metric: Metric, items: Item[]): Promise<MetricCell> {
+  // 第一级桶:同一 (eval × 快照) 的 attempt 折成一个题级值
+  const buckets = new Map<string, number[]>();
+  const refs: AttemptRef[] = [];
+  let samples = 0;
+  for (const item of items) {
+    const value = await evaluateMetric(metric, item.attempt);
+    if (value === null) continue;
+    samples += 1;
+    refs.push(item.attempt.ref); // 证据引用由 niceeval/results 的句柄直供,不反查下标
+    const bucketKey = `${item.attempt.result.id}${KEY_SEP}${snapshotIdentity(item.snapshot)}`;
+    const bucket = buckets.get(bucketKey);
+    if (bucket) bucket.push(value);
+    else buckets.set(bucketKey, [value]);
+  }
+  const perEval = metric.aggregate?.perEval ?? "mean";
+  const across = metric.aggregate?.across ?? "mean";
+  const evalValues = [...buckets.values()].map((values) => applyAggregator(perEval, values));
+  const value = evalValues.length === 0 ? null : applyAggregator(across, evalValues);
+  return { value, display: displayValue(metric, value), samples, total: items.length, refs };
+}
+
+export function toColumn(metric: Metric): MetricColumn {
+  return {
+    key: metric.name,
+    label: metric.label ?? metric.name,
+    unit: metric.unit,
+    better: metric.better,
+  };
+}
+
+export function assertUniqueMetricNames(metrics: Metric[], where: string): void {
+  const seen = new Set<string>();
+  for (const metric of metrics) {
+    if (seen.has(metric.name)) {
+      throw new Error(
+        `Duplicate metric name "${metric.name}" in ${where}. Metric names must be unique within one computation; rename one via defineMetric.`,
+      );
+    }
+    seen.add(metric.name);
+  }
+}

package/src/report/compute.ts

@@ -0,0 +1,405 @@
+// 计算函数:快照 → 一份组件数据。跑在 Node 侧,产物是算好的、可序列化的普通 JSON
+// (终值 + 渲染提示,不含公式);前端只做渲染。
+//
+// 共同约定(docs/reports.md「边界与不变量」):
+// - 聚合前按身份键去重(dedupeAttempts);
+// - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
+// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
+
+import type { SnapshotHandle } from "../results/types.ts";
+import type {
+  CaseListData,
+  DeltaData,
+  Dimension,
+  MatrixData,
+  Metric,
+  MetricCell,
+  OverviewData,
+  ScatterData,
+  ScoreboardData,
+  TableData,
+} from "./types.ts";
+import {
+  applyAggregator,
+  assertUniqueMetricNames,
+  computeCell,
+  dedupeAttempts,
+  dimensionKey,
+  dimensionName,
+  displayValue,
+  evalGroupOf,
+  evalPrefixPredicate,
+  evaluateMetric,
+  experimentIdOf,
+  filterItems,
+  groupItems,
+  toColumn,
+  type Item,
+} from "./aggregate.ts";
+import { attemptCostUSD, examScore } from "./metrics.ts";
+import { formatPlainNumber } from "./format.ts";
+
+// ───────────────────────── table ─────────────────────────
+
+export interface TableOptions {
+  /** 行维度。 */
+  rows: Dimension;
+  /** 每列一个指标。 */
+  columns: Metric[];
+  /** 构建时排序,方向随 better(higher 降序,「好」的一头在上);缺数据行沉底。 */
+  sort?: Metric;
+  /** eval id 前缀过滤,同 CLI 位置参数语义。 */
+  evals?: string | string[];
+}
+
+export async function table(snapshots: SnapshotHandle[], opts: TableOptions): Promise<TableData> {
+  assertUniqueMetricNames(opts.columns, "table columns");
+  const items = filterItems(dedupeAttempts(snapshots), opts.evals);
+  const groups = groupItems(items, opts.rows);
+  const rows: TableData["rows"] = [];
+  const sortCells = new Map<string, MetricCell>();
+  for (const [key, group] of groups) {
+    const cells: Record<string, MetricCell> = {};
+    for (const metric of opts.columns) cells[metric.name] = await computeCell(metric, group);
+    if (opts.sort) {
+      // sort 指标不在 columns 里时单独算一遍,只用于排序、不进输出
+      sortCells.set(key, cells[opts.sort.name] ?? (await computeCell(opts.sort, group)));
+    }
+    rows.push({ key, cells });
+  }
+  if (opts.sort) {
+    const better = opts.sort.better ?? "higher";
+    rows.sort((a, b) => {
+      const va = sortCells.get(a.key)?.value ?? null;
+      const vb = sortCells.get(b.key)?.value ?? null;
+      if (va === null && vb === null) return 0;
+      if (va === null) return 1; // 缺数据沉底
+      if (vb === null) return -1;
+      return better === "lower" ? va - vb : vb - va;
+    });
+  }
+  return { dimension: dimensionName(opts.rows), columns: opts.columns.map(toColumn), rows };
+}
+
+// ───────────────────────── matrix ─────────────────────────
+
+export interface MatrixOptions {
+  rows: Dimension;
+  columns: Dimension;
+  cell: Metric;
+}
+
+export async function matrix(snapshots: SnapshotHandle[], opts: MatrixOptions): Promise<MatrixData> {
+  const items = dedupeAttempts(snapshots);
+  // 稀疏分组:只有真有 attempt 的 (row, column) 组合成格;没有样本的格子不出现
+  const groups = new Map<string, { row: string; column: string; items: Item[] }>();
+  for (const item of items) {
+    const row = dimensionKey(opts.rows, item);
+    const column = dimensionKey(opts.columns, item);
+    const key = JSON.stringify([row, column]);
+    const group = groups.get(key);
+    if (group) group.items.push(item);
+    else groups.set(key, { row, column, items: [item] });
+  }
+  const cells: MatrixData["cells"] = [];
+  for (const group of groups.values()) {
+    cells.push({ row: group.row, column: group.column, cell: await computeCell(opts.cell, group.items) });
+  }
+  return {
+    rows: dimensionName(opts.rows),
+    columns: dimensionName(opts.columns),
+    metric: toColumn(opts.cell),
+    cells,
+  };
+}
+
+// ───────────────────────── scoreboard ─────────────────────────
+
+export interface ScoreboardOptions {
+  /** 给谁打分(被打分的维度)。 */
+  of: Dimension;
+  /** 按什么分科;默认 "evalGroup"(考试里的「科目」)。 */
+  subjects?: Dimension;
+  /** eval id 前缀 → 每题分值;未列默认 1;前缀重叠时最长的生效。 */
+  weights?: Record<string, number>;
+  /** 折算满分;默认 100。 */
+  fullMarks?: number;
+  /** 每题得分指标;缺省即 examScore,可换自定义(如「答对但超预算扣分」)。 */
+  score?: Metric;
+  /** 选中范围:eval id 前缀过滤;题集(分母)只遍历这个范围。 */
+  evals?: string | string[];
+}
+
+/**
+ * 逐题分值制,分母对所有被打分者恒定:
+ *   题分值 = 命中的权重(默认 1)   题得分 = score 指标的题级值(perEval 折叠后)
+ *   总分   = fullMarks × Σ(题得分 × 题分值) / Σ(题分值)   Σ 遍历选中范围内全部题
+ * 没跑到的题挣 0 分但留在分母里,missing 如实报 —— 这是显式的考试契约,不是「null ≠ 0」的例外。
+ */
+export async function scoreboard(
+  snapshots: SnapshotHandle[],
+  opts: ScoreboardOptions,
+): Promise<ScoreboardData> {
+  const fullMarks = opts.fullMarks ?? 100;
+  const scoreMetric = opts.score ?? examScore;
+  const subjectsDim: Dimension = opts.subjects ?? "evalGroup";
+  const match = evalPrefixPredicate(opts.evals);
+  const items = filterItems(dedupeAttempts(snapshots), opts.evals);
+
+  // 题集(固定分母):选中范围内、任一快照声明覆盖或实际出现过的全部题
+  const universe = new Set<string>();
+  for (const snapshot of snapshots) {
+    for (const id of snapshot.evalIds) if (match(id)) universe.add(id);
+  }
+  for (const item of items) universe.add(item.attempt.result.id);
+  const sortedUniverse = [...universe].sort();
+
+  // 每题的科目:先从任一 attempt 解析(自定义 subjects 维度也能算);
+  // 全程无 attempt 的题按内置规则兜底,自定义维度无从计算时如实标 "(unknown)"
+  const subjectByEval = new Map<string, string>();
+  for (const item of items) {
+    const id = item.attempt.result.id;
+    if (!subjectByEval.has(id)) subjectByEval.set(id, dimensionKey(subjectsDim, item));
+  }
+  const subjectOf = (id: string): string => {
+    const known = subjectByEval.get(id);
+    if (known !== undefined) return known;
+    if (subjectsDim === "eval") return id;
+    if (subjectsDim === "evalGroup") return evalGroupOf(id);
+    return "(unknown)";
+  };
+
+  // 权重:最长前缀生效(排序后线性找第一个命中即最长)
+  const weights = Object.entries(opts.weights ?? {})
+    .map(([prefix, weight]) => ({ prefix, weight }))
+    .sort((a, b) => b.prefix.length - a.prefix.length);
+  const weightOf = (id: string): number =>
+    weights.find((w) => id.startsWith(w.prefix))?.weight ?? 1;
+
+  const groups = groupItems(items, opts.of);
+  const rows: ScoreboardData["rows"] = [];
+  for (const [key, group] of groups) {
+    // 题得分:perEval 折叠(同 eval × 快照 内);同题出现在多个快照时取快照级值的均值。
+    // TODO(results-lib):多快照同题的口径待与选择器对齐;常规姿势(latestPerExperiment)每题只有一个快照。
+    const perSnapshot = new Map<string, Map<string, number[]>>(); // evalId → snapshot 身份 → 原始值
+    for (const item of group) {
+      const value = await evaluateMetric(scoreMetric, item.attempt);
+      if (value === null) continue; // 测不了的 attempt 不进题得分;整题无样本 → missing
+      const id = item.attempt.result.id;
+      const snapKey = `${item.snapshot.experimentId} @ ${item.snapshot.startedAt}`;
+      let bySnap = perSnapshot.get(id);
+      if (!bySnap) perSnapshot.set(id, (bySnap = new Map()));
+      const bucket = bySnap.get(snapKey);
+      if (bucket) bucket.push(value);
+      else bySnap.set(snapKey, [value]);
+    }
+    const perEvalAgg = scoreMetric.aggregate?.perEval ?? "mean";
+    const scoreByEval = new Map<string, number>();
+    for (const [id, bySnap] of perSnapshot) {
+      const snapValues = [...bySnap.values()].map((values) => applyAggregator(perEvalAgg, values));
+      scoreByEval.set(id, snapValues.reduce((a, b) => a + b, 0) / snapValues.length);
+    }
+
+    // 科目累计:固定分母 —— 没跑的题 0 分挣、留在分母、计入 missing
+    const subjects = new Map<
+      string,
+      { key: string; earned: number; possible: number; evals: number; missing: number }
+    >();
+    for (const id of sortedUniverse) {
+      const subjectKey = subjectOf(id);
+      let subject = subjects.get(subjectKey);
+      if (!subject) subjects.set(subjectKey, (subject = { key: subjectKey, earned: 0, possible: 0, evals: 0, missing: 0 }));
+      const weight = weightOf(id);
+      const got = scoreByEval.get(id);
+      subject.earned += (got ?? 0) * weight;
+      subject.possible += weight;
+      subject.evals += 1;
+      if (got === undefined) subject.missing += 1;
+    }
+    let earned = 0;
+    let possible = 0;
+    for (const subject of subjects.values()) {
+      earned += subject.earned;
+      possible += subject.possible;
+    }
+    const value = possible === 0 ? 0 : (fullMarks * earned) / possible;
+    rows.push({ key, total: { value, display: formatPlainNumber(value) }, subjects: [...subjects.values()] });
+  }
+
+  return { of: dimensionName(opts.of), fullMarks, weights, rows };
+}
+
+// ───────────────────────── scatter ─────────────────────────
+
+export interface ScatterOptions {
+  /** 点维度:每个点 = 该组 attempt 的聚合。 */
+  points: Dimension;
+  /** 可选:同系列的点连成线;省略 = 纯散点。 */
+  series?: Dimension;
+  x: Metric;
+  y: Metric;
+}
+
+export async function scatter(snapshots: SnapshotHandle[], opts: ScatterOptions): Promise<ScatterData> {
+  const items = dedupeAttempts(snapshots);
+  const groups = groupItems(items, opts.points);
+  const rows: ScatterData["rows"] = [];
+  for (const [key, group] of groups) {
+    rows.push({
+      key,
+      // 组内取第一条解析系列:点维度细于系列维度时(experiment ⊂ agent)天然一致
+      series: opts.series ? dimensionKey(opts.series, group[0]) : undefined,
+      x: await computeCell(opts.x, group),
+      y: await computeCell(opts.y, group), // 任一轴 null 的点留在 rows 里:组件不画,但注脚要报的数就从这里数
+    });
+  }
+  return {
+    points: dimensionName(opts.points),
+    series: opts.series ? dimensionName(opts.series) : undefined,
+    x: toColumn(opts.x),
+    y: toColumn(opts.y),
+    rows,
+  };
+}
+
+// ───────────────────────── overview ─────────────────────────
+
+export interface OverviewOptions {
+  /** 选择器(latestPerExperiment 等)的警告,原样透传给 RunOverview 渲染。 */
+  warnings?: string[];
+}
+
+export async function overview(
+  snapshots: SnapshotHandle[],
+  opts?: OverviewOptions,
+): Promise<OverviewData> {
+  const items = dedupeAttempts(snapshots);
+  const evalIds = new Set<string>();
+  let passed = 0;
+  let failed = 0;
+  let errored = 0;
+  let skipped = 0;
+  let durationMs = 0;
+  let costUSD: number | null = null; // 任一 attempt 报了成本才有;全缺 = null,不编 0
+  for (const { attempt } of items) {
+    const result = attempt.result;
+    evalIds.add(result.id);
+    switch (result.outcome) {
+      case "passed":
+        passed += 1;
+        break;
+      case "failed":
+        failed += 1;
+        break;
+      case "errored":
+        errored += 1;
+        break;
+      case "skipped":
+        skipped += 1;
+        break;
+    }
+    durationMs += result.durationMs;
+    const cost = attemptCostUSD(result);
+    if (cost !== null) costUSD = (costUSD ?? 0) + cost;
+  }
+  return {
+    snapshots: snapshots.map((s) => ({
+      experimentId: s.experimentId,
+      agent: s.agent,
+      model: s.model,
+      startedAt: s.startedAt,
+    })),
+    totals: { evals: evalIds.size, attempts: items.length, passed, failed, errored, skipped, costUSD, durationMs },
+    warnings: [...(opts?.warnings ?? [])],
+  };
+}
+
+// ───────────────────────── delta ─────────────────────────
+
+export interface DeltaPair {
+  /** 基线侧 experimentId。 */
+  a: string;
+  /** 对比侧 experimentId。 */
+  b: string;
+  label?: string;
+}
+
+export interface DeltaOptions {
+  /** 每行一对:B 相对 A。 */
+  pairs: DeltaPair[];
+  metrics: Metric[];
+}
+
+export async function delta(snapshots: SnapshotHandle[], opts: DeltaOptions): Promise<DeltaData> {
+  assertUniqueMetricNames(opts.metrics, "delta metrics");
+  const items = dedupeAttempts(snapshots);
+  const rows: DeltaData["rows"] = [];
+  for (const pair of opts.pairs) {
+    const aItems = items.filter((item) => experimentIdOf(item) === pair.a);
+    const bItems = items.filter((item) => experimentIdOf(item) === pair.b);
+    const cells: DeltaData["rows"][number]["cells"] = {};
+    for (const metric of opts.metrics) {
+      const a = await computeCell(metric, aItems);
+      const b = await computeCell(metric, bItems);
+      const d = a.value === null || b.value === null ? null : b.value - a.value;
+      cells[metric.name] = { a, b, delta: d, display: deltaDisplay(metric, d) };
+    }
+    rows.push({
+      key: pair.label ?? `${pair.a} vs ${pair.b}`,
+      a: { experimentId: pair.a },
+      b: { experimentId: pair.b },
+      cells,
+    });
+  }
+  return { columns: opts.metrics.map(toColumn), rows };
+}
+
+function deltaDisplay(metric: Metric, delta: number | null): string {
+  if (delta === null) return "—"; // 任一侧缺数据:Δ 显示为缺,不硬算
+  const text = displayValue(metric, delta); // 负号由格式化自带
+  return delta > 0 ? `+${text}` : text;
+}
+
+// ───────────────────────── cases ─────────────────────────
+
+export interface CasesOptions {
+  /** 要列出的判决;默认 failed + errored。 */
+  outcomes?: ("failed" | "errored")[];
+  /** 超出如实报 truncated,不静默截断。 */
+  limit?: number;
+  /** 自由文本(error / 断言 detail / judge evidence)的发布消毒钩子;身份字段不经它。 */
+  redact?: (text: string) => string;
+}
+
+export async function cases(snapshots: SnapshotHandle[], opts?: CasesOptions): Promise<CaseListData> {
+  const wanted = new Set<"failed" | "errored">(opts?.outcomes ?? ["failed", "errored"]);
+  const redact = opts?.redact ?? ((text: string) => text);
+  const selected = dedupeAttempts(snapshots).filter((item) => {
+    const outcome = item.attempt.result.outcome;
+    return (outcome === "failed" || outcome === "errored") && wanted.has(outcome);
+  });
+  const shown = opts?.limit === undefined ? selected : selected.slice(0, opts.limit);
+  const rows: CaseListData["rows"] = shown.map((item) => {
+    const result = item.attempt.result;
+    const cost = attemptCostUSD(result);
+    return {
+      eval: result.id,
+      experimentId: experimentIdOf(item),
+      agent: result.agent,
+      outcome: result.outcome as "failed" | "errored",
+      error: result.error === undefined ? undefined : redact(result.error),
+      failedAssertions: result.assertions
+        .filter((assertion) => !assertion.passed)
+        .map((assertion) => ({
+          name: assertion.name,
+          score: assertion.score,
+          detail: assertion.detail === undefined ? undefined : redact(assertion.detail),
+          evidence: assertion.evidence === undefined ? undefined : redact(assertion.evidence),
+        })),
+      durationMs: result.durationMs,
+      costUSD: cost ?? undefined,
+      ref: item.attempt.ref,
+    };
+  });
+  return { rows, truncated: selected.length - shown.length };
+}

package/src/report/format.ts

@@ -0,0 +1,47 @@
+// unit 驱动的内置格式化(docs/reports.md「指标与聚合」):
+//   "%" → 87%    "ms" → 1.2s    "$" → $0.31    其余 → 1.2k 缩写(带 unit 后缀)
+// metric.display 可整体覆盖;这里只负责默认。
+
+/** 一位小数、去掉无意义的 ".0" 尾巴。 */
+function trimmed(n: number): string {
+  const s = n.toFixed(1);
+  return s.endsWith(".0") ? s.slice(0, -2) : s;
+}
+
+/** 1.2k / 3.4M / 5.6B 式缩写(输入为非负数)。 */
+function abbreviate(abs: number): string {
+  if (abs >= 1e9) return `${trimmed(abs / 1e9)}B`;
+  if (abs >= 1e6) return `${trimmed(abs / 1e6)}M`;
+  if (abs >= 1e3) return `${trimmed(abs / 1e3)}k`;
+  return Number.isInteger(abs) ? String(abs) : trimmed(abs);
+}
+
+function formatDuration(absMs: number): string {
+  if (absMs < 1000) return `${Math.round(absMs)}ms`;
+  if (absMs < 60_000) return `${trimmed(absMs / 1000)}s`;
+  const totalSeconds = Math.round(absMs / 1000);
+  return `${Math.floor(totalSeconds / 60)}m ${totalSeconds % 60}s`;
+}
+
+function formatDollars(abs: number): string {
+  if (abs >= 1000) return abbreviate(abs);
+  if (abs >= 0.01 || abs === 0) return abs.toFixed(2);
+  // 小额成本保留有效位,不四舍成 "$0.00" 假零
+  return abs.toFixed(4);
+}
+
+export function formatMetricValue(value: number, unit?: string): string {
+  const sign = value < 0 ? "-" : "";
+  const abs = Math.abs(value);
+  if (unit === "%") return `${sign}${trimmed(Math.round(abs * 1000) / 10)}%`;
+  if (unit === "ms") return sign + formatDuration(abs);
+  if (unit === "$") return `${sign}$${formatDollars(abs)}`;
+  const n = abbreviate(abs);
+  return unit ? `${sign}${n} ${unit}` : `${sign}${n}`;
+}
+
+/** 无单位纯数字(scoreboard 总分等):一位小数,去尾零。 */
+export function formatPlainNumber(value: number): string {
+  const sign = value < 0 ? "-" : "";
+  return sign + trimmed(Math.round(Math.abs(value) * 10) / 10);
+}