niceeval 0.5.4 → 0.6.1

package/src/report/compute.ts

@@ -1,19 +1,24 @@
-// 计算函数:选集 → 一份组件数据。跑在 Node 侧,产物是算好的、可序列化的普通 JSON
+// 计算函数:Selection → 一份组件数据。跑在 Node 侧,产物是算好的、可序列化的普通 JSON
 // (终值 + 渲染提示,不含公式);渲染面(web/text)只做展示。
 //
 // 这些函数不做顶层导出,而是挂在对应组件上(MetricTable.data / Scoreboard.data …,
 // 见 components.tsx):配对打点即发现,泛化名不占顶层导出。
 //
-// 共同约定(docs/reports.md「边界与不变量」):
+// 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
 // - 第一参收 Selection | Snapshot[];收 Selection 时 warnings 随行进 OverviewData;
 // - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
 // - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
 // - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
 
 import type {
-  CaseListData,
+  AttemptListItem,
+  AttemptLocator,
   DeltaData,
   DimensionInput,
+  EvalListItem,
+  ExperimentListEvalRow,
+  ExperimentListItem,
+  GroupSummaryData,
   LineData,
   MatrixData,
   Metric,
@@ -23,7 +28,11 @@ import type {
   ScatterData,
   ScoreboardData,
   TableData,
+  TableRowMeta,
 } from "./types.ts";
+import type { AssertionResult, EvalResult } from "../types.ts";
+import type { AttemptHandle } from "../results/types.ts";
+import { evalLevelStats, foldEvalVerdict } from "../shared/verdict.ts";
 import {
   applyAggregator,
   assertUniqueMetricNames,
@@ -40,13 +49,14 @@ import {
   filterItems,
   groupItems,
   flagAxisValue,
+  locatorOf,
   resolveInput,
   snapshotKeyOf,
   toColumn,
   type Item,
   type SnapshotsInput,
 } from "./aggregate.ts";
-import { attemptCostUSD, examScore } from "./metrics.ts";
+import { attemptCostUSD, costUSD, durationMs, examScore, passRate, tokens } from "./metrics.ts";
 import { formatMetricValue, formatPlainNumber } from "./format.ts";
 
 // ───────────────────────── MetricTable.data ─────────────────────────
@@ -62,6 +72,132 @@ export interface TableDataOptions<M extends readonly Metric[]> {
   evals?: string | string[];
 }
 
+// 一组 Item 的 eval 全身份键:experimentId + eval id。单 experiment 场景(如 experimentRowMeta,
+// 一组本就只有一个 experimentId)下退化为只按 eval id 折叠,与旧行为一致;多 experiment 场景
+// (GroupSummary 的组可能跨多个 experiment)下避免两个 experiment 里同名 eval 被误合并成一道题。
+// 分隔符是 NUL(同 aggregate.ts 的 KEY_SEP 手法):不会出现在 eval id / experimentId 里,拼接键不串味。
+const GROUP_KEY_SEP = "\u0000";
+function fullEvalKey(item: Item): string {
+  return `${experimentIdOf(item)}${GROUP_KEY_SEP}${evalIdOf(item)}`;
+}
+
+/**
+ * 一批 Item 的组级统计:eval 级折叠计票(evalLevelStats,与 view 榜单 / `TableRowMeta.verdicts`
+ * 同一套 foldEvalVerdict 口径,按完整身份键折叠)、experiment/eval/attempt 数量、总成本
+ * (null-safe 求和)、最后运行时间(组内快照 startedAt 的最大值)。`experimentRowMeta` 与
+ * `groupSummaryData` 共用这一份实现,不各自拼装 evalLevelStats。
+ *
+ * 内部纯函数,不导出、不进 index.ts:对外只经 `experimentRowMeta`(挑 verdicts)与
+ * `groupSummaryData`(挑全部字段,包成 `GroupSummaryData`)暴露,调用方拿不到 `Item[]`
+ * 本身,所以这里也不用担心被越权复用。
+ */
+function summarizeItems(items: Item[]): {
+  experiments: number;
+  evals: number;
+  attempts: number;
+  verdicts: { passed: number; failed: number; errored: number; skipped: number };
+  /** 折叠后代表每个「已跑」(非 skipped)eval 的一条 attempt 引用,与 ran 同序同数。 */
+  refs: AttemptLocator[];
+  /** 计入通过率分母的 eval 数(passed + failed + errored,不含 skipped)。 */
+  ran: number;
+  totalCostUSD: number | null;
+  lastRunAt: string | undefined;
+} {
+  const experimentIds = new Set<string>();
+  for (const item of items) experimentIds.add(experimentIdOf(item));
+
+  const byEval = new Map<string, Item[]>();
+  for (const item of items) {
+    const key = fullEvalKey(item);
+    const list = byEval.get(key);
+    if (list) list.push(item);
+    else byEval.set(key, [item]);
+  }
+  const stats = evalLevelStats(
+    items.map((item) => ({ verdict: item.attempt.result.verdict, key: fullEvalKey(item) })),
+    (r) => r.key,
+  );
+  // 折叠代表 attempt:每个已跑的 eval 挑一条与折叠判定一致的 attempt 做证据引用,
+  // skipped 的 eval 不进分母、不出证据。
+  const refs: AttemptLocator[] = [];
+  for (const group of byEval.values()) {
+    const verdict = foldEvalVerdict(group.map((item) => item.attempt.result));
+    if (verdict === "skipped") continue;
+    const rep = group.find((item) => item.attempt.result.verdict === verdict) ?? group[0]!;
+    refs.push(locatorOf(rep));
+  }
+
+  let totalCostUSD: number | null = null;
+  for (const item of items) {
+    const cost = attemptCostUSD(item.attempt.result);
+    if (cost !== null) totalCostUSD = (totalCostUSD ?? 0) + cost;
+  }
+
+  let lastRunAt: string | undefined;
+  for (const item of items) {
+    const startedAt = item.snapshot.startedAt;
+    if (lastRunAt === undefined || startedAt > lastRunAt) lastRunAt = startedAt;
+  }
+
+  return {
+    experiments: experimentIds.size,
+    evals: stats.evals,
+    attempts: items.length,
+    verdicts: { passed: stats.passed, failed: stats.failed, errored: stats.errored, skipped: stats.skipped },
+    refs,
+    ran: stats.passed + stats.failed + stats.errored,
+    totalCostUSD,
+    lastRunAt,
+  };
+}
+
+/**
+ * experiment 行的元信息:agent/model 身份(组内去重后拼接)+ eval 级折叠计票 + eval/attempt
+ * 数量与最后运行时间(summarizeItems,即 view 榜单 / ExperimentList 的同一套 foldEvalVerdict
+ * 口径)。其它行维度(agent/eval/自定义…)没有唯一身份,不携带。
+ */
+function experimentRowMeta(group: Item[]): TableRowMeta {
+  const agents = new Set<string>();
+  const models = new Set<string>();
+  for (const item of group) {
+    agents.add(item.attempt.result.agent);
+    const model = item.attempt.result.model ?? item.snapshot.model;
+    if (model !== undefined) models.add(model);
+  }
+  const stats = summarizeItems(group);
+  return {
+    ...(agents.size > 0 ? { agent: [...agents].join(", ") } : {}),
+    ...(models.size > 0 ? { model: [...models].join(", ") } : {}),
+    verdicts: stats.verdicts,
+    evals: stats.evals,
+    attempts: stats.attempts,
+    ...(stats.lastRunAt !== undefined ? { lastRunAt: stats.lastRunAt } : {}),
+  };
+}
+
+/**
+ * 一次 attempt 未通过的 gate 断言,原始声明顺序不变;soft 断言不参与判定,不算「失败原因」,
+ * 只影响得分,永不出现在这份列表里。`EvalList` / `ExperimentList` 的失败诊断与 `AttemptList`
+ * 的断言列表共用这同一份材料,保证同一个 attempt 在各处给出同一个原因。
+ */
+export function failingGateAssertions(result: EvalResult): AssertionResult[] {
+  return result.assertions.filter((a) => !a.passed && a.severity === "gate");
+}
+
+/**
+ * 一次 attempt 的失败原因文案,按优先级取第一个在场的:
+ * `error` → `skipReason` → 未通过的 gate 断言(原始声明顺序,`name`,detail 在场则
+ * `"name: detail"`,多条用「, 」连接)→ 都缺席则无原因(如某道题恰好没有失败信号)。
+ * soft 断言永不进入这份原因文案,soft 得分是独立概念,不与 reason 混用同一个字段。
+ */
+export function reasonFor(result: EvalResult): string | undefined {
+  if (result.error !== undefined) return result.error;
+  if (result.skipReason !== undefined) return result.skipReason;
+  const gates = failingGateAssertions(result);
+  if (gates.length === 0) return undefined;
+  return gates.map((a) => (a.detail ? `${a.name}: ${a.detail}` : a.name)).join(", ");
+}
+
 export async function tableData<const M extends readonly Metric[]>(
   input: SnapshotsInput,
   opts: TableDataOptions<M>,
@@ -70,7 +206,7 @@ export async function tableData<const M extends readonly Metric[]>(
   const { snapshots } = resolveInput(input);
   const items = filterItems(collectItems(snapshots), opts.evals);
   const groups = groupItems(items, opts.rows);
-  const rows: { key: string; cells: Record<string, MetricCell> }[] = [];
+  const rows: TableData["rows"] = [];
   const sortCells = new Map<string, MetricCell>();
   for (const [key, group] of groups) {
     const cells: Record<string, MetricCell> = {};
@@ -79,7 +215,12 @@ export async function tableData<const M extends readonly Metric[]>(
       // sort 指标不在 columns 里时单独算一遍,只用于排序、不进输出
       sortCells.set(key, cells[opts.sort.name] ?? (await computeCell(opts.sort, group)));
     }
-    rows.push({ key, cells });
+    const meta: TableRowMeta = opts.rows === "experiment" ? experimentRowMeta(group) : {};
+    rows.push({
+      key,
+      cells,
+      ...(Object.keys(meta).length > 0 ? { meta } : {}),
+    });
   }
   if (opts.sort) {
     const better = opts.sort.better ?? "higher";
@@ -99,6 +240,147 @@ export async function tableData<const M extends readonly Metric[]>(
   } as TableData<M[number]["name"]>;
 }
 
+// ───────────────────────── ExperimentList.data / EvalList.data / AttemptList.data ─────────────────────────
+//
+// 三个实体列表逐级下钻(experiment → experimentId × eval → attempt),固定展示实体事实,
+// 没有列配置;过滤是报告作者对返回数组调用 .filter()/.slice() 的事,不进这里
+// (docs/feature/reports/library.md「实体列表」)。AttemptListItem 是三者共用的叶子形状——
+// ExperimentList / EvalList 的下钻数组直接复用它,不各自精简一份。
+
+/** 自由文本(error / 断言 detail / evidence)的发布消毒钩子;身份字段(name/severity/loc)不经它。 */
+function redactAssertions(assertions: AssertionResult[], redact: (text: string) => string): AssertionResult[] {
+  if (assertions.length === 0) return assertions;
+  return assertions.map((a) => ({
+    ...a,
+    ...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
+    ...(a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
+  }));
+}
+
+/** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
+function attemptListItemOf(item: Item, redact: (text: string) => string): AttemptListItem {
+  const result = item.attempt.result;
+  const cost = attemptCostUSD(result);
+  return {
+    evalId: evalIdOf(item),
+    experimentId: experimentIdOf(item),
+    attempt: result.attempt,
+    agent: result.agent,
+    verdict: result.verdict,
+    ...(result.error !== undefined ? { error: redact(result.error) } : {}),
+    assertions: redactAssertions(result.assertions, redact),
+    durationMs: result.durationMs,
+    ...(cost !== null ? { costUSD: cost } : {}),
+    locator: locatorOf(item),
+  };
+}
+
+const identityRedact = (text: string): string => text;
+
+export interface AttemptListDataOptions {
+  /** 发布消毒:error / 断言 detail / evidence 经这个钩子;身份字段(experimentId/evalId/locator…)不经它。 */
+  redact?: (text: string) => string;
+}
+
+/** `AttemptList.data(selection)`:每个 Attempt 一项,顺序取自 Selection 展平顺序(不重排)。 */
+export async function attemptListData(
+  input: SnapshotsInput,
+  opts?: AttemptListDataOptions,
+): Promise<AttemptListItem[]> {
+  const { snapshots } = resolveInput(input);
+  const redact = opts?.redact ?? identityRedact;
+  const items = collectItems(snapshots);
+  return items.map((item) => attemptListItemOf(item, redact));
+}
+
+/** `EvalList.data(selection)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
+export async function evalListData(input: SnapshotsInput): Promise<EvalListItem[]> {
+  const { snapshots } = resolveInput(input);
+  const items = collectItems(snapshots);
+  const groups = new Map<string, Item[]>();
+  for (const item of items) {
+    const key = fullEvalKey(item);
+    const list = groups.get(key);
+    if (list) list.push(item);
+    else groups.set(key, [item]);
+  }
+  const out: EvalListItem[] = [];
+  for (const group of groups.values()) {
+    const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
+    const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
+    const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
+    const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
+    out.push({
+      evalId: evalIdOf(sorted[0]!),
+      experimentId: experimentIdOf(sorted[0]!),
+      verdict,
+      reason: reasonFor(representative.attempt.result),
+      score: await computeCell(examScore, sorted),
+      duration: await computeCell(durationMs, sorted),
+      cost: await computeCell(costUSD, sorted),
+      attempts,
+    });
+  }
+  out.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
+  return out;
+}
+
+/** `ExperimentList.data(selection)`:每个 experiment 一项,按 experimentId 升序;展开到每道 Eval。 */
+export async function experimentListData(input: SnapshotsInput): Promise<ExperimentListItem[]> {
+  const { snapshots } = resolveInput(input);
+  const items = collectItems(snapshots);
+  const groups = groupItems(items, "experiment");
+  const out: ExperimentListItem[] = [];
+  for (const [experimentId, group] of groups) {
+    const stats = summarizeItems(group);
+    const newest = [...group].sort((a, b) => b.snapshot.startedAt.localeCompare(a.snapshot.startedAt))[0]!;
+    const evalGroups = groupItems(group, "eval");
+    const evalRows: ExperimentListEvalRow[] = [];
+    for (const [evalId, evalItems] of evalGroups) {
+      const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
+      const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
+      const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
+      const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
+      evalRows.push({
+        evalId,
+        verdict,
+        reason: reasonFor(representative.attempt.result),
+        duration: await computeCell(durationMs, sorted),
+        cost: await computeCell(costUSD, sorted),
+        attempts,
+      });
+    }
+    evalRows.sort((a, b) => a.evalId.localeCompare(b.evalId));
+    const experiment = newest.snapshot.experiment ?? newest.attempt.result.experiment;
+    out.push({
+      experimentId,
+      agent: newest.snapshot.agent,
+      ...((newest.attempt.result.model ?? newest.snapshot.model) !== undefined
+        ? { model: newest.attempt.result.model ?? newest.snapshot.model }
+        : {}),
+      ...(experiment?.flags ? { flags: experiment.flags } : {}),
+      verdicts: stats.verdicts,
+      passRate: await computeCell(passRate, group),
+      cost: await computeCell(costUSD, group),
+      duration: await computeCell(durationMs, group),
+      tokens: await computeCell(tokens, group),
+      evals: stats.evals,
+      attempts: stats.attempts,
+      lastRunAt: stats.lastRunAt!,
+      evalRows,
+    });
+  }
+  // ExperimentList 是默认实验比较表:初始态按成功率从高到低,缺数据沉底;
+  // 同分时按 experiment id 稳定排序。web 增强可临时重排,text 面沿用同一基准顺序。
+  out.sort((a, b) => {
+    if (a.passRate.value === null && b.passRate.value === null) return a.experimentId.localeCompare(b.experimentId);
+    if (a.passRate.value === null) return 1;
+    if (b.passRate.value === null) return -1;
+    return b.passRate.value - a.passRate.value || a.experimentId.localeCompare(b.experimentId);
+  });
+  return out;
+}
+
 // ───────────────────────── MetricMatrix.data(= MetricBars.data)─────────────────────────
 
 export interface MatrixDataOptions {
@@ -326,7 +608,7 @@ export async function lineData(input: SnapshotsInput, opts: LineDataOptions): Pr
 
 // ───────────────────────── RunOverview.data ─────────────────────────
 
-/** 选集的 warnings 随行进 OverviewData,RunOverview 直接渲染 —— 诚实不靠使用者记得接线。 */
+/** Selection 的 warnings 随行进 OverviewData,RunOverview 直接渲染 —— 诚实不靠使用者记得接线。 */
 export async function overviewData(input: SnapshotsInput): Promise<OverviewData> {
   const { snapshots, warnings } = resolveInput(input);
   const items = collectItems(snapshots);
@@ -340,7 +622,7 @@ export async function overviewData(input: SnapshotsInput): Promise<OverviewData>
   for (const item of items) {
     const result = item.attempt.result;
     evalIds.add(evalIdOf(item));
-    switch (result.outcome) {
+    switch (result.verdict) {
       case "passed":
         passed += 1;
         break;
@@ -358,6 +640,9 @@ export async function overviewData(input: SnapshotsInput): Promise<OverviewData>
     const cost = attemptCostUSD(result);
     if (cost !== null) costUSD = (costUSD ?? 0) + cost;
   }
+  // 通过率的唯一官方口径:两级聚合(computeCell),不是从上面四个 verdict 计票现场重算——
+  // 一道题内 attempt 部分通过要算部分 credit,不是二元投票。
+  const passRateCell = await computeCell(passRate, items);
   return {
     snapshots: snapshots.map((s) => ({
       experimentId: s.experimentId,
@@ -365,11 +650,52 @@ export async function overviewData(input: SnapshotsInput): Promise<OverviewData>
       model: s.model,
       startedAt: s.startedAt,
     })),
-    totals: { evals: evalIds.size, attempts: items.length, passed, failed, errored, skipped, costUSD, durationMs },
+    totals: {
+      evals: evalIds.size,
+      attempts: items.length,
+      passed,
+      failed,
+      errored,
+      skipped,
+      passRate: passRateCell,
+      costUSD,
+      durationMs,
+    },
     warnings: [...warnings],
   };
 }
 
+// ───────────────────────── GroupSummary.data ─────────────────────────
+
+/**
+ * 一组 experiment 的摘要:experiment/eval/attempt 数量、eval 级折叠计票、通过率(旧
+ * `GroupSelector` 卡片口径,见 summarizeItems)、总成本(null-safe 求和)、最后运行时间
+ * (组内快照 startedAt 最大值)。`input` 就是调用方已经收窄好的组 Selection(如自定义报告
+ * 按 experiment 组前缀 filter 出来的那份)——本函数不再自己分组。
+ */
+export async function groupSummaryData(input: SnapshotsInput): Promise<GroupSummaryData> {
+  const { snapshots } = resolveInput(input);
+  const items = collectItems(snapshots);
+  const summary = summarizeItems(items);
+  const ratio = summary.ran > 0 ? summary.verdicts.passed / summary.ran : null; // 分母为 0 → 缺数据,不编 0%
+  const passRateCell: MetricCell = {
+    value: ratio,
+    display: ratio === null ? "—" : formatMetricValue(ratio, "%"),
+    samples: summary.ran,
+    total: summary.evals,
+    refs: summary.refs,
+  };
+  return {
+    experiments: summary.experiments,
+    evals: summary.evals,
+    attempts: summary.attempts,
+    verdicts: summary.verdicts,
+    passRate: passRateCell,
+    totalCostUSD: summary.totalCostUSD,
+    ...(summary.lastRunAt !== undefined ? { lastRunAt: summary.lastRunAt } : {}),
+  };
+}
+
 // ───────────────────────── DeltaTable.data ─────────────────────────
 
 export interface DeltaPair {
@@ -423,48 +749,3 @@ function deltaDisplay(metric: Metric, delta: number | null): string {
   const text = displayValue(metric, delta); // 负号由格式化自带
   return delta > 0 ? `+${text}` : text;
 }
-
-// ───────────────────────── CaseList.data ─────────────────────────
-
-export interface CaseListDataOptions {
-  /** 要列出的判决;默认 failed + errored。 */
-  outcomes?: ("failed" | "errored")[];
-  /** 超出如实报 truncated,不静默截断。 */
-  limit?: number;
-  /** 自由文本(error / 断言 detail / judge evidence)的发布消毒钩子;身份字段不经它。 */
-  redact?: (text: string) => string;
-}
-
-export async function caseListData(input: SnapshotsInput, opts?: CaseListDataOptions): Promise<CaseListData> {
-  const { snapshots } = resolveInput(input);
-  const wanted = new Set<"failed" | "errored">(opts?.outcomes ?? ["failed", "errored"]);
-  const redact = opts?.redact ?? ((text: string) => text);
-  const selected = collectItems(snapshots).filter((item) => {
-    const outcome = item.attempt.result.outcome;
-    return (outcome === "failed" || outcome === "errored") && wanted.has(outcome);
-  });
-  const shown = opts?.limit === undefined ? selected : selected.slice(0, opts.limit);
-  const rows: CaseListData["rows"] = shown.map((item) => {
-    const result = item.attempt.result;
-    const cost = attemptCostUSD(result);
-    return {
-      eval: evalIdOf(item),
-      experimentId: experimentIdOf(item),
-      agent: result.agent,
-      outcome: result.outcome as "failed" | "errored",
-      error: result.error === undefined ? undefined : redact(result.error),
-      failedAssertions: result.assertions
-        .filter((assertion) => !assertion.passed)
-        .map((assertion) => ({
-          name: assertion.name,
-          score: assertion.score,
-          detail: assertion.detail === undefined ? undefined : redact(assertion.detail),
-          evidence: assertion.evidence === undefined ? undefined : redact(assertion.evidence),
-        })),
-      durationMs: result.durationMs,
-      costUSD: cost ?? undefined,
-      ref: item.attempt.ref,
-    };
-  });
-  return { rows, truncated: selected.length - shown.length };
-}