niceeval 0.4.6 → 0.5.1

package/src/report/aggregate.ts

@@ -4,75 +4,77 @@
 // 和重试策略纠缠在一起(eval A=[1]、eval B=[0,0,0] 平铺 = 0.25,两级宏平均 = 0.5)。
 // 自定义维度把同一道题的 attempt 分进不同组时,第一级折叠发生在各组内部。
 
-import type { EvalResult } from "../types.ts";
-import type { AttemptHandle, SnapshotHandle } from "../results/types.ts";
+import {
+  dedupeAttempts,
+  type AttemptHandle,
+  type Selection,
+  type SelectionWarning,
+  type Snapshot,
+} from "../results/index.ts";
 import type {
   Aggregator,
   AttemptRef,
-  Dimension,
+  DimensionInput,
   Metric,
   MetricCell,
   MetricColumn,
+  ParamRef,
 } from "./types.ts";
 import { formatMetricValue } from "./format.ts";
 
 // 复合键分隔符:NUL 不会出现在 eval id / experimentId / ISO 时间里,拼接键不会串味
 const KEY_SEP = "\u0000";
 
+/** param 未声明时的组名:不猜,如实归一组。 */
+export const PARAM_UNSET = "(unset)";
+
+/** 计算函数的第一参:选集(warnings 随行)或手工挑的快照数组(没有挑选过程,自然无警告)。 */
+export type SnapshotsInput = Selection | Snapshot[];
+
+export function resolveInput(input: SnapshotsInput): {
+  snapshots: Snapshot[];
+  warnings: SelectionWarning[];
+} {
+  if (Array.isArray(input)) return { snapshots: input, warnings: [] };
+  return { snapshots: input.snapshots, warnings: input.warnings };
+}
+
 /** 展平后的一条样本:attempt + 它所属的快照(维度解析与题级折叠都需要快照身份)。 */
 export interface Item {
-  snapshot: SnapshotHandle;
+  snapshot: Snapshot;
   attempt: AttemptHandle;
 }
 
 export function experimentIdOf(item: Item): string {
-  return item.attempt.result.experimentId ?? item.snapshot.experimentId;
+  return item.attempt.experimentId || item.snapshot.experimentId;
 }
 
-/** 快照身份:与 view Compare 同口径的 (experimentId, startedAt)。 */
-function snapshotIdentity(snapshot: SnapshotHandle): string {
-  return `${snapshot.experimentId}${KEY_SEP}${snapshot.startedAt}`;
+export function evalIdOf(item: Item): string {
+  return item.attempt.evalId || item.attempt.result.id;
 }
 
-/** 身份键 (experimentId, evalId, attempt, startedAt);startedAt 缺失 → null(宁可不去重也不误删)。 */
-function identityKey(snapshot: SnapshotHandle, result: EvalResult): string | null {
-  if (!result.startedAt) return null;
-  return [
-    result.experimentId ?? snapshot.experimentId,
-    result.id,
-    result.attempt,
-    result.startedAt,
-  ].join(KEY_SEP);
+/** 快照键:与 view Compare 同口径的 "<experimentId> @ <startedAt>"。 */
+export function snapshotKeyOf(snapshot: Snapshot): string {
+  return `${snapshot.experimentId} @ ${snapshot.startedAt}`;
 }
 
 /**
- * 跨快照聚合前按身份键去重:--resume 会把上一轮已通过的结果原样合入新 run 的 summary,
- * 同一 attempt 因此存在于多份落盘。重复时保留最新 run(按 run.summary.startedAt)里的那份。
- *
- * TODO(results-lib):这是「去重是消费方义务」的本地实现(docs/results-lib.md「身份键与去重」);
- * writer 给合入结果打标之后,按 niceeval/results 的规则收编,本函数随 handles.ts 一起换掉。
+ * 展平 + 聚合前去重(niceeval/results 的 dedupeAttempts,身份键
+ * (experimentId, evalId, attempt, startedAt))。missing-startedAt 的警告不透出:
+ * 官方产出永不缺 startedAt,缺失只可能来自 legacy 落盘,「不去重、如实保留重复」即终稿。
  */
-export function dedupeAttempts(snapshots: SnapshotHandle[]): Item[] {
-  // 第一遍:按身份键选赢家(run.startedAt 是 ISO 字符串,字典序即时间序;并列保留后出现的)。
-  const winner = new Map<string, { attempt: AttemptHandle; runStartedAt: string }>();
+export function collectItems(snapshots: Snapshot[]): Item[] {
+  const snapshotByAttempt = new Map<AttemptHandle, Snapshot>();
+  const flattened: AttemptHandle[] = [];
   for (const snapshot of snapshots) {
     for (const attempt of snapshot.attempts) {
-      const key = identityKey(snapshot, attempt.result);
-      if (key === null) continue;
-      const runStartedAt = snapshot.run.summary.startedAt ?? "";
-      const prev = winner.get(key);
-      if (!prev || runStartedAt >= prev.runStartedAt) winner.set(key, { attempt, runStartedAt });
+      // 同一 handle 对象出现在两个快照里时以首次归属为准(手工挑重叠快照的极端场景)
+      if (!snapshotByAttempt.has(attempt)) snapshotByAttempt.set(attempt, snapshot);
+      flattened.push(attempt);
     }
   }
-  // 第二遍:按原始遍历顺序输出,只放行赢家;无 startedAt 的一律放行(不去重)。
-  const items: Item[] = [];
-  for (const snapshot of snapshots) {
-    for (const attempt of snapshot.attempts) {
-      const key = identityKey(snapshot, attempt.result);
-      if (key === null || winner.get(key)?.attempt === attempt) items.push({ snapshot, attempt });
-    }
-  }
-  return items;
+  const { attempts } = dedupeAttempts(flattened);
+  return attempts.map((attempt) => ({ attempt, snapshot: snapshotByAttempt.get(attempt)! }));
 }
 
 /**
@@ -88,12 +90,12 @@ export function evalPrefixPredicate(evals?: string | string[]): (id: string) =>
 export function filterItems(items: Item[], evals?: string | string[]): Item[] {
   if (evals === undefined) return items;
   const match = evalPrefixPredicate(evals);
-  return items.filter((item) => match(item.attempt.result.id));
+  return items.filter((item) => match(evalIdOf(item)));
 }
 
 // ───────────────────────── 维度 ─────────────────────────
 
-export function dimensionName(dimension: Dimension): string {
+export function dimensionName(dimension: DimensionInput): string {
   return typeof dimension === "string" ? dimension : dimension.name;
 }
 
@@ -103,8 +105,35 @@ export function evalGroupOf(id: string): string {
   return slash === -1 ? id : id.slice(0, slash);
 }
 
-export function dimensionKey(dimension: Dimension, item: Item): string {
-  if (typeof dimension !== "string") return dimension.of(item.attempt);
+function isParamRef(dimension: DimensionInput): dimension is ParamRef {
+  return typeof dimension === "object" && "kind" in dimension && dimension.kind === "param";
+}
+
+/** experiment 声明的 params(经 runner 原样透传进持久化字段 ExperimentRunInfo.params)。 */
+function paramsOf(attempt: AttemptHandle): Record<string, unknown> | undefined {
+  return attempt.result.experiment?.params;
+}
+
+/** param 声明值 → 组标签:label 函数优先,其余 String();未声明 → PARAM_UNSET。 */
+export function paramGroupKey(ref: ParamRef, item: Item): string {
+  const value = paramsOf(item.attempt)?.[ref.name];
+  if (value === undefined) return PARAM_UNSET;
+  // 持久化字段是 Record<string, unknown>;声明侧的合法值就是这三种标量
+  if (typeof ref.label === "function") return ref.label(value as string | number | boolean);
+  return String(value);
+}
+
+/** param 作轴:要求数值;未声明或非数值 → null(点不画,注脚报数)。 */
+export function paramAxisValue(ref: ParamRef, item: Item): number | null {
+  const value = paramsOf(item.attempt)?.[ref.name];
+  return typeof value === "number" ? value : null;
+}
+
+export function dimensionKey(dimension: DimensionInput, item: Item): string {
+  if (typeof dimension !== "string") {
+    if (isParamRef(dimension)) return paramGroupKey(dimension, item);
+    return dimension.of(item.attempt);
+  }
   const result = item.attempt.result;
   switch (dimension) {
     case "agent":
@@ -114,11 +143,11 @@ export function dimensionKey(dimension: Dimension, item: Item): string {
     case "experiment":
       return experimentIdOf(item);
     case "eval":
-      return result.id;
+      return evalIdOf(item);
     case "evalGroup":
-      return evalGroupOf(result.id);
+      return evalGroupOf(evalIdOf(item));
     case "snapshot":
-      return `${item.snapshot.experimentId} @ ${item.snapshot.startedAt}`;
+      return snapshotKeyOf(item.snapshot);
     default: {
       // 穷尽检查:新增内置维度而漏改这里时编译期报错
       const exhausted: never = dimension;
@@ -128,7 +157,7 @@ export function dimensionKey(dimension: Dimension, item: Item): string {
 }
 
 /** 按维度分组,保持首次出现顺序(无 sort 时表格行序即此序)。 */
-export function groupItems(items: Item[], dimension: Dimension): Map<string, Item[]> {
+export function groupItems(items: Item[], dimension: DimensionInput): Map<string, Item[]> {
   const groups = new Map<string, Item[]>();
   for (const item of items) {
     const key = dimensionKey(dimension, item);
@@ -180,8 +209,8 @@ export async function computeCell(metric: Metric, items: Item[]): Promise<Metric
     const value = await evaluateMetric(metric, item.attempt);
     if (value === null) continue;
     samples += 1;
-    refs.push(item.attempt.ref); // 证据引用由 niceeval/results 的句柄直供,不反查下标
-    const bucketKey = `${item.attempt.result.id}${KEY_SEP}${snapshotIdentity(item.snapshot)}`;
+    refs.push(item.attempt.ref); // 证据引用由句柄直供,不反查下标
+    const bucketKey = `${evalIdOf(item)}${KEY_SEP}${snapshotKeyOf(item.snapshot)}`;
     const bucket = buckets.get(bucketKey);
     if (bucket) bucket.push(value);
     else buckets.set(bucketKey, [value]);
@@ -202,7 +231,7 @@ export function toColumn(metric: Metric): MetricColumn {
   };
 }
 
-export function assertUniqueMetricNames(metrics: Metric[], where: string): void {
+export function assertUniqueMetricNames(metrics: readonly Metric[], where: string): void {
   const seen = new Set<string>();
   for (const metric of metrics) {
     if (seen.has(metric.name)) {

package/src/report/components.tsx

@@ -0,0 +1,190 @@
+// 官方双面组件的装配点:web 面(./react/ 的纯 React 组件)+ text 面(./text/faces.ts)
+// + 挂在组件上的 data 计算函数。faces 两键必填 —— 配对是结构义务,不是配对表;
+// MetricBars.data 就是 MetricMatrix.data(同一份矩阵数据的另一种摆法),别名显式化。
+//
+// 官方组件在宿主里自动接上证据室:web 面的 attemptHref 缺省取 ctx.attemptHref
+// (宿主注入的证据室深链);显式传 prop 可覆盖(嵌进自己应用时自定去处)。
+
+import { defineComponent, isHostWebContextActive } from "./tree.ts";
+import type { AttemptRef } from "../results/index.ts";
+import type {
+  CaseListData,
+  DeltaData,
+  LineData,
+  MatrixData,
+  OverviewData,
+  ScatterData,
+  ScoreboardData,
+  TableData,
+} from "./types.ts";
+import {
+  caseListData,
+  deltaData,
+  lineData,
+  matrixData,
+  overviewData,
+  scatterData,
+  scoreboardData,
+  tableData,
+} from "./compute.ts";
+import {
+  barsText,
+  caseListText,
+  deltaText,
+  lineText,
+  matrixText,
+  overviewText,
+  scatterText,
+  scoreboardText,
+  tableText,
+} from "./text/faces.ts";
+import { RunOverview as RunOverviewWeb } from "./react/RunOverview.tsx";
+import { MetricTable as MetricTableWeb } from "./react/MetricTable.tsx";
+import { MetricMatrix as MetricMatrixWeb } from "./react/MetricMatrix.tsx";
+import { MetricBars as MetricBarsWeb } from "./react/MetricBars.tsx";
+import { Scoreboard as ScoreboardWeb } from "./react/Scoreboard.tsx";
+import { MetricScatter as MetricScatterWeb } from "./react/MetricScatter.tsx";
+import { MetricLine as MetricLineWeb } from "./react/MetricLine.tsx";
+import { DeltaTable as DeltaTableWeb } from "./react/DeltaTable.tsx";
+import { CaseList as CaseListWeb } from "./react/CaseList.tsx";
+
+// ───────────────────────── props ─────────────────────────
+
+export interface RunOverviewProps {
+  data: OverviewData;
+  className?: string;
+}
+
+export interface MetricTableProps {
+  data: TableData;
+  /** 传了,格子可点、下钻去处你定;不传,宿主里走证据室深链,宿主外纯展示。 */
+  attemptHref?: (ref: AttemptRef) => string;
+  className?: string;
+}
+
+export interface MetricMatrixProps {
+  data: MatrixData;
+  attemptHref?: (ref: AttemptRef) => string;
+  className?: string;
+}
+
+export interface ScoreboardProps {
+  data: ScoreboardData;
+  className?: string;
+}
+
+export interface MetricScatterProps {
+  data: ScatterData;
+  /** 点一个点 → 该配置的下钻页。 */
+  pointHref?: (row: ScatterData["rows"][number]) => string;
+  className?: string;
+}
+
+export interface MetricLineProps {
+  data: LineData;
+  pointHref?: (row: LineData["rows"][number]) => string;
+  className?: string;
+}
+
+export interface DeltaTableProps {
+  data: DeltaData;
+  className?: string;
+}
+
+export interface CaseListProps {
+  data: CaseListData;
+  attemptHref?: (ref: AttemptRef) => string;
+  className?: string;
+}
+
+// ───────────────────────── 装配 ─────────────────────────
+
+/** 页头 KPI 条:何时跑的、几个配置、几道题、通过率、总成本;选集的警告随行显示在条内。 */
+export const RunOverview = Object.assign(
+  defineComponent<RunOverviewProps>({
+    web: (props) => <RunOverviewWeb {...props} />,
+    text: ({ data }) => overviewText(data),
+  }),
+  { data: overviewData },
+);
+RunOverview.displayName = "RunOverview";
+
+/** 榜单:一行一个维度值、一列一个指标,回答「谁整体更好」。 */
+export const MetricTable = Object.assign(
+  defineComponent<MetricTableProps>({
+    web: (props, ctx) => <MetricTableWeb {...props} attemptHref={props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined)} />,
+    text: ({ data }) => tableText(data),
+  }),
+  { data: tableData },
+);
+MetricTable.displayName = "MetricTable";
+
+/** 逐题格子:行 × 列两个维度、格子里一个指标,回答「哪道题谁挂了」。 */
+export const MetricMatrix = Object.assign(
+  defineComponent<MetricMatrixProps>({
+    web: (props, ctx) => <MetricMatrixWeb {...props} attemptHref={props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined)} />,
+    text: ({ data }) => matrixText(data),
+  }),
+  { data: matrixData },
+);
+MetricMatrix.displayName = "MetricMatrix";
+
+/** 分组条形:同一份矩阵数据的另一种摆法;MetricBars.data 就是 MetricMatrix.data 的别名。 */
+export const MetricBars = Object.assign(
+  defineComponent<MetricMatrixProps>({
+    web: (props, ctx) => <MetricBarsWeb {...props} attemptHref={props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined)} />,
+    text: ({ data }) => barsText(data),
+  }),
+  { data: matrixData },
+);
+MetricBars.displayName = "MetricBars";
+
+/** 考试成绩单:总分 + 分科小计,固定分母、missing 如实报。 */
+export const Scoreboard = Object.assign(
+  defineComponent<ScoreboardProps>({
+    web: (props) => <ScoreboardWeb {...props} />,
+    text: ({ data }) => scoreboardText(data),
+  }),
+  { data: scoreboardData },
+);
+Scoreboard.displayName = "Scoreboard";
+
+/** 质量 × 成本 frontier:每个点一个配置、两个指标各占一轴,「好」的角落恒在右上。 */
+export const MetricScatter = Object.assign(
+  defineComponent<MetricScatterProps>({
+    web: (props) => <MetricScatterWeb {...props} />,
+    text: ({ data }, ctx) => scatterText(data, ctx),
+  }),
+  { data: scatterData },
+);
+MetricScatter.displayName = "MetricScatter";
+
+/** 趋势线:x 是 experiment 声明的 param(param()),同系列按 x 排序连线。 */
+export const MetricLine = Object.assign(
+  defineComponent<MetricLineProps>({
+    web: (props) => <MetricLineWeb {...props} />,
+    text: ({ data }, ctx) => lineText(data, ctx),
+  }),
+  { data: lineData },
+);
+MetricLine.displayName = "MetricLine";
+
+/** 成对对比:每行一对配置、格子里 A、B、Δ 三个值,涨跌好坏由 better 判定。 */
+export const DeltaTable = Object.assign(
+  defineComponent<DeltaTableProps>({
+    web: (props) => <DeltaTableWeb {...props} />,
+    text: ({ data }) => deltaText(data),
+  }),
+  { data: deltaData },
+);
+DeltaTable.displayName = "DeltaTable";
+
+/** 失败案例清单:榜单回答「多少」,它回答「为什么」;truncated 如实报剩余。 */
+export const CaseList = Object.assign(
+  defineComponent<CaseListProps>({
+    web: (props, ctx) => <CaseListWeb {...props} attemptHref={props.attemptHref ?? (isHostWebContextActive() ? ctx.attemptHref : undefined)} />,
+    text: (props, ctx) => caseListText(props.data, ctx),
+  }),
+  { data: caseListData },
+);
+CaseList.displayName = "CaseList";