niceeval 0.6.1 → 0.6.2

package/src/report/built-ins/experiment-comparison.tsx

@@ -1,19 +1,179 @@
-// ExperimentComparison:niceeval view 裸跑时的内置 experiment 比较报告。
+// ExperimentComparison:裸 niceeval show / view 首页渲染的内置默认报告,同时是可整体引用的
+// 官方组合件(docs/feature/reports/library.md「ExperimentComparison」)。
 //
-// 名字描述整份报告回答的问题,不绑定其中某一张图。报告由成本 × 通过率散点和固定列
-// ExperimentList 组成；它是一份普通 ReportDefinition,没有 renderer 私有通道。
+// 名字描述整份报告回答的问题,不绑定其中某一张图。它是组合件而非新的数据源——每组的
+// GroupSummary、成本 × 端到端成功率散点、固定列 ExperimentList 都消费各自公开 .data() 的结果。
+// 两种用法共享同一份口径:
+//   · 默认报告:宿主把它当普通 ReportDefinition(build 面),没有 renderer 私有通道;
+//   · 报告组件:`<ExperimentComparison data={await ExperimentComparison.data(selection)} />`,
+//     在自定义报告或用户自己的 React 页面里整体引用。
 
-import { Col } from "../primitives.tsx";
-import { ExperimentList, MetricScatter } from "../components.tsx";
-import { costUSD, passRate } from "../metrics.ts";
-import { defineReport } from "../report.ts";
+import { Col, Section, Table } from "../primitives.tsx";
+import { ExperimentList, GroupSummary, MetricScatter } from "../components.tsx";
+import { costUSD, endToEndPassRate } from "../metrics.ts";
+import { defineReport, type ReportDefinition } from "../report.ts";
+import { defineComponent, type ReportComponent, type TextContext } from "../tree.ts";
+import { localeText, type ReportLocale } from "../locale.ts";
+import type { ExperimentListItem, GroupSummaryData, ScatterData } from "../types.ts";
+import type { ScatterDataOptions } from "../compute.ts";
+import { resolveInput, type SnapshotsInput } from "../aggregate.ts";
+import { ExperimentComparisonView } from "../react/ExperimentComparison.tsx";
+import { wrapDisplay } from "../text/layout.ts";
+import { formatUSD } from "../format.ts";
 
-export const ExperimentComparison = defineReport(async ({ selection }) => {
-  const experiments = await ExperimentList.data(selection);
+// 每组散点的唯一口径：默认 definition 与公开 `.data()` 共用，不各写一份。
+const SCATTER_OPTIONS: ScatterDataOptions = {
+  points: "experiment",
+  series: "agent",
+  x: costUSD,
+  y: endToEndPassRate,
+};
+
+/**
+ * 一个可比组的数据。三个子块都只消费本组快照，不能含其它父目录的引用。
+ */
+export interface ExperimentComparisonGroupData {
+  /** experiment id 的完整父路径；根目录 experiment 使用自己的完整 id。 */
+  key: string;
+  summary: GroupSummaryData;
+  /** 成本 × 端到端成功率散点(`MetricScatter.data` 的口径:points=experiment、series=agent)。 */
+  scatter: ScatterData;
+  /** 固定列 experiment 比较表(`ExperimentList.data` 的口径,已按端到端成功率排序)。 */
+  experiments: ExperimentListItem[];
+}
+
+/** `ExperimentComparison.data(selection)` 的穷尽产物：按 key 排序的独立可比组。 */
+export interface ExperimentComparisonData {
+  groups: ExperimentComparisonGroupData[];
+}
+
+/** 完整父路径是组键；没有父路径的 experiment 不能互相比，自己形成单例组。 */
+export function experimentComparisonGroupKey(experimentId: string): string {
+  const slash = experimentId.lastIndexOf("/");
+  return slash === -1 ? experimentId : experimentId.slice(0, slash);
+}
+
+/** 组合件的数据计算：先分区，再让三个官方子块在各组内并行计算。 */
+async function experimentComparisonData(input: SnapshotsInput): Promise<ExperimentComparisonData> {
+  const { snapshots } = resolveInput(input);
+  const snapshotsByGroup = new Map<string, typeof snapshots>();
+  for (const snapshot of snapshots) {
+    const key = experimentComparisonGroupKey(snapshot.experimentId);
+    const group = snapshotsByGroup.get(key);
+    if (group) group.push(snapshot);
+    else snapshotsByGroup.set(key, [snapshot]);
+  }
+
+  const groups = await Promise.all(
+    [...snapshotsByGroup.entries()]
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(async ([key, groupSnapshots]): Promise<ExperimentComparisonGroupData> => {
+        const [summary, scatter, experiments] = await Promise.all([
+          GroupSummary.data(groupSnapshots),
+          MetricScatter.data(groupSnapshots, SCATTER_OPTIONS),
+          ExperimentList.data(groupSnapshots),
+        ]);
+        return { key, summary, scatter, experiments };
+      }),
+  );
+  return { groups };
+}
+
+export interface ExperimentComparisonProps {
+  data: ExperimentComparisonData;
+  /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
+  locale?: ReportLocale;
+  className?: string;
+}
+
+/** 单组详情：散点与实验列表全部来自同一个预分区结果。 */
+function groupComposition(group: ExperimentComparisonGroupData, locale?: ReportLocale, className?: string) {
   return (
-    <Col>
-      <MetricScatter selection={selection} points="experiment" series="agent" x={costUSD} y={passRate} />
-      <ExperimentList items={experiments} filter />
+    <Col className={className}>
+      <MetricScatter data={group.scatter} locale={locale} />
+      <ExperimentList items={group.experiments} filter locale={locale} relativeTo={group.key} />
     </Col>
   );
-});
+}
+
+function shellQuote(value: string): string {
+  return /^[A-Za-z0-9._/-]+$/.test(value) ? value : `'${value.replaceAll("'", `'"'"'`)}'`;
+}
+
+function verdictsText(group: ExperimentComparisonGroupData, locale: ReportLocale): string {
+  const parts: string[] = [];
+  for (const verdict of ["passed", "failed", "errored", "skipped"] as const) {
+    const count = group.summary.verdicts[verdict];
+    if (count > 0) parts.push(`${count} ${localeText(locale, `verdict.${verdict}`)}`);
+  }
+  return parts.length > 0 ? parts.join(" / ") : "—";
+}
+
+function textFace(data: ExperimentComparisonData, className: string | undefined, ctx: TextContext): string {
+  if (data.groups.length === 0) return localeText(ctx.locale, "experimentComparison.empty");
+  if (data.groups.length === 1) {
+    const group = data.groups[0]!;
+    return ctx.render(
+      <Section title={group.key} className={className}>
+        {groupComposition(group)}
+      </Section>,
+    );
+  }
+
+  const table = ctx.render(
+    <Table
+      columns={[
+        { key: "group", header: localeText(ctx.locale, "experimentComparison.group") },
+        { key: "experiments", header: localeText(ctx.locale, "groupSummary.experiments"), align: "right" },
+        { key: "evals", header: localeText(ctx.locale, "overview.evals"), align: "right" },
+        { key: "passRate", header: localeText(ctx.locale, "overview.passRate"), align: "right" },
+        { key: "results", header: localeText(ctx.locale, "experimentComparison.results") },
+        { key: "cost", header: localeText(ctx.locale, "overview.totalCost"), align: "right" },
+        { key: "lastRun", header: localeText(ctx.locale, "experimentComparison.lastRun") },
+      ]}
+      rows={data.groups.map((group) => ({
+        key: group.key,
+        cells: {
+          group: group.key,
+          experiments: String(group.summary.experiments),
+          evals: String(group.summary.evals),
+          passRate: group.summary.passRate.display,
+          results: verdictsText(group, ctx.locale),
+          cost: group.summary.totalCostUSD === null ? null : formatUSD(group.summary.totalCostUSD),
+          lastRun: group.summary.lastRunAt ?? null,
+        },
+      }))}
+    />,
+  );
+  const commands = data.groups
+    .map((group) => {
+      const command = `niceeval show --experiment ${shellQuote(group.key)}`;
+      return wrapDisplay(localeText(ctx.locale, "experimentComparison.command", { command }), ctx.width).join("\n");
+    })
+    .join("\n");
+  return `${localeText(ctx.locale, "experimentComparison.groups")}\n\n${table}\n\n${commands}`;
+}
+
+/**
+ * 内置默认报告兼官方组合件:作为组件时收 `data`(`.data()` 的产物),两个渲染面直接摆
+ * 数据形态的子组件;作为默认报告时经 build 面走与用户 `--report` 完全相同的管线。
+ */
+export const ExperimentComparison: ReportComponent<ExperimentComparisonProps> &
+  ReportDefinition & { data: typeof experimentComparisonData } = Object.assign(
+  defineComponent<ExperimentComparisonProps>({
+    web: (props, ctx) => (
+      <ExperimentComparisonView
+        data={props.data}
+        locale={props.locale ?? ctx.locale}
+        className={props.className}
+      />
+    ),
+    text: ({ data, className }, ctx) => textFace(data, className, ctx),
+  }),
+  defineReport(async ({ selection }) => {
+    const data = await experimentComparisonData(selection);
+    return <ExperimentComparison data={data} />;
+  }),
+  { data: experimentComparisonData },
+);
+ExperimentComparison.displayName = "ExperimentComparison";

package/src/report/built-ins/index.ts

@@ -1,2 +1,7 @@
-// 内置报告目录:一个文件一个按展示内容命名的普通 ReportDefinition,显式导出值,不建字符串 registry。
+// 内置报告目录:一个文件一个按展示内容命名的组合件(组件兼默认报告),显式导出值,不建字符串 registry。
 export { ExperimentComparison } from "./experiment-comparison.tsx";
+export type {
+  ExperimentComparisonData,
+  ExperimentComparisonGroupData,
+  ExperimentComparisonProps,
+} from "./experiment-comparison.tsx";

package/src/report/components.tsx

@@ -92,6 +92,12 @@ export interface ExperimentListProps {
   items: ExperimentListItem[];
   /** web 面在比较表前显示实验过滤框；text 面忽略。 */
   filter?: boolean;
+  /**
+   * 可选父路径:两面的行标签去掉与它相同的前缀,只显示 experiment id 末段,避免在已经以组为
+   * 标题的上下文里重复文件夹名(默认 `ExperimentComparison` 给每组传组键)。完整 id 仍是排序 /
+   * 着色 / 过滤 / 折叠的键;不传或它不是前缀(如根目录单例组)时显示完整 id。
+   */
+  relativeTo?: string;
   /** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
   locale?: ReportLocale;
   className?: string;
@@ -235,7 +241,7 @@ export const ExperimentList: ReportComponent<ExperimentListProps> & { data: type
   Object.assign(
     defineComponent<ExperimentListProps>({
       web: (props, ctx) => <ExperimentListWeb {...props} locale={props.locale ?? ctx.locale} attemptHref={ctx.attemptHref} />,
-      text: ({ items }, ctx) => experimentListText(items, ctx),
+      text: ({ items, relativeTo }, ctx) => experimentListText(items, ctx, relativeTo),
     }),
     { data: experimentListData },
   );
@@ -255,8 +261,8 @@ export const EvalList: ReportComponent<EvalListProps> & { data: typeof evalListD
 EvalList.displayName = "EvalList";
 
 /**
- * Attempt 列表:实体列表的叶子层,每项一个 Attempt,固定展示判定、断言、error、Judge 评语
- * (assertions 的 detail/evidence)与证据引用(locator)。它不预设只看失败;
+ * Attempt 列表:实体列表的叶子层,每项一个 Attempt,固定展示判定、一条主失败断言摘要或
+ * error 一层摘要与证据引用(locator)。完整 assertions / evidence 经 locator 下钻。它不预设只看失败;
  * 报告作者过滤 `AttemptListItem[]`、用 `.slice()` 限量,`total` 让渲染面如实报告剩余数量。
  */
 export const AttemptList: ReportComponent<AttemptListProps> & { data: typeof attemptListData } = Object.assign(

package/src/report/compute.ts

@@ -13,6 +13,7 @@
 import type {
   AttemptListItem,
   AttemptLocator,
+  AxisInput,
   DeltaData,
   DimensionInput,
   EvalListItem,
@@ -24,18 +25,18 @@ import type {
   Metric,
   MetricCell,
   OverviewData,
-  FlagRef,
   ScatterData,
   ScoreboardData,
   TableData,
   TableRowMeta,
 } from "./types.ts";
-import type { AssertionResult, EvalResult } from "../types.ts";
+import type { AssertionResult, AttemptError, DiagnosticRecord, EvalResult } from "../types.ts";
 import type { AttemptHandle } from "../results/types.ts";
 import { evalLevelStats, foldEvalVerdict } from "../shared/verdict.ts";
 import {
   applyAggregator,
   assertUniqueMetricNames,
+  axisValue,
   collectItems,
   computeCell,
   dimensionKey,
@@ -48,7 +49,6 @@ import {
   experimentIdOf,
   filterItems,
   groupItems,
-  flagAxisValue,
   locatorOf,
   resolveInput,
   snapshotKeyOf,
@@ -56,8 +56,9 @@ import {
   type Item,
   type SnapshotsInput,
 } from "./aggregate.ts";
-import { attemptCostUSD, costUSD, durationMs, examScore, passRate, tokens } from "./metrics.ts";
+import { attemptCostUSD, costUSD, durationMs, endToEndPassRate, examScore, tokens } from "./metrics.ts";
 import { formatMetricValue, formatPlainNumber } from "./format.ts";
+import { compactAssertionSummary, primaryAssertionSummary } from "../scoring/display.ts";
 
 // ───────────────────────── MetricTable.data ─────────────────────────
 
@@ -175,27 +176,12 @@ function experimentRowMeta(group: Item[]): TableRowMeta {
   };
 }
 
-/**
- * 一次 attempt 未通过的 gate 断言,原始声明顺序不变;soft 断言不参与判定,不算「失败原因」,
- * 只影响得分,永不出现在这份列表里。`EvalList` / `ExperimentList` 的失败诊断与 `AttemptList`
- * 的断言列表共用这同一份材料,保证同一个 attempt 在各处给出同一个原因。
- */
-export function failingGateAssertions(result: EvalResult): AssertionResult[] {
-  return result.assertions.filter((a) => !a.passed && a.severity === "gate");
-}
-
-/**
- * 一次 attempt 的失败原因文案,按优先级取第一个在场的:
- * `error` → `skipReason` → 未通过的 gate 断言(原始声明顺序,`name`,detail 在场则
- * `"name: detail"`,多条用「, 」连接)→ 都缺席则无原因(如某道题恰好没有失败信号)。
- * soft 断言永不进入这份原因文案,soft 得分是独立概念,不与 reason 混用同一个字段。
- */
+/** 一次 attempt 的有界结果摘要：error → skipReason → 一条主失败断言 + 其余失败计数。 */
 export function reasonFor(result: EvalResult): string | undefined {
-  if (result.error !== undefined) return result.error;
+  if (result.error !== undefined) return result.error.message;
   if (result.skipReason !== undefined) return result.skipReason;
-  const gates = failingGateAssertions(result);
-  if (gates.length === 0) return undefined;
-  return gates.map((a) => (a.detail ? `${a.name}: ${a.detail}` : a.name)).join(", ");
+  const summary = primaryAssertionSummary(result.assertions, result.verdict);
+  return summary === undefined ? undefined : compactAssertionSummary(summary);
 }
 
 export async function tableData<const M extends readonly Metric[]>(
@@ -247,13 +233,53 @@ export async function tableData<const M extends readonly Metric[]>(
 // (docs/feature/reports/library.md「实体列表」)。AttemptListItem 是三者共用的叶子形状——
 // ExperimentList / EvalList 的下钻数组直接复用它,不各自精简一份。
 
-/** 自由文本(error / 断言 detail / evidence)的发布消毒钩子;身份字段(name/severity/loc)不经它。 */
+/** 自由文本(断言 detail / evidence)的展示层遮蔽钩子;身份字段(name/severity/loc)不经它。 */
 function redactAssertions(assertions: AssertionResult[], redact: (text: string) => string): AssertionResult[] {
   if (assertions.length === 0) return assertions;
   return assertions.map((a) => ({
     ...a,
     ...(a.detail !== undefined ? { detail: redact(a.detail) } : {}),
-    ...(a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
+    ...(a.outcome !== "unavailable" && a.evidence !== undefined ? { evidence: redact(a.evidence) } : {}),
+    ...(a.outcome !== "unavailable" && a.expected !== undefined ? { expected: redact(a.expected) } : {}),
+    ...(a.outcome !== "unavailable" && a.received !== undefined ? { received: redact(a.received) } : {}),
+  }));
+}
+
+/**
+ * 结构化 error 的遮蔽:message / stack / cause.message 是自由文本,经钩子;
+ * code / operation / cause.name / cause.code 是分类与身份字段,原样保留
+ * (与 copySnapshots({ redact }) 的改写范围约定一致)。
+ */
+function redactError(error: AttemptError, redact: (text: string) => string): AttemptError {
+  return {
+    ...error,
+    message: redact(error.message),
+    ...(error.stack !== undefined ? { stack: redact(error.stack) } : {}),
+    ...(error.cause !== undefined ? { cause: { ...error.cause, message: redact(error.cause.message) } } : {}),
+  };
+}
+
+/** JsonValue 树里逐个字符串值经钩子;结构与非字符串标量原样。 */
+function redactJsonValue<T>(value: T, redact: (text: string) => string): T {
+  if (typeof value === "string") return redact(value) as unknown as T;
+  if (Array.isArray(value)) return value.map((v) => redactJsonValue(v, redact)) as unknown as T;
+  if (typeof value === "object" && value !== null) {
+    return Object.fromEntries(
+      Object.entries(value).map(([k, v]) => [k, redactJsonValue(v, redact)]),
+    ) as unknown as T;
+  }
+  return value;
+}
+
+/** diagnostics 的遮蔽:message 与 data(自由内容)经钩子;code / operation / level / count 原样。 */
+function redactDiagnostics(
+  diagnostics: readonly DiagnosticRecord[],
+  redact: (text: string) => string,
+): DiagnosticRecord[] {
+  return diagnostics.map((d) => ({
+    ...d,
+    message: redact(d.message),
+    ...(d.data !== undefined ? { data: redactJsonValue(d.data, redact) } : {}),
   }));
 }
 
@@ -267,7 +293,10 @@ function attemptListItemOf(item: Item, redact: (text: string) => string): Attemp
     attempt: result.attempt,
     agent: result.agent,
     verdict: result.verdict,
-    ...(result.error !== undefined ? { error: redact(result.error) } : {}),
+    ...(result.error !== undefined ? { error: redactError(result.error, redact) } : {}),
+    ...(result.diagnostics !== undefined && result.diagnostics.length > 0
+      ? { diagnostics: redactDiagnostics(result.diagnostics, redact) }
+      : {}),
     assertions: redactAssertions(result.assertions, redact),
     durationMs: result.durationMs,
     ...(cost !== null ? { costUSD: cost } : {}),
@@ -278,7 +307,11 @@ function attemptListItemOf(item: Item, redact: (text: string) => string): Attemp
 const identityRedact = (text: string): string => text;
 
 export interface AttemptListDataOptions {
-  /** 发布消毒:error / 断言 detail / evidence 经这个钩子;身份字段(experimentId/evalId/locator…)不经它。 */
+  /**
+   * 展示层遮蔽:error 的 message/cause/stack、diagnostic 的 message/data、断言 detail 与
+   * evidence 经这个钩子;experimentId、evalId、locator、error/diagnostic code 与 lifecycle
+   * operation 等身份和分类字段不经它。只作用于这次计算产出的组件数据,不改盘上 artifact。
+   */
   redact?: (text: string) => string;
 }
 
@@ -308,13 +341,11 @@ export async function evalListData(input: SnapshotsInput): Promise<EvalListItem[
   for (const group of groups.values()) {
     const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
     const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
-    const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
     const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
     out.push({
       evalId: evalIdOf(sorted[0]!),
       experimentId: experimentIdOf(sorted[0]!),
       verdict,
-      reason: reasonFor(representative.attempt.result),
       score: await computeCell(examScore, sorted),
       duration: await computeCell(durationMs, sorted),
       cost: await computeCell(costUSD, sorted),
@@ -339,12 +370,10 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
     for (const [evalId, evalItems] of evalGroups) {
       const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
       const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
-      const representative = sorted.find((item) => item.attempt.result.verdict === verdict) ?? sorted[0]!;
       const attempts = sorted.map((item) => attemptListItemOf(item, identityRedact));
       evalRows.push({
         evalId,
         verdict,
-        reason: reasonFor(representative.attempt.result),
         duration: await computeCell(durationMs, sorted),
         cost: await computeCell(costUSD, sorted),
         attempts,
@@ -360,7 +389,7 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
         : {}),
       ...(experiment?.flags ? { flags: experiment.flags } : {}),
       verdicts: stats.verdicts,
-      passRate: await computeCell(passRate, group),
+      passRate: await computeCell(endToEndPassRate, group),
       cost: await computeCell(costUSD, group),
       duration: await computeCell(durationMs, group),
       tokens: await computeCell(tokens, group),
@@ -370,7 +399,7 @@ export async function experimentListData(input: SnapshotsInput): Promise<Experim
       evalRows,
     });
   }
-  // ExperimentList 是默认实验比较表:初始态按成功率从高到低,缺数据沉底;
+  // ExperimentList 是默认实验比较表:初始态按端到端成功率(endToEndPassRate)从高到低,缺数据沉底;
   // 同分时按 experiment id 稳定排序。web 增强可临时重排,text 面沿用同一基准顺序。
   out.sort((a, b) => {
     if (a.passRate.value === null && b.passRate.value === null) return a.experimentId.localeCompare(b.experimentId);
@@ -571,10 +600,10 @@ export async function scatterData(input: SnapshotsInput, opts: ScatterDataOption
 // ───────────────────────── MetricLine.data ─────────────────────────
 
 export interface LineDataOptions {
-  /** x 轴:experiment 声明的 flag(数值),不解析 experiment 命名。 */
-  x: FlagRef;
+  /** x 轴:experiment 声明的 flag 或顶层运行配置 config()(要求数值),不解析 experiment 命名。 */
+  x: AxisInput;
   y: Metric;
-  /** 可选:每个系列一条线(flag 或普通维度);省略 = 单系列。 */
+  /** 可选:每个系列一条线(flag / config 或普通维度);省略 = 单系列。 */
   series?: DimensionInput;
 }
 
@@ -585,7 +614,7 @@ export async function lineData(input: SnapshotsInput, opts: LineDataOptions): Pr
   const groups = groupItems(items, "experiment");
   const rows: LineData["rows"] = [];
   for (const [key, group] of groups) {
-    const x = flagAxisValue(opts.x, group[0]); // flag 是 experiment 级声明,组内一致
+    const x = axisValue(opts.x, group[0]); // flag / config 都是 experiment 级声明,组内一致
     rows.push({
       key,
       series: opts.series ? dimensionKey(opts.series, group[0]) : undefined,
@@ -640,9 +669,10 @@ export async function overviewData(input: SnapshotsInput): Promise<OverviewData>
     const cost = attemptCostUSD(result);
     if (cost !== null) costUSD = (costUSD ?? 0) + cost;
   }
-  // 通过率的唯一官方口径:两级聚合(computeCell),不是从上面四个 verdict 计票现场重算——
-  // 一道题内 attempt 部分通过要算部分 credit,不是二元投票。
-  const passRateCell = await computeCell(passRate, items);
+  // 默认成功率的唯一官方口径:endToEndPassRate 的两级聚合(computeCell),不是从上面四个
+  // verdict 计票现场重算——一道题内 attempt 部分通过要算部分 credit,不是二元投票;
+  // failed / errored 都记 0,只有 skipped 不进聚合。
+  const passRateCell = await computeCell(endToEndPassRate, items);
   return {
     snapshots: snapshots.map((s) => ({
       experimentId: s.experimentId,