niceeval 0.3.0 → 0.4.0

package/src/report/react/RunOverview.tsx

@@ -0,0 +1,87 @@
+// RunOverview:页头 KPI 条——「这批数据是什么」。
+// 数字下面标注数据来源(几个快照、何时跑的);warnings 有内容时直接显示在条内,
+// 诚实不靠使用者记得渲染(docs/reports.md「第一档」行为清单)。
+
+import type { ReactElement } from "react";
+import type { OverviewData } from "./data.ts";
+import { MISSING_TEXT, cx, formatDurationMs, formatPercent, formatUSD } from "./format.ts";
+
+export function RunOverview({
+  data,
+  className,
+}: {
+  data: OverviewData;
+  className?: string;
+}): ReactElement {
+  const { totals } = data;
+  // 通过率口径与内置 passRate 指标一致:skipped → null 不进分母,errored/failed 计 0
+  const judged = totals.passed + totals.failed + totals.errored;
+  const passRate = judged > 0 ? formatPercent(totals.passed / judged) : null;
+
+  return (
+    <header className={cx("nre", "nre-overview", className)}>
+      <dl className="nre-kpis">
+        <div className="nre-kpi">
+          <dt>Snapshots</dt>
+          <dd>{data.snapshots.length}</dd>
+        </div>
+        <div className="nre-kpi">
+          <dt>Evals</dt>
+          <dd>{totals.evals}</dd>
+        </div>
+        <div className="nre-kpi">
+          <dt>attempts</dt>
+          <dd>{totals.attempts}</dd>
+        </div>
+        <div className="nre-kpi">
+          <dt>Pass rate</dt>
+          <dd>{passRate ?? <span className="nre-missing">{MISSING_TEXT}</span>}</dd>
+        </div>
+        <div className="nre-kpi">
+          <dt>Total cost</dt>
+          {/* costUSD 全缺 = null:显示缺数据,不编 $0 */}
+          <dd>
+            {totals.costUSD === null ? (
+              <span className="nre-missing">{MISSING_TEXT}</span>
+            ) : (
+              formatUSD(totals.costUSD)
+            )}
+          </dd>
+        </div>
+        <div className="nre-kpi">
+          <dt>Total duration</dt>
+          <dd>{formatDurationMs(totals.durationMs)}</dd>
+        </div>
+      </dl>
+
+      <p className="nre-outcomes">
+        <span className="nre-outcome nre-outcome-passed">passed {totals.passed}</span>
+        <span className="nre-outcome nre-outcome-failed">failed {totals.failed}</span>
+        <span className="nre-outcome nre-outcome-errored">errored {totals.errored}</span>
+        <span className="nre-outcome nre-outcome-skipped">skipped {totals.skipped}</span>
+      </p>
+
+      {/* 数据来源:哪些快照、何时跑的——报告的数字都从这里来 */}
+      <p className="nre-source">
+        Source: {data.snapshots.length} snapshots
+        {data.snapshots.map((s) => (
+          <span key={`${s.experimentId}@${s.startedAt}`} className="nre-source-snapshot">
+            {s.experimentId}({s.agent}
+            {s.model ? ` · ${s.model}` : ""})@ {s.startedAt}
+          </span>
+        ))}
+      </p>
+
+      {/* 选择器的警告(残缺快照等)直接渲染在条内,不静默 */}
+      {data.warnings.length > 0 && (
+        <ul className="nre-warnings">
+          {data.warnings.map((w, i) => (
+            <li key={i} className="nre-warning">
+              {w}
+            </li>
+          ))}
+        </ul>
+      )}
+    </header>
+  );
+}

package/src/report/react/Scoreboard.tsx

@@ -0,0 +1,87 @@
+// Scoreboard:考试成绩单——总分 + 分科小计。
+// 固定分母的口径不藏:没跑的题挣 0 分但留在分母里,科目行如实报 missing;
+// weights 是「实际生效的权重表」,渲染出来让成绩单可审计(docs/reports.md scoreboard 公式一节)。
+
+import type { ReactElement } from "react";
+import type { ScoreboardData } from "./data.ts";
+import { colorClassForKey } from "./colors.ts";
+import { cx } from "./format.ts";
+
+export function Scoreboard({
+  data,
+  className,
+}: {
+  data: ScoreboardData;
+  className?: string;
+}): ReactElement {
+  // 科目列 = 各行 subjects 的并集,按首次出现顺序;固定分母下各行本应一致,这里防御性合并
+  const subjectKeys: string[] = [];
+  for (const row of data.rows) {
+    for (const subject of row.subjects) {
+      if (!subjectKeys.includes(subject.key)) subjectKeys.push(subject.key);
+    }
+  }
+
+  return (
+    <section className={cx("nre", "nre-scoreboard", className)}>
+      <table className="nre-scoreboard-table">
+        <thead>
+          <tr>
+            <th scope="col" className="nre-dimension">
+              {data.of}
+            </th>
+            <th scope="col" className="nre-total-col">
+              Total<span className="nre-full-marks">/ {data.fullMarks}</span>
+            </th>
+            {subjectKeys.map((key) => (
+              <th scope="col" key={key} className="nre-subject-col">
+                {key}
+              </th>
+            ))}
+          </tr>
+        </thead>
+        <tbody>
+          {data.rows.map((row) => (
+            <tr key={row.key}>
+              {/* 被打分者(如 agent):稳定散列上色,跨块同键同色 */}
+              <th scope="row" className={cx("nre-row-key", "nre-key", colorClassForKey(row.key))}>
+                {row.key}
+              </th>
+              <td className="nre-total">{row.total.display}</td>
+              {subjectKeys.map((key) => {
+                const subject = row.subjects.find((s) => s.key === key);
+                if (!subject) return <td key={key} className="nre-td-empty" />;
+                return (
+                  <td key={key} className="nre-subject">
+                    <span
+                      className="nre-subject-score"
+                      title={`${subject.evals} evals, weighted ${subject.earned} of ${subject.possible}`}
+                    >
+                      {subject.earned}/{subject.possible}
+                    </span>
+                    {/* 固定分母的如实注脚:没跑、按 0 计的题数 */}
+                    {subject.missing > 0 && (
+                      <span className="nre-subject-missing">{subject.missing} {subject.missing === 1 ? "eval" : "evals"} missing, scored 0</span>
+                    )}
+                  </td>
+                );
+              })}
+            </tr>
+          ))}
+        </tbody>
+      </table>
+      {/* 实际生效的权重表:成绩单可审计 */}
+      <p className="nre-weights">
+        权重:
+        {data.weights.length === 0
+          ? "all evals ×1"
+          : data.weights.map((w) => (
+              <span key={w.prefix} className="nre-weight">
+                {w.prefix} ×{w.weight}
+              </span>
+            ))}
+        {data.weights.length > 0 && <span className="nre-weight-rest">others ×1</span>}
+      </p>
+    </section>
+  );
+}

package/src/report/react/cell.tsx

@@ -0,0 +1,49 @@
+// MetricCell 的统一渲染:MetricTable / MetricMatrix / DeltaTable 共用,
+// 保证三处对同一份数据长得一致——覆盖率角标、缺数据文案、证据链接只实现一次。
+// 纯渲染、零 hooks;交互只有普通 <a>(下钻由使用者的 attemptHref 决定去处)。
+
+import type { ReactElement } from "react";
+import type { AttemptRef, MetricCell } from "./data.ts";
+import { MISSING_TEXT } from "./format.ts";
+
+export function MetricCellView({
+  cell,
+  attemptHref,
+}: {
+  cell: MetricCell;
+  attemptHref?: (ref: AttemptRef) => string;
+}): ReactElement {
+  // 全 null(没有任何有效样本)→ 缺数据文案,绝不画 0;total 仍如实入 title
+  if (cell.value === null) {
+    return (
+      <span className="nre-cell nre-cell-missing">
+        <span className="nre-missing" title={`0/${cell.total} attempts measurable`}>
+          {MISSING_TEXT}
+        </span>
+      </span>
+    );
+  }
+  return (
+    <span className="nre-cell">
+      <span className="nre-value" title={`${cell.samples}/${cell.total} attempts measured`}>
+        {cell.display}
+      </span>
+      {/* samples < total:有 attempt 测不了这个指标,覆盖率角标如实标出 */}
+      {cell.samples < cell.total && (
+        <sup className="nre-coverage" title={`coverage ${cell.samples}/${cell.total}: this metric is null for the remaining attempts`}>
+          {cell.samples}/{cell.total}
+        </sup>
+      )}
+      {/* refs + attemptHref:格子可点,「给我看那次 attempt」就在手边 */}
+      {attemptHref && cell.refs && cell.refs.length > 0 && (
+        <span className="nre-refs">
+          {cell.refs.map((ref, i) => (
+            <a key={`${ref.run}:${ref.result}`} className="nre-ref" href={attemptHref(ref)}>
+              #{i + 1}
+            </a>
+          ))}
+        </span>
+      )}
+    </span>
+  );
+}

package/src/report/react/colors.ts

@@ -0,0 +1,42 @@
+// 跨块配色一致(docs/reports.md 四条跨组件契约之二):
+// 系列/维度键 → 固定调色板下标,用稳定散列而不是「按出现顺序分配」。
+// 这样同一个 agent 在 scatter 的线、DeltaTable 的行、matrix 的列头永远同色,
+// 不需要 Provider、不需要手工配置,甚至不需要两个组件见过同一份数据。
+// 调色板的十六进制值以这里为准:SVG(scatter 的线和点)直接内联 hex,
+// 保证漏加载 styles.css 时图形仍然分得清系列;styles.css 里的 --nre-c0..c7
+// 是同一组值的拷贝(供 .nre-cN 文本类用),改色时两边一起改。
+
+/** 固定调色板;下标即 nre-cN 的 N,与 styles.css 的 --nre-cN 逐个对应。 */
+export const NRE_PALETTE = [
+  "#2563eb", // c0 蓝
+  "#db2777", // c1 玫红
+  "#059669", // c2 绿
+  "#d97706", // c3 橙
+  "#7c3aed", // c4 紫
+  "#0891b2", // c5 青
+  "#dc2626", // c6 红
+  "#65a30d", // c7 橄榄
+] as const;
+
+export const NRE_PALETTE_SIZE = NRE_PALETTE.length;
+
+/** FNV-1a 32 位散列:输入相同永远得到相同下标,与运行顺序无关。 */
+export function colorIndexForKey(key: string): number {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < key.length; i++) {
+    hash ^= key.charCodeAt(i);
+    // FNV prime 乘法,拆成移位加法保持 32 位整数运算
+    hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
+  }
+  return hash % NRE_PALETTE_SIZE;
+}
+
+/** 键对应的稳定 class 名("nre-c3"),配 styles.css 的 .nre-cN 上色。 */
+export function colorClassForKey(key: string): string {
+  return `nre-c${colorIndexForKey(key)}`;
+}
+
+/** 键对应的十六进制颜色,给 SVG 内联 stroke/fill 用(不依赖 CSS 加载)。 */
+export function colorHexForKey(key: string): string {
+  return NRE_PALETTE[colorIndexForKey(key)];
+}

package/src/report/react/data.ts

@@ -0,0 +1,17 @@
+// niceeval/report/react 的数据契约(组件 props 类型)。
+//
+// 契约的家在计算层(../types.ts,照 docs/reports.md「计算函数与数据契约」),这里 re-export,
+// 「算」与「画」两侧共用同一份类型;本文件零运行时代码——组件不做任何聚合或数据操作。
+
+export type {
+  AttemptRef,
+  MetricColumn,
+  MetricCell,
+  TableData,
+  MatrixData,
+  ScoreboardData,
+  ScatterData,
+  OverviewData,
+  DeltaData,
+  CaseListData,
+} from "../types.ts";

package/src/report/react/fixtures.ts

@@ -0,0 +1,236 @@
+// 手工 fixture:模拟计算层(niceeval/report 的 table()/matrix()/… )的产物。
+// 仅供 render.test.tsx 与 scripts/report-react-demo.tsx 使用,不从入口导出。
+// 数字刻意覆盖诚实细节:coverage 角标(samples<total)、全 null 格子、
+// 稀疏矩阵、缺数据的散点、delta 的 null 不硬算、truncated 计数。
+
+import type {
+  CaseListData,
+  DeltaData,
+  MatrixData,
+  MetricColumn,
+  OverviewData,
+  ScatterData,
+  ScoreboardData,
+  TableData,
+} from "./data.ts";
+
+export const passRateColumn: MetricColumn = { key: "pass-rate", label: "通过率", unit: "%", better: "higher" };
+export const codeLinesColumn: MetricColumn = { key: "code-lines", label: "代码行数", unit: "lines", better: "lower" };
+export const costColumn: MetricColumn = { key: "cost", label: "成本", unit: "$", better: "lower" };
+
+export const overviewData: OverviewData = {
+  snapshots: [
+    { experimentId: "compare/bub", agent: "bub", model: "gpt-5.4", startedAt: "2026-07-01T10:00:00Z" },
+    { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T11:30:00Z" },
+  ],
+  totals: {
+    evals: 12,
+    attempts: 48,
+    passed: 36,
+    failed: 8,
+    errored: 2,
+    skipped: 2,
+    costUSD: null, // 全部 attempt 都没报成本:null,组件必须显示缺数据而不是 $0
+    durationMs: 261_000,
+  },
+  warnings: ["compare/bub 快照缺 3 个 eval 的结果"],
+};
+
+export const overviewWithCost: OverviewData = {
+  ...overviewData,
+  totals: { ...overviewData.totals, costUSD: 1.234 },
+  warnings: [],
+};
+
+export const tableData: TableData = {
+  dimension: "agent",
+  // 行顺序故意不按 passRate 排:组件必须按传入顺序渲染,不重排
+  columns: [passRateColumn, codeLinesColumn],
+  rows: [
+    {
+      key: "codex",
+      cells: {
+        "pass-rate": { value: 0.5, display: "50%", samples: 6, total: 6 },
+        // 全 null:一个有效样本都没有 → 缺数据文案,绝不画 0
+        "code-lines": { value: null, display: "", samples: 0, total: 6 },
+      },
+    },
+    {
+      key: "bub",
+      cells: {
+        "pass-rate": {
+          value: 0.87,
+          display: "87%",
+          samples: 6,
+          total: 6,
+          refs: [{ run: "run-a", result: 0 }],
+        },
+        // samples < total:有 attempt 测不了 → 覆盖率角标 5/6
+        "code-lines": { value: 120, display: "120 lines", samples: 5, total: 6 },
+      },
+    },
+  ],
+};
+
+export const matrixData: MatrixData = {
+  rows: "eval",
+  columns: "agent",
+  metric: passRateColumn,
+  // 稀疏:geometry/angles × codex 没有样本,数据里不出现 → 格子空着
+  cells: [
+    {
+      row: "algebra/quadratic",
+      column: "bub",
+      cell: { value: 1, display: "100%", samples: 2, total: 2, refs: [{ run: "run-b", result: 3 }, { run: "run-b", result: 7 }] },
+    },
+    {
+      row: "algebra/quadratic",
+      column: "codex",
+      cell: { value: 0, display: "0%", samples: 3, total: 3 },
+    },
+    {
+      row: "geometry/angles",
+      column: "bub",
+      cell: { value: 0.5, display: "50%", samples: 2, total: 2 },
+    },
+  ],
+};
+
+export const scoreboardData: ScoreboardData = {
+  of: "agent",
+  fullMarks: 100,
+  weights: [{ prefix: "algebra/", weight: 2 }],
+  rows: [
+    {
+      key: "bub",
+      total: { value: 78.5, display: "78.5" },
+      subjects: [
+        // missing 1:一题没跑、按 0 计——固定分母的如实注脚
+        { key: "algebra", earned: 14, possible: 16, evals: 8, missing: 1 },
+        { key: "geometry", earned: 3, possible: 4, evals: 4, missing: 0 },
+      ],
+    },
+    {
+      key: "codex",
+      total: { value: 52, display: "52" },
+      subjects: [
+        { key: "algebra", earned: 9, possible: 16, evals: 8, missing: 0 },
+        { key: "geometry", earned: 1.4, possible: 4, evals: 4, missing: 2 },
+      ],
+    },
+  ],
+};
+
+export const scatterData: ScatterData = {
+  points: "experiment",
+  series: "agent",
+  x: costColumn, // better: "lower" → 轴反向,便宜的一端在右
+  y: passRateColumn,
+  rows: [
+    {
+      key: "compare/bub-low",
+      series: "bub",
+      x: { value: 5, display: "$5.00", samples: 6, total: 6 },
+      y: { value: 0.5, display: "50%", samples: 6, total: 6 },
+    },
+    {
+      key: "compare/bub-high",
+      series: "bub",
+      x: { value: 10, display: "$10.00", samples: 6, total: 6 },
+      y: { value: 0.9, display: "90%", samples: 6, total: 6 },
+    },
+    {
+      key: "compare/codex-mid",
+      series: "codex",
+      x: { value: 7, display: "$7.00", samples: 6, total: 6 },
+      y: { value: 0.6, display: "60%", samples: 6, total: 6 },
+    },
+    {
+      // x 缺数据:这个点不画,注脚报 1 个点缺数据
+      key: "compare/codex-broken",
+      series: "codex",
+      x: { value: null, display: "", samples: 0, total: 6 },
+      y: { value: 0.7, display: "70%", samples: 6, total: 6 },
+    },
+  ],
+};
+
+export const deltaData: DeltaData = {
+  columns: [passRateColumn, costColumn],
+  rows: [
+    {
+      key: "bub",
+      a: { experimentId: "compare/bub" },
+      b: { experimentId: "compare/bub--agents-md" },
+      cells: {
+        // 通过率 +12pp:better higher → 好(绿)
+        "pass-rate": {
+          a: { value: 0.5, display: "50%", samples: 6, total: 6 },
+          b: { value: 0.62, display: "62%", samples: 6, total: 6 },
+          delta: 0.12,
+          display: "+12pp",
+        },
+        // 成本 +$0.15:better lower → 坏(红)
+        cost: {
+          a: { value: 0.2, display: "$0.20", samples: 6, total: 6 },
+          b: { value: 0.35, display: "$0.35", samples: 6, total: 6 },
+          delta: 0.15,
+          display: "+$0.15",
+        },
+      },
+    },
+    {
+      key: "codex",
+      a: { experimentId: "compare/codex" },
+      b: { experimentId: "compare/codex--agents-md" },
+      cells: {
+        "pass-rate": {
+          a: { value: 0.4, display: "40%", samples: 6, total: 6 },
+          b: { value: 0.4, display: "40%", samples: 6, total: 6 },
+          delta: 0,
+          display: "±0pp",
+        },
+        // A 侧缺数据 → delta null:显示缺,不硬算
+        cost: {
+          a: { value: null, display: "", samples: 0, total: 6 },
+          b: { value: 0.3, display: "$0.30", samples: 6, total: 6 },
+          delta: null,
+          display: "",
+        },
+      },
+    },
+  ],
+};
+
+export const caseListData: CaseListData = {
+  rows: [
+    {
+      eval: "algebra/quadratic",
+      experimentId: "compare/bub",
+      agent: "bub",
+      outcome: "failed",
+      failedAssertions: [
+        {
+          name: "roots-correct",
+          score: 0,
+          detail: "期望 x=2,得到 x=3",
+          evidence: "judge: 求根公式代入时符号写反",
+        },
+      ],
+      durationMs: 32_000,
+      costUSD: 0.12,
+      ref: { run: "run-a", result: 4 },
+    },
+    {
+      eval: "geometry/angles",
+      experimentId: "compare/codex",
+      agent: "codex",
+      outcome: "errored",
+      error: "TypeError: cannot read properties of undefined (reading 'foo')",
+      failedAssertions: [],
+      durationMs: 4_500,
+      ref: { run: "run-c", result: 1 },
+    },
+  ],
+  truncated: 2,
+};

package/src/report/react/format.ts

@@ -0,0 +1,34 @@
+// 展示层的小格式化。注意边界:MetricCell 一律自带 display(格式化发生在计算侧),
+// 组件不重算;这里只服务 OverviewData 这类携带裸数字的字段(耗时、成本、比率),
+// 以及 class 名拼接。缺数据的统一文案也钉在这里,保证各组件同词。
+
+/** 全 null / 无样本的统一文案。绝不画 0(docs/reports.md「null ≠ 0」)。 */
+export const MISSING_TEXT = "no data";
+
+/** 拼 class 名:过滤空值,末尾接使用者透传的 className。 */
+export function cx(...parts: (string | undefined | false)[]): string {
+  return parts.filter(Boolean).join(" ");
+}
+
+/** 毫秒 → 人读耗时("850ms" / "1.2s" / "4m 20s" / "1h 4m")。 */
+export function formatDurationMs(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  const s = ms / 1000;
+  if (s < 60) return `${s.toFixed(1)}s`;
+  const m = Math.floor(s / 60);
+  if (m < 60) return `${m}m ${Math.round(s % 60)}s`;
+  const h = Math.floor(m / 60);
+  return `${h}h ${m % 60}m`;
+}
+
+/** 美元金额;小额保留更多位数,不四舍五入成 $0.00 的假零。 */
+export function formatUSD(usd: number): string {
+  if (usd === 0) return "$0";
+  const digits = Math.abs(usd) >= 0.01 ? 2 : 4;
+  return `$${usd.toFixed(digits)}`;
+}
+
+/** 0..1 的比率 → 整数百分比。 */
+export function formatPercent(ratio: number): string {
+  return `${Math.round(ratio * 100)}%`;
+}

package/src/report/react/index.tsx

@@ -0,0 +1,34 @@
+// niceeval/report/react —— 纯渲染的报告积木(docs/reports.md 第一档,原型实现)。
+//
+// 契约:
+//   - 组件只认「算好的可序列化数据」:零 IO、零 hooks、零数据操作,可进 "use client";
+//   - 不 hydrate 也完整:renderToStaticMarkup 的产物即成品——排序在数据侧预排,
+//     hover 退化为 title,下钻是普通 <a>,展开折叠是 <details>;
+//   - 样式随包发布:配套 ./styles.css(nre-* 稳定类名),使用者在其后加载覆盖即可;
+//   - 跨块配色一致:维度键 → 稳定散列 → 固定调色板下标(colors.ts)。
+//
+// 数据从哪来:niceeval/report 的计算函数(table()/matrix()/scoreboard()/scatter()/
+// overview()/delta()/cases())。计算层在并行实验中实现;类型暂由 ./data.ts 声明,
+// 合并时切换为从计算层 re-export。
+
+export { RunOverview } from "./RunOverview.tsx";
+export { MetricTable } from "./MetricTable.tsx";
+export { MetricMatrix } from "./MetricMatrix.tsx";
+export { Scoreboard } from "./Scoreboard.tsx";
+export { MetricScatter } from "./MetricScatter.tsx";
+export { DeltaTable } from "./DeltaTable.tsx";
+export { CaseList } from "./CaseList.tsx";
+
+// 数据契约类型(临时家在 ./data.ts,见该文件头部说明)
+export type {
+  AttemptRef,
+  MetricColumn,
+  MetricCell,
+  TableData,
+  MatrixData,
+  ScoreboardData,
+  ScatterData,
+  OverviewData,
+  DeltaData,
+  CaseListData,
+} from "./data.ts";