niceeval 0.3.0 → 0.4.0

package/src/report/react/render.test.tsx

@@ -0,0 +1,303 @@
+// 「不 hydrate 也完整」的验收测试:每个组件过 renderToStaticMarkup,
+// 断言纯静态 HTML 里就有全部关键内容——数字、覆盖率角标、缺数据文案、
+// 散点的 SVG 与系列名、truncated 行、attemptHref 链接。
+// 另外锁两条契约:源码零 hooks;维度键跨组件同色。
+
+import { readFileSync, readdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { renderToStaticMarkup } from "react-dom/server";
+import { describe, expect, it } from "vitest";
+
+import {
+  CaseList,
+  DeltaTable,
+  MetricMatrix,
+  MetricScatter,
+  MetricTable,
+  RunOverview,
+  Scoreboard,
+} from "./index.tsx";
+import { NRE_PALETTE, colorClassForKey, colorIndexForKey } from "./colors.ts";
+import {
+  caseListData,
+  deltaData,
+  matrixData,
+  overviewData,
+  overviewWithCost,
+  scatterData,
+  scoreboardData,
+  tableData,
+} from "./fixtures.ts";
+
+const attemptHref = (ref: { run: string; result: number }) => `/attempts/${ref.run}/${ref.result}`;
+
+describe("RunOverview", () => {
+  const html = renderToStaticMarkup(<RunOverview data={overviewData} />);
+
+  it("KPI 条:快照数、题数、attempts、通过率、耗时", () => {
+    expect(html).toContain("nre-overview");
+    expect(html).toContain("<dd>12</dd>"); // 题目
+    expect(html).toContain("<dd>48</dd>"); // attempts
+    expect(html).toContain("78%"); // 36 / (36+8+2),skipped 不进分母
+    expect(html).toContain("4m 21s"); // 261000ms
+  });
+
+  it("costUSD 全缺 = null:显示缺数据,不编 $0", () => {
+    expect(html).toContain("no data");
+    expect(html).not.toContain("$0");
+    const withCost = renderToStaticMarkup(<RunOverview data={overviewWithCost} />);
+    expect(withCost).toContain("$1.23");
+  });
+
+  it("数据来源与 warnings 直接渲染在条内", () => {
+    expect(html).toContain("2 snapshots");
+    expect(html).toContain("compare/bub");
+    expect(html).toContain("2026-07-01T10:00:00Z");
+    expect(html).toContain("compare/bub 快照缺 3 个 eval 的结果");
+    expect(html).toContain("nre-warnings");
+  });
+});
+
+describe("MetricTable", () => {
+  const html = renderToStaticMarkup(<MetricTable data={tableData} attemptHref={attemptHref} />);
+
+  it("按传入顺序渲染行,不重排(排序在数据侧)", () => {
+    // fixture 里 codex(50%)在 bub(87%)前面,输出必须保持
+    expect(html.indexOf(">codex<")).toBeGreaterThan(-1);
+    expect(html.indexOf(">codex<")).toBeLessThan(html.indexOf(">bub<"));
+  });
+
+  it("列头带 label、unit 与 better 方向", () => {
+    expect(html).toContain("通过率");
+    expect(html).toContain("(%)");
+    expect(html).toContain("代码行数");
+    expect(html).toContain("↑");
+    expect(html).toContain("↓");
+  });
+
+  it("数字与覆盖率角标:samples < total 如实标出", () => {
+    expect(html).toContain("87%");
+    expect(html).toContain("120 lines");
+    expect(html).toMatch(/<sup class="nre-coverage"[^>]*>5\/6<\/sup>/);
+  });
+
+  it("全 null 渲染缺数据文案,绝不画 0", () => {
+    expect(html).toContain("no data");
+    expect(html).toContain("nre-cell-missing");
+  });
+
+  it("refs + attemptHref 出普通 <a>", () => {
+    expect(html).toContain('href="/attempts/run-a/0"');
+  });
+});
+
+describe("MetricMatrix", () => {
+  const html = renderToStaticMarkup(<MetricMatrix data={matrixData} attemptHref={attemptHref} />);
+
+  it("caption 标出指标与行列维度", () => {
+    expect(html).toContain("通过率");
+    expect(html).toContain("eval × agent");
+  });
+
+  it("稀疏格子:没有样本的格子空着(恰好一个)", () => {
+    expect(html.match(/nre-td-empty/g)).toHaveLength(1);
+  });
+
+  it("格子数字与 refs 下钻链接", () => {
+    expect(html).toContain("100%");
+    expect(html).toContain("0%");
+    expect(html).toContain('href="/attempts/run-b/3"');
+    expect(html).toContain('href="/attempts/run-b/7"');
+  });
+
+  it("列头(维度键)带稳定散列配色 class", () => {
+    expect(html).toContain(colorClassForKey("bub"));
+    expect(html).toContain(colorClassForKey("codex"));
+  });
+});
+
+describe("Scoreboard", () => {
+  const html = renderToStaticMarkup(<Scoreboard data={scoreboardData} />);
+
+  it("总分 + 满分口径", () => {
+    expect(html).toContain("78.5");
+    expect(html).toContain("52");
+    expect(html).toContain("/ 100");
+  });
+
+  it("分科小计 earned/possible", () => {
+    expect(html).toContain("14/16");
+    expect(html).toContain("3/4");
+  });
+
+  it("missing 注脚:没跑按 0 计的题数如实展示", () => {
+    expect(html).toContain("1 eval missing, scored 0");
+    expect(html).toContain("2 evals missing, scored 0");
+  });
+
+  it("实际生效的权重表可审计", () => {
+    expect(html).toContain("algebra/ ×2");
+    expect(html).toContain("others ×1");
+  });
+});
+
+describe("MetricScatter", () => {
+  const html = renderToStaticMarkup(<MetricScatter data={scatterData} pointHref={(row) => `/exp/${row.key}`} />);
+
+  const cxOf = (key: string): number => {
+    const m = html.match(new RegExp(`data-key="${key}"[^>]*\\bcx="([\\d.]+)"`));
+    expect(m, `circle for ${key}`).toBeTruthy();
+    return Number(m![1]);
+  };
+  const cyOf = (key: string): number => {
+    const m = html.match(new RegExp(`data-key="${key}"[^>]*\\bcy="([\\d.]+)"`));
+    expect(m, `circle for ${key}`).toBeTruthy();
+    return Number(m![1]);
+  };
+
+  it("内联 SVG + 轴标签", () => {
+    expect(html).toContain("<svg");
+    expect(html).toContain("成本($)");
+    expect(html).toContain("通过率(%)");
+  });
+
+  it("better:lower 的 x 轴反向:便宜($5)在贵($10)右边,好的角落恒在右上", () => {
+    expect(cxOf("compare/bub-low")).toBeGreaterThan(cxOf("compare/bub-high"));
+    // y 轴 better:higher:通过率高(90%)在低(50%)上方(SVG y 向下增长)
+    expect(cyOf("compare/bub-high")).toBeLessThan(cyOf("compare/bub-low"));
+  });
+
+  it("同系列点连线并标系列名,线色来自稳定散列调色板", () => {
+    expect(html).toContain("<polyline");
+    expect(html).toContain(">bub</text>");
+    expect(html).toContain(NRE_PALETTE[colorIndexForKey("bub")]);
+    // codex 只有 1 个可画点:不出线,但系列名仍标出
+    expect(html).toContain(">codex</text>");
+  });
+
+  it("null 点不画,底部注脚如实报数", () => {
+    expect(html).not.toContain('data-key="compare/codex-broken"');
+    expect(html).toContain("1 point missing data");
+  });
+
+  it("hover 退化为 <title>:display 与 samples/total", () => {
+    expect(html).toContain("<title>");
+    expect(html).toContain("50%(6/6)");
+  });
+
+  it("pointHref:点包普通 <a>", () => {
+    expect(html).toContain('href="/exp/compare/bub-low"');
+  });
+
+  it("全部点都缺数据时不画空坐标系", () => {
+    const empty = renderToStaticMarkup(
+      <MetricScatter
+        data={{
+          ...scatterData,
+          rows: scatterData.rows.map((r) => ({ ...r, x: { ...r.x, value: null } })),
+        }}
+      />,
+    );
+    expect(empty).not.toContain("<svg");
+    expect(empty).toContain("no data");
+    expect(empty).toContain("4 points missing data");
+  });
+});
+
+describe("DeltaTable", () => {
+  const html = renderToStaticMarkup(<DeltaTable data={deltaData} />);
+
+  it("每格 A/B/Δ 三值", () => {
+    expect(html).toContain("50%");
+    expect(html).toContain("62%");
+    expect(html).toContain("+12pp");
+    expect(html).toContain(">A</span>");
+    expect(html).toContain(">B</span>");
+    expect(html).toContain(">Δ</span>");
+  });
+
+  it("涨跌好坏按 better 配色:通过率涨=好,成本涨=坏,0=平", () => {
+    expect(html).toContain("nre-delta-good");
+    expect(html).toContain("nre-delta-bad");
+    expect(html).toContain("nre-delta-flat");
+  });
+
+  it("任一侧缺数据:Δ 显示为缺,不硬算", () => {
+    expect(html).toContain("nre-delta-missing");
+    expect(html).toContain("no data");
+  });
+
+  it("每行标出 A → B 的 experimentId", () => {
+    expect(html).toContain("compare/bub → compare/bub--agents-md");
+  });
+});
+
+describe("CaseList", () => {
+  const html = renderToStaticMarkup(<CaseList data={caseListData} attemptHref={attemptHref} />);
+
+  it("逐条失败断言:name、score、detail、evidence", () => {
+    expect(html).toContain("roots-correct");
+    expect(html).toContain("score 0");
+    expect(html).toContain("期望 x=2,得到 x=3");
+    expect(html).toContain("judge: 求根公式代入时符号写反");
+  });
+
+  it("errored 的 error 摘要", () => {
+    expect(html).toContain("TypeError: cannot read properties of undefined");
+  });
+
+  it("truncated 如实报「还有 n 条没列」", () => {
+    expect(html).toContain("and 2 more not shown");
+  });
+
+  it("每条案例带 attemptHref 下钻链接", () => {
+    expect(html).toContain('href="/attempts/run-a/4"');
+    expect(html).toContain('href="/attempts/run-c/1"');
+  });
+
+  it("长文本收进 <details>,零 JS 可展开", () => {
+    expect(html).toContain("<details");
+    expect(html).toContain("<summary>");
+  });
+});
+
+describe("跨组件契约", () => {
+  it("同一维度键在所有块里同色(稳定散列,与渲染顺序无关)", () => {
+    const cls = colorClassForKey("bub");
+    const table = renderToStaticMarkup(<MetricTable data={tableData} />);
+    const matrix = renderToStaticMarkup(<MetricMatrix data={matrixData} />);
+    const board = renderToStaticMarkup(<Scoreboard data={scoreboardData} />);
+    const delta = renderToStaticMarkup(<DeltaTable data={deltaData} />);
+    const cases = renderToStaticMarkup(<CaseList data={caseListData} />);
+    for (const html of [table, matrix, board, delta, cases]) {
+      expect(html).toContain(cls);
+    }
+    const scatter = renderToStaticMarkup(<MetricScatter data={scatterData} />);
+    expect(scatter).toContain(NRE_PALETTE[colorIndexForKey("bub")]);
+  });
+
+  it("静态输出不含 <script>:交互只靠 <a>/<details>/CSS", () => {
+    const all = [
+      renderToStaticMarkup(<RunOverview data={overviewData} />),
+      renderToStaticMarkup(<MetricTable data={tableData} attemptHref={attemptHref} />),
+      renderToStaticMarkup(<MetricMatrix data={matrixData} attemptHref={attemptHref} />),
+      renderToStaticMarkup(<Scoreboard data={scoreboardData} />),
+      renderToStaticMarkup(<MetricScatter data={scatterData} />),
+      renderToStaticMarkup(<DeltaTable data={deltaData} />),
+      renderToStaticMarkup(<CaseList data={caseListData} />),
+    ].join("");
+    expect(all).not.toContain("<script");
+  });
+
+  it("组件源码零 hooks(本实验的「不 hydrate 也完整」用最笨的方式保证)", () => {
+    const dir = dirname(fileURLToPath(import.meta.url));
+    const sources = readdirSync(dir)
+      .filter((f) => (f.endsWith(".tsx") || f.endsWith(".ts")) && !f.includes(".test."))
+      .map((f) => readFileSync(join(dir, f), "utf8"));
+    expect(sources.length).toBeGreaterThanOrEqual(12);
+    for (const src of sources) {
+      expect(src).not.toMatch(/\buse[A-Z][A-Za-z]*\s*\(/);
+    }
+  });
+});

package/src/report/react/styles.css

@@ -0,0 +1,302 @@
+/*
+ * niceeval/report/react/styles.css —— 随包发布的默认样式。
+ * 契约(docs/reports.md「四条跨组件契约」):
+ *   - 所有节点都有稳定的 nre-* 类名;这份 CSS + 类名 = 静态页零依赖成立;
+ *   - 要定制,在本文件之后加载自己的覆盖即可,不需要构建工具;
+ *   - 调色板 --nre-c0..c7 的值与 src/report/react/colors.ts 的 NRE_PALETTE
+ *     逐个对应(SVG 内联 hex 用那份),改色时两边一起改。
+ * 视觉刻意极简:结构与诚实细节优先,品牌化交给使用者的覆盖样式。
+ */
+
+.nre {
+  /* 调色板:键 → 稳定散列下标 → 同键同色(colors.ts) */
+  --nre-c0: #2563eb;
+  --nre-c1: #db2777;
+  --nre-c2: #059669;
+  --nre-c3: #d97706;
+  --nre-c4: #7c3aed;
+  --nre-c5: #0891b2;
+  --nre-c6: #dc2626;
+  --nre-c7: #65a30d;
+
+  font-family: system-ui, -apple-system, sans-serif;
+  font-size: 14px;
+  line-height: 1.5;
+  color: #171717;
+  margin: 1rem 0;
+}
+
+/* ---- 维度键上色:同一个 key 在任何块里同色 ---- */
+.nre .nre-c0 { color: var(--nre-c0); }
+.nre .nre-c1 { color: var(--nre-c1); }
+.nre .nre-c2 { color: var(--nre-c2); }
+.nre .nre-c3 { color: var(--nre-c3); }
+.nre .nre-c4 { color: var(--nre-c4); }
+.nre .nre-c5 { color: var(--nre-c5); }
+.nre .nre-c6 { color: var(--nre-c6); }
+.nre .nre-c7 { color: var(--nre-c7); }
+.nre .nre-key { font-weight: 600; }
+
+/* ---- 表格通用 ---- */
+table.nre,
+.nre table {
+  border-collapse: collapse;
+  width: 100%;
+}
+.nre th,
+.nre td {
+  border: 1px solid #e5e5e5;
+  padding: 0.4rem 0.6rem;
+  text-align: left;
+  vertical-align: top;
+}
+.nre thead th {
+  background: #fafafa;
+  font-weight: 600;
+}
+.nre .nre-dimension {
+  color: #737373;
+  font-weight: 500;
+}
+.nre .nre-unit {
+  color: #a3a3a3;
+  font-weight: 400;
+  margin-left: 0.25em;
+}
+.nre .nre-better {
+  color: #a3a3a3;
+  margin-left: 0.25em;
+}
+
+/* ---- MetricCell:缺数据 / 覆盖率角标 / 证据链接 ---- */
+.nre .nre-missing {
+  color: #a3a3a3;
+  font-style: italic;
+}
+.nre .nre-coverage {
+  color: #b45309;
+  font-size: 0.72em;
+  margin-left: 0.3em;
+}
+.nre .nre-refs {
+  margin-left: 0.4em;
+}
+.nre .nre-ref {
+  color: #2563eb;
+  font-size: 0.85em;
+  text-decoration: none;
+  margin-right: 0.25em;
+}
+.nre .nre-ref:hover {
+  text-decoration: underline;
+}
+.nre .nre-td-empty {
+  background: #fafafa;
+}
+
+/* ---- RunOverview ---- */
+.nre-overview .nre-kpis {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 1.5rem;
+  margin: 0 0 0.5rem;
+}
+.nre-overview .nre-kpi dt {
+  color: #737373;
+  font-size: 0.85em;
+}
+.nre-overview .nre-kpi dd {
+  margin: 0;
+  font-size: 1.4em;
+  font-weight: 600;
+}
+.nre-overview .nre-outcomes {
+  margin: 0.25rem 0;
+  color: #525252;
+}
+.nre-overview .nre-outcome {
+  margin-right: 1em;
+}
+.nre .nre-outcome-passed { color: #059669; }
+.nre .nre-outcome-failed { color: #dc2626; }
+.nre .nre-outcome-errored { color: #b45309; }
+.nre .nre-outcome-skipped { color: #a3a3a3; }
+.nre-overview .nre-source {
+  color: #737373;
+  font-size: 0.85em;
+  margin: 0.25rem 0;
+}
+.nre-overview .nre-source-snapshot {
+  margin-left: 0.75em;
+}
+.nre .nre-warnings {
+  list-style: none;
+  margin: 0.5rem 0 0;
+  padding: 0.5rem 0.75rem;
+  background: #fffbeb;
+  border: 1px solid #fde68a;
+  color: #92400e;
+}
+
+/* ---- MetricMatrix ---- */
+.nre-metric-matrix .nre-matrix-caption {
+  caption-side: top;
+  text-align: left;
+  font-weight: 600;
+  padding-bottom: 0.4rem;
+}
+.nre-metric-matrix .nre-matrix-axes {
+  color: #a3a3a3;
+  font-weight: 400;
+  margin-left: 0.75em;
+}
+
+/* ---- Scoreboard ---- */
+.nre-scoreboard .nre-total {
+  font-weight: 700;
+}
+.nre-scoreboard .nre-full-marks {
+  color: #a3a3a3;
+  font-weight: 400;
+  margin-left: 0.25em;
+}
+.nre-scoreboard .nre-subject-missing {
+  display: block;
+  color: #b45309;
+  font-size: 0.8em;
+}
+.nre-scoreboard .nre-weights {
+  color: #737373;
+  font-size: 0.85em;
+  margin: 0.4rem 0 0;
+}
+.nre-scoreboard .nre-weight,
+.nre-scoreboard .nre-weight-rest {
+  margin-left: 0.5em;
+}
+
+/* ---- MetricScatter ---- */
+.nre-metric-scatter {
+  margin: 1rem 0;
+}
+.nre-metric-scatter .nre-scatter-svg {
+  max-width: 640px;
+  width: 100%;
+  height: auto;
+  display: block;
+}
+.nre-metric-scatter .nre-scatter-missing {
+  color: #b45309;
+  font-size: 0.85em;
+  margin: 0.25rem 0 0;
+}
+.nre-metric-scatter .nre-scatter-point-link circle:hover {
+  stroke: #171717;
+  stroke-width: 1.5;
+}
+
+/* ---- DeltaTable ---- */
+.nre-delta-table .nre-pair-ids {
+  display: block;
+  color: #a3a3a3;
+  font-size: 0.8em;
+  font-weight: 400;
+}
+.nre-delta-table .nre-delta-side {
+  display: block;
+  white-space: nowrap;
+}
+.nre-delta-table .nre-delta-tag {
+  display: inline-block;
+  width: 1.2em;
+  color: #a3a3a3;
+  font-size: 0.8em;
+}
+.nre .nre-delta-good { color: #059669; }
+.nre .nre-delta-bad { color: #dc2626; }
+.nre .nre-delta-flat { color: #737373; }
+.nre .nre-delta-neutral { color: #171717; }
+.nre .nre-delta-missing { color: #a3a3a3; }
+
+/* ---- CaseList ---- */
+.nre-case-list .nre-cases {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+}
+.nre-case-list .nre-case {
+  border: 1px solid #e5e5e5;
+  border-left-width: 3px;
+  padding: 0.6rem 0.75rem;
+  margin-bottom: 0.5rem;
+}
+.nre-case-list .nre-case-failed { border-left-color: #dc2626; }
+.nre-case-list .nre-case-errored { border-left-color: #b45309; }
+.nre-case-list .nre-case-head {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 0.75em;
+  align-items: baseline;
+}
+.nre-case-list .nre-case-outcome {
+  font-weight: 700;
+  text-transform: uppercase;
+  font-size: 0.8em;
+}
+.nre-case-list .nre-case-eval {
+  font-weight: 600;
+}
+.nre-case-list .nre-case-experiment,
+.nre-case-list .nre-case-duration,
+.nre-case-list .nre-case-cost {
+  color: #737373;
+  font-size: 0.9em;
+}
+.nre-case-list .nre-case-link {
+  margin-left: auto;
+  color: #2563eb;
+}
+.nre-case-list .nre-case-error {
+  margin: 0.4rem 0 0;
+  padding: 0.3rem 0.5rem;
+  background: #fef2f2;
+  color: #991b1b;
+  font-family: ui-monospace, monospace;
+  font-size: 0.85em;
+  white-space: pre-wrap;
+}
+.nre-case-list .nre-assertions {
+  margin: 0.4rem 0 0;
+  padding-left: 1.25rem;
+}
+.nre-case-list .nre-assertion-name {
+  font-family: ui-monospace, monospace;
+}
+.nre-case-list .nre-assertion-score {
+  color: #737373;
+  margin-left: 0.75em;
+  font-size: 0.9em;
+}
+.nre-case-list .nre-assertion-more summary {
+  cursor: pointer;
+  color: #737373;
+  font-size: 0.85em;
+}
+.nre-case-list .nre-assertion-detail {
+  margin: 0.25rem 0;
+  white-space: pre-wrap;
+}
+.nre-case-list .nre-assertion-evidence {
+  margin: 0.25rem 0 0.25rem 0.75rem;
+  padding-left: 0.5rem;
+  border-left: 2px solid #d4d4d4;
+  color: #525252;
+  white-space: pre-wrap;
+}
+.nre-case-list .nre-truncated {
+  color: #b45309;
+  font-size: 0.9em;
+}
+.nre-case-list .nre-case-empty {
+  color: #737373;
+}