niceeval 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.zh.md +11 -0
  2. package/docs/README.md +2 -1
  3. package/docs/adapters/collection.md +1 -1
  4. package/docs/adapters/contract.md +6 -5
  5. package/docs/adapters/reference/claude-code-otel-telemetry.md +1 -1
  6. package/docs/assertions.md +0 -1
  7. package/docs/capabilities-by-construction.md +2 -2
  8. package/docs/cli.md +2 -2
  9. package/docs/concepts.md +2 -2
  10. package/docs/e2e-ci.md +135 -98
  11. package/docs/eval-authoring.md +3 -3
  12. package/docs/experiments.md +7 -4
  13. package/docs/observability.md +9 -5
  14. package/docs/origin-integration.md +18 -18
  15. package/docs/references.md +1 -1
  16. package/docs/reports.md +551 -0
  17. package/docs/results-format.md +3 -3
  18. package/docs/results-lib.md +191 -0
  19. package/docs/runner.md +1 -1
  20. package/docs/sandbox.md +1 -1
  21. package/docs/source-map.md +22 -2
  22. package/docs/tier-sync.md +74 -58
  23. package/docs/view.md +39 -5
  24. package/package.json +29 -3
  25. package/src/agents/ai-sdk.ts +3 -0
  26. package/src/agents/claude-code.ts +18 -1
  27. package/src/agents/codex.ts +3 -2
  28. package/src/agents/index.ts +16 -0
  29. package/src/agents/openai-compat.test.ts +56 -0
  30. package/src/agents/openai-compat.ts +151 -0
  31. package/src/agents/types.ts +3 -1
  32. package/src/cli.ts +3 -2
  33. package/src/context/context.test.ts +78 -1
  34. package/src/context/context.ts +19 -11
  35. package/src/context/session.ts +2 -0
  36. package/src/context/types.ts +11 -5
  37. package/src/i18n/en.ts +3 -4
  38. package/src/i18n/zh-CN.ts +2 -3
  39. package/src/o11y/cost.test.ts +40 -0
  40. package/src/o11y/cost.ts +27 -5
  41. package/src/o11y/derive.ts +2 -2
  42. package/src/o11y/otlp/mappers/claude-code.test.ts +31 -0
  43. package/src/o11y/otlp/mappers/claude-code.ts +24 -0
  44. package/src/o11y/otlp/mappers/index.ts +1 -0
  45. package/src/o11y/otlp/parse.test.ts +127 -0
  46. package/src/o11y/prices.json +176 -119
  47. package/src/o11y/types.ts +2 -1
  48. package/src/report/aggregate.ts +215 -0
  49. package/src/report/compute.ts +405 -0
  50. package/src/report/format.ts +47 -0
  51. package/src/report/index.ts +40 -0
  52. package/src/report/metrics.ts +96 -0
  53. package/src/report/react/CaseList.tsx +70 -0
  54. package/src/report/react/DeltaTable.tsx +79 -0
  55. package/src/report/react/MetricMatrix.tsx +75 -0
  56. package/src/report/react/MetricScatter.tsx +217 -0
  57. package/src/report/react/MetricTable.tsx +71 -0
  58. package/src/report/react/RunOverview.tsx +87 -0
  59. package/src/report/react/Scoreboard.tsx +87 -0
  60. package/src/report/react/cell.tsx +49 -0
  61. package/src/report/react/colors.ts +42 -0
  62. package/src/report/react/data.ts +17 -0
  63. package/src/report/react/fixtures.ts +236 -0
  64. package/src/report/react/format.ts +34 -0
  65. package/src/report/react/index.tsx +34 -0
  66. package/src/report/react/render.test.tsx +303 -0
  67. package/src/report/react/styles.css +302 -0
  68. package/src/report/report.test.ts +667 -0
  69. package/src/report/types.ts +210 -0
  70. package/src/results/copy.ts +200 -0
  71. package/src/results/format.ts +75 -0
  72. package/src/results/index.ts +27 -0
  73. package/src/results/open.ts +207 -0
  74. package/src/results/results.test.ts +359 -0
  75. package/src/results/select.ts +101 -0
  76. package/src/results/types.ts +98 -0
  77. package/src/runner/attempt.ts +3 -1
  78. package/src/runner/report.test.ts +111 -0
  79. package/src/runner/report.ts +52 -1
  80. package/src/runner/reporters/artifacts.ts +1 -1
  81. package/src/runner/reporters/braintrust.test.ts +106 -0
  82. package/src/runner/reporters/braintrust.ts +197 -0
  83. package/src/runner/reporters/index.ts +3 -1
  84. package/src/runner/run.ts +30 -19
  85. package/src/runner/types.ts +17 -0
  86. package/src/sandbox/types.ts +3 -3
  87. package/src/view/app/App.tsx +82 -17
  88. package/src/view/app/components/CostScoreChart.tsx +127 -0
  89. package/src/view/app/i18n.ts +10 -1
  90. package/src/view/app/lib/attempt-route.test.ts +89 -0
  91. package/src/view/app/lib/attempt-route.ts +52 -0
  92. package/src/view/app/lib/chart.ts +72 -0
  93. package/src/view/app/lib/outcome.ts +1 -1
  94. package/src/view/app/types.ts +4 -5
  95. package/src/view/client-dist/app.css +1 -1
  96. package/src/view/client-dist/app.js +18 -18
  97. package/src/view/index.ts +12 -18
  98. package/src/view/loader.ts +14 -7
  99. package/src/view/shared/types.ts +14 -1
  100. package/src/view/styles.css +43 -0
  101. package/src/view/template.html +0 -1
package/src/runner/run.ts CHANGED
@@ -70,7 +70,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
70
70
  for (const r of opts.priorResults) {
71
71
  if (!r.experimentId || !priorRunKeys.has(`${r.experimentId}|${r.id}`)) continue;
72
72
  // artifactsDir 是相对"本次 run 自己目录"的路径,新 summary 换了目录就会失效,必须清掉。
73
- // artifactBase 不一样:它已经是 loadMostRecentResults(经 withArtifactBases)拼好的、
73
+ // artifactBase 不一样:它已经是 loadMostRecentResults(经 withViewRefs)拼好的、
74
74
  // 相对稳定的 .niceeval 根的路径,指向旧 run 的产物目录依然可解析,原样带过来
75
75
  // ——不然 view 就再也找不到这条携带结果的源码/转录/trace 了。
76
76
  carriedResults.push({ ...r, artifactsDir: undefined });
@@ -201,12 +201,14 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
201
201
  // 早停:同 key 已通过,或已 errored(重跑只会重复同一个框架错误)且开了 earlyExit
202
202
  // → 跳过未启动的 attempt。
203
203
  if (a.run.earlyExit && (passedKeys.has(a.key) || erroredKeys.has(a.key))) {
204
- yield* Effect.promise(() =>
205
- emitReporterEvent(opts.reporters, {
206
- type: "run:earlyExit",
207
- evalId: a.evalDef.id,
208
- experimentId: a.run.experimentId,
209
- }),
204
+ yield* reportMutex.withPermits(1)(
205
+ Effect.promise(() =>
206
+ emitReporterEvent(opts.reporters, {
207
+ type: "run:earlyExit",
208
+ evalId: a.evalDef.id,
209
+ experimentId: a.run.experimentId,
210
+ }),
211
+ ),
210
212
  );
211
213
  return;
212
214
  }
@@ -222,8 +224,10 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
222
224
  if (s.spent >= budget) {
223
225
  if (!budgetReported.has(budgetKey)) {
224
226
  budgetReported.add(budgetKey);
225
- yield* Effect.promise(() =>
226
- emitReporterEvent(opts.reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
227
+ yield* reportMutex.withPermits(1)(
228
+ Effect.promise(() =>
229
+ emitReporterEvent(opts.reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
230
+ ),
227
231
  );
228
232
  }
229
233
  return;
@@ -255,15 +259,17 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
255
259
  ? AbortSignal.any([opts.signal, evalAc.signal])
256
260
  : (evalAc?.signal ?? opts.signal);
257
261
 
258
- yield* Effect.promise(() =>
259
- emitReporterEvent(opts.reporters, {
260
- type: "eval:start",
261
- eval: { id: a.evalDef.id },
262
- agent: a.run.agent,
263
- model: a.run.model,
264
- attempt: a.attempt,
265
- experimentId: a.run.experimentId,
266
- }),
262
+ yield* reportMutex.withPermits(1)(
263
+ Effect.promise(() =>
264
+ emitReporterEvent(opts.reporters, {
265
+ type: "eval:start",
266
+ eval: { id: a.evalDef.id },
267
+ agent: a.run.agent,
268
+ model: a.run.model,
269
+ attempt: a.attempt,
270
+ experimentId: a.run.experimentId,
271
+ }),
272
+ ),
267
273
  );
268
274
  const result = yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal).pipe(
269
275
  Effect.ensuring(
@@ -310,7 +316,12 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
310
316
  ),
311
317
  ),
312
318
  );
313
- yield* Effect.promise(() => emitReporterEvent(opts.reporters, { type: "eval:complete", result }));
319
+ // 和上面的 onEvalComplete 同一把 reportMutex:两条回调路径都要串行化,否则并发
320
+ // attempt 各自触发的 eval:complete 会绕开 permit=1 直接并发跑,和文档承诺的
321
+ // 「报告回调串行化」不一致。
322
+ yield* reportMutex.withPermits(1)(
323
+ Effect.promise(() => emitReporterEvent(opts.reporters, { type: "eval:complete", result })),
324
+ );
314
325
  }),
315
326
  { concurrency: opts.maxConcurrency, discard: true },
316
327
  ).pipe(
@@ -153,6 +153,8 @@ export interface ExperimentDef {
153
153
  agent: Agent;
154
154
  /** 单个模型(agent 留空时实验决定);省略=用 agent 原生默认。跨模型对比写多个实验文件,别用数组。 */
155
155
  model?: string;
156
+ /** 模型推理努力程度(如 "low"/"medium"/"high",取值由具体模型/adapter 决定);省略=用 agent 原生默认。经 ctx.reasoningEffort 透给 adapter 与 eval。 */
157
+ reasoningEffort?: string;
156
158
  flags?: Record<string, unknown>;
157
159
  runs?: number;
158
160
  earlyExit?: boolean;
@@ -191,6 +193,20 @@ export interface Config {
191
193
  * 的远程接入,在这里覆盖。
192
194
  */
193
195
  telemetry?: { host?: string; port?: number };
196
+ /**
197
+ * 内置价格表(`o11y/prices.json`)之上的用户覆盖 / 补充,按 model 查(见 Observability
198
+ * · 用量与成本)。key 支持精确 model 名或 `provider/*` 通配(自托管/网关折扣按 provider 批量覆盖);
199
+ * 精确 key 优先于通配。只在没有网关实测成本(`usage.costUSD`)时才会用到——实测优先于估算恒成立。
200
+ */
201
+ pricing?: Record<string, PriceOverride>;
202
+ }
203
+
204
+ /** 每百万 token 的美元单价;省略的桶退回 `inputPerMTok`(cache token 本质也是 input)。 */
205
+ export interface PriceOverride {
206
+ inputPerMTok: number;
207
+ outputPerMTok: number;
208
+ cacheReadPerMTok?: number;
209
+ cacheWritePerMTok?: number;
194
210
  }
195
211
 
196
212
  // ───────────────────────── 调度编排 ─────────────────────────
@@ -199,6 +215,7 @@ export interface Config {
199
215
  export interface AgentRun {
200
216
  agent: Agent;
201
217
  model?: string;
218
+ reasoningEffort?: string;
202
219
  flags: Record<string, unknown>;
203
220
  runs: number;
204
221
  earlyExit: boolean;
@@ -54,8 +54,8 @@ export type SandboxRuntime = "node20" | "node24";
54
54
  /**
55
55
  * Sandbox 的「数据结构」定义 —— 与 agent 一样可带参数(见 docs/sandbox.md)。
56
56
  * 必须用工厂函数构造(`dockerSandbox()` / `vercelSandbox()` / `e2bSandbox()` / `defineSandbox()`),
57
- * 放进 config / experiment 的 `sandbox` 字段 —— 字段类型只接受这个数据结构,不接受裸字符串,
58
- * 省略字段 = 按环境自动探测后端。各后端的参数互不相同 —— 这是个按 `backend` 区分的可辨识联合(discriminated union)。
57
+ * 放进 config / experiment 的 `sandbox` 字段 —— 字段类型只接受这个数据结构,不接受裸字符串。
58
+ * 各后端的参数互不相同 —— 这是个按 `backend` 区分的可辨识联合(discriminated union)。
59
59
  */
60
60
  export interface DockerSandboxSpec {
61
61
  readonly backend: "docker";
@@ -89,7 +89,7 @@ export interface CustomSandboxSpec {
89
89
 
90
90
  export type SandboxSpec = DockerSandboxSpec | VercelSandboxSpec | E2BSandboxSpec | CustomSandboxSpec;
91
91
 
92
- /** config / experiment 的 `sandbox` 字段:必须是工厂函数产出的 spec 数据结构,省略 = 自动探测后端。 */
92
+ /** config / experiment 的 `sandbox` 字段:必须是工厂函数产出的 spec 数据结构;沙箱型 agent 不能省略。 */
93
93
  export type SandboxOption = SandboxSpec;
94
94
 
95
95
  export interface CommandOptions {
@@ -1,11 +1,13 @@
1
- import { useCallback, useEffect, useMemo, useState } from "react";
1
+ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
2
2
  import { detectLocale, makeTranslator, persistLocale, setDocumentLocale } from "./i18n.ts";
3
3
  import type { MessageKey } from "./i18n.ts";
4
4
  import type { Locale, LocalizedText, SortKey, SortState, Tab, ViewData, ViewResult, ViewRow } from "./types.ts";
5
5
  import { buildGroupMap, compareRows, resultFromUrl } from "./lib/rows.ts";
6
+ import { formatAttemptHash, parseAttemptHash, resolveAttemptRef, unresolvedAttemptWarning } from "./lib/attempt-route.ts";
6
7
  import { formatCost, formatDateTime, formatDuration, formatPercent } from "./lib/format.ts";
7
8
  import { Metric } from "./components/primitives.tsx";
8
9
  import { GroupSelector } from "./components/GroupSelector.tsx";
10
+ import { CostScoreChart } from "./components/CostScoreChart.tsx";
9
11
  import { ExperimentTable } from "./components/ExperimentTable.tsx";
10
12
  import { CopyAllErrors } from "./components/CopyControls.tsx";
11
13
  import { AttemptModal } from "./components/AttemptModal.tsx";
@@ -26,6 +28,19 @@ export function localizedText(text: LocalizedText | undefined, locale: Locale):
26
28
  return text[locale] ?? text.en ?? Object.values(text)[0];
27
29
  }
28
30
 
31
+ /** 初始 URL → 直接打开的 attempt:先认 #/attempt/<run>/<result> 深链,回退旧版 ?modal= 参数。 */
32
+ function modalResultFromLocation(rows: ViewRow[]): ViewResult | null {
33
+ const ref = parseAttemptHash(location.hash);
34
+ if (ref) {
35
+ const found = resolveAttemptRef(rows, ref);
36
+ if (found) return found;
37
+ // 定位不到(run 不在、下标越界、旧格式数据):不开空 modal,页面照常渲染。
38
+ console.warn(unresolvedAttemptWarning(location.hash));
39
+ return null;
40
+ }
41
+ return resultFromUrl(rows);
42
+ }
43
+
29
44
  export function App({ data }: { data: ViewData }) {
30
45
  const rows = data.rows ?? [];
31
46
  const [locale, setLocale] = useState<Locale>(() => detectLocale());
@@ -38,7 +53,11 @@ export function App({ data }: { data: ViewData }) {
38
53
  const groups = [...new Set(rows.map((r) => r.group).filter(Boolean))].sort();
39
54
  return groups[0] ?? null;
40
55
  });
41
- const [modalResult, setModalResult] = useState<ViewResult | null>(() => resultFromUrl(rows));
56
+ const [modalResult, setModalResult] = useState<ViewResult | null>(() => modalResultFromLocation(rows));
57
+ // 当前 modal 的 hash 历史条目前面是否还有本页条目(本页 push 的 / 前进键回到的):
58
+ // true → UI 关闭走 history.back(),前进键还能重新打开;false(深链直接落地)→ 原地抹 hash,
59
+ // 免得 back 把用户弹出站外。
60
+ const modalOwnsHistory = useRef(false);
42
61
 
43
62
  useEffect(() => {
44
63
  setDocumentLocale(locale);
@@ -47,18 +66,61 @@ export function App({ data }: { data: ViewData }) {
47
66
 
48
67
  const openModal = useCallback((result: ViewResult) => {
49
68
  setModalResult(result);
50
- const p = new URLSearchParams();
51
- p.set("modal", result.id);
52
- if (result.experimentId) p.set("exp", result.experimentId);
53
- p.set("a", String(result.attempt));
54
- history.replaceState(null, "", "?" + p.toString());
69
+ const ref = result.attemptRef;
70
+ // 旧格式烘焙的数据没有 attemptRef:modal 照常打开,只是这条 attempt 产不出可分享链接。
71
+ if (!ref) return;
72
+ try {
73
+ // pushState/replaceState 不触发 hashchange,不会和下面的监听器重复开合。
74
+ if (parseAttemptHash(location.hash)) {
75
+ // 已经在某条 attempt 深链上(防御浏览器导航竞态):替换当前条目,不叠历史。
76
+ history.replaceState(null, "", formatAttemptHash(ref));
77
+ } else {
78
+ history.pushState(null, "", formatAttemptHash(ref));
79
+ modalOwnsHistory.current = true;
80
+ }
81
+ } catch {
82
+ // file:// 等受限环境可能拒绝写 history;URL 同步失败不影响打开 modal。
83
+ }
55
84
  }, []);
56
85
 
57
86
  const closeModal = useCallback(() => {
58
87
  setModalResult(null);
59
- history.replaceState(null, "", location.pathname);
88
+ if (modalOwnsHistory.current) {
89
+ modalOwnsHistory.current = false;
90
+ history.back();
91
+ return;
92
+ }
93
+ try {
94
+ // 深链直接落地 / 旧版 ?modal= 链接:没有可回退的本页条目,原地还原成无 modal 的 URL。
95
+ history.replaceState(null, "", location.pathname);
96
+ } catch {
97
+ // 还原 URL 失败不影响关闭。
98
+ }
60
99
  }, []);
61
100
 
101
+ // 浏览器前进/后退、手改 hash、页内 attempt 链接统一从 hashchange 开合 modal。
102
+ useEffect(() => {
103
+ const onHashChange = () => {
104
+ const ref = parseAttemptHash(location.hash);
105
+ if (!ref) {
106
+ modalOwnsHistory.current = false;
107
+ setModalResult(null);
108
+ return;
109
+ }
110
+ const found = resolveAttemptRef(rows, ref);
111
+ if (!found) {
112
+ console.warn(unresolvedAttemptWarning(location.hash));
113
+ setModalResult(null);
114
+ return;
115
+ }
116
+ // 经浏览器导航打开:前一条历史仍是本页,UI 关闭可以安全 back()。
117
+ modalOwnsHistory.current = true;
118
+ setModalResult(found);
119
+ };
120
+ window.addEventListener("hashchange", onHashChange);
121
+ return () => window.removeEventListener("hashchange", onHashChange);
122
+ }, [rows]);
123
+
62
124
  const groupMap = useMemo<Map<string, ViewRow[]>>(() => buildGroupMap(rows), [rows]);
63
125
  const pool = selectedGroup ? groupMap.get(selectedGroup) ?? [] : rows;
64
126
  const filtered = useMemo(() => {
@@ -181,15 +243,18 @@ export function App({ data }: { data: ViewData }) {
181
243
  </div>
182
244
  </div>
183
245
  {rows.length ? (
184
- <ExperimentTable
185
- rows={filtered}
186
- sort={sort}
187
- setSortKey={setSortKey}
188
- openRows={openRows}
189
- toggleRow={toggleRow}
190
- openModal={openModal}
191
- t={t}
192
- />
246
+ <>
247
+ <CostScoreChart rows={filtered} t={t} />
248
+ <ExperimentTable
249
+ rows={filtered}
250
+ sort={sort}
251
+ setSortKey={setSortKey}
252
+ openRows={openRows}
253
+ toggleRow={toggleRow}
254
+ openModal={openModal}
255
+ t={t}
256
+ />
257
+ </>
193
258
  ) : (
194
259
  <div className="empty">
195
260
  {t("empty.summary")}
@@ -0,0 +1,127 @@
1
+ import { useState } from "react";
2
+ import type { T } from "../shared.ts";
3
+ import type { ViewRow } from "../types.ts";
4
+ import { formatCost, formatPercent } from "../lib/format.ts";
5
+ import { layoutLabelOffsets, niceTicks, seriesColor } from "../lib/chart.ts";
6
+
7
+ const WIDTH = 640;
8
+ const HEIGHT = 300;
9
+ const MARGIN = { top: 16, right: 24, bottom: 34, left: 46 };
10
+
11
+ /** formatCost(0) 显示 "—"(表示"没测出成本"),但坐标轴原点确实就是 $0,不能沿用那个语义。 */
12
+ function formatAxisCost(value: number): string {
13
+ return value === 0 ? "$0" : formatCost(value);
14
+ }
15
+
16
+ /**
17
+ * 一个 group 内「成本 vs 通过率」散点图:一行(一个实验/config)= 一个点。
18
+ * 只在 group 内至少两条 row 有 estimatedCostUSD 时渲染,否则没有比较意义。
19
+ */
20
+ export function CostScoreChart({ rows, t }: { rows: ViewRow[]; t: T }) {
21
+ const [hoverKey, setHoverKey] = useState<string | null>(null);
22
+ const points = rows.filter((r): r is ViewRow & { estimatedCostUSD: number } => typeof r.estimatedCostUSD === "number" && Number.isFinite(r.estimatedCostUSD));
23
+ if (points.length < 2) return null;
24
+
25
+ const plotW = WIDTH - MARGIN.left - MARGIN.right;
26
+ const plotH = HEIGHT - MARGIN.top - MARGIN.bottom;
27
+
28
+ const xTicks = niceTicks(0, Math.max(...points.map((p) => p.estimatedCostUSD)), 5);
29
+ const xMax = xTicks[xTicks.length - 1] || 1;
30
+
31
+ const rates = points.map((p) => p.passRate);
32
+ const rateMin = Math.min(...rates);
33
+ const rateMax = Math.max(...rates);
34
+ const pad = Math.max(0.03, (rateMax - rateMin) * 0.25);
35
+ const yTicksRaw = niceTicks(Math.max(0, rateMin - pad), Math.min(1, rateMax + pad), 5);
36
+ const yMin = Math.max(0, yTicksRaw[0] ?? 0);
37
+ const yMax = Math.min(1, yTicksRaw[yTicksRaw.length - 1] ?? 1);
38
+ const yTicks = yTicksRaw.filter((v) => v >= yMin - 1e-9 && v <= yMax + 1e-9);
39
+
40
+ const xScale = (cost: number) => MARGIN.left + (cost / xMax) * plotW;
41
+ const yScale = (rate: number) => MARGIN.top + (1 - (rate - yMin) / (yMax - yMin)) * plotH;
42
+
43
+ const hovered = points.find((p) => p.key === hoverKey);
44
+
45
+ const positions = points.map((p) => {
46
+ const cx = xScale(p.estimatedCostUSD);
47
+ const cy = yScale(p.passRate);
48
+ const anchorLeft = cx >= MARGIN.left + plotW * 0.72;
49
+ return { cx, cy, anchorLeft, width: p.label.length * 6.4 + 10 };
50
+ });
51
+ const labelOffsets = layoutLabelOffsets(positions);
52
+
53
+ return (
54
+ <div className="cost-score-chart">
55
+ <div className="csc-head">{t("chart.costVsScore")}</div>
56
+ <div className="csc-svg-wrap">
57
+ <svg className="csc-svg" viewBox={`0 0 ${WIDTH} ${HEIGHT}`} role="img" aria-label={t("chart.costVsScore")}>
58
+ <g className="csc-grid">
59
+ {yTicks.map((tick) => (
60
+ <line key={`gy-${tick}`} x1={MARGIN.left} x2={WIDTH - MARGIN.right} y1={yScale(tick)} y2={yScale(tick)} />
61
+ ))}
62
+ {xTicks.map((tick) => (
63
+ <line key={`gx-${tick}`} y1={MARGIN.top} y2={HEIGHT - MARGIN.bottom} x1={xScale(tick)} x2={xScale(tick)} />
64
+ ))}
65
+ </g>
66
+ <g className="csc-axis csc-axis-y">
67
+ {yTicks.map((tick) => (
68
+ <text key={`ay-${tick}`} x={MARGIN.left - 8} y={yScale(tick)} textAnchor="end" dominantBaseline="middle">
69
+ {formatPercent(tick)}
70
+ </text>
71
+ ))}
72
+ </g>
73
+ <g className="csc-axis csc-axis-x">
74
+ {xTicks.map((tick) => (
75
+ <text key={`ax-${tick}`} x={xScale(tick)} y={HEIGHT - MARGIN.bottom + 18} textAnchor="middle">
76
+ {formatAxisCost(tick)}
77
+ </text>
78
+ ))}
79
+ <text x={MARGIN.left + plotW / 2} y={HEIGHT - 4} textAnchor="middle">
80
+ {t("chart.axisCost")}
81
+ </text>
82
+ </g>
83
+ <g className="csc-points">
84
+ {points.map((p, i) => {
85
+ const { cx, cy, anchorLeft } = positions[i]!;
86
+ const labelY = cy + 4 + labelOffsets[i]!;
87
+ const color = seriesColor(i);
88
+ const labelX = cx + (anchorLeft ? -10 : 10);
89
+ return (
90
+ <g
91
+ key={p.key}
92
+ className="csc-point"
93
+ tabIndex={0}
94
+ role="button"
95
+ aria-label={`${p.label}: ${formatCost(p.estimatedCostUSD)}, ${formatPercent(p.passRate)}`}
96
+ onMouseEnter={() => setHoverKey(p.key)}
97
+ onMouseLeave={() => setHoverKey((k) => (k === p.key ? null : k))}
98
+ onFocus={() => setHoverKey(p.key)}
99
+ onBlur={() => setHoverKey((k) => (k === p.key ? null : k))}
100
+ >
101
+ {/* 标签被挤开时补一条 leader line,避免脱离原点看不出对应关系。 */}
102
+ {labelOffsets[i] ? <line className="csc-leader" x1={cx} y1={cy} x2={labelX} y2={labelY - 4} /> : null}
103
+ <circle className="csc-hit" cx={cx} cy={cy} r={13} />
104
+ <circle className="csc-dot" cx={cx} cy={cy} r={5} style={{ fill: color }} />
105
+ <text x={labelX} y={labelY} textAnchor={anchorLeft ? "end" : "start"}>
106
+ {p.label}
107
+ </text>
108
+ </g>
109
+ );
110
+ })}
111
+ </g>
112
+ </svg>
113
+ {hovered ? (
114
+ <div
115
+ className="csc-tooltip"
116
+ style={{ left: `${(xScale(hovered.estimatedCostUSD) / WIDTH) * 100}%`, top: `${(yScale(hovered.passRate) / HEIGHT) * 100}%` }}
117
+ >
118
+ <b>{formatPercent(hovered.passRate)}</b> {t("chart.axisScore")}
119
+ <div className="csc-tooltip-meta">
120
+ {formatCost(hovered.estimatedCostUSD)} · {hovered.model || hovered.agent}
121
+ </div>
122
+ </div>
123
+ ) : null}
124
+ </div>
125
+ </div>
126
+ );
127
+ }
@@ -109,7 +109,10 @@ export type MessageKey =
109
109
  | "outcome.skipped"
110
110
  | "banner.skippedTitle"
111
111
  | "banner.skipped.incompatible"
112
- | "banner.skipped.malformed";
112
+ | "banner.skipped.malformed"
113
+ | "chart.costVsScore"
114
+ | "chart.axisCost"
115
+ | "chart.axisScore";
113
116
 
114
117
  type Dictionary = Record<MessageKey, string>;
115
118
 
@@ -221,6 +224,9 @@ const dictionaries: Record<Locale, Dictionary> = {
221
224
  "banner.skippedTitle": "Some runs could not be loaded and are not shown here:",
222
225
  "banner.skipped.incompatible": "written by niceeval {{producer}} (schemaVersion {{schemaVersion}}) — view it with the command on the right",
223
226
  "banner.skipped.malformed": "unreadable report ({{detail}}) — it may be corrupted; re-run the eval or delete this run directory",
227
+ "chart.costVsScore": "Cost vs. Score",
228
+ "chart.axisCost": "Avg cost per eval",
229
+ "chart.axisScore": "Pass rate",
224
230
  },
225
231
  "zh-CN": {
226
232
  "app.title": "niceeval 实验查看器",
@@ -329,6 +335,9 @@ const dictionaries: Record<Locale, Dictionary> = {
329
335
  "banner.skippedTitle": "以下 run 读取失败,此处不展示:",
330
336
  "banner.skipped.incompatible": "由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}})—— 用右侧命令查看",
331
337
  "banner.skipped.malformed": "报告读不了({{detail}})—— 可能已损坏;重跑该 eval 或删除这个 run 目录",
338
+ "chart.costVsScore": "成本 × 通过率",
339
+ "chart.axisCost": "平均每个 eval 成本",
340
+ "chart.axisScore": "通过率",
332
341
  },
333
342
  };
334
343
 
@@ -0,0 +1,89 @@
1
+ // #/attempt/<run>/<result> 深链的纯函数单测:解析 / 格式化往返、坏输入、按 attemptRef 定位。
2
+
3
+ import { describe, expect, it } from "vitest";
4
+ import { formatAttemptHash, parseAttemptHash, resolveAttemptRef } from "./attempt-route.ts";
5
+ import type { ViewResult, ViewRow } from "../types.ts";
6
+
7
+ const attempt = (run: string, index: number): ViewResult => ({
8
+ id: `demo/eval-${index}`,
9
+ agent: "demo-agent",
10
+ outcome: "passed",
11
+ attempt: 0,
12
+ durationMs: 1,
13
+ assertions: [],
14
+ attemptRef: { run, result: index },
15
+ });
16
+
17
+ // 榜单行只有 results 参与定位,其余聚合字段与路由无关。
18
+ const row = (results: ViewResult[]): ViewRow => ({ results }) as ViewRow;
19
+
20
+ describe("parseAttemptHash", () => {
21
+ it("parses the canonical run-dir + index shape", () => {
22
+ expect(parseAttemptHash("#/attempt/2026-07-02T03-10-24-123Z/4")).toEqual({
23
+ run: "2026-07-02T03-10-24-123Z",
24
+ result: 4,
25
+ });
26
+ });
27
+
28
+ it("round-trips through formatAttemptHash, including runs that need encoding", () => {
29
+ for (const ref of [
30
+ { run: "2026-07-02T03-10-24-123Z", result: 0 },
31
+ { run: "nested/2026-01-01T00-00-00-000Z", result: 12 },
32
+ { run: "with space", result: 3 },
33
+ { run: ".", result: 7 }, // 单文件入口的 run 占位
34
+ ]) {
35
+ expect(parseAttemptHash(formatAttemptHash(ref))).toEqual(ref);
36
+ }
37
+ });
38
+
39
+ it("treats the last segment as the index so hand-written nested runs still parse", () => {
40
+ expect(parseAttemptHash("#/attempt/nested/2026-01-01T00-00-00-000Z/12")).toEqual({
41
+ run: "nested/2026-01-01T00-00-00-000Z",
42
+ result: 12,
43
+ });
44
+ });
45
+
46
+ it("rejects non-attempt hashes and malformed shapes", () => {
47
+ for (const hash of [
48
+ "",
49
+ "#",
50
+ "#/",
51
+ "#tab-experiments", // 页内锚点不归这条路由
52
+ "#/compare/a/b",
53
+ "#/attempt",
54
+ "#/attempt/",
55
+ "#/attempt/run-only",
56
+ "#/attempt//3",
57
+ "#/attempt/run/",
58
+ "#/attempt/run/abc",
59
+ "#/attempt/run/1.5",
60
+ "#/attempt/run/-1",
61
+ "#/attempt/run/1/", // 末段为空
62
+ "#/attempt/%zz/1", // 非法 % 转义
63
+ ]) {
64
+ expect(parseAttemptHash(hash), hash).toBeNull();
65
+ }
66
+ });
67
+ });
68
+
69
+ describe("resolveAttemptRef", () => {
70
+ const runA = "2026-07-01T10-00-00-000Z";
71
+ const runB = "2026-07-02T10-00-00-000Z";
72
+ const rows = [row([attempt(runA, 0), attempt(runB, 0)]), row([attempt(runA, 1)])];
73
+
74
+ it("finds the attempt whose injected ref matches run + index", () => {
75
+ expect(resolveAttemptRef(rows, { run: runA, result: 1 })).toBe(rows[1]!.results[0]);
76
+ expect(resolveAttemptRef(rows, { run: runB, result: 0 })).toBe(rows[0]!.results[1]);
77
+ });
78
+
79
+ it("returns null for unknown runs and out-of-range indexes", () => {
80
+ expect(resolveAttemptRef(rows, { run: "no-such-run", result: 0 })).toBeNull();
81
+ expect(resolveAttemptRef(rows, { run: runA, result: 99 })).toBeNull();
82
+ });
83
+
84
+ it("returns null when results predate attemptRef injection (old baked data)", () => {
85
+ const legacy = attempt(runA, 0);
86
+ delete (legacy as { attemptRef?: unknown }).attemptRef;
87
+ expect(resolveAttemptRef([row([legacy])], { run: runA, result: 0 })).toBeNull();
88
+ });
89
+ });
@@ -0,0 +1,52 @@
1
+ // attempt 级 hash 深链:`#/attempt/<run>/<result>`(docs/view.md「用 Reports 积木重建 view」)。
2
+ // 路由参数就是 AttemptRef(run 目录名 + summary.results 下标),由 loader 注入到每条 result 上;
3
+ // 这里只做纯解析 / 格式化 / 匹配,不碰 location / history,方便单测。
4
+ // hash 目前只有这一种路由:tab 切换是纯组件 state,旧版 modal 深链走 ?modal= 查询参数,互不占用。
5
+
6
+ import type { AttemptRef, ViewResult, ViewRow } from "../types.ts";
7
+
8
+ export const ATTEMPT_HASH_PREFIX = "#/attempt/";
9
+
10
+ /** AttemptRef → 可分享的 hash。run 整体编码(嵌套 run 目录里的 "/" 也编进去),下标恒为末段。 */
11
+ export function formatAttemptHash(ref: AttemptRef): string {
12
+ return `${ATTEMPT_HASH_PREFIX}${encodeURIComponent(ref.run)}/${ref.result}`;
13
+ }
14
+
15
+ /**
16
+ * hash → AttemptRef;不是本路由 / 形状不对返回 null(由调用方决定 warn 与否)。
17
+ * 手写链接可能不编码 run 里的 "/"(嵌套 run 目录),所以按「最后一段是下标」切,而不是按段数。
18
+ */
19
+ export function parseAttemptHash(hash: string): AttemptRef | null {
20
+ if (!hash.startsWith(ATTEMPT_HASH_PREFIX)) return null;
21
+ const rest = hash.slice(ATTEMPT_HASH_PREFIX.length);
22
+ const cut = rest.lastIndexOf("/");
23
+ if (cut <= 0) return null; // 没有下标段,或 run 为空
24
+ const indexPart = rest.slice(cut + 1);
25
+ if (!/^\d+$/.test(indexPart)) return null;
26
+ let run: string;
27
+ try {
28
+ run = decodeURIComponent(rest.slice(0, cut));
29
+ } catch {
30
+ return null; // 非法 % 转义
31
+ }
32
+ if (!run) return null;
33
+ return { run, result: parseInt(indexPart, 10) };
34
+ }
35
+
36
+ /** 在榜单行里找 AttemptRef 指向的 attempt;旧格式烘焙的数据没有 attemptRef,自然找不到。 */
37
+ export function resolveAttemptRef(rows: ViewRow[], ref: AttemptRef): ViewResult | null {
38
+ for (const row of rows) {
39
+ for (const result of row.results ?? []) {
40
+ if (result.attemptRef?.run === ref.run && result.attemptRef.result === ref.result) return result;
41
+ }
42
+ }
43
+ return null;
44
+ }
45
+
46
+ /** 深链定位不到时的提示(console.warn 用,英文);页面照常渲染,不开空 modal。 */
47
+ export function unresolvedAttemptWarning(hash: string): string {
48
+ return (
49
+ `[niceeval view] Ignoring attempt link "${hash}": no matching attempt in this view ` +
50
+ `(run not loaded, result index out of range, or the data was baked without attempt refs).`
51
+ );
52
+ }
@@ -0,0 +1,72 @@
1
+ // 通用图表数值工具:round-number 刻度生成,无第三方依赖(遵循 view/ 不引图表库的现状)。
2
+
3
+ function niceNumber(range: number, round: boolean): number {
4
+ if (range <= 0) return 1;
5
+ const exponent = Math.floor(Math.log10(range));
6
+ const fraction = range / 10 ** exponent;
7
+ let niceFraction: number;
8
+ if (round) {
9
+ if (fraction < 1.5) niceFraction = 1;
10
+ else if (fraction < 3) niceFraction = 2;
11
+ else if (fraction < 7) niceFraction = 5;
12
+ else niceFraction = 10;
13
+ } else {
14
+ if (fraction <= 1) niceFraction = 1;
15
+ else if (fraction <= 2) niceFraction = 2;
16
+ else if (fraction <= 5) niceFraction = 5;
17
+ else niceFraction = 10;
18
+ }
19
+ return niceFraction * 10 ** exponent;
20
+ }
21
+
22
+ /** [min, max] 区间上生成 count 个左右的“整齐”刻度(Heckbert nice-numbers)。 */
23
+ export function niceTicks(min: number, max: number, count = 5): number[] {
24
+ if (min === max) {
25
+ min -= 1;
26
+ max += 1;
27
+ }
28
+ const range = niceNumber(max - min, false);
29
+ const step = niceNumber(range / Math.max(1, count - 1), true);
30
+ const niceMin = Math.floor(min / step) * step;
31
+ const niceMax = Math.ceil(max / step) * step;
32
+ const ticks: number[] = [];
33
+ for (let v = niceMin; v <= niceMax + step * 0.5; v += step) ticks.push(Number(v.toFixed(10)));
34
+ return ticks;
35
+ }
36
+
37
+ /** 六色分类色板之外的第 7+ 个系列一律回退到中性色,避免同色误导身份(见 dataviz 分类色板规则)。 */
38
+ export function seriesColor(index: number): string {
39
+ return index < 6 ? `var(--series-${index + 1})` : "var(--muted)";
40
+ }
41
+
42
+ export interface LabelInput {
43
+ cx: number;
44
+ cy: number;
45
+ width: number;
46
+ /** true = 文字锚在点左侧(text-anchor: end),false = 锚在右侧(text-anchor: start)。 */
47
+ anchorLeft: boolean;
48
+ }
49
+
50
+ /**
51
+ * 贪心地把互相重叠的直接标签往下推开(按 y 排序、逐个查重叠再下移),点本身位置不变。
52
+ * 散点常见同分不同价的聚簇,这一步避免标签叠成一团糊字。
53
+ */
54
+ export function layoutLabelOffsets(items: LabelInput[], lineHeight = 14, padX = 4): number[] {
55
+ const offsets = new Array(items.length).fill(0);
56
+ const order = [...items.keys()].sort((a, b) => items[a].cy - items[b].cy || items[a].cx - items[b].cx);
57
+ const placed: { x0: number; x1: number; y: number }[] = [];
58
+ for (const i of order) {
59
+ const it = items[i];
60
+ const x0 = it.anchorLeft ? it.cx - it.width : it.cx;
61
+ const x1 = it.anchorLeft ? it.cx : it.cx + it.width;
62
+ let y = it.cy;
63
+ let guard = 0;
64
+ while (guard < 40 && placed.some((p) => x0 < p.x1 + padX && x1 > p.x0 - padX && Math.abs(y - p.y) < lineHeight)) {
65
+ y += lineHeight;
66
+ guard++;
67
+ }
68
+ placed.push({ x0, x1, y });
69
+ offsets[i] = y - it.cy;
70
+ }
71
+ return offsets;
72
+ }