niceeval 0.2.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +34 -34
  2. package/README.zh.md +29 -15
  3. package/docs/README.md +2 -1
  4. package/docs/adapters/collection.md +1 -1
  5. package/docs/adapters/contract.md +6 -5
  6. package/docs/adapters/reference/claude-code-otel-telemetry.md +1 -1
  7. package/docs/architecture.md +20 -20
  8. package/docs/assertions.md +0 -1
  9. package/docs/capabilities-by-construction.md +2 -2
  10. package/docs/cli.md +2 -2
  11. package/docs/concepts.md +2 -2
  12. package/docs/e2e-ci.md +135 -98
  13. package/docs/eval-authoring.md +3 -3
  14. package/docs/experiments.md +7 -4
  15. package/docs/observability.md +9 -5
  16. package/docs/origin-integration.md +18 -18
  17. package/docs/references.md +1 -1
  18. package/docs/reports.md +551 -0
  19. package/docs/results-format.md +3 -3
  20. package/docs/results-lib.md +191 -0
  21. package/docs/runner.md +1 -1
  22. package/docs/sandbox.md +1 -1
  23. package/docs/source-map.md +22 -2
  24. package/docs/tier-sync.md +74 -58
  25. package/docs/view.md +41 -4
  26. package/package.json +29 -3
  27. package/src/agents/ai-sdk.ts +3 -0
  28. package/src/agents/claude-code.ts +18 -1
  29. package/src/agents/codex.ts +3 -2
  30. package/src/agents/index.ts +16 -0
  31. package/src/agents/openai-compat.test.ts +56 -0
  32. package/src/agents/openai-compat.ts +151 -0
  33. package/src/agents/types.ts +3 -1
  34. package/src/cli.ts +2 -1
  35. package/src/context/context.test.ts +78 -1
  36. package/src/context/context.ts +19 -11
  37. package/src/context/session.ts +2 -0
  38. package/src/context/types.ts +11 -5
  39. package/src/i18n/en.ts +8 -4
  40. package/src/i18n/zh-CN.ts +6 -3
  41. package/src/o11y/cost.test.ts +40 -0
  42. package/src/o11y/cost.ts +27 -5
  43. package/src/o11y/derive.ts +2 -2
  44. package/src/o11y/otlp/mappers/claude-code.test.ts +31 -0
  45. package/src/o11y/otlp/mappers/claude-code.ts +24 -0
  46. package/src/o11y/otlp/mappers/index.ts +1 -0
  47. package/src/o11y/otlp/parse.test.ts +127 -0
  48. package/src/o11y/prices.json +176 -119
  49. package/src/o11y/types.ts +2 -1
  50. package/src/report/aggregate.ts +215 -0
  51. package/src/report/compute.ts +405 -0
  52. package/src/report/format.ts +47 -0
  53. package/src/report/index.ts +40 -0
  54. package/src/report/metrics.ts +96 -0
  55. package/src/report/react/CaseList.tsx +70 -0
  56. package/src/report/react/DeltaTable.tsx +79 -0
  57. package/src/report/react/MetricMatrix.tsx +75 -0
  58. package/src/report/react/MetricScatter.tsx +217 -0
  59. package/src/report/react/MetricTable.tsx +71 -0
  60. package/src/report/react/RunOverview.tsx +87 -0
  61. package/src/report/react/Scoreboard.tsx +87 -0
  62. package/src/report/react/cell.tsx +49 -0
  63. package/src/report/react/colors.ts +42 -0
  64. package/src/report/react/data.ts +17 -0
  65. package/src/report/react/fixtures.ts +236 -0
  66. package/src/report/react/format.ts +34 -0
  67. package/src/report/react/index.tsx +34 -0
  68. package/src/report/react/render.test.tsx +303 -0
  69. package/src/report/react/styles.css +302 -0
  70. package/src/report/report.test.ts +667 -0
  71. package/src/report/types.ts +210 -0
  72. package/src/results/copy.ts +200 -0
  73. package/src/results/format.ts +75 -0
  74. package/src/results/index.ts +27 -0
  75. package/src/results/open.ts +207 -0
  76. package/src/results/results.test.ts +359 -0
  77. package/src/results/select.ts +101 -0
  78. package/src/results/types.ts +98 -0
  79. package/src/runner/attempt.ts +3 -1
  80. package/src/runner/report.test.ts +111 -0
  81. package/src/runner/report.ts +52 -1
  82. package/src/runner/reporters/artifacts.ts +1 -1
  83. package/src/runner/reporters/braintrust.test.ts +106 -0
  84. package/src/runner/reporters/braintrust.ts +197 -0
  85. package/src/runner/reporters/console.ts +3 -3
  86. package/src/runner/reporters/index.ts +3 -1
  87. package/src/runner/reporters/live.ts +17 -5
  88. package/src/runner/reporters/shared.ts +3 -0
  89. package/src/runner/reporters/table.ts +1 -0
  90. package/src/runner/run.ts +31 -18
  91. package/src/runner/types.ts +20 -1
  92. package/src/sandbox/types.ts +3 -3
  93. package/src/shared/format.ts +8 -2
  94. package/src/view/app/App.tsx +82 -17
  95. package/src/view/app/components/AttemptModal.tsx +9 -2
  96. package/src/view/app/components/CodeView.tsx +2 -2
  97. package/src/view/app/components/CostScoreChart.tsx +127 -0
  98. package/src/view/app/components/LazyArtifact.tsx +2 -1
  99. package/src/view/app/i18n.ts +13 -1
  100. package/src/view/app/lib/artifact-url.ts +6 -0
  101. package/src/view/app/lib/attempt-route.test.ts +89 -0
  102. package/src/view/app/lib/attempt-route.ts +52 -0
  103. package/src/view/app/lib/chart.ts +72 -0
  104. package/src/view/app/lib/outcome.ts +1 -1
  105. package/src/view/app/types.ts +4 -5
  106. package/src/view/client-dist/app.css +1 -1
  107. package/src/view/client-dist/app.js +18 -18
  108. package/src/view/index.ts +36 -7
  109. package/src/view/loader.ts +14 -7
  110. package/src/view/server.ts +7 -0
  111. package/src/view/shared/types.ts +14 -1
  112. package/src/view/styles.css +43 -0
  113. package/src/view/template.html +0 -1
@@ -2,10 +2,10 @@
2
2
  // spinner 每 80ms 刷新;attempt 完成后行内显示 ✓/✗/~ 符号。
3
3
  // onRunComplete 时清除状态表,打印和网页榜单同口径的表格报告。
4
4
 
5
- import type { Reporter, RunShape, RunSummary } from "../../types.ts";
5
+ import type { Reporter, ReporterEvent, RunShape, RunSummary } from "../../types.ts";
6
6
  import { t } from "../../i18n/index.ts";
7
7
  import { renderRunReport } from "./table.ts";
8
- import { outcomeSymbol } from "./shared.ts";
8
+ import { outcomeSymbol, WAITING_SYM } from "./shared.ts";
9
9
 
10
10
  const SPINNER = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
11
11
 
@@ -42,6 +42,8 @@ interface RowState extends LiveRow {
42
42
  completed: number;
43
43
  lastMsg: string;
44
44
  dominantOutcome: string | undefined;
45
+ /** true 一旦拿到并发名额、attempt effect 真正开始跑;之前是排队等待,不该转圈误导。 */
46
+ started: boolean;
45
47
  }
46
48
 
47
49
  export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
@@ -51,7 +53,7 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
51
53
  for (const r of rows) {
52
54
  const k = `${r.evalId}|${r.who}`;
53
55
  if (!stateMap.has(k)) {
54
- stateMap.set(k, { ...r, completed: 0, lastMsg: "", dominantOutcome: undefined });
56
+ stateMap.set(k, { ...r, completed: 0, lastMsg: "", dominantOutcome: undefined, started: false });
55
57
  keyOrder.push(k);
56
58
  } else {
57
59
  // 同一 (evalId, who) 可能在多个 agentRun 里出现(不应发生,但做防御)
@@ -71,7 +73,9 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
71
73
  const done = state.completed >= state.total;
72
74
  const sym = done
73
75
  ? outcomeSymbol(state.dominantOutcome ?? "")
74
- : SPINNER[frame % SPINNER.length];
76
+ : state.started
77
+ ? SPINNER[frame % SPINNER.length]
78
+ : WAITING_SYM;
75
79
 
76
80
  const evalCol = state.evalId.slice(0, 24).padEnd(24);
77
81
  const whoCol = `[${state.who}]`.slice(0, 26).padEnd(26);
@@ -79,7 +83,7 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
79
83
 
80
84
  const prefix = ` ${sym} ${evalCol} ${whoCol} ${cntCol} `;
81
85
  const budget = Math.max(0, cols() - prefix.length - 1);
82
- const msg = done ? "" : state.lastMsg.slice(0, budget);
86
+ const msg = done ? "" : state.started ? state.lastMsg.slice(0, budget) : t("live.waiting").slice(0, budget);
83
87
 
84
88
  return `\x1B[2K${prefix}${msg}`;
85
89
  }
@@ -90,6 +94,7 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
90
94
  totalRuns: shape.totalRuns,
91
95
  evals: shape.evals,
92
96
  configs: shape.configs,
97
+ concurrency: shape.maxConcurrency,
93
98
  completed: totalCompleted,
94
99
  total: totalAttempts,
95
100
  })
@@ -132,6 +137,13 @@ export function Live(rows: LiveRow[], totalAttempts: number): LiveReporter {
132
137
  // 不在这里 draw();由 interval 驱动,避免每条日志都刷屏
133
138
  },
134
139
 
140
+ onEvent(event: ReporterEvent) {
141
+ if (event.type !== "eval:start") return;
142
+ const who = event.model ? `${event.agent.name}/${event.model}` : event.agent.name;
143
+ const state = stateMap.get(`${event.eval.id}|${who}`);
144
+ if (state) state.started = true;
145
+ },
146
+
135
147
  onRunStart(_evals, _agent, s) {
136
148
  shape = s;
137
149
  // 初始渲染:让用户看到行表
@@ -13,3 +13,6 @@ export const OUTCOME_SYM: Record<ResultOutcome, string> = {
13
13
  export function outcomeSymbol(outcome: string): string {
14
14
  return OUTCOME_SYM[outcome as ResultOutcome] ?? "?";
15
15
  }
16
+
17
+ /** live 表格里「还没抢到并发名额」的行:和转圈的 SPINNER、完成后的 OUTCOME_SYM 三态区分开。 */
18
+ export const WAITING_SYM = "·";
@@ -104,6 +104,7 @@ export function renderRunReport(summary: RunSummary): string {
104
104
  tokens: tokStr,
105
105
  cost,
106
106
  }).trimEnd());
107
+ lines.push(t("report.viewHint").trimEnd());
107
108
 
108
109
  return `${lines.join("\n")}\n\n`;
109
110
  }
package/src/runner/run.ts CHANGED
@@ -70,7 +70,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
70
70
  for (const r of opts.priorResults) {
71
71
  if (!r.experimentId || !priorRunKeys.has(`${r.experimentId}|${r.id}`)) continue;
72
72
  // artifactsDir 是相对"本次 run 自己目录"的路径,新 summary 换了目录就会失效,必须清掉。
73
- // artifactBase 不一样:它已经是 loadMostRecentResults(经 withArtifactBases)拼好的、
73
+ // artifactBase 不一样:它已经是 loadMostRecentResults(经 withViewRefs)拼好的、
74
74
  // 相对稳定的 .niceeval 根的路径,指向旧 run 的产物目录依然可解析,原样带过来
75
75
  // ——不然 view 就再也找不到这条携带结果的源码/转录/trace 了。
76
76
  carriedResults.push({ ...r, artifactsDir: undefined });
@@ -113,6 +113,7 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
113
113
  evals: runningEvals.length,
114
114
  configs: opts.agentRuns.length,
115
115
  totalRuns: attempts.length,
116
+ maxConcurrency: opts.maxConcurrency,
116
117
  };
117
118
  for (const r of opts.reporters) {
118
119
  // reporter 只是结果消费方:单个 reporter 抛错记 diagnostic,不能让整次调度崩(P2)。
@@ -200,12 +201,14 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
200
201
  // 早停:同 key 已通过,或已 errored(重跑只会重复同一个框架错误)且开了 earlyExit
201
202
  // → 跳过未启动的 attempt。
202
203
  if (a.run.earlyExit && (passedKeys.has(a.key) || erroredKeys.has(a.key))) {
203
- yield* Effect.promise(() =>
204
- emitReporterEvent(opts.reporters, {
205
- type: "run:earlyExit",
206
- evalId: a.evalDef.id,
207
- experimentId: a.run.experimentId,
208
- }),
204
+ yield* reportMutex.withPermits(1)(
205
+ Effect.promise(() =>
206
+ emitReporterEvent(opts.reporters, {
207
+ type: "run:earlyExit",
208
+ evalId: a.evalDef.id,
209
+ experimentId: a.run.experimentId,
210
+ }),
211
+ ),
209
212
  );
210
213
  return;
211
214
  }
@@ -221,8 +224,10 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
221
224
  if (s.spent >= budget) {
222
225
  if (!budgetReported.has(budgetKey)) {
223
226
  budgetReported.add(budgetKey);
224
- yield* Effect.promise(() =>
225
- emitReporterEvent(opts.reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
227
+ yield* reportMutex.withPermits(1)(
228
+ Effect.promise(() =>
229
+ emitReporterEvent(opts.reporters, { type: "run:budgetExceeded", budget, spent: s.spent }),
230
+ ),
226
231
  );
227
232
  }
228
233
  return;
@@ -254,14 +259,17 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
254
259
  ? AbortSignal.any([opts.signal, evalAc.signal])
255
260
  : (evalAc?.signal ?? opts.signal);
256
261
 
257
- yield* Effect.promise(() =>
258
- emitReporterEvent(opts.reporters, {
259
- type: "eval:start",
260
- eval: { id: a.evalDef.id },
261
- agent: a.run.agent,
262
- attempt: a.attempt,
263
- experimentId: a.run.experimentId,
264
- }),
262
+ yield* reportMutex.withPermits(1)(
263
+ Effect.promise(() =>
264
+ emitReporterEvent(opts.reporters, {
265
+ type: "eval:start",
266
+ eval: { id: a.evalDef.id },
267
+ agent: a.run.agent,
268
+ model: a.run.model,
269
+ attempt: a.attempt,
270
+ experimentId: a.run.experimentId,
271
+ }),
272
+ ),
265
273
  );
266
274
  const result = yield* runAttemptEffect(a, opts, sandboxSem, attemptSignal).pipe(
267
275
  Effect.ensuring(
@@ -308,7 +316,12 @@ export async function runEvals(opts: RunOptions): Promise<RunSummary> {
308
316
  ),
309
317
  ),
310
318
  );
311
- yield* Effect.promise(() => emitReporterEvent(opts.reporters, { type: "eval:complete", result }));
319
+ // 和上面的 onEvalComplete 同一把 reportMutex:两条回调路径都要串行化,否则并发
320
+ // attempt 各自触发的 eval:complete 会绕开 permit=1 直接并发跑,和文档承诺的
321
+ // 「报告回调串行化」不一致。
322
+ yield* reportMutex.withPermits(1)(
323
+ Effect.promise(() => emitReporterEvent(opts.reporters, { type: "eval:complete", result })),
324
+ );
312
325
  }),
313
326
  { concurrency: opts.maxConcurrency, discard: true },
314
327
  ).pipe(
@@ -97,6 +97,8 @@ export interface RunShape {
97
97
  configs: number;
98
98
  /** 总 attempt 数(evals × configs × runs);逐行输出与汇总计数都按它。 */
99
99
  totalRuns: number;
100
+ /** 本次运行实际生效的并发数(flag/env/experiment/config/sandbox 默认值解析后的结果)。 */
101
+ maxConcurrency: number;
100
102
  }
101
103
 
102
104
  export interface Reporter {
@@ -108,7 +110,7 @@ export interface Reporter {
108
110
 
109
111
  export type ReporterEvent =
110
112
  | { type: "run:start"; evals: { id: string }[]; agent: Agent; shape: RunShape }
111
- | { type: "eval:start"; eval: { id: string }; agent: Agent; attempt: number; experimentId?: string }
113
+ | { type: "eval:start"; eval: { id: string }; agent: Agent; model?: string; attempt: number; experimentId?: string }
112
114
  | { type: "eval:complete"; result: EvalResult }
113
115
  | { type: "run:earlyExit"; evalId: string; experimentId?: string }
114
116
  | { type: "run:budgetExceeded"; budget: number; spent: number }
@@ -151,6 +153,8 @@ export interface ExperimentDef {
151
153
  agent: Agent;
152
154
  /** 单个模型(agent 留空时实验决定);省略=用 agent 原生默认。跨模型对比写多个实验文件,别用数组。 */
153
155
  model?: string;
156
+ /** 模型推理努力程度(如 "low"/"medium"/"high",取值由具体模型/adapter 决定);省略=用 agent 原生默认。经 ctx.reasoningEffort 透给 adapter 与 eval。 */
157
+ reasoningEffort?: string;
154
158
  flags?: Record<string, unknown>;
155
159
  runs?: number;
156
160
  earlyExit?: boolean;
@@ -189,6 +193,20 @@ export interface Config {
189
193
  * 的远程接入,在这里覆盖。
190
194
  */
191
195
  telemetry?: { host?: string; port?: number };
196
+ /**
197
+ * 内置价格表(`o11y/prices.json`)之上的用户覆盖 / 补充,按 model 查(见 Observability
198
+ * · 用量与成本)。key 支持精确 model 名或 `provider/*` 通配(自托管/网关折扣按 provider 批量覆盖);
199
+ * 精确 key 优先于通配。只在没有网关实测成本(`usage.costUSD`)时才会用到——实测优先于估算恒成立。
200
+ */
201
+ pricing?: Record<string, PriceOverride>;
202
+ }
203
+
204
+ /** 每百万 token 的美元单价;省略的桶退回 `inputPerMTok`(cache token 本质也是 input)。 */
205
+ export interface PriceOverride {
206
+ inputPerMTok: number;
207
+ outputPerMTok: number;
208
+ cacheReadPerMTok?: number;
209
+ cacheWritePerMTok?: number;
192
210
  }
193
211
 
194
212
  // ───────────────────────── 调度编排 ─────────────────────────
@@ -197,6 +215,7 @@ export interface Config {
197
215
  export interface AgentRun {
198
216
  agent: Agent;
199
217
  model?: string;
218
+ reasoningEffort?: string;
200
219
  flags: Record<string, unknown>;
201
220
  runs: number;
202
221
  earlyExit: boolean;
@@ -54,8 +54,8 @@ export type SandboxRuntime = "node20" | "node24";
54
54
  /**
55
55
  * Sandbox 的「数据结构」定义 —— 与 agent 一样可带参数(见 docs/sandbox.md)。
56
56
  * 必须用工厂函数构造(`dockerSandbox()` / `vercelSandbox()` / `e2bSandbox()` / `defineSandbox()`),
57
- * 放进 config / experiment 的 `sandbox` 字段 —— 字段类型只接受这个数据结构,不接受裸字符串,
58
- * 省略字段 = 按环境自动探测后端。各后端的参数互不相同 —— 这是个按 `backend` 区分的可辨识联合(discriminated union)。
57
+ * 放进 config / experiment 的 `sandbox` 字段 —— 字段类型只接受这个数据结构,不接受裸字符串。
58
+ * 各后端的参数互不相同 —— 这是个按 `backend` 区分的可辨识联合(discriminated union)。
59
59
  */
60
60
  export interface DockerSandboxSpec {
61
61
  readonly backend: "docker";
@@ -89,7 +89,7 @@ export interface CustomSandboxSpec {
89
89
 
90
90
  export type SandboxSpec = DockerSandboxSpec | VercelSandboxSpec | E2BSandboxSpec | CustomSandboxSpec;
91
91
 
92
- /** config / experiment 的 `sandbox` 字段:必须是工厂函数产出的 spec 数据结构,省略 = 自动探测后端。 */
92
+ /** config / experiment 的 `sandbox` 字段:必须是工厂函数产出的 spec 数据结构;沙箱型 agent 不能省略。 */
93
93
  export type SandboxOption = SandboxSpec;
94
94
 
95
95
  export interface CommandOptions {
@@ -15,6 +15,12 @@ export function formatDuration(ms?: number): string {
15
15
  }
16
16
 
17
17
  export function formatCost(value?: number): string {
18
- if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return "$0";
19
- return "$" + value.toFixed(value < 1 ? 3 : 2);
18
+ // 查不到价 / 没有 model 时上游传 undefined(见 o11y/cost.ts) —— 显示 "—" 而不是騙人的 $0,
19
+ // 否则「真实但极小的花费」和「压根没算出成本」在界面上长得一模一样。
20
+ if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return "—";
21
+ if (value >= 1) return "$" + value.toFixed(2);
22
+ if (value >= 0.001) return "$" + value.toFixed(3);
23
+ // 便宜模型 + 小样本(如 tier1 示例)常见的真实成本,3 位小数会整个舍成 0.000,
24
+ // 这里退到 2 位有效数字,保留"确实花了一点点钱"的信号。
25
+ return "$" + value.toPrecision(2);
20
26
  }
@@ -1,11 +1,13 @@
1
- import { useCallback, useEffect, useMemo, useState } from "react";
1
+ import { useCallback, useEffect, useMemo, useRef, useState } from "react";
2
2
  import { detectLocale, makeTranslator, persistLocale, setDocumentLocale } from "./i18n.ts";
3
3
  import type { MessageKey } from "./i18n.ts";
4
4
  import type { Locale, LocalizedText, SortKey, SortState, Tab, ViewData, ViewResult, ViewRow } from "./types.ts";
5
5
  import { buildGroupMap, compareRows, resultFromUrl } from "./lib/rows.ts";
6
+ import { formatAttemptHash, parseAttemptHash, resolveAttemptRef, unresolvedAttemptWarning } from "./lib/attempt-route.ts";
6
7
  import { formatCost, formatDateTime, formatDuration, formatPercent } from "./lib/format.ts";
7
8
  import { Metric } from "./components/primitives.tsx";
8
9
  import { GroupSelector } from "./components/GroupSelector.tsx";
10
+ import { CostScoreChart } from "./components/CostScoreChart.tsx";
9
11
  import { ExperimentTable } from "./components/ExperimentTable.tsx";
10
12
  import { CopyAllErrors } from "./components/CopyControls.tsx";
11
13
  import { AttemptModal } from "./components/AttemptModal.tsx";
@@ -26,6 +28,19 @@ export function localizedText(text: LocalizedText | undefined, locale: Locale):
26
28
  return text[locale] ?? text.en ?? Object.values(text)[0];
27
29
  }
28
30
 
31
+ /** 初始 URL → 直接打开的 attempt:先认 #/attempt/<run>/<result> 深链,回退旧版 ?modal= 参数。 */
32
+ function modalResultFromLocation(rows: ViewRow[]): ViewResult | null {
33
+ const ref = parseAttemptHash(location.hash);
34
+ if (ref) {
35
+ const found = resolveAttemptRef(rows, ref);
36
+ if (found) return found;
37
+ // 定位不到(run 不在、下标越界、旧格式数据):不开空 modal,页面照常渲染。
38
+ console.warn(unresolvedAttemptWarning(location.hash));
39
+ return null;
40
+ }
41
+ return resultFromUrl(rows);
42
+ }
43
+
29
44
  export function App({ data }: { data: ViewData }) {
30
45
  const rows = data.rows ?? [];
31
46
  const [locale, setLocale] = useState<Locale>(() => detectLocale());
@@ -38,7 +53,11 @@ export function App({ data }: { data: ViewData }) {
38
53
  const groups = [...new Set(rows.map((r) => r.group).filter(Boolean))].sort();
39
54
  return groups[0] ?? null;
40
55
  });
41
- const [modalResult, setModalResult] = useState<ViewResult | null>(() => resultFromUrl(rows));
56
+ const [modalResult, setModalResult] = useState<ViewResult | null>(() => modalResultFromLocation(rows));
57
+ // 当前 modal 的 hash 历史条目前面是否还有本页条目(本页 push 的 / 前进键回到的):
58
+ // true → UI 关闭走 history.back(),前进键还能重新打开;false(深链直接落地)→ 原地抹 hash,
59
+ // 免得 back 把用户弹出站外。
60
+ const modalOwnsHistory = useRef(false);
42
61
 
43
62
  useEffect(() => {
44
63
  setDocumentLocale(locale);
@@ -47,18 +66,61 @@ export function App({ data }: { data: ViewData }) {
47
66
 
48
67
  const openModal = useCallback((result: ViewResult) => {
49
68
  setModalResult(result);
50
- const p = new URLSearchParams();
51
- p.set("modal", result.id);
52
- if (result.experimentId) p.set("exp", result.experimentId);
53
- p.set("a", String(result.attempt));
54
- history.replaceState(null, "", "?" + p.toString());
69
+ const ref = result.attemptRef;
70
+ // 旧格式烘焙的数据没有 attemptRef:modal 照常打开,只是这条 attempt 产不出可分享链接。
71
+ if (!ref) return;
72
+ try {
73
+ // pushState/replaceState 不触发 hashchange,不会和下面的监听器重复开合。
74
+ if (parseAttemptHash(location.hash)) {
75
+ // 已经在某条 attempt 深链上(防御浏览器导航竞态):替换当前条目,不叠历史。
76
+ history.replaceState(null, "", formatAttemptHash(ref));
77
+ } else {
78
+ history.pushState(null, "", formatAttemptHash(ref));
79
+ modalOwnsHistory.current = true;
80
+ }
81
+ } catch {
82
+ // file:// 等受限环境可能拒绝写 history;URL 同步失败不影响打开 modal。
83
+ }
55
84
  }, []);
56
85
 
57
86
  const closeModal = useCallback(() => {
58
87
  setModalResult(null);
59
- history.replaceState(null, "", location.pathname);
88
+ if (modalOwnsHistory.current) {
89
+ modalOwnsHistory.current = false;
90
+ history.back();
91
+ return;
92
+ }
93
+ try {
94
+ // 深链直接落地 / 旧版 ?modal= 链接:没有可回退的本页条目,原地还原成无 modal 的 URL。
95
+ history.replaceState(null, "", location.pathname);
96
+ } catch {
97
+ // 还原 URL 失败不影响关闭。
98
+ }
60
99
  }, []);
61
100
 
101
+ // 浏览器前进/后退、手改 hash、页内 attempt 链接统一从 hashchange 开合 modal。
102
+ useEffect(() => {
103
+ const onHashChange = () => {
104
+ const ref = parseAttemptHash(location.hash);
105
+ if (!ref) {
106
+ modalOwnsHistory.current = false;
107
+ setModalResult(null);
108
+ return;
109
+ }
110
+ const found = resolveAttemptRef(rows, ref);
111
+ if (!found) {
112
+ console.warn(unresolvedAttemptWarning(location.hash));
113
+ setModalResult(null);
114
+ return;
115
+ }
116
+ // 经浏览器导航打开:前一条历史仍是本页,UI 关闭可以安全 back()。
117
+ modalOwnsHistory.current = true;
118
+ setModalResult(found);
119
+ };
120
+ window.addEventListener("hashchange", onHashChange);
121
+ return () => window.removeEventListener("hashchange", onHashChange);
122
+ }, [rows]);
123
+
62
124
  const groupMap = useMemo<Map<string, ViewRow[]>>(() => buildGroupMap(rows), [rows]);
63
125
  const pool = selectedGroup ? groupMap.get(selectedGroup) ?? [] : rows;
64
126
  const filtered = useMemo(() => {
@@ -181,15 +243,18 @@ export function App({ data }: { data: ViewData }) {
181
243
  </div>
182
244
  </div>
183
245
  {rows.length ? (
184
- <ExperimentTable
185
- rows={filtered}
186
- sort={sort}
187
- setSortKey={setSortKey}
188
- openRows={openRows}
189
- toggleRow={toggleRow}
190
- openModal={openModal}
191
- t={t}
192
- />
246
+ <>
247
+ <CostScoreChart rows={filtered} t={t} />
248
+ <ExperimentTable
249
+ rows={filtered}
250
+ sort={sort}
251
+ setSortKey={setSortKey}
252
+ openRows={openRows}
253
+ toggleRow={toggleRow}
254
+ openModal={openModal}
255
+ t={t}
256
+ />
257
+ </>
193
258
  ) : (
194
259
  <div className="empty">
195
260
  {t("empty.summary")}
@@ -1,6 +1,7 @@
1
1
  import { useEffect, useState } from "react";
2
2
  import type { ArtifactLoadState, T } from "../shared.ts";
3
3
  import type { ViewResult } from "../types.ts";
4
+ import { artifactUrl } from "../lib/artifact-url.ts";
4
5
  import { asEvents, asSources } from "../lib/guards.ts";
5
6
  import { outcomeClass, outcomeLabel } from "../lib/outcome.ts";
6
7
  import { CodeView, NoSourceBody } from "./CodeView.tsx";
@@ -19,7 +20,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
19
20
  let alive = true;
20
21
  const grab = (name: string, has?: boolean): Promise<unknown> =>
21
22
  has
22
- ? fetch("/artifact?p=" + encodeURIComponent(`${base}/${name}`))
23
+ ? fetch(artifactUrl(`${base}/${name}`))
23
24
  .then((r) => (r.ok ? r.json() : null))
24
25
  .catch(() => null)
25
26
  : Promise.resolve(null);
@@ -56,7 +57,13 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
56
57
  {hasCode ? (
57
58
  <CodeView sources={data.sources ?? []} events={data.events || []} assertions={allAssertions} t={t} />
58
59
  ) : data.status !== "loading" ? (
59
- <NoSourceBody assertions={allAssertions} events={data.events || []} t={t} />
60
+ // hasSources 为真却取不到 源码捕获过,是工件文件在当前托管里缺失;和「从未捕获」分开提示。
61
+ <NoSourceBody
62
+ assertions={allAssertions}
63
+ events={data.events || []}
64
+ message={t(result.hasSources && base ? "code.sourceUnavailable" : "code.noSource")}
65
+ t={t}
66
+ />
60
67
  ) : null}
61
68
  {result.hasTrace && base ? (
62
69
  <LazyArtifact type="trace" src={`${base}/trace.json`} t={t} />
@@ -245,12 +245,12 @@ export function AssertDetail({ asserts, t }: { asserts: Assertion[]; t: T }) {
245
245
  * 没源码可叠时(此 run 早于 source-loc,或源码不可读:远程沙箱等)。不退回老的分组视图——
246
246
  * 用代码视图同一套视觉语言:一句说明 + checks(绿过/红不过)+ 原始会话流。重跑即可看到代码视图。
247
247
  */
248
- export function NoSourceBody({ assertions, events, t }: { assertions: Assertion[]; events: TranscriptEvent[]; t: T }) {
248
+ export function NoSourceBody({ assertions, events, message, t }: { assertions: Assertion[]; events: TranscriptEvent[]; message?: string; t: T }) {
249
249
  const checks = assertions || [];
250
250
  return (
251
251
  <div className="nosource">
252
252
  <div className="nosource-note">
253
- {t("code.noSource")}
253
+ {message ?? t("code.noSource")}
254
254
  </div>
255
255
  {checks.length ? (
256
256
  <div className="nosource-block">
@@ -0,0 +1,127 @@
1
+ import { useState } from "react";
2
+ import type { T } from "../shared.ts";
3
+ import type { ViewRow } from "../types.ts";
4
+ import { formatCost, formatPercent } from "../lib/format.ts";
5
+ import { layoutLabelOffsets, niceTicks, seriesColor } from "../lib/chart.ts";
6
+
7
+ const WIDTH = 640;
8
+ const HEIGHT = 300;
9
+ const MARGIN = { top: 16, right: 24, bottom: 34, left: 46 };
10
+
11
+ /** formatCost(0) 显示 "—"(表示"没测出成本"),但坐标轴原点确实就是 $0,不能沿用那个语义。 */
12
+ function formatAxisCost(value: number): string {
13
+ return value === 0 ? "$0" : formatCost(value);
14
+ }
15
+
16
+ /**
17
+ * 一个 group 内「成本 vs 通过率」散点图:一行(一个实验/config)= 一个点。
18
+ * 只在 group 内至少两条 row 有 estimatedCostUSD 时渲染,否则没有比较意义。
19
+ */
20
+ export function CostScoreChart({ rows, t }: { rows: ViewRow[]; t: T }) {
21
+ const [hoverKey, setHoverKey] = useState<string | null>(null);
22
+ const points = rows.filter((r): r is ViewRow & { estimatedCostUSD: number } => typeof r.estimatedCostUSD === "number" && Number.isFinite(r.estimatedCostUSD));
23
+ if (points.length < 2) return null;
24
+
25
+ const plotW = WIDTH - MARGIN.left - MARGIN.right;
26
+ const plotH = HEIGHT - MARGIN.top - MARGIN.bottom;
27
+
28
+ const xTicks = niceTicks(0, Math.max(...points.map((p) => p.estimatedCostUSD)), 5);
29
+ const xMax = xTicks[xTicks.length - 1] || 1;
30
+
31
+ const rates = points.map((p) => p.passRate);
32
+ const rateMin = Math.min(...rates);
33
+ const rateMax = Math.max(...rates);
34
+ const pad = Math.max(0.03, (rateMax - rateMin) * 0.25);
35
+ const yTicksRaw = niceTicks(Math.max(0, rateMin - pad), Math.min(1, rateMax + pad), 5);
36
+ const yMin = Math.max(0, yTicksRaw[0] ?? 0);
37
+ const yMax = Math.min(1, yTicksRaw[yTicksRaw.length - 1] ?? 1);
38
+ const yTicks = yTicksRaw.filter((v) => v >= yMin - 1e-9 && v <= yMax + 1e-9);
39
+
40
+ const xScale = (cost: number) => MARGIN.left + (cost / xMax) * plotW;
41
+ const yScale = (rate: number) => MARGIN.top + (1 - (rate - yMin) / (yMax - yMin)) * plotH;
42
+
43
+ const hovered = points.find((p) => p.key === hoverKey);
44
+
45
+ const positions = points.map((p) => {
46
+ const cx = xScale(p.estimatedCostUSD);
47
+ const cy = yScale(p.passRate);
48
+ const anchorLeft = cx >= MARGIN.left + plotW * 0.72;
49
+ return { cx, cy, anchorLeft, width: p.label.length * 6.4 + 10 };
50
+ });
51
+ const labelOffsets = layoutLabelOffsets(positions);
52
+
53
+ return (
54
+ <div className="cost-score-chart">
55
+ <div className="csc-head">{t("chart.costVsScore")}</div>
56
+ <div className="csc-svg-wrap">
57
+ <svg className="csc-svg" viewBox={`0 0 ${WIDTH} ${HEIGHT}`} role="img" aria-label={t("chart.costVsScore")}>
58
+ <g className="csc-grid">
59
+ {yTicks.map((tick) => (
60
+ <line key={`gy-${tick}`} x1={MARGIN.left} x2={WIDTH - MARGIN.right} y1={yScale(tick)} y2={yScale(tick)} />
61
+ ))}
62
+ {xTicks.map((tick) => (
63
+ <line key={`gx-${tick}`} y1={MARGIN.top} y2={HEIGHT - MARGIN.bottom} x1={xScale(tick)} x2={xScale(tick)} />
64
+ ))}
65
+ </g>
66
+ <g className="csc-axis csc-axis-y">
67
+ {yTicks.map((tick) => (
68
+ <text key={`ay-${tick}`} x={MARGIN.left - 8} y={yScale(tick)} textAnchor="end" dominantBaseline="middle">
69
+ {formatPercent(tick)}
70
+ </text>
71
+ ))}
72
+ </g>
73
+ <g className="csc-axis csc-axis-x">
74
+ {xTicks.map((tick) => (
75
+ <text key={`ax-${tick}`} x={xScale(tick)} y={HEIGHT - MARGIN.bottom + 18} textAnchor="middle">
76
+ {formatAxisCost(tick)}
77
+ </text>
78
+ ))}
79
+ <text x={MARGIN.left + plotW / 2} y={HEIGHT - 4} textAnchor="middle">
80
+ {t("chart.axisCost")}
81
+ </text>
82
+ </g>
83
+ <g className="csc-points">
84
+ {points.map((p, i) => {
85
+ const { cx, cy, anchorLeft } = positions[i]!;
86
+ const labelY = cy + 4 + labelOffsets[i]!;
87
+ const color = seriesColor(i);
88
+ const labelX = cx + (anchorLeft ? -10 : 10);
89
+ return (
90
+ <g
91
+ key={p.key}
92
+ className="csc-point"
93
+ tabIndex={0}
94
+ role="button"
95
+ aria-label={`${p.label}: ${formatCost(p.estimatedCostUSD)}, ${formatPercent(p.passRate)}`}
96
+ onMouseEnter={() => setHoverKey(p.key)}
97
+ onMouseLeave={() => setHoverKey((k) => (k === p.key ? null : k))}
98
+ onFocus={() => setHoverKey(p.key)}
99
+ onBlur={() => setHoverKey((k) => (k === p.key ? null : k))}
100
+ >
101
+ {/* 标签被挤开时补一条 leader line,避免脱离原点看不出对应关系。 */}
102
+ {labelOffsets[i] ? <line className="csc-leader" x1={cx} y1={cy} x2={labelX} y2={labelY - 4} /> : null}
103
+ <circle className="csc-hit" cx={cx} cy={cy} r={13} />
104
+ <circle className="csc-dot" cx={cx} cy={cy} r={5} style={{ fill: color }} />
105
+ <text x={labelX} y={labelY} textAnchor={anchorLeft ? "end" : "start"}>
106
+ {p.label}
107
+ </text>
108
+ </g>
109
+ );
110
+ })}
111
+ </g>
112
+ </svg>
113
+ {hovered ? (
114
+ <div
115
+ className="csc-tooltip"
116
+ style={{ left: `${(xScale(hovered.estimatedCostUSD) / WIDTH) * 100}%`, top: `${(yScale(hovered.passRate) / HEIGHT) * 100}%` }}
117
+ >
118
+ <b>{formatPercent(hovered.passRate)}</b> {t("chart.axisScore")}
119
+ <div className="csc-tooltip-meta">
120
+ {formatCost(hovered.estimatedCostUSD)} · {hovered.model || hovered.agent}
121
+ </div>
122
+ </div>
123
+ ) : null}
124
+ </div>
125
+ </div>
126
+ );
127
+ }
@@ -1,5 +1,6 @@
1
1
  import { useEffect, useState } from "react";
2
2
  import type { LazyArtifactType, T } from "../shared.ts";
3
+ import { artifactUrl } from "../lib/artifact-url.ts";
3
4
  import { asEvents, asSpans } from "../lib/guards.ts";
4
5
  import { Trace } from "./Trace.tsx";
5
6
  import { Transcript } from "./Transcript.tsx";
@@ -14,7 +15,7 @@ export function LazyArtifact({ type, src, autoLoad = false, t }: { type: LazyArt
14
15
  if (loaded) return;
15
16
  setLoaded(true);
16
17
  try {
17
- const resp = await fetch("/artifact?p=" + encodeURIComponent(src));
18
+ const resp = await fetch(artifactUrl(src));
18
19
  if (!resp.ok) throw new Error("HTTP " + resp.status);
19
20
  const body = await resp.json();
20
21
  setContent(body);