niceeval 0.4.6 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +7 -7
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/agent-feedback-loop.mdx +64 -21
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +243 -0
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/report-components.mdx +252 -0
  10. package/docs-site/zh/guides/reporters.mdx +1 -1
  11. package/docs-site/zh/guides/results-data.mdx +224 -0
  12. package/docs-site/zh/guides/runner.mdx +1 -1
  13. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  14. package/docs-site/zh/guides/viewing-results.mdx +147 -77
  15. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  16. package/docs-site/zh/guides/write-send.mdx +6 -6
  17. package/docs-site/zh/index.mdx +1 -1
  18. package/docs-site/zh/introduction.mdx +1 -1
  19. package/docs-site/zh/reference/cli.mdx +11 -3
  20. package/docs-site/zh/reference/define-agent.mdx +5 -5
  21. package/docs-site/zh/reference/define-config.mdx +1 -1
  22. package/docs-site/zh/reference/define-eval.mdx +3 -3
  23. package/package.json +3 -2
  24. package/src/agents/ai-sdk.test.ts +1 -1
  25. package/src/agents/ai-sdk.ts +2 -2
  26. package/src/agents/claude-code.ts +1 -1
  27. package/src/agents/codex.ts +2 -2
  28. package/src/agents/streaming.test.ts +1 -1
  29. package/src/agents/types.ts +4 -4
  30. package/src/agents/ui-message-stream.test.ts +1 -1
  31. package/src/cli.ts +120 -18
  32. package/src/context/context.test.ts +1 -1
  33. package/src/context/context.ts +3 -3
  34. package/src/context/session.ts +2 -2
  35. package/src/context/types.ts +2 -2
  36. package/src/i18n/en.ts +29 -7
  37. package/src/i18n/zh-CN.ts +24 -2
  38. package/src/report/aggregate.ts +79 -50
  39. package/src/report/components.tsx +190 -0
  40. package/src/report/compute.ts +149 -84
  41. package/src/report/default-report.tsx +222 -0
  42. package/src/report/dual-face.test.tsx +527 -0
  43. package/src/report/format.ts +29 -0
  44. package/src/report/index.ts +79 -24
  45. package/src/report/load.ts +67 -0
  46. package/src/report/metrics.ts +2 -1
  47. package/src/report/param.ts +18 -0
  48. package/src/report/primitives.tsx +117 -0
  49. package/src/report/react/CaseList.tsx +2 -2
  50. package/src/report/react/DeltaTable.tsx +2 -2
  51. package/src/report/react/MetricBars.tsx +109 -0
  52. package/src/report/react/MetricLine.tsx +200 -0
  53. package/src/report/react/MetricMatrix.tsx +1 -1
  54. package/src/report/react/MetricScatter.tsx +1 -1
  55. package/src/report/react/MetricTable.tsx +1 -1
  56. package/src/report/react/RunOverview.tsx +4 -4
  57. package/src/report/react/Scoreboard.tsx +3 -3
  58. package/src/report/react/cell.tsx +1 -1
  59. package/src/report/react/fixtures.ts +98 -36
  60. package/src/report/react/format.ts +3 -28
  61. package/src/report/react/index.tsx +41 -24
  62. package/src/report/react/render.test.tsx +8 -8
  63. package/src/report/react/styles.css +77 -0
  64. package/src/report/report.test.ts +248 -128
  65. package/src/report/report.ts +69 -0
  66. package/src/report/text/faces.ts +356 -0
  67. package/src/report/text/layout.ts +125 -0
  68. package/src/report/text/plot.ts +127 -0
  69. package/src/report/tree.ts +220 -0
  70. package/src/report/types.ts +75 -27
  71. package/src/report/web.ts +40 -0
  72. package/src/results/copy.ts +95 -41
  73. package/src/results/format.ts +12 -11
  74. package/src/results/index.ts +34 -17
  75. package/src/results/open.ts +201 -81
  76. package/src/results/results.test.ts +595 -191
  77. package/src/results/select.ts +107 -55
  78. package/src/results/types.ts +137 -45
  79. package/src/results/writer.ts +258 -0
  80. package/src/runner/attempt.ts +3 -3
  81. package/src/runner/fingerprint.ts +1 -1
  82. package/src/runner/reporters/artifacts.ts +38 -78
  83. package/src/runner/reporters/braintrust.test.ts +2 -2
  84. package/src/runner/reporters/braintrust.ts +3 -3
  85. package/src/runner/run.ts +1 -1
  86. package/src/runner/types.ts +18 -12
  87. package/src/show/compose.ts +215 -0
  88. package/src/show/index.ts +263 -0
  89. package/src/show/render.ts +456 -0
  90. package/src/show/show.test.ts +505 -0
  91. package/src/view/app/App.tsx +85 -24
  92. package/src/view/app/components/CostScoreChart.tsx +29 -10
  93. package/src/view/app/components/ExperimentTable.tsx +27 -9
  94. package/src/view/app/components/GroupSelector.tsx +1 -1
  95. package/src/view/app/i18n.ts +18 -3
  96. package/src/view/app/lib/attempt-route.test.ts +9 -9
  97. package/src/view/app/lib/attempt-route.ts +5 -5
  98. package/src/view/app/lib/outcome.ts +1 -3
  99. package/src/view/app/lib/rows.ts +95 -15
  100. package/src/view/app/main.tsx +18 -6
  101. package/src/view/app/pages/RunsPage.tsx +4 -7
  102. package/src/view/app/pages/TracesPage.tsx +3 -8
  103. package/src/view/app/types.ts +50 -6
  104. package/src/view/client-dist/app.css +1 -1
  105. package/src/view/client-dist/app.js +20 -20
  106. package/src/view/data.test.ts +254 -0
  107. package/src/view/data.ts +336 -0
  108. package/src/view/index.ts +64 -10
  109. package/src/view/server.ts +31 -9
  110. package/src/view/shared/types.ts +47 -41
  111. package/src/view/styles.css +5 -0
  112. package/src/view/template.html +1 -0
  113. package/src/view/view-report.test.ts +308 -0
  114. package/docs-site/.mintignore +0 -13
  115. package/docs-site/AGENTS.md +0 -46
  116. package/docs-site/concepts/adapter.mdx +0 -287
  117. package/docs-site/concepts/assert.mdx +0 -243
  118. package/docs-site/concepts/drive.mdx +0 -118
  119. package/docs-site/concepts/evals.mdx +0 -108
  120. package/docs-site/concepts/experiment.mdx +0 -37
  121. package/docs-site/concepts/hitl.mdx +0 -83
  122. package/docs-site/concepts/judge.mdx +0 -129
  123. package/docs-site/concepts/overview.mdx +0 -98
  124. package/docs-site/concepts/tier.mdx +0 -42
  125. package/docs-site/docs-ref/00-index.md +0 -23
  126. package/docs-site/docs-ref/01-page-types.md +0 -43
  127. package/docs-site/docs-ref/03-style-rules.md +0 -45
  128. package/docs-site/docs-ref/04-code-examples.md +0 -37
  129. package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
  130. package/docs-site/docs-ref/06-checklists.md +0 -53
  131. package/docs-site/docs.json +0 -256
  132. package/docs-site/example/ai-agent-application.mdx +0 -148
  133. package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
  134. package/docs-site/example/claude-code-codex-skill.mdx +0 -147
  135. package/docs-site/example/showcase.mdx +0 -39
  136. package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
  137. package/docs-site/example/tier1-claude-sdk.mdx +0 -119
  138. package/docs-site/example/tier1-codex-sdk.mdx +0 -111
  139. package/docs-site/example/tier1-langgraph.mdx +0 -119
  140. package/docs-site/example/tier1-pi-sdk.mdx +0 -119
  141. package/docs-site/favicon.svg +0 -5
  142. package/docs-site/github-diff.css +0 -135
  143. package/docs-site/github-diff.js +0 -11
  144. package/docs-site/guides/authoring.mdx +0 -129
  145. package/docs-site/guides/ci-integration.mdx +0 -97
  146. package/docs-site/guides/connect-otel.mdx +0 -209
  147. package/docs-site/guides/connect-your-agent.mdx +0 -221
  148. package/docs-site/guides/dataset-fanout.mdx +0 -98
  149. package/docs-site/guides/experiments.mdx +0 -71
  150. package/docs-site/guides/fixtures.mdx +0 -147
  151. package/docs-site/guides/reporters.mdx +0 -113
  152. package/docs-site/guides/runner.mdx +0 -79
  153. package/docs-site/guides/sandbox-agent.mdx +0 -138
  154. package/docs-site/guides/sandbox-backends.mdx +0 -64
  155. package/docs-site/guides/scoring-guide.mdx +0 -92
  156. package/docs-site/guides/viewing-results.mdx +0 -195
  157. package/docs-site/guides/write-experiment.mdx +0 -81
  158. package/docs-site/guides/write-send.mdx +0 -347
  159. package/docs-site/index.mdx +0 -181
  160. package/docs-site/introduction.mdx +0 -141
  161. package/docs-site/quickstart.mdx +0 -136
  162. package/docs-site/reference/builtin-agents.mdx +0 -161
  163. package/docs-site/reference/capabilities.mdx +0 -76
  164. package/docs-site/reference/cli.mdx +0 -120
  165. package/docs-site/reference/define-agent.mdx +0 -168
  166. package/docs-site/reference/define-config.mdx +0 -41
  167. package/docs-site/reference/define-eval.mdx +0 -160
  168. package/docs-site/reference/events.mdx +0 -131
  169. package/docs-site/reference/expect.mdx +0 -112
  170. package/docs-site/tracker.js +0 -8
  171. package/src/report/react/data.ts +0 -17
  172. package/src/view/aggregate.ts +0 -103
  173. package/src/view/loader.test.ts +0 -61
  174. package/src/view/loader.ts +0 -257
package/src/cli.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  // niceeval CLI 入口。执行 eval 必须以 experiment 为单位;位置参数只在 exp 后筛 eval id 前缀。
2
2
  // niceeval exp [组|配置] [pattern] 跑实验
3
+ // niceeval show [pattern] 终端读结果:榜单 / 单 eval / 证据切面 / 时间轴 / --report
3
4
  // niceeval list 只列出发现到的 eval
4
5
  // niceeval clean 删除 .niceeval/ 历史运行工件
5
6
 
@@ -19,15 +20,27 @@ import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
19
20
  import { JUnit } from "./runner/reporters/json.ts";
20
21
  import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
21
22
  import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
22
- import { buildView, startViewServer, loadLatestResultsPerEval, IncompatibleResultsError } from "./view/index.ts";
23
+ import {
24
+ buildView,
25
+ startViewServer,
26
+ loadLatestResultsPerEval,
27
+ resolveViewInput,
28
+ IncompatibleResultsError,
29
+ ViewInputError,
30
+ } from "./view/index.ts";
31
+ import { ReportLoadError } from "./report/load.ts";
32
+ import { runShow } from "./show/index.ts";
23
33
  import { t } from "./i18n/index.ts";
24
34
  import { formatThrown, upsertManagedBlock } from "./util.ts";
25
35
  import type { Config, DiscoveredExperiment, Reporter } from "./types.ts";
26
36
 
27
- /** `niceeval view <summary.json>` 指向版本不同的报告时:打印 npx 提示后退出,不抛堆栈。 */
28
- function exitOnIncompatibleResults(e: unknown): never {
29
- if (e instanceof IncompatibleResultsError) {
30
- process.stderr.write(e.message);
37
+ /**
38
+ * view 的可预期用户错误:版本不同的报告(npx 提示)、位置参数/组合语义错误、
39
+ * --report 装载失败。打一句直说问题与下一步后退出,不抛堆栈。
40
+ */
41
+ function exitOnViewUserError(e: unknown): never {
42
+ if (e instanceof IncompatibleResultsError || e instanceof ViewInputError || e instanceof ReportLoadError) {
43
+ process.stderr.write(e.message.endsWith("\n") ? e.message : `${e.message}\n`);
31
44
  process.exit(1);
32
45
  }
33
46
  throw e;
@@ -51,6 +64,17 @@ interface Flags {
51
64
  out?: string;
52
65
  port?: number;
53
66
  help: boolean;
67
+ // ── show 专属(位置参数仍是 eval id 前缀;这些 flag 选「怎么看」)──
68
+ transcript: boolean;
69
+ trace: boolean;
70
+ diff: boolean;
71
+ /** --diff=<路径>(必须 = 连写;空格形式会把路径当 eval id 前缀,按文档如此)。 */
72
+ diffPath?: string;
73
+ history: boolean;
74
+ experiment?: string;
75
+ attempt?: number;
76
+ run?: string;
77
+ report?: string;
54
78
  }
55
79
 
56
80
  // 表驱动的 flag 定义(node:util parseArgs)。--no-x 显式声明,不依赖 allowNegative(需 Node 20.14+,
@@ -72,6 +96,17 @@ const FLAG_OPTIONS = {
72
96
  // 免得照旧文档写的脚本直接崩;真正实现前不接任何行为。
73
97
  watch: { type: "boolean" },
74
98
  json: { type: "boolean" },
99
+ // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/guides/viewing-results.mdx)。
100
+ transcript: { type: "boolean" },
101
+ trace: { type: "boolean" },
102
+ // --diff 是布尔;--diff=<路径> 在 parseArgs 前预扫成 diffPath(路径必须 = 连写,
103
+ // 空格形式的下一个 token 仍是位置参数 = eval id 前缀,与文档一致)。
104
+ diff: { type: "boolean" },
105
+ history: { type: "boolean" },
106
+ experiment: { type: "string" },
107
+ attempt: { type: "string" },
108
+ run: { type: "string" },
109
+ report: { type: "string" },
75
110
  dry: { type: "boolean" },
76
111
  quiet: { type: "boolean" },
77
112
  force: { type: "boolean" },
@@ -96,6 +131,17 @@ function numberFlag(name: string, raw: string | undefined): number | undefined {
96
131
  function parseArgs(argv: string[]): { command: string; positionals: string[]; flags: Flags } {
97
132
  if (argv[0] === "--") argv = argv.slice(1);
98
133
 
134
+ // --diff=<路径> 预扫:diff 本体是布尔(裸 --diff = 文件级摘要),路径只接受 = 连写。
135
+ let diffPath: string | undefined;
136
+ argv = argv.map((arg) => {
137
+ if (arg.startsWith("--diff=")) {
138
+ const path = arg.slice("--diff=".length);
139
+ if (path) diffPath = path;
140
+ return "--diff";
141
+ }
142
+ return arg;
143
+ });
144
+
99
145
  let values: Record<string, string | boolean | undefined>;
100
146
  let rawPositionals: string[];
101
147
  try {
@@ -113,7 +159,7 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
113
159
  }
114
160
 
115
161
  // 第一个位置参数若是已知命令,则为命令;其余是 eval id 前缀 / view 输入。
116
- const commands = new Set(["exp", "list", "view", "clean", "init", "watch", "run"]);
162
+ const commands = new Set(["exp", "show", "list", "view", "clean", "init", "watch", "run"]);
117
163
  let command = "run";
118
164
  let positionals = rawPositionals;
119
165
  if (rawPositionals[0] && commands.has(rawPositionals[0])) {
@@ -139,6 +185,15 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
139
185
  earlyExit: values["no-early-exit"] === true ? false : values["early-exit"] === true ? true : undefined,
140
186
  open: values["no-open"] === true ? false : values.open === true ? true : undefined,
141
187
  help: values.help === true,
188
+ transcript: values.transcript === true,
189
+ trace: values.trace === true,
190
+ diff: values.diff === true && diffPath === undefined,
191
+ diffPath,
192
+ history: values.history === true,
193
+ experiment: values.experiment as string | undefined,
194
+ attempt: numberFlag("attempt", values.attempt as string | undefined),
195
+ run: values.run as string | undefined,
196
+ report: values.report as string | undefined,
142
197
  };
143
198
  return { command, positionals, flags };
144
199
  }
@@ -184,19 +239,35 @@ async function loadConfig(cwd: string): Promise<Config> {
184
239
  return mod.default;
185
240
  }
186
241
 
187
- // AGENTS.md 托管区块:告诉在这个项目里干活的 coding agent「niceeval 不在你的训练数据里,
188
- // 先读随包文档,跑完读结构化结果」。init 时写入/刷新;标记之外的用户内容永不触碰。
242
+ // AGENTS.md/CLAUDE.md 托管区块:告诉在这个项目里干活的 coding agent「niceeval 不在你的训练数据里,
243
+ // 先读随包文档,跑完读结构化结果」。随包只发中文准绳版文档(英文站是手工同步、可能滞后,
244
+ // 不进包,见 package.json 的 files);init 时写入/刷新;标记之外的用户内容永不触碰。
189
245
  const AGENT_RULES_BEGIN = "<!-- BEGIN:niceeval-agent-rules -->";
190
246
  const AGENT_RULES_END = "<!-- END:niceeval-agent-rules -->";
191
247
  const AGENT_RULES_CONTENT = [
192
248
  "# niceeval is NOT in your training data",
193
249
  "",
194
250
  "Its APIs and conventions may differ from anything you have seen. Read the relevant",
195
- "guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under",
196
- "`zh/`) before writing any eval, experiment, adapter, or niceeval config. After a run,",
197
- "read the `summary.json` path the CLI prints for structured results.",
251
+ "guide in `node_modules/niceeval/docs-site/zh/` before writing any eval, experiment,",
252
+ "adapter, or niceeval config. The bundled docs are Chinese-only — that is the single",
253
+ "authoritative, always-current version; read it regardless of your working language.",
254
+ "After a run, drill into failures with `niceeval show <eval id>` (add `--transcript` /",
255
+ "`--trace` / `--diff` for evidence); the `summary.json` path the CLI prints and the",
256
+ "artifact files it references are the structured source of truth.",
198
257
  ].join("\n");
199
258
 
259
+ // 优先复用已有的 AGENTS.md;项目只有 CLAUDE.md(没有 AGENTS.md)时改写 CLAUDE.md 本身,
260
+ // 不再另建一份重复文件;两者都没有则新建 AGENTS.md。CLAUDE.md 是指向 AGENTS.md 的符号链接时,
261
+ // existsSync 会 follow 链接——目标存在则直接算作「AGENTS.md 已存在」,写入的还是同一份内容,
262
+ // 不会产生分裂;目标不存在(悬空链接)则落回新建 AGENTS.md,写入后链接自然生效。
263
+ function resolveAgentDocPath(cwd: string): string {
264
+ const agentsPath = join(cwd, "AGENTS.md");
265
+ if (existsSync(agentsPath)) return agentsPath;
266
+ const claudePath = join(cwd, "CLAUDE.md");
267
+ if (existsSync(claudePath)) return claudePath;
268
+ return agentsPath;
269
+ }
270
+
200
271
  async function initProject(cwd: string): Promise<void> {
201
272
  await mkdir(join(cwd, "evals"), { recursive: true });
202
273
  const configPath = join(cwd, "niceeval.config.ts");
@@ -214,10 +285,10 @@ async function initProject(cwd: string): Promise<void> {
214
285
  "utf-8",
215
286
  );
216
287
  }
217
- const agentsPath = join(cwd, "AGENTS.md");
218
- const existing = existsSync(agentsPath) ? await readFile(agentsPath, "utf-8") : "";
288
+ const agentDocPath = resolveAgentDocPath(cwd);
289
+ const existing = existsSync(agentDocPath) ? await readFile(agentDocPath, "utf-8") : "";
219
290
  const next = upsertManagedBlock(existing, AGENT_RULES_BEGIN, AGENT_RULES_END, AGENT_RULES_CONTENT);
220
- if (next !== existing) await writeFile(agentsPath, next, "utf-8");
291
+ if (next !== existing) await writeFile(agentDocPath, next, "utf-8");
221
292
  }
222
293
 
223
294
  async function openBrowser(url: string): Promise<boolean> {
@@ -265,12 +336,27 @@ async function main(): Promise<void> {
265
336
  }
266
337
 
267
338
  if (command === "view") {
339
+ // 位置参数 = eval id 前缀(收窄报告槽选集);存在的文件路径 = 单文件模式;
340
+ // 结果目录经 --run 递入;--report 整槽替换报告槽(与 show --report 吃同一个文件)。
341
+ let viewInput: { input?: string; patterns: string[] };
342
+ try {
343
+ viewInput = resolveViewInput(cwd, positionals, flags.run);
344
+ } catch (e) {
345
+ exitOnViewUserError(e);
346
+ }
347
+ const scan = {
348
+ patterns: viewInput.patterns,
349
+ ...(flags.experiment !== undefined ? { experiment: flags.experiment } : {}),
350
+ ...(flags.report !== undefined ? { report: { path: flags.report, cwd } } : {}),
351
+ };
268
352
  if (flags.out) {
269
- const out = await buildView({ input: positionals[0], out: flags.out }).catch(exitOnIncompatibleResults);
353
+ const out = await buildView({ input: viewInput.input, out: flags.out, scan }).catch(exitOnViewUserError);
270
354
  process.stdout.write(t("cli.view.exportedDir", { out }));
271
355
  process.exit(0);
272
356
  }
273
- const server = await startViewServer({ input: positionals[0], port: flags.port }).catch(exitOnIncompatibleResults);
357
+ const server = await startViewServer({ input: viewInput.input, port: flags.port, scan }).catch(
358
+ exitOnViewUserError,
359
+ );
274
360
  process.stdout.write(t("cli.view.url", { url: server.url }));
275
361
  if (flags.open !== false) {
276
362
  const opened = await openBrowser(server.url);
@@ -280,6 +366,22 @@ async function main(): Promise<void> {
280
366
  await new Promise(() => {});
281
367
  }
282
368
 
369
+ if (command === "show") {
370
+ // show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --run 指定目录)的落盘结果。
371
+ const code = await runShow(cwd, positionals, {
372
+ transcript: flags.transcript,
373
+ trace: flags.trace,
374
+ diff: flags.diff,
375
+ diffPath: flags.diffPath,
376
+ history: flags.history,
377
+ experiment: flags.experiment,
378
+ attempt: flags.attempt,
379
+ run: flags.run,
380
+ report: flags.report,
381
+ });
382
+ process.exit(code);
383
+ }
384
+
283
385
  if (command === "clean") {
284
386
  await rm(join(cwd, ".niceeval"), { recursive: true, force: true });
285
387
  process.stdout.write(t("cli.clean.done"));
@@ -333,7 +435,7 @@ async function main(): Promise<void> {
333
435
  agent: exp.agent,
334
436
  model: exp.model,
335
437
  reasoningEffort: exp.reasoningEffort,
336
- flags: exp.flags ?? {},
438
+ params: exp.params ?? {},
337
439
  runs: flags.runs ?? envNumber("NICEEVAL_RUNS") ?? exp.runs ?? 1,
338
440
  earlyExit: flags.earlyExit ?? exp.earlyExit ?? true,
339
441
  sandbox: exp.sandbox ?? config.sandbox,
@@ -349,7 +451,7 @@ async function main(): Promise<void> {
349
451
  }
350
452
  } else {
351
453
  // 裸 run / `niceeval <eval>` 不再执行。运行配置必须来自 experiments/,
352
- // 这样 agent/model/flags/runs/budget 与结果聚合都有可签入的身份。
454
+ // 这样 agent/model/params/runs/budget 与结果聚合都有可签入的身份。
353
455
  const experiments = await discoverExperiments(cwd);
354
456
  const asExp = experiments.filter((e) =>
355
457
  positionals.some((p) => e.group === p || e.id === p || e.id.startsWith(p + "/")),
@@ -57,7 +57,7 @@ function makeContext(agent: Agent, sandbox = fakeSandbox(), evalBaseDir?: string
57
57
  return createEvalContext({
58
58
  agent,
59
59
  sandbox,
60
- flags: {},
60
+ params: {},
61
61
  signal: new AbortController().signal,
62
62
  log: () => {},
63
63
  judge: undefined,
@@ -61,7 +61,7 @@ export interface ContextDeps {
61
61
  sandbox: Sandbox;
62
62
  model?: string;
63
63
  reasoningEffort?: string;
64
- flags: Record<string, unknown>;
64
+ params: Record<string, unknown>;
65
65
  signal: AbortSignal;
66
66
  log(msg: string): void;
67
67
  judge: JudgeConfig | undefined;
@@ -92,7 +92,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
92
92
  sandbox: deps.sandbox,
93
93
  model: deps.model,
94
94
  reasoningEffort: deps.reasoningEffort,
95
- flags: deps.flags,
95
+ params: deps.params,
96
96
  signal: deps.signal,
97
97
  log: deps.log,
98
98
  telemetry: deps.telemetry,
@@ -284,7 +284,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
284
284
  signal: deps.signal,
285
285
  model: deps.model,
286
286
  reasoningEffort: deps.reasoningEffort,
287
- flags: deps.flags,
287
+ params: deps.params,
288
288
  log: deps.log,
289
289
  skip: (reason: string) => {
290
290
  if (reason.trim().length === 0) throw new Error(t("context.skipEmpty"));
@@ -88,7 +88,7 @@ export interface SessionDeps {
88
88
  sandbox: Sandbox;
89
89
  model?: string;
90
90
  reasoningEffort?: string;
91
- flags: Record<string, unknown>;
91
+ params: Record<string, unknown>;
92
92
  signal: AbortSignal;
93
93
  log(msg: string): void;
94
94
  /** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
@@ -138,7 +138,7 @@ export class SessionManager {
138
138
  signal: this.deps.signal,
139
139
  model: this.deps.model,
140
140
  reasoningEffort: this.deps.reasoningEffort,
141
- flags: this.deps.flags,
141
+ params: this.deps.params,
142
142
  sandbox: this.deps.sandbox,
143
143
  session,
144
144
  telemetry: this.deps.telemetry,
@@ -274,8 +274,8 @@ export interface TestContext {
274
274
  readonly model?: string;
275
275
  /** 本次 attempt 的推理努力程度(如 "low"/"medium"/"high",取值由 adapter/模型决定)。 */
276
276
  readonly reasoningEffort?: string;
277
- /** 本次 attempt 生效的 flags(experiment.flags 与 CLI flag 合并后的只读视图)。 */
278
- readonly flags: Readonly<Record<string, unknown>>;
277
+ /** 本次 attempt 生效的 params(experiment.params 与 CLI flag 合并后的只读视图)。 */
278
+ readonly params: Readonly<Record<string, unknown>>;
279
279
  /** 打一行调试日志;有 live 进度回调时走该回调,否则落到 stderr,不出现在最终结果里。 */
280
280
  log(msg: string): void;
281
281
  /** 立即中止本 eval 并标记为 skipped(outcome / EvalResult.skipReason),reason 不能为空。 */
package/src/i18n/en.ts CHANGED
@@ -20,7 +20,7 @@ export const en = {
20
20
  "Ways to fix:\n" +
21
21
  " - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
22
22
  " - [cd] Run from the project root that contains niceeval.config.ts\n" +
23
- " Docs: node_modules/niceeval/docs-site/quickstart.mdx",
23
+ " Docs: node_modules/niceeval/docs-site/zh/quickstart.mdx",
24
24
  "cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
25
25
  "cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
26
26
  "cli.dry.noMatches": "(no matches)",
@@ -31,7 +31,7 @@ export const en = {
31
31
  "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
32
32
  "judge.modelMissing":
33
33
  "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
34
- " Docs: node_modules/niceeval/docs-site/guides/scoring-guide.mdx",
34
+ " Docs: node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
35
35
  "loaders.yamlMissing":
36
36
  "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
37
37
  "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
@@ -40,8 +40,18 @@ export const en = {
40
40
  "niceeval — agent-native evals\n\n" +
41
41
  "Usage:\n" +
42
42
  " niceeval exp [group|experiment] [eval-id-prefix…] run experiments\n" +
43
+ " niceeval show [eval-id-prefix…] read results in the terminal\n" +
44
+ " bare: current verdicts per experiment (composed across runs);\n" +
45
+ " a single eval id: attempts + assertion details\n" +
46
+ " --transcript / --trace / --diff[=file] evidence slices of one eval\n" +
47
+ " --history cross-run timeline (mutually exclusive with --report)\n" +
48
+ " --run <dir> pin a results dir --experiment <id> one experiment\n" +
49
+ " --attempt <n> pick an attempt --report <file> custom report\n" +
43
50
  " niceeval list list discovered evals\n" +
44
- " niceeval view [summary.json|dir] [--out dir] [--port n] [--no-open]\n" +
51
+ " niceeval view [eval-id-prefix…|summary.json] [--out dir] [--port n] [--no-open]\n" +
52
+ " report slot + evidence rooms; --report <file> swaps in your report\n" +
53
+ " (same file as show); --run <dir> pins a results dir;\n" +
54
+ " --experiment <id> one experiment\n" +
45
55
  " --out <dir> exports a static site: index.html plus the viewer\n" +
46
56
  " artifacts, ready for any static host\n" +
47
57
  " niceeval clean delete .niceeval/ artifacts\n" +
@@ -53,6 +63,17 @@ export const en = {
53
63
  "Positional args only select which evals to run (id prefixes); which agent and\n" +
54
64
  "how to run come from experiments/ + flags. Env overrides (flag > env > config):\n" +
55
65
  " NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
66
+ "cli.show.noResults": "No results found under {{root}}. Run `niceeval exp` first, then `niceeval show`.\n",
67
+ "cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
68
+ "cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
69
+ "cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
70
+ "cli.show.historyReportConflict":
71
+ "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
72
+ "cli.show.evidenceNeedsEval":
73
+ "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
74
+ "cli.show.attemptNeedsEval":
75
+ "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
76
+ "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
56
77
  "cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
57
78
  "cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
58
79
  "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
@@ -61,7 +82,7 @@ export const en = {
61
82
  "cli.experimentGroup": " group",
62
83
  "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
63
84
  "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
64
- "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site).\n",
85
+ "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site/zh).\n",
65
86
  "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
66
87
  "cli.list.header": "Discovered {{count}} evals:\n",
67
88
  "cli.noAgent": "No agent specified (use --agent <name>).\n",
@@ -70,17 +91,18 @@ export const en = {
70
91
  "cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
71
92
  "cli.run.experimentRequired":
72
93
  "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
73
- " Docs: node_modules/niceeval/docs-site/guides/write-experiment.mdx\n",
94
+ " Docs: node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
74
95
  "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
75
96
  "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
76
97
  "cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
77
98
  "cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
78
99
  "cli.view.exportedDir": "Exported static report site: {{out}} (serve the whole directory with any static host; opening index.html via file:// cannot fetch artifacts)\n",
79
100
  "cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
101
+ "cli.view.incompatibleForeign": "{{dir}}: written by {{name}} {{version}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nOpen this report with the tool that produced it.\n",
80
102
  "cli.view.url": "niceeval view: {{url}}\n",
81
103
  "context.capabilityMissing":
82
104
  "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
83
- " Docs: node_modules/niceeval/docs-site/guides/sandbox-agent.mdx",
105
+ " Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-agent.mdx",
84
106
  "context.skipEmpty": "skip() requires a non-empty reason.",
85
107
  "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
86
108
  "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -172,7 +194,7 @@ export const en = {
172
194
  "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
173
195
  "sandbox.missingSpec":
174
196
  "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
175
- " Docs: node_modules/niceeval/docs-site/guides/sandbox-backends.mdx",
197
+ " Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
176
198
  "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
177
199
  "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
178
200
  "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
package/src/i18n/zh-CN.ts CHANGED
@@ -38,8 +38,17 @@ export const zhCN = {
38
38
  "niceeval — agent-native evals\n\n" +
39
39
  "用法:\n" +
40
40
  " niceeval exp [组|实验] [eval-id 前缀…] 跑实验\n" +
41
+ " niceeval show [eval-id 前缀…] 终端读结果\n" +
42
+ " 裸跑:每个 experiment 的现刻判决(跨 run 合成)\n" +
43
+ " 单个 eval id:attempt 与断言明细\n" +
44
+ " --transcript / --trace / --diff[=文件] 单 eval 的证据切面\n" +
45
+ " --history 跨 run 时间轴(与 --report 互斥)\n" +
46
+ " --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
47
+ " --attempt <n> 指定 attempt --report <文件> 自定义报告\n" +
41
48
  " niceeval list 列出发现到的 eval\n" +
42
- " niceeval view [summary.json|目录] [--out 目录] [--port n] [--no-open]\n" +
49
+ " niceeval view [eval-id 前缀…|summary.json] [--out 目录] [--port n] [--no-open]\n" +
50
+ " 报告槽 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
51
+ " --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
43
52
  " --out <目录> 静态导出:index.html 连同查看器工件,可直接静态托管\n" +
44
53
  " niceeval clean 删除 .niceeval/ 历史工件\n" +
45
54
  " niceeval init 脚手架 config + evals/\n\n" +
@@ -50,6 +59,18 @@ export const zhCN = {
50
59
  "位置参数只选「跑哪些 eval」(id 前缀);对着哪个 agent、怎么跑来自 experiments/ 与\n" +
51
60
  "标志。环境变量覆盖(标志 > 环境变量 > config):\n" +
52
61
  " NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
62
+ // show 的错误文案保持英文(错误文案英文的仓库约定);noResults 是提示,翻译。
63
+ "cli.show.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval show`。\n",
64
+ "cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
65
+ "cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
66
+ "cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
67
+ "cli.show.historyReportConflict":
68
+ "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
69
+ "cli.show.evidenceNeedsEval":
70
+ "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
71
+ "cli.show.attemptNeedsEval":
72
+ "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
73
+ "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
53
74
  "cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
54
75
  "cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
55
76
  "cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
@@ -58,7 +79,7 @@ export const zhCN = {
58
79
  "cli.experimentGroup": "组",
59
80
  "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
60
81
  "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
61
- "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site)。\n",
82
+ "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site/zh)。\n",
62
83
  "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
63
84
  "cli.list.header": "发现 {{count}} 个 eval:\n",
64
85
  "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
@@ -74,6 +95,7 @@ export const zhCN = {
74
95
  "cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
75
96
  "cli.view.exportedDir": "已导出静态查看站:{{out}}(整个目录可直接静态托管;本地打开 {{out}}/index.html 需经 http 服务,file:// 下工件 fetch 不可用)\n",
76
97
  "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
98
+ "cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n",
77
99
  "cli.view.url": "niceeval view: {{url}}\n",
78
100
  "context.capabilityMissing":
79
101
  "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +