niceeval 0.4.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +6 -6
  2. package/docs-site/zh/concepts/experiment.mdx +7 -7
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/agent-feedback-loop.mdx +63 -20
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +243 -0
  8. package/docs-site/zh/guides/experiments.mdx +3 -3
  9. package/docs-site/zh/guides/report-components.mdx +252 -0
  10. package/docs-site/zh/guides/reporters.mdx +1 -1
  11. package/docs-site/zh/guides/results-data.mdx +224 -0
  12. package/docs-site/zh/guides/runner.mdx +1 -1
  13. package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
  14. package/docs-site/zh/guides/viewing-results.mdx +147 -77
  15. package/docs-site/zh/guides/write-experiment.mdx +9 -9
  16. package/docs-site/zh/guides/write-send.mdx +6 -6
  17. package/docs-site/zh/index.mdx +1 -1
  18. package/docs-site/zh/introduction.mdx +1 -1
  19. package/docs-site/zh/reference/cli.mdx +10 -2
  20. package/docs-site/zh/reference/define-agent.mdx +5 -5
  21. package/docs-site/zh/reference/define-config.mdx +1 -1
  22. package/docs-site/zh/reference/define-eval.mdx +3 -3
  23. package/package.json +3 -2
  24. package/src/agents/ai-sdk.test.ts +1 -1
  25. package/src/agents/ai-sdk.ts +2 -2
  26. package/src/agents/bub.ts +14 -4
  27. package/src/agents/claude-code.ts +1 -1
  28. package/src/agents/codex.ts +2 -2
  29. package/src/agents/streaming.test.ts +1 -1
  30. package/src/agents/types.ts +4 -4
  31. package/src/agents/ui-message-stream.test.ts +1 -1
  32. package/src/cli.ts +104 -14
  33. package/src/context/context.test.ts +1 -1
  34. package/src/context/context.ts +3 -3
  35. package/src/context/session.ts +2 -2
  36. package/src/context/types.ts +2 -2
  37. package/src/i18n/en.ts +29 -7
  38. package/src/i18n/zh-CN.ts +24 -2
  39. package/src/report/aggregate.ts +79 -50
  40. package/src/report/components.tsx +190 -0
  41. package/src/report/compute.ts +149 -84
  42. package/src/report/default-report.tsx +222 -0
  43. package/src/report/dual-face.test.tsx +527 -0
  44. package/src/report/format.ts +29 -0
  45. package/src/report/index.ts +79 -24
  46. package/src/report/load.ts +67 -0
  47. package/src/report/metrics.ts +2 -1
  48. package/src/report/param.ts +18 -0
  49. package/src/report/primitives.tsx +117 -0
  50. package/src/report/react/CaseList.tsx +2 -2
  51. package/src/report/react/DeltaTable.tsx +2 -2
  52. package/src/report/react/MetricBars.tsx +109 -0
  53. package/src/report/react/MetricLine.tsx +200 -0
  54. package/src/report/react/MetricMatrix.tsx +1 -1
  55. package/src/report/react/MetricScatter.tsx +1 -1
  56. package/src/report/react/MetricTable.tsx +1 -1
  57. package/src/report/react/RunOverview.tsx +4 -4
  58. package/src/report/react/Scoreboard.tsx +3 -3
  59. package/src/report/react/cell.tsx +1 -1
  60. package/src/report/react/fixtures.ts +98 -36
  61. package/src/report/react/format.ts +3 -28
  62. package/src/report/react/index.tsx +41 -24
  63. package/src/report/react/render.test.tsx +8 -8
  64. package/src/report/react/styles.css +77 -0
  65. package/src/report/report.test.ts +248 -128
  66. package/src/report/report.ts +69 -0
  67. package/src/report/text/faces.ts +356 -0
  68. package/src/report/text/layout.ts +125 -0
  69. package/src/report/text/plot.ts +127 -0
  70. package/src/report/tree.ts +220 -0
  71. package/src/report/types.ts +75 -27
  72. package/src/report/web.ts +40 -0
  73. package/src/results/copy.ts +95 -41
  74. package/src/results/format.ts +12 -11
  75. package/src/results/index.ts +34 -17
  76. package/src/results/open.ts +201 -81
  77. package/src/results/results.test.ts +595 -191
  78. package/src/results/select.ts +107 -55
  79. package/src/results/types.ts +137 -45
  80. package/src/results/writer.ts +258 -0
  81. package/src/runner/attempt.ts +3 -3
  82. package/src/runner/fingerprint.ts +1 -1
  83. package/src/runner/reporters/artifacts.ts +38 -78
  84. package/src/runner/reporters/braintrust.test.ts +2 -2
  85. package/src/runner/reporters/braintrust.ts +3 -3
  86. package/src/runner/run.ts +1 -1
  87. package/src/runner/types.ts +18 -12
  88. package/src/show/compose.ts +215 -0
  89. package/src/show/index.ts +263 -0
  90. package/src/show/render.ts +456 -0
  91. package/src/show/show.test.ts +505 -0
  92. package/src/view/app/App.tsx +85 -24
  93. package/src/view/app/components/CostScoreChart.tsx +29 -10
  94. package/src/view/app/components/ExperimentTable.tsx +27 -9
  95. package/src/view/app/components/GroupSelector.tsx +1 -1
  96. package/src/view/app/i18n.ts +18 -3
  97. package/src/view/app/lib/attempt-route.test.ts +9 -9
  98. package/src/view/app/lib/attempt-route.ts +5 -5
  99. package/src/view/app/lib/outcome.ts +1 -3
  100. package/src/view/app/lib/rows.ts +95 -15
  101. package/src/view/app/main.tsx +18 -6
  102. package/src/view/app/pages/RunsPage.tsx +4 -7
  103. package/src/view/app/pages/TracesPage.tsx +3 -8
  104. package/src/view/app/types.ts +50 -6
  105. package/src/view/client-dist/app.css +1 -1
  106. package/src/view/client-dist/app.js +20 -20
  107. package/src/view/data.test.ts +254 -0
  108. package/src/view/data.ts +336 -0
  109. package/src/view/index.ts +64 -10
  110. package/src/view/server.ts +31 -9
  111. package/src/view/shared/types.ts +47 -41
  112. package/src/view/styles.css +5 -0
  113. package/src/view/template.html +1 -0
  114. package/src/view/view-report.test.ts +308 -0
  115. package/docs-site/.mintignore +0 -13
  116. package/docs-site/AGENTS.md +0 -46
  117. package/docs-site/concepts/adapter.mdx +0 -287
  118. package/docs-site/concepts/assert.mdx +0 -243
  119. package/docs-site/concepts/drive.mdx +0 -118
  120. package/docs-site/concepts/evals.mdx +0 -108
  121. package/docs-site/concepts/experiment.mdx +0 -37
  122. package/docs-site/concepts/hitl.mdx +0 -83
  123. package/docs-site/concepts/judge.mdx +0 -129
  124. package/docs-site/concepts/overview.mdx +0 -98
  125. package/docs-site/concepts/tier.mdx +0 -42
  126. package/docs-site/docs-ref/00-index.md +0 -23
  127. package/docs-site/docs-ref/01-page-types.md +0 -43
  128. package/docs-site/docs-ref/03-style-rules.md +0 -45
  129. package/docs-site/docs-ref/04-code-examples.md +0 -37
  130. package/docs-site/docs-ref/05-dx-failure-paths.md +0 -45
  131. package/docs-site/docs-ref/06-checklists.md +0 -53
  132. package/docs-site/docs.json +0 -256
  133. package/docs-site/example/ai-agent-application.mdx +0 -148
  134. package/docs-site/example/claude-code-codex-plugin.mdx +0 -162
  135. package/docs-site/example/claude-code-codex-skill.mdx +0 -147
  136. package/docs-site/example/showcase.mdx +0 -39
  137. package/docs-site/example/tier1-ai-sdk-v7.mdx +0 -136
  138. package/docs-site/example/tier1-claude-sdk.mdx +0 -119
  139. package/docs-site/example/tier1-codex-sdk.mdx +0 -111
  140. package/docs-site/example/tier1-langgraph.mdx +0 -119
  141. package/docs-site/example/tier1-pi-sdk.mdx +0 -119
  142. package/docs-site/favicon.svg +0 -5
  143. package/docs-site/github-diff.css +0 -135
  144. package/docs-site/github-diff.js +0 -11
  145. package/docs-site/guides/authoring.mdx +0 -129
  146. package/docs-site/guides/ci-integration.mdx +0 -97
  147. package/docs-site/guides/connect-otel.mdx +0 -209
  148. package/docs-site/guides/connect-your-agent.mdx +0 -221
  149. package/docs-site/guides/dataset-fanout.mdx +0 -98
  150. package/docs-site/guides/experiments.mdx +0 -71
  151. package/docs-site/guides/fixtures.mdx +0 -147
  152. package/docs-site/guides/reporters.mdx +0 -113
  153. package/docs-site/guides/runner.mdx +0 -79
  154. package/docs-site/guides/sandbox-agent.mdx +0 -138
  155. package/docs-site/guides/sandbox-backends.mdx +0 -64
  156. package/docs-site/guides/scoring-guide.mdx +0 -92
  157. package/docs-site/guides/viewing-results.mdx +0 -195
  158. package/docs-site/guides/write-experiment.mdx +0 -81
  159. package/docs-site/guides/write-send.mdx +0 -347
  160. package/docs-site/index.mdx +0 -181
  161. package/docs-site/introduction.mdx +0 -141
  162. package/docs-site/quickstart.mdx +0 -136
  163. package/docs-site/reference/builtin-agents.mdx +0 -161
  164. package/docs-site/reference/capabilities.mdx +0 -76
  165. package/docs-site/reference/cli.mdx +0 -120
  166. package/docs-site/reference/define-agent.mdx +0 -168
  167. package/docs-site/reference/define-config.mdx +0 -41
  168. package/docs-site/reference/define-eval.mdx +0 -160
  169. package/docs-site/reference/events.mdx +0 -131
  170. package/docs-site/reference/expect.mdx +0 -112
  171. package/docs-site/tracker.js +0 -8
  172. package/src/report/react/data.ts +0 -17
  173. package/src/view/aggregate.ts +0 -103
  174. package/src/view/loader.test.ts +0 -61
  175. package/src/view/loader.ts +0 -257
@@ -155,12 +155,12 @@ export interface AgentContext {
155
155
  /** 模型推理努力程度;归属同 model——实验决定,省略时不覆盖 agent 原生默认。 */
156
156
  readonly reasoningEffort?: string;
157
157
  /**
158
- * experiment 的 `flags` 字段原样透传,内容和结构完全由 experiment 作者自定义
158
+ * experiment 的 `params` 字段原样透传,内容和结构完全由 experiment 作者自定义
159
159
  * (如 `{ webResearch: true }`、`{ systemPrompt: "..." }`)。adapter 按自己的约定
160
- * 读取其中的字段;框架本身不解释、不校验它的内容。与 CLI 解析出的同名 `flags`
161
- * (跑法层面的 --timeout/--budget 等)是两个不相关的概念。
160
+ * 读取其中的字段;框架本身不解释、不校验它的内容。命名特意避开 CLI 解析出的
161
+ * `flag`(跑法层面的 --timeout/--budget 等),两者是不相关的概念。
162
162
  */
163
- readonly flags: Readonly<Record<string, unknown>>;
163
+ readonly params: Readonly<Record<string, unknown>>;
164
164
  /** 仅沙箱型 agent 有(运行器按 --sandbox 备好)。 */
165
165
  readonly sandbox: Sandbox;
166
166
  readonly session: AgentSession;
@@ -73,7 +73,7 @@ function ctx(overrides: Partial<{ model: string }> = {}): AgentContext {
73
73
  return {
74
74
  signal: new AbortController().signal,
75
75
  model: overrides.model,
76
- flags: {},
76
+ params: {},
77
77
  sandbox: undefined as never,
78
78
  // 同一个 ctx 重复用 = 同一条会话线(续接,同一个 ctx.session);新造 = 新线。
79
79
  session: createAgentSession(),
package/src/cli.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  // niceeval CLI 入口。执行 eval 必须以 experiment 为单位;位置参数只在 exp 后筛 eval id 前缀。
2
2
  // niceeval exp [组|配置] [pattern] 跑实验
3
+ // niceeval show [pattern] 终端读结果:榜单 / 单 eval / 证据切面 / 时间轴 / --report
3
4
  // niceeval list 只列出发现到的 eval
4
5
  // niceeval clean 删除 .niceeval/ 历史运行工件
5
6
 
@@ -19,15 +20,27 @@ import { Console as ConsoleReporter } from "./runner/reporters/console.ts";
19
20
  import { JUnit } from "./runner/reporters/json.ts";
20
21
  import { Live as LiveReporter, type LiveRow } from "./runner/reporters/live.ts";
21
22
  import { Artifacts as ArtifactsReporter } from "./runner/reporters/artifacts.ts";
22
- import { buildView, startViewServer, loadLatestResultsPerEval, IncompatibleResultsError } from "./view/index.ts";
23
+ import {
24
+ buildView,
25
+ startViewServer,
26
+ loadLatestResultsPerEval,
27
+ resolveViewInput,
28
+ IncompatibleResultsError,
29
+ ViewInputError,
30
+ } from "./view/index.ts";
31
+ import { ReportLoadError } from "./report/load.ts";
32
+ import { runShow } from "./show/index.ts";
23
33
  import { t } from "./i18n/index.ts";
24
34
  import { formatThrown, upsertManagedBlock } from "./util.ts";
25
35
  import type { Config, DiscoveredExperiment, Reporter } from "./types.ts";
26
36
 
27
- /** `niceeval view <summary.json>` 指向版本不同的报告时:打印 npx 提示后退出,不抛堆栈。 */
28
- function exitOnIncompatibleResults(e: unknown): never {
29
- if (e instanceof IncompatibleResultsError) {
30
- process.stderr.write(e.message);
37
+ /**
38
+ * view 的可预期用户错误:版本不同的报告(npx 提示)、位置参数/组合语义错误、
39
+ * --report 装载失败。打一句直说问题与下一步后退出,不抛堆栈。
40
+ */
41
+ function exitOnViewUserError(e: unknown): never {
42
+ if (e instanceof IncompatibleResultsError || e instanceof ViewInputError || e instanceof ReportLoadError) {
43
+ process.stderr.write(e.message.endsWith("\n") ? e.message : `${e.message}\n`);
31
44
  process.exit(1);
32
45
  }
33
46
  throw e;
@@ -51,6 +64,17 @@ interface Flags {
51
64
  out?: string;
52
65
  port?: number;
53
66
  help: boolean;
67
+ // ── show 专属(位置参数仍是 eval id 前缀;这些 flag 选「怎么看」)──
68
+ transcript: boolean;
69
+ trace: boolean;
70
+ diff: boolean;
71
+ /** --diff=<路径>(必须 = 连写;空格形式会把路径当 eval id 前缀,按文档如此)。 */
72
+ diffPath?: string;
73
+ history: boolean;
74
+ experiment?: string;
75
+ attempt?: number;
76
+ run?: string;
77
+ report?: string;
54
78
  }
55
79
 
56
80
  // 表驱动的 flag 定义(node:util parseArgs)。--no-x 显式声明,不依赖 allowNegative(需 Node 20.14+,
@@ -72,6 +96,17 @@ const FLAG_OPTIONS = {
72
96
  // 免得照旧文档写的脚本直接崩;真正实现前不接任何行为。
73
97
  watch: { type: "boolean" },
74
98
  json: { type: "boolean" },
99
+ // show 的证据切面 / 时间轴 / 报告装载(docs-site/zh/guides/viewing-results.mdx)。
100
+ transcript: { type: "boolean" },
101
+ trace: { type: "boolean" },
102
+ // --diff 是布尔;--diff=<路径> 在 parseArgs 前预扫成 diffPath(路径必须 = 连写,
103
+ // 空格形式的下一个 token 仍是位置参数 = eval id 前缀,与文档一致)。
104
+ diff: { type: "boolean" },
105
+ history: { type: "boolean" },
106
+ experiment: { type: "string" },
107
+ attempt: { type: "string" },
108
+ run: { type: "string" },
109
+ report: { type: "string" },
75
110
  dry: { type: "boolean" },
76
111
  quiet: { type: "boolean" },
77
112
  force: { type: "boolean" },
@@ -96,6 +131,17 @@ function numberFlag(name: string, raw: string | undefined): number | undefined {
96
131
  function parseArgs(argv: string[]): { command: string; positionals: string[]; flags: Flags } {
97
132
  if (argv[0] === "--") argv = argv.slice(1);
98
133
 
134
+ // --diff=<路径> 预扫:diff 本体是布尔(裸 --diff = 文件级摘要),路径只接受 = 连写。
135
+ let diffPath: string | undefined;
136
+ argv = argv.map((arg) => {
137
+ if (arg.startsWith("--diff=")) {
138
+ const path = arg.slice("--diff=".length);
139
+ if (path) diffPath = path;
140
+ return "--diff";
141
+ }
142
+ return arg;
143
+ });
144
+
99
145
  let values: Record<string, string | boolean | undefined>;
100
146
  let rawPositionals: string[];
101
147
  try {
@@ -113,7 +159,7 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
113
159
  }
114
160
 
115
161
  // 第一个位置参数若是已知命令,则为命令;其余是 eval id 前缀 / view 输入。
116
- const commands = new Set(["exp", "list", "view", "clean", "init", "watch", "run"]);
162
+ const commands = new Set(["exp", "show", "list", "view", "clean", "init", "watch", "run"]);
117
163
  let command = "run";
118
164
  let positionals = rawPositionals;
119
165
  if (rawPositionals[0] && commands.has(rawPositionals[0])) {
@@ -139,6 +185,15 @@ function parseArgs(argv: string[]): { command: string; positionals: string[]; fl
139
185
  earlyExit: values["no-early-exit"] === true ? false : values["early-exit"] === true ? true : undefined,
140
186
  open: values["no-open"] === true ? false : values.open === true ? true : undefined,
141
187
  help: values.help === true,
188
+ transcript: values.transcript === true,
189
+ trace: values.trace === true,
190
+ diff: values.diff === true && diffPath === undefined,
191
+ diffPath,
192
+ history: values.history === true,
193
+ experiment: values.experiment as string | undefined,
194
+ attempt: numberFlag("attempt", values.attempt as string | undefined),
195
+ run: values.run as string | undefined,
196
+ report: values.report as string | undefined,
142
197
  };
143
198
  return { command, positionals, flags };
144
199
  }
@@ -185,16 +240,20 @@ async function loadConfig(cwd: string): Promise<Config> {
185
240
  }
186
241
 
187
242
  // AGENTS.md 托管区块:告诉在这个项目里干活的 coding agent「niceeval 不在你的训练数据里,
188
- // 先读随包文档,跑完读结构化结果」。init 时写入/刷新;标记之外的用户内容永不触碰。
243
+ // 先读随包文档,跑完读结构化结果」。随包只发中文准绳版文档(英文站是手工同步、可能滞后,
244
+ // 不进包,见 package.json 的 files);init 时写入/刷新;标记之外的用户内容永不触碰。
189
245
  const AGENT_RULES_BEGIN = "<!-- BEGIN:niceeval-agent-rules -->";
190
246
  const AGENT_RULES_END = "<!-- END:niceeval-agent-rules -->";
191
247
  const AGENT_RULES_CONTENT = [
192
248
  "# niceeval is NOT in your training data",
193
249
  "",
194
250
  "Its APIs and conventions may differ from anything you have seen. Read the relevant",
195
- "guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under",
196
- "`zh/`) before writing any eval, experiment, adapter, or niceeval config. After a run,",
197
- "read the `summary.json` path the CLI prints for structured results.",
251
+ "guide in `node_modules/niceeval/docs-site/zh/` before writing any eval, experiment,",
252
+ "adapter, or niceeval config. The bundled docs are Chinese-only — that is the single",
253
+ "authoritative, always-current version; read it regardless of your working language.",
254
+ "After a run, drill into failures with `niceeval show <eval id>` (add `--transcript` /",
255
+ "`--trace` / `--diff` for evidence); the `summary.json` path the CLI prints and the",
256
+ "artifact files it references are the structured source of truth.",
198
257
  ].join("\n");
199
258
 
200
259
  async function initProject(cwd: string): Promise<void> {
@@ -265,12 +324,27 @@ async function main(): Promise<void> {
265
324
  }
266
325
 
267
326
  if (command === "view") {
327
+ // 位置参数 = eval id 前缀(收窄报告槽选集);存在的文件路径 = 单文件模式;
328
+ // 结果目录经 --run 递入;--report 整槽替换报告槽(与 show --report 吃同一个文件)。
329
+ let viewInput: { input?: string; patterns: string[] };
330
+ try {
331
+ viewInput = resolveViewInput(cwd, positionals, flags.run);
332
+ } catch (e) {
333
+ exitOnViewUserError(e);
334
+ }
335
+ const scan = {
336
+ patterns: viewInput.patterns,
337
+ ...(flags.experiment !== undefined ? { experiment: flags.experiment } : {}),
338
+ ...(flags.report !== undefined ? { report: { path: flags.report, cwd } } : {}),
339
+ };
268
340
  if (flags.out) {
269
- const out = await buildView({ input: positionals[0], out: flags.out }).catch(exitOnIncompatibleResults);
341
+ const out = await buildView({ input: viewInput.input, out: flags.out, scan }).catch(exitOnViewUserError);
270
342
  process.stdout.write(t("cli.view.exportedDir", { out }));
271
343
  process.exit(0);
272
344
  }
273
- const server = await startViewServer({ input: positionals[0], port: flags.port }).catch(exitOnIncompatibleResults);
345
+ const server = await startViewServer({ input: viewInput.input, port: flags.port, scan }).catch(
346
+ exitOnViewUserError,
347
+ );
274
348
  process.stdout.write(t("cli.view.url", { url: server.url }));
275
349
  if (flags.open !== false) {
276
350
  const opened = await openBrowser(server.url);
@@ -280,6 +354,22 @@ async function main(): Promise<void> {
280
354
  await new Promise(() => {});
281
355
  }
282
356
 
357
+ if (command === "show") {
358
+ // show 不依赖 niceeval.config.ts:读的是 .niceeval/(或 --run 指定目录)的落盘结果。
359
+ const code = await runShow(cwd, positionals, {
360
+ transcript: flags.transcript,
361
+ trace: flags.trace,
362
+ diff: flags.diff,
363
+ diffPath: flags.diffPath,
364
+ history: flags.history,
365
+ experiment: flags.experiment,
366
+ attempt: flags.attempt,
367
+ run: flags.run,
368
+ report: flags.report,
369
+ });
370
+ process.exit(code);
371
+ }
372
+
283
373
  if (command === "clean") {
284
374
  await rm(join(cwd, ".niceeval"), { recursive: true, force: true });
285
375
  process.stdout.write(t("cli.clean.done"));
@@ -333,7 +423,7 @@ async function main(): Promise<void> {
333
423
  agent: exp.agent,
334
424
  model: exp.model,
335
425
  reasoningEffort: exp.reasoningEffort,
336
- flags: exp.flags ?? {},
426
+ params: exp.params ?? {},
337
427
  runs: flags.runs ?? envNumber("NICEEVAL_RUNS") ?? exp.runs ?? 1,
338
428
  earlyExit: flags.earlyExit ?? exp.earlyExit ?? true,
339
429
  sandbox: exp.sandbox ?? config.sandbox,
@@ -349,7 +439,7 @@ async function main(): Promise<void> {
349
439
  }
350
440
  } else {
351
441
  // 裸 run / `niceeval <eval>` 不再执行。运行配置必须来自 experiments/,
352
- // 这样 agent/model/flags/runs/budget 与结果聚合都有可签入的身份。
442
+ // 这样 agent/model/params/runs/budget 与结果聚合都有可签入的身份。
353
443
  const experiments = await discoverExperiments(cwd);
354
444
  const asExp = experiments.filter((e) =>
355
445
  positionals.some((p) => e.group === p || e.id === p || e.id.startsWith(p + "/")),
@@ -57,7 +57,7 @@ function makeContext(agent: Agent, sandbox = fakeSandbox(), evalBaseDir?: string
57
57
  return createEvalContext({
58
58
  agent,
59
59
  sandbox,
60
- flags: {},
60
+ params: {},
61
61
  signal: new AbortController().signal,
62
62
  log: () => {},
63
63
  judge: undefined,
@@ -61,7 +61,7 @@ export interface ContextDeps {
61
61
  sandbox: Sandbox;
62
62
  model?: string;
63
63
  reasoningEffort?: string;
64
- flags: Record<string, unknown>;
64
+ params: Record<string, unknown>;
65
65
  signal: AbortSignal;
66
66
  log(msg: string): void;
67
67
  judge: JudgeConfig | undefined;
@@ -92,7 +92,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
92
92
  sandbox: deps.sandbox,
93
93
  model: deps.model,
94
94
  reasoningEffort: deps.reasoningEffort,
95
- flags: deps.flags,
95
+ params: deps.params,
96
96
  signal: deps.signal,
97
97
  log: deps.log,
98
98
  telemetry: deps.telemetry,
@@ -284,7 +284,7 @@ export function createEvalContext(deps: ContextDeps): { context: TestContext; st
284
284
  signal: deps.signal,
285
285
  model: deps.model,
286
286
  reasoningEffort: deps.reasoningEffort,
287
- flags: deps.flags,
287
+ params: deps.params,
288
288
  log: deps.log,
289
289
  skip: (reason: string) => {
290
290
  if (reason.trim().length === 0) throw new Error(t("context.skipEmpty"));
@@ -88,7 +88,7 @@ export interface SessionDeps {
88
88
  sandbox: Sandbox;
89
89
  model?: string;
90
90
  reasoningEffort?: string;
91
- flags: Record<string, unknown>;
91
+ params: Record<string, unknown>;
92
92
  signal: AbortSignal;
93
93
  log(msg: string): void;
94
94
  /** tracing agent 的 OTLP 端点(经 send ctx 透给 adapter,用于注入导出 env)。 */
@@ -138,7 +138,7 @@ export class SessionManager {
138
138
  signal: this.deps.signal,
139
139
  model: this.deps.model,
140
140
  reasoningEffort: this.deps.reasoningEffort,
141
- flags: this.deps.flags,
141
+ params: this.deps.params,
142
142
  sandbox: this.deps.sandbox,
143
143
  session,
144
144
  telemetry: this.deps.telemetry,
@@ -274,8 +274,8 @@ export interface TestContext {
274
274
  readonly model?: string;
275
275
  /** 本次 attempt 的推理努力程度(如 "low"/"medium"/"high",取值由 adapter/模型决定)。 */
276
276
  readonly reasoningEffort?: string;
277
- /** 本次 attempt 生效的 flags(experiment.flags 与 CLI flag 合并后的只读视图)。 */
278
- readonly flags: Readonly<Record<string, unknown>>;
277
+ /** 本次 attempt 生效的 params(experiment.params 与 CLI flag 合并后的只读视图)。 */
278
+ readonly params: Readonly<Record<string, unknown>>;
279
279
  /** 打一行调试日志;有 live 进度回调时走该回调,否则落到 stderr,不出现在最终结果里。 */
280
280
  log(msg: string): void;
281
281
  /** 立即中止本 eval 并标记为 skipped(outcome / EvalResult.skipReason),reason 不能为空。 */
package/src/i18n/en.ts CHANGED
@@ -20,7 +20,7 @@ export const en = {
20
20
  "Ways to fix:\n" +
21
21
  " - [init] Run `npx niceeval init` to scaffold niceeval.config.ts and evals/\n" +
22
22
  " - [cd] Run from the project root that contains niceeval.config.ts\n" +
23
- " Docs: node_modules/niceeval/docs-site/quickstart.mdx",
23
+ " Docs: node_modules/niceeval/docs-site/zh/quickstart.mdx",
24
24
  "cli.config.noDefault": "niceeval.config.ts must default export defineConfig(...).",
25
25
  "cli.dry.header": "\n[dry] {{evals}} evals × {{configs}} run configs:\n",
26
26
  "cli.dry.noMatches": "(no matches)",
@@ -31,7 +31,7 @@ export const en = {
31
31
  "budget for {{budgetKey}}: several attempts completed without any cost data (agent reports no usage and the model is not in the price table) — the budget cannot be enforced for this agent; continuing without the guard.\n",
32
32
  "judge.modelMissing":
33
33
  "No judge model configured. Set it in defineConfig({ judge: { model: \"...\" } }), the eval's judge config, or the NICEEVAL_JUDGE_MODEL environment variable (there is no built-in default model).\n" +
34
- " Docs: node_modules/niceeval/docs-site/guides/scoring-guide.mdx",
34
+ " Docs: node_modules/niceeval/docs-site/zh/guides/scoring-guide.mdx",
35
35
  "loaders.yamlMissing":
36
36
  "loadYaml(\"{{path}}\") needs a YAML parser: run `pnpm add yaml` first (or switch to loadJson with a JSON dataset).",
37
37
  "cli.flag.parseError": "{{message}}\nRun `niceeval --help` for usage.\n",
@@ -40,8 +40,18 @@ export const en = {
40
40
  "niceeval — agent-native evals\n\n" +
41
41
  "Usage:\n" +
42
42
  " niceeval exp [group|experiment] [eval-id-prefix…] run experiments\n" +
43
+ " niceeval show [eval-id-prefix…] read results in the terminal\n" +
44
+ " bare: current verdicts per experiment (composed across runs);\n" +
45
+ " a single eval id: attempts + assertion details\n" +
46
+ " --transcript / --trace / --diff[=file] evidence slices of one eval\n" +
47
+ " --history cross-run timeline (mutually exclusive with --report)\n" +
48
+ " --run <dir> pin a results dir --experiment <id> one experiment\n" +
49
+ " --attempt <n> pick an attempt --report <file> custom report\n" +
43
50
  " niceeval list list discovered evals\n" +
44
- " niceeval view [summary.json|dir] [--out dir] [--port n] [--no-open]\n" +
51
+ " niceeval view [eval-id-prefix…|summary.json] [--out dir] [--port n] [--no-open]\n" +
52
+ " report slot + evidence rooms; --report <file> swaps in your report\n" +
53
+ " (same file as show); --run <dir> pins a results dir;\n" +
54
+ " --experiment <id> one experiment\n" +
45
55
  " --out <dir> exports a static site: index.html plus the viewer\n" +
46
56
  " artifacts, ready for any static host\n" +
47
57
  " niceeval clean delete .niceeval/ artifacts\n" +
@@ -53,6 +63,17 @@ export const en = {
53
63
  "Positional args only select which evals to run (id prefixes); which agent and\n" +
54
64
  "how to run come from experiments/ + flags. Env overrides (flag > env > config):\n" +
55
65
  " NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
66
+ "cli.show.noResults": "No results found under {{root}}. Run `niceeval exp` first, then `niceeval show`.\n",
67
+ "cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
68
+ "cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
69
+ "cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
70
+ "cli.show.historyReportConflict":
71
+ "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
72
+ "cli.show.evidenceNeedsEval":
73
+ "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
74
+ "cli.show.attemptNeedsEval":
75
+ "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
76
+ "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
56
77
  "cli.eval.noMatch": "No eval matched: {{patterns}}.\n",
57
78
  "cli.eval.noMatchHintExperiment": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
58
79
  "cli.eval.noMatchKnown": "Discovered {{count}} evals: {{evals}}\n",
@@ -61,7 +82,7 @@ export const en = {
61
82
  "cli.experimentGroup": " group",
62
83
  "cli.fallbackCleanupTimeout": "\ngraceful cleanup timed out; force-cleaning sandboxes...\n",
63
84
  "cli.forceCleanupExit": "\nForce-cleaning sandboxes and exiting...\n",
64
- "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site).\n",
85
+ "cli.init.done": "Ready: evals/, niceeval.config.ts, and the niceeval agent-rules block in AGENTS.md (points coding agents at node_modules/niceeval/docs-site/zh).\n",
65
86
  "cli.interruptCleanup": "\nInterrupted; cleaning up sandbox containers... (press again to force cleanup and exit)\n",
66
87
  "cli.list.header": "Discovered {{count}} evals:\n",
67
88
  "cli.noAgent": "No agent specified (use --agent <name>).\n",
@@ -70,17 +91,18 @@ export const en = {
70
91
  "cli.resultsPath": "Structured results: {{path}} (each result's artifactsDir holds that attempt's events.json / trace.json / diff.json)\n",
71
92
  "cli.run.experimentRequired":
72
93
  "Run evals through an experiment: use `niceeval exp [group|config] [eval id prefix]`.\n" +
73
- " Docs: node_modules/niceeval/docs-site/guides/write-experiment.mdx\n",
94
+ " Docs: node_modules/niceeval/docs-site/zh/guides/write-experiment.mdx\n",
74
95
  "cli.run.experimentRequiredHint": "Hint: \"{{pattern}}\" is an experiment{{kind}}; you probably meant: niceeval exp {{pattern}}\n",
75
96
  "cli.run.experimentRequiredKnown": "Discovered experiments: {{experiments}}\n",
76
97
  "cli.sandboxFlagRemoved": "`--sandbox` is not a CLI flag. Set `sandbox` in the experiment (or `niceeval.config.ts` as a project-wide fallback) to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n",
77
98
  "cli.unimplemented": "Command \"{{command}}\" is not implemented yet (MVP).\n",
78
99
  "cli.view.exportedDir": "Exported static report site: {{out}} (serve the whole directory with any static host; opening index.html via file:// cannot fetch artifacts)\n",
79
100
  "cli.view.incompatible": "{{dir}}: written by niceeval {{producer}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nRun `{{command}}` to view it.\n",
101
+ "cli.view.incompatibleForeign": "{{dir}}: written by {{name}} {{version}} (schemaVersion {{schemaVersion}}); this CLI reads schemaVersion {{supported}}.\nOpen this report with the tool that produced it.\n",
80
102
  "cli.view.url": "niceeval view: {{url}}\n",
81
103
  "context.capabilityMissing":
82
104
  "Agent \"{{agent}}\" is not sandbox-backed (built with defineSandboxAgent), so t.{{method}} is unavailable. Use an agent built with defineSandboxAgent, or drop this assertion.\n" +
83
- " Docs: node_modules/niceeval/docs-site/guides/sandbox-agent.mdx",
105
+ " Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-agent.mdx",
84
106
  "context.skipEmpty": "skip() requires a non-empty reason.",
85
107
  "context.turnFailed": "This send returned failed (turn status = failed): {{message}}",
86
108
  "context.turnFailedDefault": "This send returned failed (turn status = failed)",
@@ -172,7 +194,7 @@ export const en = {
172
194
  "sandbox.backendNotImplemented": "{{backend}} sandbox backend is not implemented; use docker, vercel, or e2b",
173
195
  "sandbox.missingSpec":
174
196
  "sandbox agent needs a sandbox, but none was given. niceeval no longer picks a default — set `sandbox` in defineExperiment()/defineConfig() to dockerSandbox() / vercelSandbox() / e2bSandbox() (import from \"niceeval/sandbox\").\n" +
175
- " Docs: node_modules/niceeval/docs-site/guides/sandbox-backends.mdx",
197
+ " Docs: node_modules/niceeval/docs-site/zh/guides/sandbox-backends.mdx",
176
198
  "sandbox.dependencyMissing.docker": "Docker sandbox requires 'dockerode'. Install it with: pnpm add dockerode @types/dockerode",
177
199
  "sandbox.dependencyMissing.e2b": "E2B sandbox requires 'e2b'. Install it with: pnpm add e2b",
178
200
  "sandbox.dependencyMissing.vercel": "Vercel sandbox requires '@vercel/sandbox'. Install it with: pnpm add @vercel/sandbox",
package/src/i18n/zh-CN.ts CHANGED
@@ -38,8 +38,17 @@ export const zhCN = {
38
38
  "niceeval — agent-native evals\n\n" +
39
39
  "用法:\n" +
40
40
  " niceeval exp [组|实验] [eval-id 前缀…] 跑实验\n" +
41
+ " niceeval show [eval-id 前缀…] 终端读结果\n" +
42
+ " 裸跑:每个 experiment 的现刻判决(跨 run 合成)\n" +
43
+ " 单个 eval id:attempt 与断言明细\n" +
44
+ " --transcript / --trace / --diff[=文件] 单 eval 的证据切面\n" +
45
+ " --history 跨 run 时间轴(与 --report 互斥)\n" +
46
+ " --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
47
+ " --attempt <n> 指定 attempt --report <文件> 自定义报告\n" +
41
48
  " niceeval list 列出发现到的 eval\n" +
42
- " niceeval view [summary.json|目录] [--out 目录] [--port n] [--no-open]\n" +
49
+ " niceeval view [eval-id 前缀…|summary.json] [--out 目录] [--port n] [--no-open]\n" +
50
+ " 报告槽 + 证据室;--report <文件> 整槽换成自定义报告(与 show 同一文件)\n" +
51
+ " --run <目录> 钉死结果目录 --experiment <id> 只看该实验\n" +
43
52
  " --out <目录> 静态导出:index.html 连同查看器工件,可直接静态托管\n" +
44
53
  " niceeval clean 删除 .niceeval/ 历史工件\n" +
45
54
  " niceeval init 脚手架 config + evals/\n\n" +
@@ -50,6 +59,18 @@ export const zhCN = {
50
59
  "位置参数只选「跑哪些 eval」(id 前缀);对着哪个 agent、怎么跑来自 experiments/ 与\n" +
51
60
  "标志。环境变量覆盖(标志 > 环境变量 > config):\n" +
52
61
  " NICEEVAL_RUNS NICEEVAL_MAX_CONCURRENCY NICEEVAL_TIMEOUT NICEEVAL_BUDGET\n",
62
+ // show 的错误文案保持英文(错误文案英文的仓库约定);noResults 是提示,翻译。
63
+ "cli.show.noResults": "{{root}} 下没有结果。先 `niceeval exp` 跑一轮,再 `niceeval show`。\n",
64
+ "cli.show.runDirMissing": "Results directory not found: {{dir}}\n",
65
+ "cli.show.noEvalMatch": "No results matched: {{patterns}}. Evals with results: {{evals}}\n",
66
+ "cli.show.noExperimentMatch": "No experiment matched --experiment {{arg}}. Experiments with results: {{experiments}}\n",
67
+ "cli.show.historyReportConflict":
68
+ "`--history` and `--report` are mutually exclusive: --history is the built-in trend view. For a custom trend, compose exp.snapshots inside your report file instead.\n",
69
+ "cli.show.evidenceNeedsEval":
70
+ "--transcript / --trace / --diff show one eval's evidence, but the selection matched {{matched}} evals. Narrow to a single eval id first: niceeval show <eval id> --transcript\n",
71
+ "cli.show.attemptNeedsEval":
72
+ "--attempt picks one attempt of a single eval; pass a full eval id (and --experiment when several experiments ran it).\n",
73
+ "cli.show.attemptNotFound": "Attempt {{attempt}} not found for {{evalId}}. Available attempts: {{available}}\n",
53
74
  "cli.eval.noMatch": "没有匹配的 eval:{{patterns}}。\n",
54
75
  "cli.eval.noMatchHintExperiment": "提示:\"{{pattern}}\" 是实验{{kind}},你大概想跑:niceeval exp {{pattern}}\n",
55
76
  "cli.eval.noMatchKnown": "已发现 {{count}} 个 eval:{{evals}}\n",
@@ -58,7 +79,7 @@ export const zhCN = {
58
79
  "cli.experimentGroup": "组",
59
80
  "cli.fallbackCleanupTimeout": "\ngraceful 清理超时,强制清理沙箱…\n",
60
81
  "cli.forceCleanupExit": "\n强制清理沙箱并退出…\n",
61
- "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site)。\n",
82
+ "cli.init.done": "已就绪:evals/、niceeval.config.ts,以及 AGENTS.md 里的 niceeval agent 指引区块(指向 node_modules/niceeval/docs-site/zh)。\n",
62
83
  "cli.interruptCleanup": "\n收到中断,正在清理沙箱容器…(再按一次强制清理并退出)\n",
63
84
  "cli.list.header": "发现 {{count}} 个 eval:\n",
64
85
  "cli.noAgent": "未指定 agent(用 --agent <name>)。\n",
@@ -74,6 +95,7 @@ export const zhCN = {
74
95
  "cli.unimplemented": "命令 \"{{command}}\" 暂未实现(MVP)。\n",
75
96
  "cli.view.exportedDir": "已导出静态查看站:{{out}}(整个目录可直接静态托管;本地打开 {{out}}/index.html 需经 http 服务,file:// 下工件 fetch 不可用)\n",
76
97
  "cli.view.incompatible": "{{dir}}: 由 niceeval {{producer}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n运行 `{{command}}` 查看这份报告。\n",
98
+ "cli.view.incompatibleForeign": "{{dir}}: 由 {{name}} {{version}} 写入(schemaVersion {{schemaVersion}}),当前 CLI 只读 schemaVersion {{supported}}。\n请用写出它的那个工具查看这份报告。\n",
77
99
  "cli.view.url": "niceeval view: {{url}}\n",
78
100
  "context.capabilityMissing":
79
101
  "agent \"{{agent}}\" 不是沙箱型(defineSandboxAgent 构造),t.{{method}} 这类断言只有沙箱型 agent 可用。换用 defineSandboxAgent 构造的 agent,或去掉这条断言。\n" +