niceeval 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/docs-site/zh/concepts/adapter.mdx +11 -6
  2. package/docs-site/zh/concepts/experiment.mdx +8 -6
  3. package/docs-site/zh/concepts/overview.mdx +1 -1
  4. package/docs-site/zh/concepts/tier.mdx +6 -6
  5. package/docs-site/zh/guides/ci-integration.mdx +2 -0
  6. package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
  7. package/docs-site/zh/guides/custom-reports.mdx +13 -13
  8. package/docs-site/zh/guides/experiments.mdx +4 -3
  9. package/docs-site/zh/guides/publish-report.mdx +91 -0
  10. package/docs-site/zh/guides/report-components.mdx +4 -4
  11. package/docs-site/zh/guides/reporters.mdx +1 -1
  12. package/docs-site/zh/guides/results-data.mdx +2 -2
  13. package/docs-site/zh/guides/runner.mdx +1 -1
  14. package/docs-site/zh/guides/sandbox-agent.mdx +11 -6
  15. package/docs-site/zh/guides/sandbox-backends.mdx +27 -0
  16. package/docs-site/zh/guides/viewing-results.mdx +5 -21
  17. package/docs-site/zh/guides/write-experiment.mdx +24 -10
  18. package/docs-site/zh/guides/write-send.mdx +6 -6
  19. package/docs-site/zh/index.mdx +1 -1
  20. package/docs-site/zh/introduction.mdx +1 -1
  21. package/docs-site/zh/reference/builtin-agents.mdx +4 -0
  22. package/docs-site/zh/reference/cli.mdx +9 -6
  23. package/docs-site/zh/reference/define-agent.mdx +19 -3
  24. package/docs-site/zh/reference/define-config.mdx +3 -1
  25. package/docs-site/zh/reference/define-eval.mdx +9 -3
  26. package/docs-site/zh/reference/events.mdx +4 -0
  27. package/docs-site/zh/reference/expect.mdx +4 -0
  28. package/package.json +3 -2
  29. package/src/agents/ai-sdk.test.ts +1 -1
  30. package/src/agents/ai-sdk.ts +2 -2
  31. package/src/agents/bub.ts +5 -1
  32. package/src/agents/claude-code.ts +1 -1
  33. package/src/agents/codex.ts +2 -2
  34. package/src/agents/index.ts +2 -0
  35. package/src/agents/streaming.test.ts +1 -1
  36. package/src/agents/types.ts +10 -2
  37. package/src/agents/ui-message-stream.test.ts +1 -1
  38. package/src/cli.ts +38 -14
  39. package/src/context/context.test.ts +1 -1
  40. package/src/context/context.ts +6 -3
  41. package/src/context/session.ts +5 -2
  42. package/src/context/types.ts +2 -2
  43. package/src/define.test.ts +82 -0
  44. package/src/define.ts +68 -7
  45. package/src/i18n/en.ts +4 -2
  46. package/src/i18n/zh-CN.ts +4 -2
  47. package/src/report/aggregate.ts +16 -16
  48. package/src/report/components.tsx +1 -1
  49. package/src/report/compute.ts +7 -7
  50. package/src/report/dual-face.test.tsx +1 -1
  51. package/src/report/{param.ts → flag.ts} +6 -6
  52. package/src/report/index.ts +3 -3
  53. package/src/report/react/MetricLine.tsx +1 -1
  54. package/src/report/react/fixtures.ts +1 -1
  55. package/src/report/report.test.ts +17 -17
  56. package/src/report/report.ts +1 -1
  57. package/src/report/types.ts +11 -11
  58. package/src/results/results.test.ts +2 -2
  59. package/src/runner/attempt.ts +53 -6
  60. package/src/runner/fingerprint.ts +1 -1
  61. package/src/runner/reporters/braintrust.test.ts +2 -2
  62. package/src/runner/reporters/braintrust.ts +3 -3
  63. package/src/runner/run.ts +1 -1
  64. package/src/runner/types.ts +8 -8
  65. package/src/sandbox/types.ts +45 -4
  66. package/src/show/compose.ts +1 -1
  67. package/src/view/app/App.tsx +1 -1
  68. package/src/view/app/css.d.ts +3 -0
  69. package/src/view/app/i18n.ts +3 -3
  70. package/src/view/app/lib/rows.ts +5 -5
  71. package/src/view/app/types.ts +1 -1
  72. package/src/view/client-dist/app.js +2 -2
  73. package/src/view/data.test.ts +26 -2
  74. package/src/view/data.ts +28 -4
  75. package/src/view/shared/types.ts +1 -1
@@ -7,7 +7,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
7
7
  import { tmpdir } from "node:os";
8
8
  import { join } from "node:path";
9
9
  import { afterEach, describe, expect, it } from "vitest";
10
- import { IncompatibleResultsError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
10
+ import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
11
11
  import { createRunWriter } from "../results/index.ts";
12
12
  import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
13
13
 
@@ -182,7 +182,7 @@ describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一
182
182
  });
183
183
 
184
184
  describe("loadViewScan · createRunWriter 写出的落盘直接可读(writer/reader 同一契约)", () => {
185
- it("writer roundtrip:官方榜单、快照与 attemptRef 都从写入面产物算出", async () => {
185
+ it("writer roundtrip:默认报告、快照与 attemptRef 都从写入面产物算出", async () => {
186
186
  const root = await makeRoot();
187
187
  const writer = await createRunWriter(root, { producer: { name: "niceeval", version: "0.4.6" } });
188
188
  const snap = writer.snapshot({
@@ -252,3 +252,27 @@ describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致
252
252
  expect(withArtifacts.artifactBase).toBe("2026-01-02T00-00-00/artifacts/e1/a0");
253
253
  });
254
254
  });
255
+
256
+ describe("loadViewScan · 零可读结果直说,不渲染空页面", () => {
257
+ it("目录真空:抛 ViewInputError,给「先跑一轮」提示(与 show 同文案)", async () => {
258
+ const root = await makeRoot();
259
+ await expect(loadViewScan(root)).rejects.toBeInstanceOf(ViewInputError);
260
+ await expect(loadViewScan(root)).rejects.toThrow(/niceeval exp/);
261
+ });
262
+
263
+ it("全被跳过:错误逐条列目录与原因,niceeval 落盘的 schemaVersion 场景给出可跑的 npx 命令", async () => {
264
+ const root = await makeRoot();
265
+ await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
266
+ await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
267
+
268
+ const err = await loadViewScan(root).then(
269
+ () => { throw new Error("expected ViewInputError"); },
270
+ (e) => e as Error,
271
+ );
272
+ expect(err).toBeInstanceOf(ViewInputError);
273
+ expect(err.message).toContain("2 run directories were skipped");
274
+ expect(err.message).toContain("incompatible-version, schemaVersion 999");
275
+ expect(err.message).toContain("npx niceeval@9.9.9 view ");
276
+ expect(err.message).toContain("malformed");
277
+ });
278
+ });
package/src/view/data.ts CHANGED
@@ -140,6 +140,7 @@ export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<Eval
140
140
  * 位置前缀 / --experiment / --report 在场时,报告槽选集经 composeShowSelection 合成
141
141
  * (与 `niceeval show` 同一口径,两扇门判决不分叉);全部缺省时维持 results.latest(),
142
142
  * 默认行为不变。证据室数据(快照明细 / skipped)恒为全量,深链在任何收窄下都可达。
143
+ * 零可读结果一律抛 ViewInputError,不渲染/导出空页面(server 起不来,--out 非零退出)。
143
144
  */
144
145
  export async function loadViewScan(input?: string, opts: ViewScanOptions = {}): Promise<ViewScan> {
145
146
  const target = resolve(input ?? ".niceeval");
@@ -150,9 +151,11 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
150
151
  const patterns = opts.patterns ?? [];
151
152
  const narrowed = patterns.length > 0 || opts.experiment !== undefined || opts.report !== undefined;
152
153
 
153
- // 组合语义的输入校验,与 show 同文案:匹配不到直说,不渲染一张空页面。
154
- if (opts.report && results.experiments.length === 0) {
155
- throw new ViewInputError(t("cli.show.noResults", { root: target }).trimEnd());
154
+ // 零可读结果直说,不渲染/导出一张空页面(与 show 的「匹配不到直说」同一原则;
155
+ // CI 静态发布还靠这个非零退出保住上一次部署,空报告不顶上线)。零可读最常见的
156
+ // 根因不是目录空,而是落盘整批 schemaVersion 不兼容被跳过,所以带上 skipped 摘要。
157
+ if (results.experiments.length === 0) {
158
+ throw new ViewInputError(noReadableResults(target, results.skipped));
156
159
  }
157
160
  if (
158
161
  opts.experiment !== undefined &&
@@ -248,7 +251,7 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
248
251
 
249
252
  /**
250
253
  * 报告槽渲染:装载报告文件(dev server 语义 —— 文件变更下次请求整页重算,经 mtime
251
- * cache-busting)→ 注入与官方榜单同口径的选集 → web 面 renderToStaticMarkup 成静态 HTML。
254
+ * cache-busting)→ 注入与默认报告同口径的选集 → web 面 renderToStaticMarkup 成静态 HTML。
252
255
  * react-dom 只在 --report 在场时动态加载,默认 CLI 路径不背 react。
253
256
  * attemptHref 缺省即 `#/attempt/<run>/<result>`(view 的 attempt 深链路由)。
254
257
  */
@@ -318,6 +321,27 @@ function annotateResult(
318
321
  return { annotated };
319
322
  }
320
323
 
324
+ /**
325
+ * 零可读结果的报错文案。目录真空时给「先跑一轮」的入门提示(与 show 的 noResults 同形态);
326
+ * 有 skipped 时逐条列目录与原因——schemaVersion 不兼容的 niceeval 落盘给出可跑的
327
+ * npx 命令,让「全被跳过」和「真没跑过」在错误里就能分清,不用进查看器排查。
328
+ */
329
+ function noReadableResults(target: string, skipped: SkippedRun[]): string {
330
+ if (skipped.length === 0) return t("cli.view.noResults", { root: target }).trimEnd();
331
+ const lines = skipped.map((skip) => {
332
+ const notice = toSkippedNotice(skip);
333
+ const version = notice.schemaVersion !== undefined ? `, schemaVersion ${notice.schemaVersion}` : "";
334
+ const hint = notice.command ? ` — view it with \`${notice.command.trim()}\`` : "";
335
+ return ` ${notice.dir}: ${notice.reason}${version}${hint}`;
336
+ });
337
+ const runs = skipped.length === 1 ? "1 run directory was" : `${skipped.length} run directories were`;
338
+ return [
339
+ `No readable results under ${target} — ${runs} skipped:`,
340
+ ...lines,
341
+ "Re-run your experiments with this niceeval to produce fresh results.",
342
+ ].join("\n");
343
+ }
344
+
321
345
  function toSkippedNotice(skip: SkippedRun): SkippedRunNotice {
322
346
  const dir = relative(process.cwd(), skip.dir) || ".";
323
347
  const command =
@@ -72,7 +72,7 @@ export interface ViewData {
72
72
  composedRuns: number;
73
73
  /** 官方 KPI(RunOverview.data 产物):totals + 选中快照元信息 + 选集 warnings 随行。 */
74
74
  overview: OverviewData;
75
- /** 官方榜单(MetricTable.data,rows: "experiment"):每个实验最新一次快照的口径。 */
75
+ /** 默认报告(MetricTable.data,rows: "experiment"):每个实验最新一次快照的口径。 */
76
76
  table: TableData;
77
77
  /** 整体单行(MetricTable.data,常量维度):hero 的官方通过率,同一台聚合引擎。 */
78
78
  overall: TableData;