niceeval 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +6 -6
- package/docs-site/zh/concepts/experiment.mdx +6 -6
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/ci-integration.mdx +2 -0
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +7 -7
- package/docs-site/zh/guides/experiments.mdx +3 -3
- package/docs-site/zh/guides/publish-report.mdx +91 -0
- package/docs-site/zh/guides/report-components.mdx +4 -4
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +1 -1
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/viewing-results.mdx +3 -19
- package/docs-site/zh/guides/write-experiment.mdx +9 -9
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +4 -0
- package/docs-site/zh/reference/cli.mdx +9 -6
- package/docs-site/zh/reference/define-agent.mdx +7 -3
- package/docs-site/zh/reference/define-config.mdx +3 -1
- package/docs-site/zh/reference/define-eval.mdx +9 -3
- package/docs-site/zh/reference/events.mdx +4 -0
- package/docs-site/zh/reference/expect.mdx +4 -0
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +2 -2
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +38 -14
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +3 -3
- package/src/context/session.ts +2 -2
- package/src/context/types.ts +2 -2
- package/src/i18n/en.ts +2 -2
- package/src/i18n/zh-CN.ts +2 -2
- package/src/report/aggregate.ts +16 -16
- package/src/report/components.tsx +1 -1
- package/src/report/compute.ts +7 -7
- package/src/report/dual-face.test.tsx +1 -1
- package/src/report/{param.ts → flag.ts} +6 -6
- package/src/report/index.ts +3 -3
- package/src/report/react/MetricLine.tsx +1 -1
- package/src/report/react/fixtures.ts +1 -1
- package/src/report/report.test.ts +17 -17
- package/src/report/types.ts +11 -11
- package/src/results/results.test.ts +2 -2
- package/src/runner/attempt.ts +3 -3
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +8 -8
- package/src/view/app/css.d.ts +3 -0
- package/src/view/app/i18n.ts +3 -3
- package/src/view/app/lib/rows.ts +4 -4
- package/src/view/client-dist/app.js +2 -2
- package/src/view/data.test.ts +25 -1
- package/src/view/data.ts +27 -3
package/src/view/data.test.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
|
7
7
|
import { tmpdir } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { afterEach, describe, expect, it } from "vitest";
|
|
10
|
-
import { IncompatibleResultsError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
10
|
+
import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
11
11
|
import { createRunWriter } from "../results/index.ts";
|
|
12
12
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
|
|
13
13
|
|
|
@@ -252,3 +252,27 @@ describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致
|
|
|
252
252
|
expect(withArtifacts.artifactBase).toBe("2026-01-02T00-00-00/artifacts/e1/a0");
|
|
253
253
|
});
|
|
254
254
|
});
|
|
255
|
+
|
|
256
|
+
describe("loadViewScan · 零可读结果直说,不渲染空页面", () => {
|
|
257
|
+
it("目录真空:抛 ViewInputError,给「先跑一轮」提示(与 show 同文案)", async () => {
|
|
258
|
+
const root = await makeRoot();
|
|
259
|
+
await expect(loadViewScan(root)).rejects.toBeInstanceOf(ViewInputError);
|
|
260
|
+
await expect(loadViewScan(root)).rejects.toThrow(/niceeval exp/);
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it("全被跳过:错误逐条列目录与原因,niceeval 落盘的 schemaVersion 场景给出可跑的 npx 命令", async () => {
|
|
264
|
+
const root = await makeRoot();
|
|
265
|
+
await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
|
|
266
|
+
await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
|
|
267
|
+
|
|
268
|
+
const err = await loadViewScan(root).then(
|
|
269
|
+
() => { throw new Error("expected ViewInputError"); },
|
|
270
|
+
(e) => e as Error,
|
|
271
|
+
);
|
|
272
|
+
expect(err).toBeInstanceOf(ViewInputError);
|
|
273
|
+
expect(err.message).toContain("2 run directories were skipped");
|
|
274
|
+
expect(err.message).toContain("incompatible-version, schemaVersion 999");
|
|
275
|
+
expect(err.message).toContain("npx niceeval@9.9.9 view ");
|
|
276
|
+
expect(err.message).toContain("malformed");
|
|
277
|
+
});
|
|
278
|
+
});
|
package/src/view/data.ts
CHANGED
|
@@ -140,6 +140,7 @@ export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<Eval
|
|
|
140
140
|
* 位置前缀 / --experiment / --report 在场时,报告槽选集经 composeShowSelection 合成
|
|
141
141
|
* (与 `niceeval show` 同一口径,两扇门判决不分叉);全部缺省时维持 results.latest(),
|
|
142
142
|
* 默认行为不变。证据室数据(快照明细 / skipped)恒为全量,深链在任何收窄下都可达。
|
|
143
|
+
* 零可读结果一律抛 ViewInputError,不渲染/导出空页面(server 起不来,--out 非零退出)。
|
|
143
144
|
*/
|
|
144
145
|
export async function loadViewScan(input?: string, opts: ViewScanOptions = {}): Promise<ViewScan> {
|
|
145
146
|
const target = resolve(input ?? ".niceeval");
|
|
@@ -150,9 +151,11 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
150
151
|
const patterns = opts.patterns ?? [];
|
|
151
152
|
const narrowed = patterns.length > 0 || opts.experiment !== undefined || opts.report !== undefined;
|
|
152
153
|
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
154
|
+
// 零可读结果直说,不渲染/导出一张空页面(与 show 的「匹配不到直说」同一原则;
|
|
155
|
+
// CI 静态发布还靠这个非零退出保住上一次部署,空报告不顶上线)。零可读最常见的
|
|
156
|
+
// 根因不是目录空,而是落盘整批 schemaVersion 不兼容被跳过,所以带上 skipped 摘要。
|
|
157
|
+
if (results.experiments.length === 0) {
|
|
158
|
+
throw new ViewInputError(noReadableResults(target, results.skipped));
|
|
156
159
|
}
|
|
157
160
|
if (
|
|
158
161
|
opts.experiment !== undefined &&
|
|
@@ -318,6 +321,27 @@ function annotateResult(
|
|
|
318
321
|
return { annotated };
|
|
319
322
|
}
|
|
320
323
|
|
|
324
|
+
/**
|
|
325
|
+
* 零可读结果的报错文案。目录真空时给「先跑一轮」的入门提示(与 show 的 noResults 同形态);
|
|
326
|
+
* 有 skipped 时逐条列目录与原因——schemaVersion 不兼容的 niceeval 落盘给出可跑的
|
|
327
|
+
* npx 命令,让「全被跳过」和「真没跑过」在错误里就能分清,不用进查看器排查。
|
|
328
|
+
*/
|
|
329
|
+
function noReadableResults(target: string, skipped: SkippedRun[]): string {
|
|
330
|
+
if (skipped.length === 0) return t("cli.view.noResults", { root: target }).trimEnd();
|
|
331
|
+
const lines = skipped.map((skip) => {
|
|
332
|
+
const notice = toSkippedNotice(skip);
|
|
333
|
+
const version = notice.schemaVersion !== undefined ? `, schemaVersion ${notice.schemaVersion}` : "";
|
|
334
|
+
const hint = notice.command ? ` — view it with \`${notice.command.trim()}\`` : "";
|
|
335
|
+
return ` ${notice.dir}: ${notice.reason}${version}${hint}`;
|
|
336
|
+
});
|
|
337
|
+
const runs = skipped.length === 1 ? "1 run directory was" : `${skipped.length} run directories were`;
|
|
338
|
+
return [
|
|
339
|
+
`No readable results under ${target} — ${runs} skipped:`,
|
|
340
|
+
...lines,
|
|
341
|
+
"Re-run your experiments with this niceeval to produce fresh results.",
|
|
342
|
+
].join("\n");
|
|
343
|
+
}
|
|
344
|
+
|
|
321
345
|
function toSkippedNotice(skip: SkippedRun): SkippedRunNotice {
|
|
322
346
|
const dir = relative(process.cwd(), skip.dir) || ".";
|
|
323
347
|
const command =
|