niceeval 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs-site/zh/concepts/adapter.mdx +11 -6
- package/docs-site/zh/concepts/experiment.mdx +8 -6
- package/docs-site/zh/concepts/overview.mdx +1 -1
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/guides/ci-integration.mdx +2 -0
- package/docs-site/zh/guides/connect-your-agent.mdx +7 -7
- package/docs-site/zh/guides/custom-reports.mdx +13 -13
- package/docs-site/zh/guides/experiments.mdx +4 -3
- package/docs-site/zh/guides/publish-report.mdx +91 -0
- package/docs-site/zh/guides/report-components.mdx +4 -4
- package/docs-site/zh/guides/reporters.mdx +1 -1
- package/docs-site/zh/guides/results-data.mdx +2 -2
- package/docs-site/zh/guides/runner.mdx +1 -1
- package/docs-site/zh/guides/sandbox-agent.mdx +11 -6
- package/docs-site/zh/guides/sandbox-backends.mdx +27 -0
- package/docs-site/zh/guides/viewing-results.mdx +5 -21
- package/docs-site/zh/guides/write-experiment.mdx +24 -10
- package/docs-site/zh/guides/write-send.mdx +6 -6
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/introduction.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +4 -0
- package/docs-site/zh/reference/cli.mdx +9 -6
- package/docs-site/zh/reference/define-agent.mdx +19 -3
- package/docs-site/zh/reference/define-config.mdx +3 -1
- package/docs-site/zh/reference/define-eval.mdx +9 -3
- package/docs-site/zh/reference/events.mdx +4 -0
- package/docs-site/zh/reference/expect.mdx +4 -0
- package/package.json +3 -2
- package/src/agents/ai-sdk.test.ts +1 -1
- package/src/agents/ai-sdk.ts +2 -2
- package/src/agents/bub.ts +5 -1
- package/src/agents/claude-code.ts +1 -1
- package/src/agents/codex.ts +2 -2
- package/src/agents/index.ts +2 -0
- package/src/agents/streaming.test.ts +1 -1
- package/src/agents/types.ts +10 -2
- package/src/agents/ui-message-stream.test.ts +1 -1
- package/src/cli.ts +38 -14
- package/src/context/context.test.ts +1 -1
- package/src/context/context.ts +6 -3
- package/src/context/session.ts +5 -2
- package/src/context/types.ts +2 -2
- package/src/define.test.ts +82 -0
- package/src/define.ts +68 -7
- package/src/i18n/en.ts +4 -2
- package/src/i18n/zh-CN.ts +4 -2
- package/src/report/aggregate.ts +16 -16
- package/src/report/components.tsx +1 -1
- package/src/report/compute.ts +7 -7
- package/src/report/dual-face.test.tsx +1 -1
- package/src/report/{param.ts → flag.ts} +6 -6
- package/src/report/index.ts +3 -3
- package/src/report/react/MetricLine.tsx +1 -1
- package/src/report/react/fixtures.ts +1 -1
- package/src/report/report.test.ts +17 -17
- package/src/report/report.ts +1 -1
- package/src/report/types.ts +11 -11
- package/src/results/results.test.ts +2 -2
- package/src/runner/attempt.ts +53 -6
- package/src/runner/fingerprint.ts +1 -1
- package/src/runner/reporters/braintrust.test.ts +2 -2
- package/src/runner/reporters/braintrust.ts +3 -3
- package/src/runner/run.ts +1 -1
- package/src/runner/types.ts +8 -8
- package/src/sandbox/types.ts +45 -4
- package/src/show/compose.ts +1 -1
- package/src/view/app/App.tsx +1 -1
- package/src/view/app/css.d.ts +3 -0
- package/src/view/app/i18n.ts +3 -3
- package/src/view/app/lib/rows.ts +5 -5
- package/src/view/app/types.ts +1 -1
- package/src/view/client-dist/app.js +2 -2
- package/src/view/data.test.ts +26 -2
- package/src/view/data.ts +28 -4
- package/src/view/shared/types.ts +1 -1
package/src/view/data.test.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
|
7
7
|
import { tmpdir } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { afterEach, describe, expect, it } from "vitest";
|
|
10
|
-
import { IncompatibleResultsError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
10
|
+
import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
11
11
|
import { createRunWriter } from "../results/index.ts";
|
|
12
12
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type RunSummary } from "../types.ts";
|
|
13
13
|
|
|
@@ -182,7 +182,7 @@ describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一
|
|
|
182
182
|
});
|
|
183
183
|
|
|
184
184
|
describe("loadViewScan · createRunWriter 写出的落盘直接可读(writer/reader 同一契约)", () => {
|
|
185
|
-
it("writer roundtrip
|
|
185
|
+
it("writer roundtrip:默认报告、快照与 attemptRef 都从写入面产物算出", async () => {
|
|
186
186
|
const root = await makeRoot();
|
|
187
187
|
const writer = await createRunWriter(root, { producer: { name: "niceeval", version: "0.4.6" } });
|
|
188
188
|
const snap = writer.snapshot({
|
|
@@ -252,3 +252,27 @@ describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致
|
|
|
252
252
|
expect(withArtifacts.artifactBase).toBe("2026-01-02T00-00-00/artifacts/e1/a0");
|
|
253
253
|
});
|
|
254
254
|
});
|
|
255
|
+
|
|
256
|
+
describe("loadViewScan · 零可读结果直说,不渲染空页面", () => {
|
|
257
|
+
it("目录真空:抛 ViewInputError,给「先跑一轮」提示(与 show 同文案)", async () => {
|
|
258
|
+
const root = await makeRoot();
|
|
259
|
+
await expect(loadViewScan(root)).rejects.toBeInstanceOf(ViewInputError);
|
|
260
|
+
await expect(loadViewScan(root)).rejects.toThrow(/niceeval exp/);
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
it("全被跳过:错误逐条列目录与原因,niceeval 落盘的 schemaVersion 场景给出可跑的 npx 命令", async () => {
|
|
264
|
+
const root = await makeRoot();
|
|
265
|
+
await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([], { schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } }));
|
|
266
|
+
await writeRun(root, "2026-07-04T08-00-00-000Z", "{not json");
|
|
267
|
+
|
|
268
|
+
const err = await loadViewScan(root).then(
|
|
269
|
+
() => { throw new Error("expected ViewInputError"); },
|
|
270
|
+
(e) => e as Error,
|
|
271
|
+
);
|
|
272
|
+
expect(err).toBeInstanceOf(ViewInputError);
|
|
273
|
+
expect(err.message).toContain("2 run directories were skipped");
|
|
274
|
+
expect(err.message).toContain("incompatible-version, schemaVersion 999");
|
|
275
|
+
expect(err.message).toContain("npx niceeval@9.9.9 view ");
|
|
276
|
+
expect(err.message).toContain("malformed");
|
|
277
|
+
});
|
|
278
|
+
});
|
package/src/view/data.ts
CHANGED
|
@@ -140,6 +140,7 @@ export async function loadLatestResultsPerEval(root = ".niceeval"): Promise<Eval
|
|
|
140
140
|
* 位置前缀 / --experiment / --report 在场时,报告槽选集经 composeShowSelection 合成
|
|
141
141
|
* (与 `niceeval show` 同一口径,两扇门判决不分叉);全部缺省时维持 results.latest(),
|
|
142
142
|
* 默认行为不变。证据室数据(快照明细 / skipped)恒为全量,深链在任何收窄下都可达。
|
|
143
|
+
* 零可读结果一律抛 ViewInputError,不渲染/导出空页面(server 起不来,--out 非零退出)。
|
|
143
144
|
*/
|
|
144
145
|
export async function loadViewScan(input?: string, opts: ViewScanOptions = {}): Promise<ViewScan> {
|
|
145
146
|
const target = resolve(input ?? ".niceeval");
|
|
@@ -150,9 +151,11 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
150
151
|
const patterns = opts.patterns ?? [];
|
|
151
152
|
const narrowed = patterns.length > 0 || opts.experiment !== undefined || opts.report !== undefined;
|
|
152
153
|
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
154
|
+
// 零可读结果直说,不渲染/导出一张空页面(与 show 的「匹配不到直说」同一原则;
|
|
155
|
+
// CI 静态发布还靠这个非零退出保住上一次部署,空报告不顶上线)。零可读最常见的
|
|
156
|
+
// 根因不是目录空,而是落盘整批 schemaVersion 不兼容被跳过,所以带上 skipped 摘要。
|
|
157
|
+
if (results.experiments.length === 0) {
|
|
158
|
+
throw new ViewInputError(noReadableResults(target, results.skipped));
|
|
156
159
|
}
|
|
157
160
|
if (
|
|
158
161
|
opts.experiment !== undefined &&
|
|
@@ -248,7 +251,7 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
248
251
|
|
|
249
252
|
/**
|
|
250
253
|
* 报告槽渲染:装载报告文件(dev server 语义 —— 文件变更下次请求整页重算,经 mtime
|
|
251
|
-
* cache-busting)→
|
|
254
|
+
* cache-busting)→ 注入与默认报告同口径的选集 → web 面 renderToStaticMarkup 成静态 HTML。
|
|
252
255
|
* react-dom 只在 --report 在场时动态加载,默认 CLI 路径不背 react。
|
|
253
256
|
* attemptHref 缺省即 `#/attempt/<run>/<result>`(view 的 attempt 深链路由)。
|
|
254
257
|
*/
|
|
@@ -318,6 +321,27 @@ function annotateResult(
|
|
|
318
321
|
return { annotated };
|
|
319
322
|
}
|
|
320
323
|
|
|
324
|
+
/**
|
|
325
|
+
* 零可读结果的报错文案。目录真空时给「先跑一轮」的入门提示(与 show 的 noResults 同形态);
|
|
326
|
+
* 有 skipped 时逐条列目录与原因——schemaVersion 不兼容的 niceeval 落盘给出可跑的
|
|
327
|
+
* npx 命令,让「全被跳过」和「真没跑过」在错误里就能分清,不用进查看器排查。
|
|
328
|
+
*/
|
|
329
|
+
function noReadableResults(target: string, skipped: SkippedRun[]): string {
|
|
330
|
+
if (skipped.length === 0) return t("cli.view.noResults", { root: target }).trimEnd();
|
|
331
|
+
const lines = skipped.map((skip) => {
|
|
332
|
+
const notice = toSkippedNotice(skip);
|
|
333
|
+
const version = notice.schemaVersion !== undefined ? `, schemaVersion ${notice.schemaVersion}` : "";
|
|
334
|
+
const hint = notice.command ? ` — view it with \`${notice.command.trim()}\`` : "";
|
|
335
|
+
return ` ${notice.dir}: ${notice.reason}${version}${hint}`;
|
|
336
|
+
});
|
|
337
|
+
const runs = skipped.length === 1 ? "1 run directory was" : `${skipped.length} run directories were`;
|
|
338
|
+
return [
|
|
339
|
+
`No readable results under ${target} — ${runs} skipped:`,
|
|
340
|
+
...lines,
|
|
341
|
+
"Re-run your experiments with this niceeval to produce fresh results.",
|
|
342
|
+
].join("\n");
|
|
343
|
+
}
|
|
344
|
+
|
|
321
345
|
function toSkippedNotice(skip: SkippedRun): SkippedRunNotice {
|
|
322
346
|
const dir = relative(process.cwd(), skip.dir) || ".";
|
|
323
347
|
const command =
|
package/src/view/shared/types.ts
CHANGED
|
@@ -72,7 +72,7 @@ export interface ViewData {
|
|
|
72
72
|
composedRuns: number;
|
|
73
73
|
/** 官方 KPI(RunOverview.data 产物):totals + 选中快照元信息 + 选集 warnings 随行。 */
|
|
74
74
|
overview: OverviewData;
|
|
75
|
-
/**
|
|
75
|
+
/** 默认报告(MetricTable.data,rows: "experiment"):每个实验最新一次快照的口径。 */
|
|
76
76
|
table: TableData;
|
|
77
77
|
/** 整体单行(MetricTable.data,常量维度):hero 的官方通过率,同一台聚合引擎。 */
|
|
78
78
|
overall: TableData;
|