niceeval 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +321 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/experiment-comparison.d.ts +1 -0
- package/dist/report/built-ins/experiment-comparison.js +13 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +180 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +600 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +26 -0
- package/dist/report/format.js +109 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +94 -0
- package/dist/report/locale.js +183 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +19 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +11 -0
- package/dist/report/react/ExperimentList.js +67 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +96 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +398 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +541 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +408 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +7 -7
- package/docs-site/zh/concepts/assert.mdx +19 -18
- package/docs-site/zh/concepts/evals.mdx +12 -11
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +3 -3
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +237 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +4 -2
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +4 -4
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +61 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +3 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +12 -12
- package/docs-site/zh/guides/viewing-results.mdx +200 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +4 -3
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +15 -13
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +9 -3
- package/docs-site/zh/reference/expect.mdx +13 -8
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +59 -3
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +33 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +640 -0
- package/src/report/built-ins/experiment-comparison.tsx +19 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +225 -50
- package/src/report/compute.ts +335 -54
- package/src/report/dual-render.test.tsx +1129 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +35 -2
- package/src/report/index.ts +47 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +229 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +113 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +220 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +126 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +168 -0
- package/src/report/react/fixtures.ts +169 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +21 -6
- package/src/report/react/render.test.tsx +180 -31
- package/src/report/react/styles.css +432 -213
- package/src/report/report.test.ts +398 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +344 -96
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +184 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +119 -95
- package/src/show/render.ts +462 -159
- package/src/show/show.test.ts +471 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
package/src/view/data.test.ts
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
|
-
// view 数据层(data.ts)的单测:loader 收编到 openResults
|
|
2
|
-
// 守护三件事——skipped 三种原因如实进 viewData(producer 感知的 npx 提示)
|
|
3
|
-
//
|
|
1
|
+
// view 数据层(data.ts)的单测:loader 收编到 openResults、统计整体住进报告槽之后,
|
|
2
|
+
// 守护三件事——skipped 三种原因如实进 viewData(producer 感知的 npx 提示)、报告槽是
|
|
3
|
+
// 现刻水位口径(裸跑经 selectCurrentResults 跨快照合成每 experiment × eval 的最新判定,
|
|
4
|
+
// 与 show 同一函数;en / zh-CN 双语渲染)、跨快照去重让 Runs/Traces 不被 --resume 复印件
|
|
5
|
+
// 灌票。viewData 只携带证据室数据(快照明细 + skipped + 壳元信息),不再有 overview /
|
|
6
|
+
// table / overall 统计产物。
|
|
4
7
|
// 另含 loadLatestResultsPerEval 的续跑携带语义(从旧 loader.test.ts 移植,口径不变)。
|
|
8
|
+
//
|
|
9
|
+
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
10
|
+
// 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
|
|
5
11
|
|
|
6
12
|
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
7
13
|
import { tmpdir } from "node:os";
|
|
8
14
|
import { join } from "node:path";
|
|
9
15
|
import { afterEach, describe, expect, it } from "vitest";
|
|
10
16
|
import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
17
|
+
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
18
|
+
import { encodeAttemptLocator } from "../results/locator.ts";
|
|
13
19
|
|
|
14
20
|
const roots: string[] = [];
|
|
15
21
|
async function makeRoot(): Promise<string> {
|
|
@@ -21,52 +27,90 @@ afterEach(async () => {
|
|
|
21
27
|
await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
|
|
22
28
|
});
|
|
23
29
|
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
type AttemptFixture = Pick<EvalResult, "id" | "verdict"> &
|
|
31
|
+
Partial<Pick<EvalResult, "attempt" | "durationMs" | "assertions" | "fingerprint" | "startedAt" | "artifactBase" | "hasEvents">>;
|
|
32
|
+
|
|
33
|
+
function res(id: string, verdict: Verdict, extra: Partial<AttemptFixture> = {}): AttemptFixture {
|
|
34
|
+
return { id, verdict, attempt: 0, durationMs: 1000, assertions: [], ...extra };
|
|
26
35
|
}
|
|
27
36
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return
|
|
31
|
-
format: RESULTS_FORMAT,
|
|
32
|
-
schemaVersion: RESULTS_SCHEMA_VERSION,
|
|
33
|
-
producer: { name: "niceeval", version: "0.4.0" },
|
|
34
|
-
agent: results[0]?.agent ?? "agent",
|
|
35
|
-
startedAt: "2026-07-01T08:00:00.000Z",
|
|
36
|
-
completedAt: "2026-07-01T08:10:00.000Z",
|
|
37
|
-
passed: count("passed"),
|
|
38
|
-
failed: count("failed"),
|
|
39
|
-
skipped: count("skipped"),
|
|
40
|
-
errored: count("errored"),
|
|
41
|
-
durationMs: 60_000,
|
|
42
|
-
results,
|
|
43
|
-
...over,
|
|
44
|
-
};
|
|
37
|
+
/** 实验目录名的清洗:与 docs/feature/results/architecture.md 一致(/ 与非 [\w.@-] 换成 _)。 */
|
|
38
|
+
function cleanDirName(id: string): string {
|
|
39
|
+
return id.replace(/[^\w.@-]/g, "_");
|
|
45
40
|
}
|
|
46
41
|
|
|
47
|
-
|
|
48
|
-
|
|
42
|
+
interface SnapshotOpts {
|
|
43
|
+
experimentId: string;
|
|
44
|
+
agent?: string;
|
|
45
|
+
model?: string;
|
|
46
|
+
startedAt: string;
|
|
47
|
+
completedAt?: string;
|
|
48
|
+
schemaVersion?: number;
|
|
49
|
+
producer?: { name: string; version?: string };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 写一份新布局快照:snapshot.json + 各 attempt 的 result.json。返回快照目录绝对路径。 */
|
|
53
|
+
async function writeSnapshot(
|
|
54
|
+
root: string,
|
|
55
|
+
expDirName: string,
|
|
56
|
+
snapDirName: string,
|
|
57
|
+
opts: SnapshotOpts,
|
|
58
|
+
results: AttemptFixture[],
|
|
59
|
+
): Promise<string> {
|
|
60
|
+
const dir = join(root, expDirName, snapDirName);
|
|
49
61
|
await mkdir(dir, { recursive: true });
|
|
50
|
-
|
|
62
|
+
const meta = {
|
|
63
|
+
format: RESULTS_FORMAT,
|
|
64
|
+
schemaVersion: opts.schemaVersion ?? RESULTS_SCHEMA_VERSION,
|
|
65
|
+
producer: opts.producer ?? { name: "niceeval", version: "0.4.0" },
|
|
66
|
+
experimentId: opts.experimentId,
|
|
67
|
+
agent: opts.agent ?? "agent",
|
|
68
|
+
...(opts.model !== undefined ? { model: opts.model } : {}),
|
|
69
|
+
startedAt: opts.startedAt,
|
|
70
|
+
...(opts.completedAt !== undefined ? { completedAt: opts.completedAt } : {}),
|
|
71
|
+
};
|
|
72
|
+
await writeFile(join(dir, "snapshot.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
73
|
+
for (const r of results) {
|
|
74
|
+
const attemptDir = join(dir, r.id, `a${r.attempt ?? 0}`);
|
|
75
|
+
await mkdir(attemptDir, { recursive: true });
|
|
76
|
+
await writeFile(join(attemptDir, "result.json"), JSON.stringify(r, null, 2), "utf-8");
|
|
77
|
+
}
|
|
51
78
|
return dir;
|
|
52
79
|
}
|
|
53
80
|
|
|
54
81
|
describe("loadViewScan · skipped 三种原因进 viewData", () => {
|
|
55
82
|
it("incompatible-version / malformed / incomplete 都进 skippedRuns;niceeval 落盘拼 npx 命令,第三方如实报名字不拼", async () => {
|
|
56
83
|
const root = await makeRoot();
|
|
57
|
-
//
|
|
58
|
-
await
|
|
84
|
+
// 正常快照,页面照常渲染。
|
|
85
|
+
await writeSnapshot(root, "exp_a", "2026-07-01T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
86
|
+
res("q1", "passed"),
|
|
87
|
+
]);
|
|
59
88
|
// 版本不同:niceeval 写的。
|
|
60
|
-
await
|
|
89
|
+
await writeSnapshot(
|
|
90
|
+
root,
|
|
91
|
+
"exp_b",
|
|
92
|
+
"2026-07-02T08-00-00-000Z",
|
|
93
|
+
{ experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
|
|
94
|
+
[],
|
|
95
|
+
);
|
|
61
96
|
// 版本不同:第三方 harness 写的。
|
|
62
|
-
await
|
|
97
|
+
await writeSnapshot(
|
|
98
|
+
root,
|
|
99
|
+
"exp_c",
|
|
100
|
+
"2026-07-03T08-00-00-000Z",
|
|
101
|
+
{ experimentId: "exp/c", startedAt: "2026-07-03T08:00:00.000Z", schemaVersion: 999, producer: { name: "otherharness", version: "1.2.3" } },
|
|
102
|
+
[],
|
|
103
|
+
);
|
|
63
104
|
// 坏 JSON。
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
await
|
|
67
|
-
|
|
105
|
+
const malformedDir = join(root, "exp_d", "2026-07-04T08-00-00-000Z");
|
|
106
|
+
await mkdir(malformedDir, { recursive: true });
|
|
107
|
+
await writeFile(join(malformedDir, "snapshot.json"), "{not json", "utf-8");
|
|
108
|
+
// incomplete:有 attempt 落盘、没有 snapshot.json(快照目录建好、元数据没写完的极窄窗口)。
|
|
109
|
+
const incompleteDir = join(root, "exp_e", "2026-07-05T08-00-00-000Z");
|
|
110
|
+
await mkdir(join(incompleteDir, "q1", "a0"), { recursive: true });
|
|
111
|
+
await writeFile(join(incompleteDir, "q1", "a0", "events.json"), "[]", "utf-8");
|
|
68
112
|
|
|
69
|
-
const { viewData } = await loadViewScan(root);
|
|
113
|
+
const { viewData, reportHtml } = await loadViewScan(root);
|
|
70
114
|
const byReason = new Map(viewData.skippedRuns!.map((s) => [s.dir, s]));
|
|
71
115
|
expect(viewData.skippedRuns).toHaveLength(4);
|
|
72
116
|
|
|
@@ -85,171 +129,190 @@ describe("loadViewScan · skipped 三种原因进 viewData", () => {
|
|
|
85
129
|
const incomplete = [...byReason.values()].find((s) => s.reason === "incomplete")!;
|
|
86
130
|
expect(incomplete.dir).toContain("2026-07-05T08-00-00-000Z");
|
|
87
131
|
|
|
88
|
-
//
|
|
89
|
-
expect(
|
|
132
|
+
// 正常快照照常进报告槽(默认报告的榜单),坏快照不拖垮整页。
|
|
133
|
+
expect(reportHtml.en).toContain("exp/a");
|
|
134
|
+
expect(viewData.snapshots.map((s) => s.experimentId)).toEqual(["exp/a"]);
|
|
90
135
|
});
|
|
91
136
|
|
|
92
137
|
it("单文件模式指向版本不同的报告:抛 IncompatibleResultsError(CLI 打印提示退出)", async () => {
|
|
93
138
|
const root = await makeRoot();
|
|
94
|
-
const dir = await
|
|
95
|
-
|
|
139
|
+
const dir = await writeSnapshot(
|
|
140
|
+
root,
|
|
141
|
+
"exp_b",
|
|
142
|
+
"2026-07-02T08-00-00-000Z",
|
|
143
|
+
{ experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
|
|
144
|
+
[],
|
|
145
|
+
);
|
|
146
|
+
await expect(loadViewScan(join(dir, "snapshot.json"))).rejects.toBeInstanceOf(IncompatibleResultsError);
|
|
96
147
|
});
|
|
97
148
|
});
|
|
98
149
|
|
|
99
|
-
describe("loadViewScan ·
|
|
100
|
-
it("
|
|
150
|
+
describe("loadViewScan · 报告槽是现刻水位口径,裸跑与局部收窄合成规则一致", () => {
|
|
151
|
+
it("同一实验两次快照:报告槽跨快照补齐每 eval 的最新判定,不残缺;历史快照仍供证据室", async () => {
|
|
101
152
|
const root = await makeRoot();
|
|
102
153
|
// 周一全量:q1 失败、q2 通过。
|
|
103
|
-
await
|
|
104
|
-
res(
|
|
105
|
-
res(
|
|
106
|
-
]
|
|
107
|
-
// 周二只补跑 q1(通过):latest 快照只盖 1/2
|
|
108
|
-
await
|
|
109
|
-
res(
|
|
110
|
-
]
|
|
154
|
+
await writeSnapshot(root, "exp_a", "2026-07-01T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
155
|
+
res("q1", "failed"),
|
|
156
|
+
res("q2", "passed"),
|
|
157
|
+
]);
|
|
158
|
+
// 周二只补跑 q1(通过):latest 快照只盖 1/2 道题,但现刻水位从周一补齐 q2。
|
|
159
|
+
await writeSnapshot(root, "exp_a", "2026-07-02T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
|
|
160
|
+
res("q1", "passed"),
|
|
161
|
+
]);
|
|
111
162
|
|
|
112
|
-
const { viewData } = await loadViewScan(root);
|
|
113
|
-
|
|
114
|
-
//
|
|
115
|
-
|
|
116
|
-
expect(
|
|
163
|
+
const { viewData, reportHtml } = await loadViewScan(root);
|
|
164
|
+
// 现刻水位:q1 取周二(更新,通过),q2 取周一(仅此一次,通过)—— 两题全过(2/2 = 100%),
|
|
165
|
+
// 不是「只看周二快照的 1/1」。深链分别指向各自的贡献快照(AttemptLocator 由身份元组——
|
|
166
|
+
// 含快照 startedAt——确定性派生,两个不同快照的 q1/q2 编出两个不同的 locator)。
|
|
167
|
+
expect(reportHtml.en).toContain("100%");
|
|
168
|
+
const q1Locator = encodeAttemptLocator({
|
|
169
|
+
experimentId: "exp/a",
|
|
170
|
+
snapshotStartedAt: "2026-07-02T08:00:00.000Z",
|
|
171
|
+
evalId: "q1",
|
|
172
|
+
attempt: 0,
|
|
173
|
+
});
|
|
174
|
+
const q2Locator = encodeAttemptLocator({
|
|
175
|
+
experimentId: "exp/a",
|
|
176
|
+
snapshotStartedAt: "2026-07-01T08:00:00.000Z",
|
|
177
|
+
evalId: "q2",
|
|
178
|
+
attempt: 0,
|
|
179
|
+
});
|
|
180
|
+
expect(reportHtml.en).toContain(`#/attempt/${q1Locator}`); // q1 来自周二
|
|
181
|
+
expect(reportHtml.en).toContain(`#/attempt/${q2Locator}`); // q2 来自周一
|
|
182
|
+
|
|
183
|
+
// 两题都有真实判定,不是伪残缺:报告槽不再出 partial-coverage 警告。
|
|
184
|
+
expect(reportHtml.en).not.toContain('data-kind="partial-coverage"');
|
|
117
185
|
|
|
118
|
-
//
|
|
186
|
+
// 证据室的 latest 标记仍按 results.latest() 口径(周二快照),与报告槽 Selection 无关。
|
|
119
187
|
const latest = viewData.snapshots.filter((s) => s.latest);
|
|
120
188
|
expect(latest).toHaveLength(1);
|
|
121
189
|
expect(latest[0]!.startedAt).toBe("2026-07-02T08:00:00.000Z");
|
|
122
|
-
expect(viewData.composedRuns).toBe(1);
|
|
123
190
|
expect(viewData.lastRunAt).toBe("2026-07-02T08:00:00.000Z");
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const kinds = viewData.overview.warnings.map((w) => w.kind);
|
|
127
|
-
expect(kinds).toContain("partial-coverage");
|
|
128
|
-
const partial = viewData.overview.warnings.find((w) => w.kind === "partial-coverage")!;
|
|
129
|
-
expect(partial.message).toContain("1 of 2");
|
|
191
|
+
// 合成 Selection 的 attempts 来自周一、周二两个物理 run。
|
|
192
|
+
expect(viewData.composedRuns).toBe(2);
|
|
130
193
|
|
|
131
194
|
// 历史快照仍在(Runs / Traces 吃全部),但不是 latest。
|
|
132
195
|
const historical = viewData.snapshots.filter((s) => !s.latest);
|
|
133
196
|
expect(historical).toHaveLength(1);
|
|
134
197
|
expect(historical[0]!.results.map((r) => r.id).sort()).toEqual(["q1", "q2"]);
|
|
135
198
|
});
|
|
136
|
-
|
|
137
|
-
it("缺 experimentId 的落盘:合成键快照 synthetic: true,synthetic-experiment-id 警告透传", async () => {
|
|
138
|
-
const root = await makeRoot();
|
|
139
|
-
await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
|
|
140
|
-
res({ id: "q1", agent: "bub", model: "gpt-5", startedAt: "2026-07-01T08:01:00.000Z" }),
|
|
141
|
-
]));
|
|
142
|
-
const { viewData } = await loadViewScan(root);
|
|
143
|
-
expect(viewData.snapshots[0]!.synthetic).toBe(true);
|
|
144
|
-
expect(viewData.snapshots[0]!.experimentId).toBe("bub/gpt-5");
|
|
145
|
-
expect(viewData.overview.warnings.map((w) => w.kind)).toContain("synthetic-experiment-id");
|
|
146
|
-
});
|
|
147
199
|
});
|
|
148
200
|
|
|
149
201
|
describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一次)", () => {
|
|
150
|
-
it("同一 attempt
|
|
202
|
+
it("同一 attempt 存在于两份落盘:只保留最新快照里的那份,locator 落在最新落盘,artifactBase 沿用原快照 artifact", async () => {
|
|
151
203
|
const root = await makeRoot();
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
startedAt: "2026-07-01T08:
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
await
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
res(
|
|
168
|
-
]
|
|
204
|
+
const oldDir = await writeSnapshot(
|
|
205
|
+
root,
|
|
206
|
+
"exp_a",
|
|
207
|
+
"2026-07-01T08-00-00-000Z",
|
|
208
|
+
{ experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" },
|
|
209
|
+
[res("q1", "passed", { hasEvents: true })],
|
|
210
|
+
);
|
|
211
|
+
await writeFile(join(oldDir, "q1", "a0", "events.json"), "[]", "utf-8");
|
|
212
|
+
// 携带条目:startedAt 锚定原快照, artifactBase 指向原快照的 attempt 目录(root 相对)。
|
|
213
|
+
await writeSnapshot(root, "exp_a", "2026-07-02T08-00-00-000Z", { experimentId: "exp/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
|
|
214
|
+
res("q1", "passed", {
|
|
215
|
+
hasEvents: true,
|
|
216
|
+
startedAt: "2026-07-01T08:00:00.000Z",
|
|
217
|
+
artifactBase: "exp_a/2026-07-01T08-00-00-000Z/q1/a0",
|
|
218
|
+
}),
|
|
219
|
+
res("q2", "passed"),
|
|
220
|
+
]);
|
|
169
221
|
|
|
170
222
|
const { viewData, artifactDirs } = await loadViewScan(root);
|
|
171
223
|
// 全部快照(Runs/Traces 的数据面)里 q1 只出现一次:复印件不灌票。
|
|
172
224
|
const allResults = viewData.snapshots.flatMap((s) => s.results);
|
|
173
225
|
expect(allResults.filter((r) => r.id === "q1")).toHaveLength(1);
|
|
174
226
|
const q1 = allResults.find((r) => r.id === "q1")!;
|
|
175
|
-
|
|
176
|
-
expect(q1.
|
|
177
|
-
|
|
178
|
-
|
|
227
|
+
// 证据身份(locator)跟着最新落盘走:身份元组里的 snapshotStartedAt 是新快照的 startedAt。
|
|
228
|
+
expect(q1.locator).toBe(
|
|
229
|
+
encodeAttemptLocator({ experimentId: "exp/a", snapshotStartedAt: "2026-07-02T08:00:00.000Z", evalId: "q1", attempt: 0 }),
|
|
230
|
+
);
|
|
231
|
+
expect(q1.artifactBase).toBe("exp_a/2026-07-01T08-00-00-000Z/q1/a0"); // artifact 仍指原快照
|
|
232
|
+
// 静态导出(--out)能把携带条目的 artifact 一并带走。
|
|
233
|
+
expect(artifactDirs.get("exp_a/2026-07-01T08-00-00-000Z/q1/a0")).toBe(join(oldDir, "q1", "a0"));
|
|
179
234
|
// 条目全被吸走的旧快照不再出现。
|
|
180
235
|
expect(viewData.snapshots.filter((s) => !s.latest)).toHaveLength(0);
|
|
181
236
|
});
|
|
182
237
|
});
|
|
183
238
|
|
|
184
|
-
describe("loadViewScan ·
|
|
185
|
-
it("
|
|
239
|
+
describe("loadViewScan · 新布局落盘直接可读(写入面 / 读取面同一契约)", () => {
|
|
240
|
+
it("快照与 locator 都从落盘产物算出", async () => {
|
|
186
241
|
const root = await makeRoot();
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
model: "gpt-5",
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
{ events: [{ type: "run.started", t: 0 } as never] },
|
|
242
|
+
await writeSnapshot(
|
|
243
|
+
root,
|
|
244
|
+
"compare_bub",
|
|
245
|
+
"2026-07-03T08-00-00-000Z",
|
|
246
|
+
{ experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-03T08:00:00.000Z" },
|
|
247
|
+
[
|
|
248
|
+
res("q1", "passed", { hasEvents: true }),
|
|
249
|
+
res("q2", "failed"),
|
|
250
|
+
],
|
|
197
251
|
);
|
|
198
|
-
await snap.writeAttempt({ id: "q2", outcome: "failed", attempt: 0, durationMs: 800, assertions: [] });
|
|
199
|
-
await writer.finish();
|
|
200
252
|
|
|
201
|
-
const { viewData } = await loadViewScan(root);
|
|
202
|
-
|
|
203
|
-
expect(
|
|
204
|
-
expect(
|
|
253
|
+
const { viewData, reportHtml } = await loadViewScan(root);
|
|
254
|
+
// 报告槽(ExperimentComparison 的 web 面):Experiment 工作台行 + 官方通过率格子(1 过 1 败 = 50%)。
|
|
255
|
+
expect(reportHtml.en).toContain("compare/bub");
|
|
256
|
+
expect(reportHtml.en).toContain("50%");
|
|
205
257
|
const snapshot = viewData.snapshots[0]!;
|
|
206
258
|
expect(snapshot.latest).toBe(true);
|
|
207
259
|
expect(snapshot.agent).toBe("bub");
|
|
208
|
-
expect(snapshot.
|
|
209
|
-
//
|
|
260
|
+
expect(snapshot.run).toBe("compare_bub/2026-07-03T08-00-00-000Z");
|
|
261
|
+
// 每条结果的 locator 都能由身份元组(experimentId/快照 startedAt/evalId/attempt 下标)独立复算,
|
|
262
|
+
// 证明它不是随手塞的占位值,而是真从落盘产物(snapshot.json + result.json)算出来的。
|
|
263
|
+
expect(snapshot.results.every((r) => r.locator === encodeAttemptLocator({
|
|
264
|
+
experimentId: snapshot.experimentId,
|
|
265
|
+
snapshotStartedAt: snapshot.startedAt,
|
|
266
|
+
evalId: r.id,
|
|
267
|
+
attempt: r.attempt,
|
|
268
|
+
}))).toBe(true);
|
|
269
|
+
// 本快照跑出的条目落盘没有 artifactBase 字段:读取面按 `${ref.snapshot}/${ref.attempt}` 现算,
|
|
270
|
+
// ref.snapshot 恒等于 snapshot.run,ref.attempt 恒是 `${evalId}/a${attempt}`。
|
|
210
271
|
const q1 = snapshot.results.find((r) => r.id === "q1")!;
|
|
211
272
|
expect(q1.hasEvents).toBe(true);
|
|
212
|
-
expect(q1.artifactBase).toBe(`${snapshot.run}
|
|
273
|
+
expect(q1.artifactBase).toBe(`${snapshot.run}/q1/a0`);
|
|
213
274
|
});
|
|
214
275
|
});
|
|
215
276
|
|
|
216
277
|
describe("loadLatestResultsPerEval(续跑携带基线,口径与旧 loader 一致)", () => {
|
|
217
|
-
it("
|
|
278
|
+
it("部分补跑快照只遮蔽它跑过的 eval,其它 eval 仍取自更早的全量快照", async () => {
|
|
218
279
|
const root = await makeRoot();
|
|
219
|
-
await
|
|
220
|
-
res(
|
|
221
|
-
res(
|
|
222
|
-
|
|
223
|
-
|
|
280
|
+
await writeSnapshot(root, "exp_a", "2026-01-01T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
|
|
281
|
+
res("e1", "passed"),
|
|
282
|
+
res("e2", "errored"),
|
|
283
|
+
]);
|
|
284
|
+
await writeSnapshot(root, "exp_b", "2026-01-01T00-00-00-000Z", { experimentId: "exp/b", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
|
|
285
|
+
res("e1", "passed"),
|
|
286
|
+
]);
|
|
224
287
|
// 部分补跑:只重跑了 exp/a 的 e2
|
|
225
|
-
await
|
|
226
|
-
res(
|
|
227
|
-
]
|
|
288
|
+
await writeSnapshot(root, "exp_a", "2026-01-02T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-02T00:00:00.000Z" }, [
|
|
289
|
+
res("e2", "passed"),
|
|
290
|
+
]);
|
|
228
291
|
|
|
229
292
|
const results = await loadLatestResultsPerEval(root);
|
|
230
|
-
const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.
|
|
231
|
-
expect(byKey.get("exp/a|e1")).toBe("passed"); //
|
|
232
|
-
expect(byKey.get("exp/a|e2")).toBe("passed"); //
|
|
293
|
+
const byKey = new Map(results.map((r) => [`${r.experimentId}|${r.id}`, r.verdict]));
|
|
294
|
+
expect(byKey.get("exp/a|e1")).toBe("passed"); // 来自旧全量快照,没被部分快照冲掉
|
|
295
|
+
expect(byKey.get("exp/a|e2")).toBe("passed"); // 来自补跑快照(最新)
|
|
233
296
|
expect(byKey.get("exp/b|e1")).toBe("passed");
|
|
234
297
|
expect(results).toHaveLength(3);
|
|
235
298
|
});
|
|
236
299
|
|
|
237
|
-
it("同 (experiment, eval) 多 attempt
|
|
300
|
+
it("同 (experiment, eval) 多 attempt 整批取自含它的最新快照,不跨快照混装;artifactBase 已拼好", async () => {
|
|
238
301
|
const root = await makeRoot();
|
|
239
|
-
await
|
|
240
|
-
res(
|
|
241
|
-
]
|
|
242
|
-
await
|
|
243
|
-
res(
|
|
244
|
-
res(
|
|
245
|
-
]
|
|
302
|
+
await writeSnapshot(root, "exp_a", "2026-01-01T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-01T00:00:00.000Z" }, [
|
|
303
|
+
res("e1", "passed", { fingerprint: "old" }),
|
|
304
|
+
]);
|
|
305
|
+
await writeSnapshot(root, "exp_a", "2026-01-02T00-00-00-000Z", { experimentId: "exp/a", agent: "a", startedAt: "2026-01-02T00:00:00.000Z" }, [
|
|
306
|
+
res("e1", "failed", { fingerprint: "new" }),
|
|
307
|
+
res("e1", "passed", { attempt: 1, fingerprint: "new" }),
|
|
308
|
+
]);
|
|
246
309
|
|
|
247
310
|
const results = await loadLatestResultsPerEval(root);
|
|
248
311
|
expect(results).toHaveLength(2);
|
|
249
312
|
expect(results.every((r) => r.fingerprint === "new")).toBe(true);
|
|
250
|
-
// runner 携带条目时依赖 artifactBase(相对结果根)可解析,view
|
|
251
|
-
const
|
|
252
|
-
expect(
|
|
313
|
+
// runner 携带条目时依赖 artifactBase(相对结果根)可解析,view 才找得回 artifact。
|
|
314
|
+
const withArtifact = results.find((r) => r.attempt === 0)!;
|
|
315
|
+
expect(withArtifact.artifactBase).toBe("exp_a/2026-01-02T00-00-00-000Z/e1/a0");
|
|
253
316
|
});
|
|
254
317
|
});
|
|
255
318
|
|
|
@@ -262,15 +325,23 @@ describe("loadViewScan · 零可读结果直说,不渲染空页面", () => {
|
|
|
262
325
|
|
|
263
326
|
it("全被跳过:错误逐条列目录与原因,niceeval 落盘的 schemaVersion 场景给出可跑的 npx 命令", async () => {
|
|
264
327
|
const root = await makeRoot();
|
|
265
|
-
await
|
|
266
|
-
|
|
328
|
+
await writeSnapshot(
|
|
329
|
+
root,
|
|
330
|
+
"exp_b",
|
|
331
|
+
"2026-07-02T08-00-00-000Z",
|
|
332
|
+
{ experimentId: "exp/b", startedAt: "2026-07-02T08:00:00.000Z", schemaVersion: 999, producer: { name: "niceeval", version: "9.9.9" } },
|
|
333
|
+
[],
|
|
334
|
+
);
|
|
335
|
+
const malformedDir = join(root, "exp_d", "2026-07-04T08-00-00-000Z");
|
|
336
|
+
await mkdir(malformedDir, { recursive: true });
|
|
337
|
+
await writeFile(join(malformedDir, "snapshot.json"), "{not json", "utf-8");
|
|
267
338
|
|
|
268
339
|
const err = await loadViewScan(root).then(
|
|
269
340
|
() => { throw new Error("expected ViewInputError"); },
|
|
270
341
|
(e) => e as Error,
|
|
271
342
|
);
|
|
272
343
|
expect(err).toBeInstanceOf(ViewInputError);
|
|
273
|
-
expect(err.message).toContain("2
|
|
344
|
+
expect(err.message).toContain("2 snapshot directories were skipped");
|
|
274
345
|
expect(err.message).toContain("incompatible-version, schemaVersion 999");
|
|
275
346
|
expect(err.message).toContain("npx niceeval@9.9.9 view ");
|
|
276
347
|
expect(err.message).toContain("malformed");
|