niceeval 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +55 -0
- package/dist/agents/types.d.ts +317 -0
- package/dist/context/types.d.ts +335 -0
- package/dist/i18n/core.d.ts +11 -0
- package/dist/i18n/en.d.ts +181 -0
- package/dist/i18n/index.d.ts +5 -0
- package/dist/i18n/zh-CN.d.ts +183 -0
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/parse.d.ts +3 -0
- package/dist/o11y/otlp/receiver.d.ts +21 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/o11y/otlp/turn-otel.d.ts +43 -0
- package/dist/o11y/types.d.ts +190 -0
- package/dist/report/aggregate.d.ts +57 -0
- package/dist/report/aggregate.js +212 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +1 -0
- package/dist/report/built-ins/cost-pass-rate-comparison.js +17 -0
- package/dist/report/built-ins/index.d.ts +1 -0
- package/dist/report/built-ins/index.js +2 -0
- package/dist/report/components.d.ts +178 -0
- package/dist/report/components.js +132 -0
- package/dist/report/compute.d.ts +106 -0
- package/dist/report/compute.js +618 -0
- package/dist/report/flag.d.ts +6 -0
- package/dist/report/flag.js +8 -0
- package/dist/report/format.d.ts +32 -0
- package/dist/report/format.js +126 -0
- package/dist/report/index.d.ts +19 -0
- package/dist/report/index.js +30 -0
- package/dist/report/load.d.ts +12 -0
- package/dist/report/load.js +56 -0
- package/dist/report/locale.d.ts +83 -0
- package/dist/report/locale.js +160 -0
- package/dist/report/metrics.d.ts +21 -0
- package/dist/report/metrics.js +115 -0
- package/dist/report/primitives.d.ts +64 -0
- package/dist/report/primitives.js +113 -0
- package/dist/report/react/AttemptList.d.ts +23 -0
- package/dist/report/react/AttemptList.js +20 -0
- package/dist/report/react/DeltaTable.d.ts +8 -0
- package/dist/report/react/DeltaTable.js +24 -0
- package/dist/report/react/EvalList.d.ts +10 -0
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentList.d.ts +10 -0
- package/dist/report/react/ExperimentList.js +15 -0
- package/dist/report/react/GroupSummary.d.ts +8 -0
- package/dist/report/react/GroupSummary.js +8 -0
- package/dist/report/react/MetricBars.d.ts +10 -0
- package/dist/report/react/MetricBars.js +48 -0
- package/dist/report/react/MetricLine.d.ts +9 -0
- package/dist/report/react/MetricLine.js +68 -0
- package/dist/report/react/MetricMatrix.d.ts +10 -0
- package/dist/report/react/MetricMatrix.js +23 -0
- package/dist/report/react/MetricScatter.d.ts +9 -0
- package/dist/report/react/MetricScatter.js +100 -0
- package/dist/report/react/MetricTable.d.ts +11 -0
- package/dist/report/react/MetricTable.js +26 -0
- package/dist/report/react/RunOverview.d.ts +8 -0
- package/dist/report/react/RunOverview.js +12 -0
- package/dist/report/react/Scoreboard.d.ts +8 -0
- package/dist/report/react/Scoreboard.js +26 -0
- package/dist/report/react/cell.d.ts +9 -0
- package/dist/report/react/cell.js +9 -0
- package/dist/report/react/chart-math.d.ts +14 -0
- package/dist/report/react/chart-math.js +68 -0
- package/dist/report/react/colors.d.ts +14 -0
- package/dist/report/react/colors.js +45 -0
- package/dist/report/react/fixtures.d.ts +19 -0
- package/dist/report/react/fixtures.js +393 -0
- package/dist/report/react/format.d.ts +3 -0
- package/dist/report/react/format.js +7 -0
- package/dist/report/react/index.d.ts +6 -0
- package/dist/report/react/index.js +14 -0
- package/dist/report/report.d.ts +24 -0
- package/dist/report/report.js +49 -0
- package/dist/report/text/faces.d.ts +29 -0
- package/dist/report/text/faces.js +503 -0
- package/dist/report/text/layout.d.ts +75 -0
- package/dist/report/text/layout.js +173 -0
- package/dist/report/text/plot.d.ts +37 -0
- package/dist/report/text/plot.js +81 -0
- package/dist/report/text/table.d.ts +8 -0
- package/dist/report/text/table.js +86 -0
- package/dist/report/tree.d.ts +99 -0
- package/dist/report/tree.js +228 -0
- package/dist/report/types.d.ts +420 -0
- package/dist/report/types.js +4 -0
- package/dist/report/web.d.ts +15 -0
- package/dist/report/web.js +37 -0
- package/dist/results/annotated-source.d.ts +61 -0
- package/dist/results/attempt-evidence.d.ts +69 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/locator.d.ts +75 -0
- package/dist/results/locator.js +0 -0
- package/dist/results/select.d.ts +45 -0
- package/dist/results/select.js +230 -0
- package/dist/results/source-hash.d.ts +9 -0
- package/dist/results/types.d.ts +204 -0
- package/dist/runner/eval-source.d.ts +22 -0
- package/dist/runner/fingerprint.d.ts +23 -0
- package/dist/runner/types.d.ts +332 -0
- package/dist/sandbox/docker-stream.d.ts +24 -0
- package/dist/sandbox/docker.d.ts +84 -0
- package/dist/sandbox/e2b.d.ts +31 -0
- package/dist/sandbox/errors.d.ts +4 -0
- package/dist/sandbox/local-files.d.ts +6 -0
- package/dist/sandbox/paths.d.ts +4 -0
- package/dist/sandbox/registry.d.ts +13 -0
- package/dist/sandbox/resolve.d.ts +40 -0
- package/dist/sandbox/retry.d.ts +10 -0
- package/dist/sandbox/shell.d.ts +11 -0
- package/dist/sandbox/source-files.d.ts +13 -0
- package/dist/sandbox/types.d.ts +203 -0
- package/dist/sandbox/vercel.d.ts +37 -0
- package/dist/scoring/types.d.ts +70 -0
- package/dist/shared/aggregate.d.ts +29 -0
- package/dist/shared/aggregate.js +55 -0
- package/dist/shared/types.d.ts +27 -0
- package/dist/shared/verdict.d.ts +27 -0
- package/dist/shared/verdict.js +34 -0
- package/dist/tty-line.d.ts +6 -0
- package/dist/types.d.ts +7 -0
- package/dist/util.d.ts +22 -0
- package/docs-site/zh/concepts/adapter.mdx +5 -5
- package/docs-site/zh/concepts/assert.mdx +8 -8
- package/docs-site/zh/concepts/evals.mdx +5 -5
- package/docs-site/zh/concepts/experiment.mdx +2 -2
- package/docs-site/zh/concepts/judge.mdx +9 -9
- package/docs-site/zh/concepts/overview.mdx +2 -2
- package/docs-site/zh/concepts/tier.mdx +6 -6
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +9 -5
- package/docs-site/zh/example/claude-code-codex-skill.mdx +12 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +235 -78
- package/docs-site/zh/guides/connect-otel.mdx +3 -3
- package/docs-site/zh/guides/connect-your-agent.mdx +2 -2
- package/docs-site/zh/guides/custom-reports.mdx +196 -36
- package/docs-site/zh/guides/experiments.mdx +7 -1
- package/docs-site/zh/guides/fixtures.mdx +1 -1
- package/docs-site/zh/guides/official-adapters.mdx +31 -9
- package/docs-site/zh/guides/publish-report.mdx +3 -3
- package/docs-site/zh/guides/report-components.mdx +194 -55
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +35 -34
- package/docs-site/zh/guides/runner.mdx +2 -2
- package/docs-site/zh/guides/sandbox-agent.mdx +2 -2
- package/docs-site/zh/guides/{sandbox-backends.mdx → sandbox-providers.mdx} +11 -11
- package/docs-site/zh/guides/viewing-results.mdx +199 -79
- package/docs-site/zh/guides/write-experiment.mdx +9 -1
- package/docs-site/zh/guides/write-send.mdx +2 -2
- package/docs-site/zh/index.mdx +4 -3
- package/docs-site/zh/introduction.mdx +2 -1
- package/docs-site/zh/quickstart.mdx +5 -3
- package/docs-site/zh/reference/builtin-agents.mdx +38 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +36 -13
- package/docs-site/zh/reference/define-agent.mdx +12 -12
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +3 -3
- package/docs-site/zh/reference/events.mdx +7 -1
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/package.json +10 -6
- package/src/agents/bub.ts +129 -42
- package/src/agents/claude-code.test.ts +152 -0
- package/src/agents/claude-code.ts +127 -13
- package/src/agents/codex.test.ts +192 -0
- package/src/agents/codex.ts +151 -12
- package/src/agents/index.ts +9 -3
- package/src/agents/manifest.ts +41 -0
- package/src/agents/plugin-config.test.ts +94 -0
- package/src/agents/shared.ts +33 -2
- package/src/agents/skills.test.ts +214 -0
- package/src/agents/skills.ts +304 -0
- package/src/agents/streaming.ts +5 -5
- package/src/agents/types.ts +54 -2
- package/src/cli.ts +74 -47
- package/src/context/context.test.ts +20 -2
- package/src/context/control-flow.ts +1 -1
- package/src/context/session.test.ts +92 -1
- package/src/context/session.ts +23 -1
- package/src/context/types.ts +4 -4
- package/src/define.test.ts +8 -8
- package/src/define.ts +11 -11
- package/src/expect/index.ts +5 -5
- package/src/i18n/en.ts +34 -15
- package/src/i18n/zh-CN.ts +37 -21
- package/src/index.ts +1 -1
- package/src/o11y/execution-tree.test.ts +452 -0
- package/src/o11y/execution-tree.ts +367 -0
- package/src/o11y/otlp/receiver.ts +1 -1
- package/src/o11y/otlp/select.ts +7 -3
- package/src/o11y/parsers/claude-code.test.ts +175 -0
- package/src/o11y/parsers/claude-code.ts +32 -0
- package/src/o11y/types.ts +6 -1
- package/src/report/aggregate.ts +26 -20
- package/src/report/built-in-user-parity.test.tsx +643 -0
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +23 -0
- package/src/report/built-ins/index.ts +2 -0
- package/src/report/components.tsx +223 -50
- package/src/report/compute.ts +358 -53
- package/src/report/dual-render.test.tsx +1125 -0
- package/src/report/flag.ts +1 -1
- package/src/report/format.ts +50 -2
- package/src/report/index.ts +48 -14
- package/src/report/load.ts +1 -1
- package/src/report/locale.ts +206 -0
- package/src/report/metrics.ts +39 -15
- package/src/report/primitives.tsx +100 -2
- package/src/report/react/AttemptList.tsx +115 -0
- package/src/report/react/DeltaTable.tsx +9 -6
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentList.tsx +108 -0
- package/src/report/react/GroupSummary.tsx +66 -0
- package/src/report/react/MetricBars.tsx +24 -13
- package/src/report/react/MetricLine.tsx +21 -21
- package/src/report/react/MetricMatrix.tsx +8 -4
- package/src/report/react/MetricScatter.tsx +136 -96
- package/src/report/react/MetricTable.tsx +115 -15
- package/src/report/react/RunOverview.tsx +48 -26
- package/src/report/react/Scoreboard.tsx +15 -7
- package/src/report/react/cell.tsx +18 -9
- package/src/{view/app/lib/chart.ts → report/react/chart-math.ts} +2 -6
- package/src/report/react/colors.ts +24 -15
- package/src/report/react/enhance.js +129 -0
- package/src/report/react/fixtures.ts +163 -34
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +22 -6
- package/src/report/react/render.test.tsx +164 -31
- package/src/report/react/styles.css +405 -213
- package/src/report/report.test.ts +403 -94
- package/src/report/report.ts +29 -11
- package/src/report/text/faces.ts +303 -95
- package/src/report/text/layout.ts +76 -11
- package/src/report/text/table.ts +98 -0
- package/src/report/tree.ts +103 -20
- package/src/report/types.ts +196 -25
- package/src/report/web.ts +46 -19
- package/src/results/annotated-source.test.ts +146 -0
- package/src/results/annotated-source.ts +128 -0
- package/src/results/attempt-evidence.test.ts +158 -0
- package/src/results/attempt-evidence.ts +158 -0
- package/src/results/attempt-source.ts +48 -0
- package/src/results/copy.ts +154 -139
- package/src/results/format.ts +54 -47
- package/src/results/host-equivalence.test.ts +527 -0
- package/src/results/index.ts +56 -14
- package/src/results/locator.test.ts +198 -0
- package/src/results/locator.ts +0 -0
- package/src/results/open.ts +392 -175
- package/src/results/results.test.ts +943 -494
- package/src/results/select.ts +129 -23
- package/src/results/skipped-notice.ts +0 -0
- package/src/results/source-hash.ts +28 -0
- package/src/results/types.ts +98 -62
- package/src/results/writer.ts +304 -176
- package/src/runner/attempt.test.ts +170 -0
- package/src/runner/attempt.ts +48 -13
- package/src/runner/discover.test.ts +65 -0
- package/src/runner/discover.ts +6 -2
- package/src/runner/eval-source.test.ts +121 -0
- package/src/runner/eval-source.ts +45 -0
- package/src/runner/fingerprint.ts +54 -1
- package/src/runner/remote-sandbox.ts +1 -1
- package/src/runner/report.test.ts +6 -8
- package/src/runner/report.ts +5 -4
- package/src/runner/reporters/artifacts.ts +27 -41
- package/src/runner/reporters/braintrust.test.ts +3 -3
- package/src/runner/reporters/braintrust.ts +1 -1
- package/src/runner/reporters/console.ts +6 -6
- package/src/runner/reporters/json.ts +4 -4
- package/src/runner/reporters/live.test.ts +56 -0
- package/src/runner/reporters/live.ts +36 -10
- package/src/runner/reporters/quiet.test.ts +66 -0
- package/src/runner/reporters/quiet.ts +49 -0
- package/src/runner/reporters/shared.ts +5 -5
- package/src/runner/reporters/table.ts +17 -17
- package/src/runner/run.ts +61 -102
- package/src/runner/types.ts +43 -32
- package/src/sandbox/checkpoint.ts +1 -1
- package/src/sandbox/docker.ts +15 -2
- package/src/sandbox/e2b.ts +11 -5
- package/src/sandbox/errors.ts +13 -0
- package/src/sandbox/index.ts +3 -3
- package/src/sandbox/local-files.ts +1 -1
- package/src/sandbox/registry.ts +6 -5
- package/src/sandbox/resolve.ts +38 -23
- package/src/sandbox/retry.test.ts +68 -0
- package/src/sandbox/retry.ts +44 -0
- package/src/sandbox/types.ts +24 -24
- package/src/sandbox/vercel.ts +17 -3
- package/src/scoring/collector.ts +3 -3
- package/src/scoring/judge.ts +2 -2
- package/src/scoring/scoped.ts +13 -1
- package/src/scoring/types.ts +7 -7
- package/src/scoring/verdict.ts +4 -4
- package/src/shared/aggregate.ts +23 -3
- package/src/shared/types.ts +2 -2
- package/src/shared/{outcome.ts → verdict.ts} +15 -15
- package/src/show/compose.ts +12 -132
- package/src/show/index.ts +125 -94
- package/src/show/render.ts +660 -160
- package/src/show/show.test.ts +472 -253
- package/src/source-loc.ts +1 -1
- package/src/tty-line.ts +32 -0
- package/src/types.ts +2 -2
- package/src/util.test.ts +21 -1
- package/src/util.ts +5 -1
- package/src/view/app/App.tsx +33 -198
- package/src/view/app/components/AttemptModal.tsx +6 -6
- package/src/view/app/components/CodeView.tsx +2 -2
- package/src/view/app/components/CopyControls.tsx +32 -36
- package/src/view/app/components/SkippedRunsBanner.tsx +140 -0
- package/src/view/app/components/ui/badge.tsx +1 -1
- package/src/view/app/components/ui/dialog.tsx +2 -2
- package/src/view/app/i18n.ts +40 -31
- package/src/view/app/lib/artifact-url.ts +1 -1
- package/src/view/app/lib/attempt-route.test.ts +37 -47
- package/src/view/app/lib/attempt-route.ts +26 -26
- package/src/view/app/lib/rows.ts +9 -112
- package/src/view/app/lib/verdict.ts +25 -0
- package/src/view/app/main.tsx +9 -12
- package/src/view/app/pages/RunsPage.tsx +4 -4
- package/src/view/app/pages/TracesPage.tsx +3 -3
- package/src/view/app/shared.ts +0 -1
- package/src/view/app/types.ts +6 -51
- package/src/view/artifact-serving.test.ts +136 -0
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +21 -21
- package/src/view/data.test.ts +214 -143
- package/src/view/data.ts +166 -120
- package/src/view/index.ts +23 -11
- package/src/view/server.ts +52 -17
- package/src/view/shared/types.ts +22 -31
- package/src/view/styles.css +117 -327
- package/src/view/view-report.test.ts +146 -98
- package/src/report/default-report.tsx +0 -222
- package/src/report/dual-face.test.tsx +0 -527
- package/src/report/react/CaseList.tsx +0 -70
- package/src/view/app/components/CostScoreChart.tsx +0 -147
- package/src/view/app/components/ExperimentTable.tsx +0 -284
- package/src/view/app/components/GroupSelector.tsx +0 -61
- package/src/view/app/components/primitives.tsx +0 -43
- package/src/view/app/lib/outcome.ts +0 -82
|
@@ -1,24 +1,34 @@
|
|
|
1
|
-
// niceeval/results 的单测:临时目录里构造最小
|
|
2
|
-
// (docs/results
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
// niceeval/results 的单测:临时目录里构造最小 snapshot.json / result.json / artifact fixture,
|
|
2
|
+
// 覆盖定稿契约(docs/feature/results/library.md、docs/feature/results/architecture.md):分层读取(含快照级字段注入)、
|
|
3
|
+
// 懒加载与 artifactBase 回退、skipped 三种原因、unfinished-snapshot、latest() 三种警告、
|
|
4
|
+
// Selection.filter 修剪、dedupeAttempts 身份键、writer(独占目录、并发快照互不干扰、
|
|
5
|
+
// snapshot.json 键形状、writeAttempt/writeAttemptFor、finish 幂等)、copySnapshots(布局、
|
|
6
|
+
// knownEvalIds 补记、has* 重算)。
|
|
7
|
+
// 读取面 fixture 的目录名/artifact 路径手写(不 import 库的路径函数),让测试独立于实现充当格式基准。
|
|
8
|
+
|
|
9
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
10
|
+
import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
9
11
|
import { tmpdir } from "node:os";
|
|
10
|
-
import { join } from "node:path";
|
|
12
|
+
import { basename, join } from "node:path";
|
|
11
13
|
import {
|
|
12
14
|
RESULTS_FORMAT,
|
|
13
15
|
RESULTS_SCHEMA_VERSION,
|
|
14
16
|
copySnapshots,
|
|
15
|
-
|
|
17
|
+
createResultsWriter,
|
|
16
18
|
dedupeAttempts,
|
|
17
19
|
openResults,
|
|
20
|
+
resolveLocator,
|
|
21
|
+
loadAnnotatedEvalSource,
|
|
22
|
+
LocatorNotFoundError,
|
|
23
|
+
MalformedLocatorError,
|
|
24
|
+
LocatorCollisionError,
|
|
25
|
+
encodeAttemptLocator,
|
|
26
|
+
type AttemptHandle,
|
|
18
27
|
type EvalResult,
|
|
19
|
-
type
|
|
28
|
+
type Results,
|
|
29
|
+
type Snapshot,
|
|
30
|
+
type SnapshotMeta,
|
|
20
31
|
} from "./index.ts";
|
|
21
|
-
import { Artifacts } from "../runner/reporters/artifacts.ts";
|
|
22
32
|
|
|
23
33
|
// ───────────────────────── fixture 工具 ─────────────────────────
|
|
24
34
|
|
|
@@ -32,38 +42,36 @@ afterEach(async () => {
|
|
|
32
42
|
await Promise.all(roots.splice(0).map((r) => rm(r, { recursive: true, force: true })));
|
|
33
43
|
});
|
|
34
44
|
|
|
35
|
-
function
|
|
36
|
-
return { outcome: "passed", attempt: 1, durationMs: 1000, assertions: [], ...over };
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function summaryOf(results: EvalResult[], over: Partial<RunSummary> = {}): RunSummary {
|
|
40
|
-
const count = (o: EvalResult["outcome"]) => results.filter((r) => r.outcome === o).length;
|
|
45
|
+
function meta(over: { experimentId: string; agent: string; startedAt: string } & Partial<SnapshotMeta>): SnapshotMeta {
|
|
41
46
|
return {
|
|
42
47
|
format: RESULTS_FORMAT,
|
|
43
48
|
schemaVersion: RESULTS_SCHEMA_VERSION,
|
|
44
49
|
producer: { name: "niceeval", version: "0.3.0" },
|
|
45
|
-
agent: results[0]?.agent ?? "agent",
|
|
46
|
-
startedAt: "2026-07-01T08:00:00.000Z",
|
|
47
|
-
completedAt: "2026-07-01T08:10:00.000Z",
|
|
48
|
-
passed: count("passed"),
|
|
49
|
-
failed: count("failed"),
|
|
50
|
-
skipped: count("skipped"),
|
|
51
|
-
errored: count("errored"),
|
|
52
|
-
durationMs: 60_000,
|
|
53
|
-
results,
|
|
54
50
|
...over,
|
|
55
51
|
};
|
|
56
52
|
}
|
|
57
53
|
|
|
58
|
-
|
|
59
|
-
|
|
54
|
+
function record(over: { id: string; attempt: number } & Record<string, unknown>): EvalResult {
|
|
55
|
+
return { verdict: "passed", durationMs: 1000, assertions: [], ...over } as unknown as EvalResult;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function writeSnapshot(root: string, expDir: string, snapDirName: string, m: SnapshotMeta): Promise<string> {
|
|
59
|
+
const dir = join(root, expDir, snapDirName);
|
|
60
60
|
await mkdir(dir, { recursive: true });
|
|
61
|
-
await writeFile(join(dir, "
|
|
61
|
+
await writeFile(join(dir, "snapshot.json"), JSON.stringify(m, null, 2), "utf-8");
|
|
62
62
|
return dir;
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
async function
|
|
66
|
-
const dir = join(
|
|
65
|
+
async function writeResultFile(snapDir: string, relAttemptDir: string, r: unknown): Promise<string> {
|
|
66
|
+
const dir = join(snapDir, relAttemptDir);
|
|
67
|
+
await mkdir(dir, { recursive: true });
|
|
68
|
+
const path = join(dir, "result.json");
|
|
69
|
+
await writeFile(path, JSON.stringify(r, null, 2), "utf-8");
|
|
70
|
+
return path;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function writeArtifactFile(snapDir: string, relAttemptDir: string, file: string, data: unknown): Promise<string> {
|
|
74
|
+
const dir = join(snapDir, relAttemptDir);
|
|
67
75
|
await mkdir(dir, { recursive: true });
|
|
68
76
|
const path = join(dir, file);
|
|
69
77
|
await writeFile(path, JSON.stringify(data), "utf-8");
|
|
@@ -82,184 +90,222 @@ async function exists(path: string): Promise<boolean> {
|
|
|
82
90
|
// ───────────────────────── 分层读取 ─────────────────────────
|
|
83
91
|
|
|
84
92
|
describe("openResults · 实验 → 快照 → eval → attempt 分层", () => {
|
|
85
|
-
it("
|
|
86
|
-
const root = await makeRoot();
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
93
|
+
it("字典序实验、最新快照在前;eval 分组、attempt 平铺;快照级字段注入进 attempt.result(缺才补)", async () => {
|
|
94
|
+
const root = await makeRoot();
|
|
95
|
+
const bubDir = await writeSnapshot(
|
|
96
|
+
root,
|
|
97
|
+
"compare_bub",
|
|
98
|
+
"2026-07-01T08-00-00-000Z-a1b2",
|
|
99
|
+
meta({
|
|
100
|
+
experimentId: "compare/bub",
|
|
101
|
+
agent: "bub",
|
|
102
|
+
model: "gpt-5",
|
|
103
|
+
startedAt: "2026-07-01T08:00:00.000Z",
|
|
104
|
+
completedAt: "2026-07-01T08:10:00.000Z",
|
|
105
|
+
}),
|
|
106
|
+
);
|
|
107
|
+
await writeResultFile(bubDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" }));
|
|
108
|
+
await writeResultFile(bubDir, "algebra/q1/a2", record({ id: "algebra/q1", attempt: 2, verdict: "failed" }));
|
|
109
|
+
await writeResultFile(bubDir, "algebra/q2/a1", record({ id: "algebra/q2", attempt: 1 }));
|
|
110
|
+
|
|
111
|
+
await writeSnapshot(
|
|
112
|
+
root,
|
|
113
|
+
"compare_codex",
|
|
114
|
+
"2026-07-01T08-05-00-000Z-c3d4",
|
|
115
|
+
meta({ experimentId: "compare/codex", agent: "codex", model: "o3", startedAt: "2026-07-01T08:05:00.000Z" }),
|
|
116
|
+
);
|
|
96
117
|
|
|
97
118
|
const results = await openResults(root);
|
|
98
119
|
expect(results.skipped).toHaveLength(0);
|
|
99
|
-
expect(results.runDirs.map((r) => r.dir)).toEqual([tuesday, monday]); // 新→旧
|
|
100
120
|
expect(results.experiments.map((e) => e.id)).toEqual(["compare/bub", "compare/codex"]); // 字典序
|
|
101
121
|
|
|
102
122
|
const bub = results.experiments[0];
|
|
103
|
-
expect(bub.snapshots
|
|
123
|
+
expect(bub.snapshots).toHaveLength(1);
|
|
104
124
|
expect(bub.latest).toBe(bub.snapshots[0]);
|
|
105
|
-
expect(bub.evalIds).toEqual(["algebra/q1", "algebra/q2"]);
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
expect(
|
|
110
|
-
expect(
|
|
111
|
-
expect(
|
|
112
|
-
expect(
|
|
113
|
-
expect(
|
|
114
|
-
expect(
|
|
115
|
-
expect(
|
|
116
|
-
expect(
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
const attempt = mondayBub.evals[0].attempts[0];
|
|
125
|
+
expect(bub.evalIds).toEqual(["algebra/q1", "algebra/q2"]);
|
|
126
|
+
|
|
127
|
+
const snap = bub.latest;
|
|
128
|
+
expect(snap.agent).toBe("bub");
|
|
129
|
+
expect(snap.model).toBe("gpt-5");
|
|
130
|
+
expect(snap.producer).toEqual({ name: "niceeval", version: "0.3.0" });
|
|
131
|
+
expect(snap.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
|
|
132
|
+
expect(snap.dir).toBe(bubDir);
|
|
133
|
+
expect(snap.completedAt).toBe("2026-07-01T08:10:00.000Z");
|
|
134
|
+
expect(snap.evals.map((e) => e.id)).toEqual(["algebra/q1", "algebra/q2"]);
|
|
135
|
+
expect(snap.evals[0].attempts).toHaveLength(2);
|
|
136
|
+
expect(snap.attempts).toHaveLength(3);
|
|
137
|
+
|
|
138
|
+
const attempt = snap.evals[0].attempts[0];
|
|
120
139
|
expect(attempt.evalId).toBe("algebra/q1");
|
|
121
140
|
expect(attempt.experimentId).toBe("compare/bub");
|
|
122
|
-
expect(attempt.ref).toEqual({
|
|
141
|
+
expect(attempt.ref).toEqual({ snapshot: "compare_bub/2026-07-01T08-00-00-000Z-a1b2", attempt: "algebra/q1/a1" });
|
|
142
|
+
// 快照级字段注入(record 没写 agent/model/experimentId)。
|
|
143
|
+
expect(attempt.result.agent).toBe("bub");
|
|
144
|
+
expect(attempt.result.model).toBe("gpt-5");
|
|
145
|
+
expect(attempt.result.experimentId).toBe("compare/bub");
|
|
146
|
+
// 缺才补:条目自带的 startedAt 优先。
|
|
123
147
|
expect(attempt.result.startedAt).toBe("2026-07-01T08:01:00.000Z");
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
it("summary.snapshots 元数据:快照级 startedAt 覆盖顶层,knownEvalIds 并进 exp.evalIds", async () => {
|
|
127
|
-
const root = await makeRoot();
|
|
128
|
-
await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
|
|
129
|
-
res({ id: "q1", agent: "bub", experimentId: "mid/a" }),
|
|
130
|
-
res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
|
|
131
|
-
], {
|
|
132
|
-
startedAt: "2026-07-01T08:00:00.000Z",
|
|
133
|
-
snapshots: {
|
|
134
|
-
"mid/b": { startedAt: "2026-06-20T08:00:00.000Z", knownEvalIds: ["q1", "q2", "q3"] },
|
|
135
|
-
},
|
|
136
|
-
}));
|
|
137
|
-
|
|
138
|
-
const results = await openResults(root);
|
|
139
|
-
const a = results.experiments.find((e) => e.id === "mid/a")!;
|
|
140
|
-
const b = results.experiments.find((e) => e.id === "mid/b")!;
|
|
141
|
-
expect(a.latest.startedAt).toBe("2026-07-01T08:00:00.000Z");
|
|
142
|
-
expect(b.latest.startedAt).toBe("2026-06-20T08:00:00.000Z");
|
|
143
|
-
expect(b.latest.knownEvalIds).toEqual(["q1", "q2", "q3"]);
|
|
144
|
-
expect(b.evalIds).toEqual(["q1", "q2", "q3"]); // 本地覆盖 ∪ 携带的 knownEvalIds
|
|
145
|
-
expect(a.evalIds).toEqual(["q1"]);
|
|
148
|
+
// 第二个 attempt 没写 startedAt,补快照的。
|
|
149
|
+
expect(snap.evals[0].attempts[1].result.startedAt).toBe("2026-07-01T08:00:00.000Z");
|
|
146
150
|
});
|
|
147
151
|
});
|
|
148
152
|
|
|
149
153
|
// ───────────────────────── 懒加载与回退 ─────────────────────────
|
|
150
154
|
|
|
151
155
|
describe("AttemptHandle · 懒加载", () => {
|
|
152
|
-
it("缺文件返回 null
|
|
153
|
-
const root = await makeRoot();
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
156
|
+
it("缺文件返回 null;读一次记忆化;attempt 目录优先、artifactBase 回退;原快照清理后如实 null", async () => {
|
|
157
|
+
const root = await makeRoot();
|
|
158
|
+
const oldSnap = await writeSnapshot(
|
|
159
|
+
root,
|
|
160
|
+
"e",
|
|
161
|
+
"2026-06-30T08-00-00-000Z-xxxx",
|
|
162
|
+
meta({ experimentId: "e", agent: "bub", startedAt: "2026-06-30T08:00:00.000Z", completedAt: "2026-06-30T08:10:00.000Z" }),
|
|
163
|
+
);
|
|
164
|
+
await writeResultFile(oldSnap, "q3/a1", record({ id: "q3", attempt: 1, hasEvents: true }));
|
|
165
|
+
await writeArtifactFile(oldSnap, "q3/a1", "events.json", [{ type: "message", text: "old" }]);
|
|
166
|
+
|
|
167
|
+
const newSnap = await writeSnapshot(
|
|
168
|
+
root,
|
|
169
|
+
"e",
|
|
170
|
+
"2026-07-01T08-00-00-000Z-yyyy",
|
|
171
|
+
meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
|
|
172
|
+
);
|
|
173
|
+
await writeResultFile(newSnap, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true }));
|
|
174
|
+
const eventsPath = await writeArtifactFile(newSnap, "q1/a1", "events.json", [{ type: "message", text: "hi" }]);
|
|
175
|
+
await writeResultFile(newSnap, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
176
|
+
await writeResultFile(
|
|
177
|
+
newSnap,
|
|
178
|
+
"q3/a1",
|
|
179
|
+
record({
|
|
180
|
+
id: "q3",
|
|
181
|
+
attempt: 1,
|
|
182
|
+
startedAt: "2026-06-30T08:01:00.000Z",
|
|
183
|
+
artifactBase: "e/2026-06-30T08-00-00-000Z-xxxx/q3/a1",
|
|
184
|
+
hasEvents: true,
|
|
185
|
+
}),
|
|
186
|
+
);
|
|
167
187
|
|
|
168
188
|
const results = await openResults(root);
|
|
169
189
|
const snap = results.experiments[0].latest;
|
|
170
|
-
const
|
|
190
|
+
const q1 = snap.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
191
|
+
const q2 = snap.evals.find((e) => e.id === "q2")!.attempts[0];
|
|
192
|
+
const q3 = snap.evals.find((e) => e.id === "q3")!.attempts[0];
|
|
171
193
|
|
|
172
194
|
const events = await q1.events();
|
|
173
|
-
expect(events).
|
|
174
|
-
//
|
|
195
|
+
expect(events).toEqual([{ type: "message", text: "hi" }]);
|
|
196
|
+
// result.json 只有 hasEvents/hasTrace/hasSources 三个标记,o11y/diff 没有标记 —— 全靠方法语义吸收。
|
|
175
197
|
expect(await q1.trace()).toBeNull();
|
|
176
198
|
expect(await q1.o11y()).toBeNull();
|
|
177
199
|
expect(await q1.diff()).toBeNull();
|
|
178
200
|
expect(await q1.sources()).toBeNull();
|
|
179
201
|
|
|
180
|
-
// 记忆化:同一 handle 读一次缓存,文件删掉后再读仍返回同一份数据。
|
|
181
202
|
await rm(eventsPath);
|
|
182
|
-
expect(await q1.events()).toBe(events);
|
|
203
|
+
expect(await q1.events()).toBe(events); // 记忆化:同一 handle 不重新读盘
|
|
183
204
|
|
|
184
|
-
|
|
185
|
-
expect(await q2.events()).toBeNull();
|
|
205
|
+
expect(await q2.events()).toBeNull(); // 无 artifactBase,不猜路径
|
|
186
206
|
|
|
187
|
-
|
|
188
|
-
expect(
|
|
189
|
-
expect(q3.ref.run).toBe("2026-07-01T08-00-00-000Z");
|
|
207
|
+
expect(await q3.events()).toEqual([{ type: "message", text: "old" }]); // artifactBase 回退到原快照
|
|
208
|
+
expect(q3.ref.snapshot).toBe("e/2026-07-01T08-00-00-000Z-yyyy"); // ref 指条目所在的落盘(新快照)
|
|
190
209
|
|
|
191
|
-
|
|
192
|
-
await rm(oldRun, { recursive: true });
|
|
210
|
+
await rm(oldSnap, { recursive: true });
|
|
193
211
|
const reopened = await openResults(root);
|
|
194
212
|
const q3Again = reopened.experiments[0].latest.evals.find((e) => e.id === "q3")!.attempts[0];
|
|
195
|
-
expect(await q3Again.events()).toBeNull();
|
|
213
|
+
expect(await q3Again.events()).toBeNull(); // 原快照清理后如实返回 null(新句柄,不吃上面的记忆化)
|
|
196
214
|
});
|
|
197
215
|
});
|
|
198
216
|
|
|
199
217
|
// ───────────────────────── skipped 三种原因 ─────────────────────────
|
|
200
218
|
|
|
201
219
|
describe("openResults · skipped", () => {
|
|
202
|
-
it("
|
|
203
|
-
const root = await makeRoot();
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
220
|
+
it("incompatible(v3 summary.json / 无信封 legacy)、malformed(坏 JSON)、incomplete(有 attempt 无 snapshot.json);无关 JSON 静默", async () => {
|
|
221
|
+
const root = await makeRoot();
|
|
222
|
+
|
|
223
|
+
// v2/v3 的 summary.json 带 format + schemaVersion(≠ 4),自然落进 incompatible 档。
|
|
224
|
+
const v3Dir = join(root, "old-exp", "2026-06-01T08-00-00-000Z");
|
|
225
|
+
await mkdir(v3Dir, { recursive: true });
|
|
226
|
+
await writeFile(
|
|
227
|
+
join(v3Dir, "summary.json"),
|
|
228
|
+
JSON.stringify({
|
|
229
|
+
format: RESULTS_FORMAT,
|
|
230
|
+
schemaVersion: 3,
|
|
231
|
+
producer: { name: "niceeval", version: "0.4.6" },
|
|
232
|
+
agent: "bub",
|
|
233
|
+
startedAt: "2026-06-01T08:00:00.000Z",
|
|
234
|
+
completedAt: "2026-06-01T08:10:00.000Z",
|
|
235
|
+
passed: 1,
|
|
236
|
+
failed: 0,
|
|
237
|
+
skipped: 0,
|
|
238
|
+
errored: 0,
|
|
239
|
+
durationMs: 1000,
|
|
240
|
+
results: [],
|
|
241
|
+
}),
|
|
242
|
+
"utf-8",
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
// legacy:引入版本信封之前的存量报告,无 format,按 schemaVersion 1 读。
|
|
246
|
+
const legacyDir = join(root, "legacy-exp", "2026-05-01T08-00-00-000Z");
|
|
247
|
+
await mkdir(legacyDir, { recursive: true });
|
|
248
|
+
await writeFile(
|
|
249
|
+
join(legacyDir, "summary.json"),
|
|
250
|
+
JSON.stringify({ agent: "bub", startedAt: "2026-05-01T08:00:00.000Z", completedAt: "2026-05-01T08:10:00.000Z", results: [] }),
|
|
251
|
+
"utf-8",
|
|
252
|
+
);
|
|
253
|
+
|
|
254
|
+
// malformed:坏 JSON。
|
|
255
|
+
const badDir = join(root, "bad-exp", "2026-07-02T08-00-00-000Z-zzzz");
|
|
256
|
+
await mkdir(badDir, { recursive: true });
|
|
257
|
+
await writeFile(join(badDir, "snapshot.json"), "not json {", "utf-8");
|
|
258
|
+
|
|
259
|
+
// incomplete:有 result.json,没有 snapshot.json —— crash 没收尾。
|
|
260
|
+
const crashDir = join(root, "crash-exp", "2026-07-03T08-00-00-000Z-wwww");
|
|
261
|
+
await writeResultFile(crashDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
262
|
+
|
|
263
|
+
// 直接存在 summary.json 但内容与 niceeval 无关(不满足 legacy 启发式)→ not-a-report,静默忽略,
|
|
264
|
+
// 且不连累父目录被判 incomplete。
|
|
265
|
+
const alienDir = join(root, "alien-exp", "2026-07-06T08-00-00-000Z-alien");
|
|
266
|
+
await mkdir(alienDir, { recursive: true });
|
|
267
|
+
await writeFile(join(alienDir, "summary.json"), JSON.stringify({ hello: 1 }), "utf-8");
|
|
268
|
+
|
|
269
|
+
// 完全无关的空目录:静默忽略。
|
|
270
|
+
await mkdir(join(root, "unrelated"), { recursive: true });
|
|
271
|
+
await writeFile(join(root, "unrelated", "hello.json"), JSON.stringify({ hello: 1 }), "utf-8");
|
|
272
|
+
|
|
273
|
+
// 一份正常快照,确认不受干扰。
|
|
274
|
+
const okDir = await writeSnapshot(root, "ok-exp", "2026-07-04T08-00-00-000Z-oooo", meta({ experimentId: "ok", agent: "bub", startedAt: "2026-07-04T08:00:00.000Z" }));
|
|
275
|
+
await writeResultFile(okDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
226
276
|
|
|
227
277
|
const results = await openResults(root);
|
|
228
|
-
expect(results.
|
|
229
|
-
expect(results.skipped).toHaveLength(
|
|
278
|
+
expect(results.experiments.map((e) => e.id)).toEqual(["ok"]);
|
|
279
|
+
expect(results.skipped).toHaveLength(4);
|
|
230
280
|
|
|
231
|
-
const
|
|
232
|
-
expect(
|
|
233
|
-
expect(
|
|
234
|
-
|
|
235
|
-
expect(versionSkip.producer).toEqual({ name: "other-harness", version: "9.9.9" });
|
|
281
|
+
const v3Skip = results.skipped.find((s) => s.dir === v3Dir)!;
|
|
282
|
+
expect(v3Skip.reason).toBe("incompatible-version");
|
|
283
|
+
expect(v3Skip.schemaVersion).toBe(3);
|
|
284
|
+
expect(v3Skip.producer).toEqual({ name: "niceeval", version: "0.4.6" });
|
|
236
285
|
|
|
237
|
-
|
|
238
|
-
expect(
|
|
286
|
+
const legacySkip = results.skipped.find((s) => s.dir === legacyDir)!;
|
|
287
|
+
expect(legacySkip.reason).toBe("incompatible-version");
|
|
288
|
+
expect(legacySkip.schemaVersion).toBe(1);
|
|
239
289
|
|
|
240
|
-
|
|
241
|
-
expect(
|
|
242
|
-
expect(
|
|
290
|
+
expect(results.skipped.find((s) => s.dir === badDir)!.reason).toBe("malformed");
|
|
291
|
+
expect(results.skipped.find((s) => s.dir === crashDir)!.reason).toBe("incomplete");
|
|
292
|
+
expect(results.skipped.find((s) => s.dir === alienDir)).toBeUndefined();
|
|
243
293
|
});
|
|
244
294
|
});
|
|
245
295
|
|
|
246
|
-
// ───────────────────────── latest()
|
|
296
|
+
// ───────────────────────── latest() Selection 与警告 ─────────────────────────
|
|
247
297
|
|
|
248
298
|
describe("results.latest() · Selection", () => {
|
|
249
299
|
it("每个实验取最新快照;experiments 前缀过滤同 CLI 语义(尾斜杠等价)", async () => {
|
|
250
300
|
const root = await makeRoot();
|
|
251
|
-
await
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
], { startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
255
|
-
const tuesday = await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
|
|
256
|
-
res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
|
|
257
|
-
], { startedAt: "2026-07-02T08:00:00.000Z" }));
|
|
301
|
+
await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
302
|
+
await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
303
|
+
const tuesday = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
258
304
|
|
|
259
305
|
const results = await openResults(root);
|
|
260
306
|
const latest = results.latest();
|
|
261
307
|
expect(latest.snapshots.map((s) => s.experimentId)).toEqual(["mid/a", "mid/b"]);
|
|
262
|
-
expect(latest.snapshots[1].
|
|
308
|
+
expect(latest.snapshots[1].dir).toBe(tuesday);
|
|
263
309
|
|
|
264
310
|
expect(results.latest({ experiments: "mid/a" }).snapshots).toHaveLength(1);
|
|
265
311
|
expect(results.latest({ experiments: "mid/" }).snapshots).toHaveLength(2);
|
|
@@ -270,14 +316,13 @@ describe("results.latest() · Selection", () => {
|
|
|
270
316
|
|
|
271
317
|
it("partial-coverage:最新快照覆盖 < 已知并集;结构化字段 + 渲染好的英文 message", async () => {
|
|
272
318
|
const root = await makeRoot();
|
|
273
|
-
await
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
await
|
|
279
|
-
|
|
280
|
-
], { startedAt: "2026-07-05T08:00:00.000Z" }));
|
|
319
|
+
const mondayDir = await writeSnapshot(root, "midterm", "2026-07-01T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
320
|
+
await writeResultFile(mondayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
|
|
321
|
+
await writeResultFile(mondayDir, "algebra/q2/a1", record({ id: "algebra/q2", attempt: 1 }));
|
|
322
|
+
await writeResultFile(mondayDir, "algebra/q3/a1", record({ id: "algebra/q3", attempt: 1 }));
|
|
323
|
+
|
|
324
|
+
const fridayDir = await writeSnapshot(root, "midterm", "2026-07-05T08-00-00-000Z", meta({ experimentId: "midterm/bub-gpt-5.4", agent: "bub", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }));
|
|
325
|
+
await writeResultFile(fridayDir, "algebra/q1/a1", record({ id: "algebra/q1", attempt: 1 }));
|
|
281
326
|
|
|
282
327
|
const latest = (await openResults(root)).latest();
|
|
283
328
|
expect(latest.snapshots).toHaveLength(1);
|
|
@@ -288,56 +333,41 @@ describe("results.latest() · Selection", () => {
|
|
|
288
333
|
);
|
|
289
334
|
});
|
|
290
335
|
|
|
291
|
-
it("stale-snapshot
|
|
336
|
+
it("stale-snapshot:早于 Selection 中最新落盘即触发(无阈值),message 带人话时距", async () => {
|
|
292
337
|
const root = await makeRoot();
|
|
293
|
-
await
|
|
294
|
-
|
|
295
|
-
], { startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
296
|
-
await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
|
|
297
|
-
res({ id: "q1", agent: "codex", experimentId: "mid/b" }),
|
|
298
|
-
], { startedAt: "2026-07-05T08:00:00.000Z" }));
|
|
338
|
+
await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
339
|
+
await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }));
|
|
299
340
|
|
|
300
341
|
const latest = (await openResults(root)).latest();
|
|
301
342
|
const stale = latest.warnings.filter((w) => w.kind === "stale-snapshot");
|
|
302
343
|
expect(stale).toHaveLength(1);
|
|
303
|
-
expect(stale[0]).toMatchObject({
|
|
304
|
-
experimentId: "mid/a",
|
|
305
|
-
startedAt: "2026-07-01T08:00:00.000Z",
|
|
306
|
-
latestStartedAt: "2026-07-05T08:00:00.000Z",
|
|
307
|
-
});
|
|
344
|
+
expect(stale[0]).toMatchObject({ experimentId: "mid/a", startedAt: "2026-07-01T08:00:00.000Z", latestStartedAt: "2026-07-05T08:00:00.000Z" });
|
|
308
345
|
expect(stale[0].message).toContain("predates the latest run in this selection by 4 days");
|
|
309
346
|
});
|
|
310
347
|
|
|
311
|
-
it("
|
|
348
|
+
it("unfinished-snapshot:选中快照缺 completedAt", async () => {
|
|
312
349
|
const root = await makeRoot();
|
|
313
|
-
const
|
|
314
|
-
res({ id: "q1", agent: "codex" }),
|
|
315
|
-
]));
|
|
350
|
+
const dir = await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
316
351
|
|
|
317
|
-
const
|
|
318
|
-
|
|
319
|
-
expect(
|
|
320
|
-
|
|
321
|
-
const latest = results.latest();
|
|
322
|
-
const synthetic = latest.warnings.find((w) => w.kind === "synthetic-experiment-id")!;
|
|
323
|
-
expect(synthetic).toMatchObject({ experimentId: "codex/default", runDir });
|
|
324
|
-
expect(synthetic.message).toContain("without experimentId");
|
|
325
|
-
// 合成键拼不出可执行的 niceeval exp 命令:partial 提示退化成中性说法(此处无 partial,仅验证不炸)。
|
|
352
|
+
const latest = (await openResults(root)).latest();
|
|
353
|
+
const warn = latest.warnings.find((w) => w.kind === "unfinished-snapshot")!;
|
|
354
|
+
expect(warn).toMatchObject({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir });
|
|
355
|
+
expect(warn.message).toContain("has no completedAt");
|
|
326
356
|
});
|
|
327
357
|
|
|
328
358
|
it("Selection.filter 只删不换:快照删减,幸存实验的警告保留、其余丢弃", async () => {
|
|
329
359
|
const root = await makeRoot();
|
|
330
|
-
|
|
331
|
-
await
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
await
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
360
|
+
const aDir = await writeSnapshot(root, "mid_a", "s1", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
361
|
+
await writeResultFile(aDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
362
|
+
await writeResultFile(aDir, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
363
|
+
const bDir = await writeSnapshot(root, "mid_b", "s1", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
364
|
+
await writeResultFile(bDir, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
365
|
+
await writeResultFile(bDir, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
366
|
+
|
|
367
|
+
const aDir2 = await writeSnapshot(root, "mid_a", "s2", meta({ experimentId: "mid/a", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
368
|
+
await writeResultFile(aDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
369
|
+
const bDir2 = await writeSnapshot(root, "mid_b", "s2", meta({ experimentId: "mid/b", agent: "codex", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
370
|
+
await writeResultFile(bDir2, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
341
371
|
|
|
342
372
|
const latest = (await openResults(root)).latest();
|
|
343
373
|
expect(latest.warnings.filter((w) => w.kind === "partial-coverage")).toHaveLength(2);
|
|
@@ -345,7 +375,7 @@ describe("results.latest() · Selection", () => {
|
|
|
345
375
|
const filtered = latest.filter((s) => s.experimentId !== "mid/b");
|
|
346
376
|
expect(filtered.snapshots.map((s) => s.experimentId)).toEqual(["mid/a"]);
|
|
347
377
|
expect(filtered.warnings.map((w) => w.experimentId)).toEqual(["mid/a"]);
|
|
348
|
-
//
|
|
378
|
+
// 原 Selection 不被改动。
|
|
349
379
|
expect(latest.snapshots).toHaveLength(2);
|
|
350
380
|
expect(latest.warnings).toHaveLength(2);
|
|
351
381
|
});
|
|
@@ -353,30 +383,50 @@ describe("results.latest() · Selection", () => {
|
|
|
353
383
|
|
|
354
384
|
// ───────────────────────── 身份键去重 ─────────────────────────
|
|
355
385
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
res({ id: "q2", agent: "bub", experimentId: "e", startedAt: "2026-07-02T08:02:00.000Z" }), // 重跑,新 startedAt
|
|
367
|
-
res({ id: "q3", agent: "bub", experimentId: "e" }),
|
|
368
|
-
], { startedAt: "2026-07-02T08:00:00.000Z" }));
|
|
386
|
+
function fakeSnapshot(over: { experimentId: string; startedAt: string; dir: string }): Snapshot {
|
|
387
|
+
return {
|
|
388
|
+
agent: "bub",
|
|
389
|
+
producer: { name: "niceeval" },
|
|
390
|
+
schemaVersion: RESULTS_SCHEMA_VERSION,
|
|
391
|
+
evals: [],
|
|
392
|
+
attempts: [],
|
|
393
|
+
...over,
|
|
394
|
+
};
|
|
395
|
+
}
|
|
369
396
|
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
397
|
+
function fakeAttempt(snapshot: Snapshot, result: EvalResult): AttemptHandle {
|
|
398
|
+
return {
|
|
399
|
+
evalId: result.id,
|
|
400
|
+
experimentId: snapshot.experimentId,
|
|
401
|
+
result,
|
|
402
|
+
ref: { snapshot: "x/y", attempt: `${result.id}/a${result.attempt}` },
|
|
403
|
+
snapshot,
|
|
404
|
+
events: async () => null,
|
|
405
|
+
trace: async () => null,
|
|
406
|
+
o11y: async () => null,
|
|
407
|
+
agentSetup: async () => null,
|
|
408
|
+
diff: async () => null,
|
|
409
|
+
sources: async () => null,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
373
412
|
|
|
374
|
-
|
|
375
|
-
|
|
413
|
+
describe("dedupeAttempts", () => {
|
|
414
|
+
it("按 (experimentId, evalId, attempt, startedAt) 去重,保留最新快照里的那份;缺 startedAt 不去重并出 missing-startedAt", () => {
|
|
415
|
+
const monday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir: "/tmp/e/monday" });
|
|
416
|
+
const tuesday = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-02T08:00:00.000Z", dir: "/tmp/e/tuesday" });
|
|
417
|
+
|
|
418
|
+
const a1 = fakeAttempt(monday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" }));
|
|
419
|
+
const a1Resumed = fakeAttempt(tuesday, record({ id: "q1", attempt: 1, startedAt: "2026-07-01T08:01:00.000Z" })); // resume 原样合入
|
|
420
|
+
const a2Mon = fakeAttempt(monday, record({ id: "q2", attempt: 1, startedAt: "2026-07-01T08:02:00.000Z" }));
|
|
421
|
+
const a2Tue = fakeAttempt(tuesday, record({ id: "q2", attempt: 1, startedAt: "2026-07-02T08:02:00.000Z" })); // 重跑,新 startedAt
|
|
422
|
+
const a3Mon = fakeAttempt(monday, record({ id: "q3", attempt: 1 })); // 缺 startedAt(携带条目缺锚的极端情况)
|
|
423
|
+
const a3Tue = fakeAttempt(tuesday, record({ id: "q3", attempt: 1 }));
|
|
424
|
+
|
|
425
|
+
const { attempts, warnings } = dedupeAttempts([a1, a2Mon, a3Mon, a1Resumed, a2Tue, a3Tue]);
|
|
376
426
|
expect(attempts).toHaveLength(5);
|
|
377
427
|
const q1 = attempts.filter((a) => a.evalId === "q1");
|
|
378
428
|
expect(q1).toHaveLength(1);
|
|
379
|
-
expect(q1[0].
|
|
429
|
+
expect(q1[0].snapshot).toBe(tuesday); // 保留最新快照
|
|
380
430
|
expect(attempts.filter((a) => a.evalId === "q2")).toHaveLength(2);
|
|
381
431
|
expect(attempts.filter((a) => a.evalId === "q3")).toHaveLength(2);
|
|
382
432
|
|
|
@@ -386,53 +436,44 @@ describe("dedupeAttempts", () => {
|
|
|
386
436
|
});
|
|
387
437
|
});
|
|
388
438
|
|
|
389
|
-
// ───────────────────────── writer
|
|
439
|
+
// ───────────────────────── writer ─────────────────────────
|
|
390
440
|
|
|
391
|
-
describe("
|
|
392
|
-
it("
|
|
441
|
+
describe("createResultsWriter", () => {
|
|
442
|
+
it("snapshot() 建目录(独占)+ 写 snapshot.json(无 completedAt);writeAttempt 拆 artifact + 回填 has*;finish 补 completedAt 并幂等", async () => {
|
|
393
443
|
const root = await makeRoot();
|
|
394
|
-
const writer =
|
|
395
|
-
expect(writer.dir.startsWith(root)).toBe(true);
|
|
444
|
+
const writer = createResultsWriter(root, { producer: { name: "my-harness", version: "1.0.0" } });
|
|
396
445
|
|
|
397
|
-
const snapA = writer.snapshot({
|
|
398
|
-
|
|
446
|
+
const snapA = await writer.snapshot({
|
|
447
|
+
experimentId: "compare/a",
|
|
399
448
|
agent: "bub",
|
|
400
449
|
model: "gpt-5",
|
|
401
450
|
startedAt: "2026-07-01T08:00:00.000Z",
|
|
402
451
|
knownEvalIds: ["q1", "q2", "q3"],
|
|
403
452
|
});
|
|
404
|
-
|
|
405
|
-
const
|
|
453
|
+
expect(snapA.dir.startsWith(root)).toBe(true);
|
|
454
|
+
const before = JSON.parse(await readFile(join(snapA.dir, "snapshot.json"), "utf-8"));
|
|
455
|
+
expect(before.completedAt).toBeUndefined();
|
|
456
|
+
expect(Object.keys(before)).toEqual(["format", "schemaVersion", "producer", "experimentId", "agent", "model", "startedAt", "knownEvalIds"]);
|
|
457
|
+
|
|
458
|
+
const events = [{ type: "message", text: "hi" }] as never[];
|
|
459
|
+
const o11yData = { toolCalls: 2 } as never;
|
|
406
460
|
await snapA.writeAttempt(
|
|
407
|
-
{
|
|
408
|
-
|
|
409
|
-
outcome: "passed",
|
|
410
|
-
attempt: 1,
|
|
411
|
-
durationMs: 100,
|
|
412
|
-
assertions: [],
|
|
413
|
-
usage: { inputTokens: 10, outputTokens: 5 },
|
|
414
|
-
estimatedCostUSD: 0.25,
|
|
415
|
-
},
|
|
416
|
-
{ events, o11y },
|
|
461
|
+
{ id: "q1", verdict: "passed", attempt: 1, durationMs: 100, assertions: [], usage: { inputTokens: 10, outputTokens: 5 }, estimatedCostUSD: 0.25 },
|
|
462
|
+
{ events, o11y: o11yData },
|
|
417
463
|
);
|
|
418
|
-
await snapA.writeAttempt({ id: "q2",
|
|
464
|
+
await snapA.writeAttempt({ id: "q2", verdict: "failed", attempt: 1, durationMs: 50, assertions: [] });
|
|
419
465
|
|
|
420
|
-
const snapB = writer.snapshot({
|
|
421
|
-
await snapB.writeAttempt(
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
);
|
|
466
|
+
const snapB = await writer.snapshot({ experimentId: "compare/b", agent: "codex", startedAt: "2026-07-02T09:00:00.000Z" });
|
|
467
|
+
await snapB.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 80, assertions: [] }, { diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] } });
|
|
468
|
+
|
|
469
|
+
await writer.finish();
|
|
470
|
+
expect(writer.snapshotDirs().map((s) => s.experimentId).sort()).toEqual(["compare/a", "compare/b"]);
|
|
425
471
|
|
|
426
|
-
const
|
|
427
|
-
|
|
428
|
-
expect(
|
|
429
|
-
|
|
430
|
-
expect(
|
|
431
|
-
expect(summary.passed).toBe(2);
|
|
432
|
-
expect(summary.failed).toBe(1);
|
|
433
|
-
expect(summary.startedAt).toBe("2026-07-01T08:00:00.000Z"); // 最早的快照时刻
|
|
434
|
-
expect(summary.usage).toEqual({ inputTokens: 10, outputTokens: 5 });
|
|
435
|
-
expect(summary.estimatedCostUSD).toBe(0.25);
|
|
472
|
+
const after = JSON.parse(await readFile(join(snapA.dir, "snapshot.json"), "utf-8"));
|
|
473
|
+
expect(typeof after.completedAt).toBe("string");
|
|
474
|
+
expect(Object.keys(after)).toEqual(["format", "schemaVersion", "producer", "experimentId", "agent", "model", "startedAt", "completedAt", "knownEvalIds"]);
|
|
475
|
+
|
|
476
|
+
await expect(writer.finish()).rejects.toThrow(/already called/);
|
|
436
477
|
|
|
437
478
|
const results = await openResults(root);
|
|
438
479
|
expect(results.skipped).toHaveLength(0);
|
|
@@ -441,22 +482,17 @@ describe("createRunWriter", () => {
|
|
|
441
482
|
const a = results.experiments[0].latest;
|
|
442
483
|
expect(a.agent).toBe("bub");
|
|
443
484
|
expect(a.model).toBe("gpt-5");
|
|
444
|
-
expect(a.startedAt).toBe("2026-07-01T08:00:00.000Z");
|
|
445
|
-
expect(a.producer).toEqual({ name: "my-harness", version: "1.0.0" });
|
|
446
|
-
expect(a.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
|
|
447
485
|
expect(a.knownEvalIds).toEqual(["q1", "q2", "q3"]);
|
|
448
486
|
expect(results.experiments[0].evalIds).toEqual(["q1", "q2", "q3"]);
|
|
449
487
|
|
|
450
|
-
// 快照级字段注入进条目(agent/model/experimentId/startedAt),attempt 级字段原样读回。
|
|
451
488
|
const q1 = a.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
452
|
-
expect(q1.experimentId).toBe("compare/a");
|
|
453
489
|
expect(q1.result).toMatchObject({
|
|
454
490
|
id: "q1",
|
|
455
491
|
agent: "bub",
|
|
456
492
|
model: "gpt-5",
|
|
457
493
|
experimentId: "compare/a",
|
|
458
494
|
startedAt: "2026-07-01T08:00:00.000Z",
|
|
459
|
-
|
|
495
|
+
verdict: "passed",
|
|
460
496
|
durationMs: 100,
|
|
461
497
|
usage: { inputTokens: 10, outputTokens: 5 },
|
|
462
498
|
estimatedCostUSD: 0.25,
|
|
@@ -465,137 +501,122 @@ describe("createRunWriter", () => {
|
|
|
465
501
|
hasSources: false,
|
|
466
502
|
});
|
|
467
503
|
expect(await q1.events()).toEqual(events);
|
|
468
|
-
expect(await q1.o11y()).toEqual(
|
|
504
|
+
expect(await q1.o11y()).toEqual(o11yData);
|
|
469
505
|
expect(await q1.trace()).toBeNull();
|
|
470
|
-
expect(await q1.diff()).toBeNull();
|
|
471
506
|
|
|
472
|
-
// 第二个快照:自己的 startedAt(≠ 顶层)经快照元数据读回;diff 工件可达。
|
|
473
507
|
const b = results.experiments[1].latest;
|
|
474
|
-
expect(b.agent).toBe("codex");
|
|
475
508
|
expect(b.model).toBeUndefined();
|
|
476
|
-
expect(b.startedAt).toBe("2026-07-02T09:00:00.000Z");
|
|
477
509
|
expect(await b.attempts[0].diff()).toEqual({ generatedFiles: { "a.txt": "1" }, deletedFiles: [] });
|
|
478
510
|
|
|
479
|
-
// knownEvalIds 是残缺检测的分母:compare/a 只写了 2/3 → partial-coverage。
|
|
480
511
|
const partial = results.latest().warnings.find((w) => w.kind === "partial-coverage")!;
|
|
481
512
|
expect(partial).toMatchObject({ experimentId: "compare/a", covered: 2, total: 3 });
|
|
482
513
|
});
|
|
483
514
|
|
|
484
|
-
it("
|
|
515
|
+
it("agentSetup:落成 agent-setup.json(不内联进 result.json),懒加载读回;没装扩展的 attempt 恒 null;copySnapshots 能带上", async () => {
|
|
485
516
|
const root = await makeRoot();
|
|
486
|
-
const writer =
|
|
487
|
-
const
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
517
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "0.12.0" } });
|
|
518
|
+
const manifest = {
|
|
519
|
+
skills: [{ kind: "repo" as const, source: "Effect-TS/skills", ref: "8f3c1a2", skills: ["effect"] }],
|
|
520
|
+
nativePlugins: [
|
|
521
|
+
{
|
|
522
|
+
agent: "claude-code" as const,
|
|
523
|
+
marketplace: { name: "acme", source: "acme/claude-code-plugins", ref: "v1.3.0" },
|
|
524
|
+
name: "safe-shell",
|
|
525
|
+
resolvedVersion: "1.3.0",
|
|
526
|
+
},
|
|
527
|
+
],
|
|
528
|
+
mcpServers: [{ name: "browser", command: "npx", args: ["-y", "@modelcontextprotocol/server-browser"] }],
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
const snap = await writer.snapshot({
|
|
532
|
+
experimentId: "skill-ab/claude-effect",
|
|
533
|
+
agent: "claude-code",
|
|
534
|
+
startedAt: "2026-07-11T08:00:00.000Z",
|
|
535
|
+
});
|
|
536
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 10, assertions: [] }, { agentSetup: manifest });
|
|
537
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 1, durationMs: 10, assertions: [] });
|
|
538
|
+
await writer.finish();
|
|
539
|
+
|
|
540
|
+
// 文件名是磁盘侧的 kebab;判决记录里不内联 manifest(它是 artifact,不是判决的一部分)
|
|
541
|
+
const attemptDir = join(snap.dir, "q1", "a1");
|
|
542
|
+
expect(await exists(join(attemptDir, "agent-setup.json"))).toBe(true);
|
|
543
|
+
expect(JSON.parse(await readFile(join(attemptDir, "result.json"), "utf-8")).agentSetup).toBeUndefined();
|
|
544
|
+
expect(await exists(join(snap.dir, "q2", "a1", "agent-setup.json"))).toBe(false);
|
|
545
|
+
|
|
493
546
|
const results = await openResults(root);
|
|
494
|
-
|
|
495
|
-
expect(
|
|
547
|
+
const [q1, q2] = results.experiments[0].latest.attempts;
|
|
548
|
+
expect(await q1.agentSetup()).toEqual(manifest);
|
|
549
|
+
expect(await q2.agentSetup()).toBeNull();
|
|
550
|
+
|
|
551
|
+
const dest = join(await makeRoot(), "published");
|
|
552
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["agentSetup"] });
|
|
553
|
+
const copied = join(dest, "skill-ab_claude-effect", basename(snap.dir), "q1", "a1", "agent-setup.json");
|
|
554
|
+
expect(JSON.parse(await readFile(copied, "utf-8"))).toEqual(manifest);
|
|
496
555
|
});
|
|
497
556
|
|
|
498
|
-
it("snapshot()
|
|
557
|
+
it("snapshot() 缺 experimentId/agent/startedAt 抛可执行错误", async () => {
|
|
499
558
|
const root = await makeRoot();
|
|
500
|
-
const writer =
|
|
501
|
-
expect(
|
|
559
|
+
const writer = createResultsWriter(root, { producer: { name: "x" } });
|
|
560
|
+
await expect(writer.snapshot({ experimentId: "", agent: "a", startedAt: "t" })).rejects.toThrow(/experimentId/);
|
|
561
|
+
await expect(writer.snapshot({ experimentId: "e", agent: "", startedAt: "t" })).rejects.toThrow(/agent/);
|
|
562
|
+
await expect(writer.snapshot({ experimentId: "e", agent: "a", startedAt: "" })).rejects.toThrow(/startedAt/);
|
|
502
563
|
});
|
|
503
|
-
});
|
|
504
564
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
const
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "events.json", [{ n: 1 }]);
|
|
516
|
-
await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "trace.json", [{ name: "turn" }]);
|
|
517
|
-
await writeArtifact(monday, "q1/bub/gpt-5/compare_bub/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
|
|
518
|
-
await writeArtifact(monday, "q1/codex/o3/compare_codex/a1", "events.json", [{ n: 2 }]);
|
|
519
|
-
// 周五只重跑了 compare/bub 的 q1:它的最新快照残缺。
|
|
520
|
-
const friday = await writeRun(root, "2026-07-05T08-00-00-000Z", summaryOf([
|
|
521
|
-
res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "compare/bub", artifactsDir: "q1/bub/gpt-5/compare_bub/a1", hasEvents: true, startedAt: "2026-07-05T08:01:00.000Z" }),
|
|
522
|
-
], { startedAt: "2026-07-05T08:00:00.000Z" }));
|
|
523
|
-
await writeArtifact(friday, "q1/bub/gpt-5/compare_bub/a1", "events.json", [{ n: 1 }, { n: 2 }]);
|
|
524
|
-
|
|
525
|
-
const results = await openResults(root);
|
|
526
|
-
const dest = join(await makeRoot(), "site/data/run");
|
|
527
|
-
const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events", "sources"] });
|
|
528
|
-
|
|
529
|
-
expect(copied.warnings).toHaveLength(0);
|
|
530
|
-
expect(copied.summary.format).toBe(RESULTS_FORMAT);
|
|
531
|
-
expect(copied.summary.schemaVersion).toBe(RESULTS_SCHEMA_VERSION);
|
|
532
|
-
expect(copied.summary.producer?.version).toBe("0.3.0");
|
|
533
|
-
expect(copied.summary.results).toHaveLength(2);
|
|
534
|
-
// 补记的覆盖事实:复制时刻该实验已知的 eval 并集。
|
|
535
|
-
expect(copied.summary.snapshots?.["compare/bub"].knownEvalIds).toEqual(["q1", "q2"]);
|
|
536
|
-
expect(copied.summary.snapshots?.["compare/codex"].knownEvalIds).toEqual(["q1"]);
|
|
537
|
-
|
|
538
|
-
// 磁盘:只有选中的工件种类被复制;存在标记按目标目录重算。
|
|
539
|
-
expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/events.json"))).toBe(true);
|
|
540
|
-
expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/trace.json"))).toBe(false);
|
|
541
|
-
expect(await exists(join(dest, "q1/bub/gpt-5/compare_bub/a1/diff.json"))).toBe(false);
|
|
542
|
-
const bubEntry = copied.summary.results.find((r) => r.experimentId === "compare/bub")!;
|
|
543
|
-
expect(bubEntry.startedAt).toBe("2026-07-05T08:01:00.000Z"); // 最新快照的那份
|
|
544
|
-
expect(bubEntry.hasEvents).toBe(true);
|
|
545
|
-
expect(bubEntry.hasTrace).toBe(false);
|
|
565
|
+
it("同一 writer 内同 experimentId 重复声明:复用同一个 SnapshotWriter,knownEvalIds 取并集", async () => {
|
|
566
|
+
const root = await makeRoot();
|
|
567
|
+
const writer = createResultsWriter(root, { producer: { name: "x" } });
|
|
568
|
+
const s1 = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] });
|
|
569
|
+
const s2 = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q2", "q3"] });
|
|
570
|
+
expect(s2).toBe(s1);
|
|
571
|
+
await writer.finish();
|
|
572
|
+
const written = JSON.parse(await readFile(join(s1.dir, "snapshot.json"), "utf-8"));
|
|
573
|
+
expect(written.knownEvalIds).toEqual(["q1", "q2", "q3"]);
|
|
574
|
+
});
|
|
546
575
|
|
|
547
|
-
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
const
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
expect(await bubAttempt.trace()).toBeNull();
|
|
559
|
-
});
|
|
560
|
-
|
|
561
|
-
it("目标目录非空即报错(不静默覆盖、不合并);artifacts 非法值报错;同实验重复快照落同一目录出 warning", async () => {
|
|
562
|
-
const root = await makeRoot();
|
|
563
|
-
await writeRun(root, "2026-07-01T08-00-00-000Z", summaryOf([
|
|
564
|
-
res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/e/a1", startedAt: "2026-07-01T08:01:00.000Z" }),
|
|
565
|
-
], { startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
566
|
-
await writeRun(root, "2026-07-02T08-00-00-000Z", summaryOf([
|
|
567
|
-
res({ id: "q1", agent: "bub", model: "gpt-5", experimentId: "e", artifactsDir: "q1/bub/gpt-5/e/a1", startedAt: "2026-07-01T08:01:00.000Z" }),
|
|
568
|
-
], { startedAt: "2026-07-02T08:00:00.000Z" }));
|
|
576
|
+
it("同一毫秒并发声明不同 experimentId 的快照:互不干扰,各自独立目录", async () => {
|
|
577
|
+
const root = await makeRoot();
|
|
578
|
+
const writer = createResultsWriter(root, { producer: { name: "x" } });
|
|
579
|
+
const now = "2026-07-01T08:00:00.000Z";
|
|
580
|
+
const [a, b, c] = await Promise.all([
|
|
581
|
+
writer.snapshot({ experimentId: "e/a", agent: "bub", startedAt: now }),
|
|
582
|
+
writer.snapshot({ experimentId: "e/b", agent: "bub", startedAt: now }),
|
|
583
|
+
writer.snapshot({ experimentId: "e/c", agent: "bub", startedAt: now }),
|
|
584
|
+
]);
|
|
585
|
+
expect(new Set([a.dir, b.dir, c.dir]).size).toBe(3);
|
|
586
|
+
await writer.finish();
|
|
569
587
|
const results = await openResults(root);
|
|
570
|
-
|
|
571
|
-
const occupied = await makeRoot();
|
|
572
|
-
await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
|
|
573
|
-
await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
|
|
574
|
-
|
|
575
|
-
await expect(
|
|
576
|
-
copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] }),
|
|
577
|
-
).rejects.toThrow(/Unknown artifact kind/);
|
|
578
|
-
|
|
579
|
-
await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
|
|
580
|
-
|
|
581
|
-
// 同一 experiment 的两个快照(未走 latest 的手工数组):同键 attempt 落同一目录 → 保留最新 + warning。
|
|
582
|
-
const both = results.experiments[0].snapshots;
|
|
583
|
-
expect(both).toHaveLength(2);
|
|
584
|
-
const dest2 = join(await makeRoot(), "run2");
|
|
585
|
-
const collided = await copySnapshots(both, dest2);
|
|
586
|
-
expect(collided.warnings).toHaveLength(1);
|
|
587
|
-
expect(collided.warnings[0]).toMatch(/multiple attempts map to/);
|
|
588
|
-
expect(collided.summary.results).toHaveLength(1);
|
|
588
|
+
expect(results.experiments.map((e) => e.id).sort()).toEqual(["e/a", "e/b", "e/c"]);
|
|
589
589
|
});
|
|
590
|
-
});
|
|
591
590
|
|
|
592
|
-
|
|
591
|
+
it("快照目录独占创建:撞名换随机后缀重试直到成功(EEXIST 不会覆盖已有目录)", async () => {
|
|
592
|
+
const root = await makeRoot();
|
|
593
|
+
vi.useFakeTimers();
|
|
594
|
+
vi.setSystemTime(new Date("2026-07-01T08:00:00.000Z"));
|
|
595
|
+
let call = 0;
|
|
596
|
+
const randomSpy = vi.spyOn(Math, "random").mockImplementation(() => {
|
|
597
|
+
call += 1;
|
|
598
|
+
return call <= 4 ? 0 : 0.03; // 第一次 randomSuffix() 产出 "aaaa",第二次产出 "bbbb"
|
|
599
|
+
});
|
|
600
|
+
try {
|
|
601
|
+
const collidingDir = join(root, "e", "2026-07-01T08-00-00-000Z-aaaa");
|
|
602
|
+
await mkdir(collidingDir, { recursive: true });
|
|
603
|
+
|
|
604
|
+
const writer = createResultsWriter(root, { producer: { name: "x" } });
|
|
605
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
606
|
+
expect(snap.dir).not.toBe(collidingDir);
|
|
607
|
+
expect(snap.dir.endsWith("-bbbb")).toBe(true);
|
|
608
|
+
expect(await exists(join(snap.dir, "snapshot.json"))).toBe(true);
|
|
609
|
+
// 被占用的目录没有被写入 snapshot.json —— 独占创建不会覆盖已有内容。
|
|
610
|
+
expect(await exists(join(collidingDir, "snapshot.json"))).toBe(false);
|
|
611
|
+
} finally {
|
|
612
|
+
randomSpy.mockRestore();
|
|
613
|
+
vi.useRealTimers();
|
|
614
|
+
}
|
|
615
|
+
});
|
|
593
616
|
|
|
594
|
-
|
|
595
|
-
it("落盘行为与 runner 直写时代逐字节等价:summary.json 键序/瘦身/携带条目原样,工件文件按需紧凑写", async () => {
|
|
617
|
+
it("writeAttemptFor:按 EvalResult.experimentId 懒建快照;正常条目拆 artifact,携带条目原样保留 startedAt/artifactBase/has*", async () => {
|
|
596
618
|
const root = await makeRoot();
|
|
597
|
-
const
|
|
598
|
-
await rep.onRunStart?.([], {} as never);
|
|
619
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "0.12.0" } });
|
|
599
620
|
|
|
600
621
|
const fresh: EvalResult = {
|
|
601
622
|
id: "algebra/q1",
|
|
@@ -603,12 +624,12 @@ describe("Artifacts reporter(writer 薄壳)", () => {
|
|
|
603
624
|
experiment: { id: "compare/bub", flags: { style: "concise" } },
|
|
604
625
|
agent: "bub",
|
|
605
626
|
model: "gpt-5.4",
|
|
606
|
-
|
|
627
|
+
verdict: "passed",
|
|
607
628
|
fingerprint: "abc",
|
|
608
629
|
attempt: 1,
|
|
609
630
|
startedAt: "2026-07-01T08:01:00.000Z",
|
|
610
631
|
durationMs: 1234,
|
|
611
|
-
assertions: [
|
|
632
|
+
assertions: [],
|
|
612
633
|
usage: { inputTokens: 10, outputTokens: 5 },
|
|
613
634
|
estimatedCostUSD: 0.5,
|
|
614
635
|
events: [{ type: "message", role: "assistant", text: "hi" } as never],
|
|
@@ -618,146 +639,574 @@ describe("Artifacts reporter(writer 薄壳)", () => {
|
|
|
618
639
|
diff: { generatedFiles: { "a.txt": "1" }, deletedFiles: [] },
|
|
619
640
|
rawTranscript: "raw",
|
|
620
641
|
};
|
|
621
|
-
const noArtifacts: EvalResult = {
|
|
622
|
-
id: "algebra/q2",
|
|
623
|
-
agent: "bub",
|
|
624
|
-
outcome: "failed",
|
|
625
|
-
attempt: 1,
|
|
626
|
-
durationMs: 10,
|
|
627
|
-
assertions: [],
|
|
628
|
-
events: [],
|
|
629
|
-
};
|
|
630
|
-
// --resume 携带条目:artifactBase 指向原 run,has* 真值原样携带,不得重算或编造 artifactsDir。
|
|
631
642
|
const carried: EvalResult = {
|
|
632
643
|
id: "algebra/q3",
|
|
633
644
|
experimentId: "compare/bub",
|
|
634
645
|
agent: "bub",
|
|
635
646
|
model: "gpt-5.4",
|
|
636
|
-
|
|
647
|
+
verdict: "passed",
|
|
637
648
|
attempt: 1,
|
|
638
649
|
startedAt: "2026-06-30T08:01:00.000Z",
|
|
639
650
|
durationMs: 99,
|
|
640
651
|
assertions: [],
|
|
641
|
-
artifactBase: "2026-06-30T08-00-00-000Z/algebra/q3/
|
|
652
|
+
artifactBase: "compare_bub/2026-06-30T08-00-00-000Z-xxxx/algebra/q3/a1",
|
|
642
653
|
hasEvents: true,
|
|
643
654
|
hasTrace: false,
|
|
644
655
|
hasSources: true,
|
|
645
656
|
};
|
|
646
657
|
|
|
647
|
-
await
|
|
648
|
-
await
|
|
649
|
-
await
|
|
650
|
-
|
|
658
|
+
await writer.writeAttemptFor(fresh);
|
|
659
|
+
await writer.writeAttemptFor(carried);
|
|
660
|
+
await writer.finish();
|
|
661
|
+
|
|
662
|
+
const dirs = writer.snapshotDirs();
|
|
663
|
+
expect(dirs).toHaveLength(1);
|
|
664
|
+
const snapDir = dirs[0].dir;
|
|
665
|
+
|
|
666
|
+
const freshRecord = JSON.parse(await readFile(join(snapDir, "algebra/q1/a1/result.json"), "utf-8"));
|
|
667
|
+
for (const key of ["agent", "model", "experimentId", "experiment", "events", "sources", "o11y", "trace", "diff", "rawTranscript"]) {
|
|
668
|
+
expect(freshRecord).not.toHaveProperty(key);
|
|
669
|
+
}
|
|
670
|
+
// startedAt 是 attempt 级事实(每条各异,view 靠它显示「何时跑的」),正常条目也原样落盘。
|
|
671
|
+
expect(freshRecord.startedAt).toBe("2026-07-01T08:01:00.000Z");
|
|
672
|
+
expect(freshRecord.hasEvents).toBe(true);
|
|
673
|
+
expect(freshRecord.hasTrace).toBe(true);
|
|
674
|
+
expect(freshRecord.hasSources).toBe(true);
|
|
675
|
+
expect(await readFile(join(snapDir, "algebra/q1/a1/events.json"), "utf-8")).toBe('[{"type":"message","role":"assistant","text":"hi"}]');
|
|
676
|
+
expect(await readFile(join(snapDir, "algebra/q1/a1/o11y.json"), "utf-8")).toBe('{"toolCalls":2}');
|
|
677
|
+
|
|
678
|
+
const carriedRecord = JSON.parse(await readFile(join(snapDir, "algebra/q3/a1/result.json"), "utf-8"));
|
|
679
|
+
expect(carriedRecord.startedAt).toBe("2026-06-30T08:01:00.000Z");
|
|
680
|
+
expect(carriedRecord.artifactBase).toBe("compare_bub/2026-06-30T08-00-00-000Z-xxxx/algebra/q3/a1");
|
|
681
|
+
expect(carriedRecord.hasEvents).toBe(true);
|
|
682
|
+
expect(carriedRecord.hasTrace).toBe(false);
|
|
683
|
+
expect(carriedRecord.hasSources).toBe(true);
|
|
684
|
+
expect(carriedRecord).not.toHaveProperty("agent");
|
|
685
|
+
expect(carriedRecord).not.toHaveProperty("experimentId");
|
|
686
|
+
expect(await exists(join(snapDir, "algebra/q3/a1/events.json"))).toBe(false); // 携带条目不写 artifact 文件
|
|
687
|
+
|
|
688
|
+
const meta = JSON.parse(await readFile(join(snapDir, "snapshot.json"), "utf-8"));
|
|
689
|
+
expect(meta.experimentId).toBe("compare/bub");
|
|
690
|
+
expect(meta.agent).toBe("bub");
|
|
691
|
+
expect(meta.model).toBe("gpt-5.4");
|
|
692
|
+
});
|
|
693
|
+
|
|
694
|
+
it("writeAttemptFor:result.experimentId 缺失时抛可执行错误(v4 布局按实验分目录)", async () => {
|
|
695
|
+
const root = await makeRoot();
|
|
696
|
+
const writer = createResultsWriter(root, { producer: { name: "x" } });
|
|
697
|
+
await expect(
|
|
698
|
+
writer.writeAttemptFor({ id: "q1", agent: "bub", verdict: "passed", attempt: 1, durationMs: 1, assertions: [] }),
|
|
699
|
+
).rejects.toThrow(/experimentId/);
|
|
700
|
+
});
|
|
701
|
+
});
|
|
702
|
+
|
|
703
|
+
// ───────────────────────── copySnapshots ─────────────────────────
|
|
704
|
+
|
|
705
|
+
describe("copySnapshots", () => {
|
|
706
|
+
it("产物是标准结果根目录(快照目录名原样保留);按指定 artifact 复制;补记 knownEvalIds;has* 按目标目录重算", async () => {
|
|
707
|
+
const root = await makeRoot();
|
|
708
|
+
const monday = await writeSnapshot(
|
|
709
|
+
root,
|
|
710
|
+
"compare_bub",
|
|
711
|
+
"2026-07-01T08-00-00-000Z-mon1",
|
|
712
|
+
meta({ experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
|
|
713
|
+
);
|
|
714
|
+
await writeResultFile(monday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true, hasTrace: true }));
|
|
715
|
+
await writeArtifactFile(monday, "q1/a1", "events.json", [{ n: 1 }]);
|
|
716
|
+
await writeArtifactFile(monday, "q1/a1", "trace.json", [{ name: "turn" }]);
|
|
717
|
+
await writeArtifactFile(monday, "q1/a1", "diff.json", { generatedFiles: {}, deletedFiles: [] });
|
|
718
|
+
await writeResultFile(monday, "q2/a1", record({ id: "q2", attempt: 1 }));
|
|
719
|
+
|
|
720
|
+
// 周五只重跑了 q1:最新快照残缺。
|
|
721
|
+
const friday = await writeSnapshot(
|
|
722
|
+
root,
|
|
723
|
+
"compare_bub",
|
|
724
|
+
"2026-07-05T08-00-00-000Z-fri1",
|
|
725
|
+
meta({ experimentId: "compare/bub", agent: "bub", model: "gpt-5", startedAt: "2026-07-05T08:00:00.000Z", completedAt: "2026-07-05T08:10:00.000Z" }),
|
|
726
|
+
);
|
|
727
|
+
await writeResultFile(friday, "q1/a1", record({ id: "q1", attempt: 1, hasEvents: true }));
|
|
728
|
+
await writeArtifactFile(friday, "q1/a1", "events.json", [{ n: 1 }, { n: 2 }]);
|
|
729
|
+
|
|
730
|
+
const results = await openResults(root);
|
|
731
|
+
const dest = join(await makeRoot(), "site/data/run");
|
|
732
|
+
const copied = await copySnapshots(results.latest(), dest, { artifacts: ["events"] });
|
|
733
|
+
|
|
734
|
+
expect(copied.warnings).toHaveLength(0);
|
|
735
|
+
expect(copied.dir).toBe(dest);
|
|
736
|
+
|
|
737
|
+
const destSnapDir = join(dest, "compare_bub", "2026-07-05T08-00-00-000Z-fri1"); // 快照目录名原样保留
|
|
738
|
+
expect(await exists(join(destSnapDir, "snapshot.json"))).toBe(true);
|
|
739
|
+
expect(await exists(join(destSnapDir, "q1/a1/events.json"))).toBe(true);
|
|
740
|
+
expect(await exists(join(destSnapDir, "q1/a1/trace.json"))).toBe(false); // 未选中的 artifact 种类不复制
|
|
741
|
+
|
|
742
|
+
const destMeta = JSON.parse(await readFile(join(destSnapDir, "snapshot.json"), "utf-8"));
|
|
743
|
+
expect(destMeta.knownEvalIds).toEqual(["q1", "q2"]); // 补记:复制时刻该实验已知的 eval 并集
|
|
744
|
+
expect(destMeta.completedAt).toBe("2026-07-05T08:10:00.000Z");
|
|
745
|
+
expect(destMeta.producer).toEqual({ name: "niceeval", version: "0.3.0" });
|
|
746
|
+
|
|
747
|
+
const destRecord = JSON.parse(await readFile(join(destSnapDir, "q1/a1/result.json"), "utf-8"));
|
|
748
|
+
expect(destRecord.hasEvents).toBe(true);
|
|
749
|
+
expect(destRecord.hasTrace).toBe(false); // 没选中 trace,目标按实际复制重算(不沿用源的 true)
|
|
750
|
+
expect(destRecord).not.toHaveProperty("artifactBase");
|
|
751
|
+
expect(destRecord).not.toHaveProperty("agent"); // 快照级字段不重复
|
|
752
|
+
|
|
753
|
+
// 发布目录上重新 openResults().latest():残缺警告被同一套机制重新算出来,不靠发布者转述。
|
|
754
|
+
const republished = await openResults(dest);
|
|
755
|
+
expect(republished.experiments[0].evalIds).toEqual(["q1", "q2"]);
|
|
756
|
+
const partial = republished.latest().warnings.find((w) => w.kind === "partial-coverage")!;
|
|
757
|
+
expect(partial).toMatchObject({ experimentId: "compare/bub", covered: 1, total: 2 });
|
|
758
|
+
});
|
|
759
|
+
|
|
760
|
+
it("目标目录非空即报错;artifacts 非法值报错;无快照报错;同实验多快照选中 → 取最新 + warning", async () => {
|
|
761
|
+
const root = await makeRoot();
|
|
762
|
+
await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z-a", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }));
|
|
763
|
+
const tuesday = await writeSnapshot(root, "e", "2026-07-02T08-00-00-000Z-b", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-02T08:00:00.000Z", completedAt: "2026-07-02T08:10:00.000Z" }));
|
|
764
|
+
await writeResultFile(tuesday, "q1/a1", record({ id: "q1", attempt: 1 }));
|
|
765
|
+
|
|
766
|
+
const results = await openResults(root);
|
|
767
|
+
|
|
768
|
+
const occupied = await makeRoot();
|
|
769
|
+
await writeFile(join(occupied, "existing.txt"), "x", "utf-8");
|
|
770
|
+
await expect(copySnapshots(results.latest(), occupied)).rejects.toThrow(/not empty/);
|
|
771
|
+
|
|
772
|
+
await expect(copySnapshots(results.latest(), join(await makeRoot(), "out"), { artifacts: ["evnets" as never] })).rejects.toThrow(/Unknown artifact kind/);
|
|
773
|
+
|
|
774
|
+
await expect(copySnapshots([], join(await makeRoot(), "out"))).rejects.toThrow(/no snapshots/);
|
|
775
|
+
|
|
776
|
+
// 手工传入同一 experiment 的两个快照(未走 latest 去重):只带最新,记 warning。
|
|
777
|
+
const dest2 = join(await makeRoot(), "run2");
|
|
778
|
+
const collided = await copySnapshots(results.experiments[0].snapshots, dest2);
|
|
779
|
+
expect(collided.warnings).toHaveLength(1);
|
|
780
|
+
expect(collided.warnings[0]).toMatch(/multiple snapshots selected/);
|
|
781
|
+
const destDirs = await readdir(join(dest2, "e"));
|
|
782
|
+
expect(destDirs).toEqual([basename(tuesday)]);
|
|
783
|
+
});
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
// ───────────────────────── AttemptLocator 集成 ─────────────────────────
|
|
787
|
+
|
|
788
|
+
describe("AttemptLocator · 落盘 / 读取 / 携带 / 撞车", () => {
|
|
789
|
+
it("非携带条目由 writer 按身份算出 locator 并落盘;确定性(同身份重开两次相同);resolveLocator 能找到", async () => {
|
|
790
|
+
const root = await makeRoot();
|
|
791
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
792
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
793
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
794
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
795
|
+
await writer.finish();
|
|
796
|
+
|
|
797
|
+
const record1 = JSON.parse(await readFile(join(snap.dir, "q1/a0/result.json"), "utf-8"));
|
|
798
|
+
expect(record1.locator).toMatch(/^@1[0-9a-z]{7}$/);
|
|
799
|
+
expect(record1.locator).toBe(
|
|
800
|
+
encodeAttemptLocator({ experimentId: "e", snapshotStartedAt: "2026-07-01T08:00:00.000Z", evalId: "q1", attempt: 0 }),
|
|
801
|
+
);
|
|
802
|
+
|
|
803
|
+
const record2 = JSON.parse(await readFile(join(snap.dir, "q2/a0/result.json"), "utf-8"));
|
|
804
|
+
expect(record2.locator).not.toBe(record1.locator); // 不同 evalId → 不同 locator
|
|
805
|
+
|
|
806
|
+
const resultsA = await openResults(root);
|
|
807
|
+
const resultsB = await openResults(root); // 独立重开一次:身份不变,locator 必须一致
|
|
808
|
+
const q1a = resultsA.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
809
|
+
const q1b = resultsB.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
810
|
+
expect(q1a.locator).toBe(record1.locator);
|
|
811
|
+
expect(q1b.locator).toBe(record1.locator);
|
|
812
|
+
|
|
813
|
+
expect(resolveLocator(resultsA, record1.locator).evalId).toBe("q1");
|
|
814
|
+
});
|
|
815
|
+
|
|
816
|
+
it("携带条目(--resume 合入)原样复制原 locator,不按新快照的 startedAt 重算", async () => {
|
|
817
|
+
const root = await makeRoot();
|
|
818
|
+
const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
819
|
+
await writer1.writeAttemptFor({
|
|
820
|
+
id: "q1",
|
|
821
|
+
experimentId: "e",
|
|
651
822
|
agent: "bub",
|
|
823
|
+
verdict: "passed",
|
|
824
|
+
attempt: 0,
|
|
652
825
|
startedAt: "2026-07-01T08:00:00.000Z",
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
826
|
+
durationMs: 1,
|
|
827
|
+
assertions: [],
|
|
828
|
+
});
|
|
829
|
+
await writer1.finish();
|
|
830
|
+
|
|
831
|
+
const opened1 = await openResults(root);
|
|
832
|
+
const original = opened1.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
833
|
+
const originalLocator = original.locator!;
|
|
834
|
+
|
|
835
|
+
// 第二轮:carry 合入 q1(artifactBase 指回第一轮的快照),locator 从上一轮读回的记录里原样带过来。
|
|
836
|
+
// q2(真正新跑的)先写:snapshot() 的 startedAt 由「该实验首条落盘结果的 attempt 时刻」锚定
|
|
837
|
+
// (writer.ts 的注释),让第二轮快照的真实 startedAt("07-02")明确不同于原快照("07-01")——
|
|
838
|
+
// 这样如果 locator 被错误地按「当前快照」重算,会得到一个可判别的不同字符串。
|
|
839
|
+
const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
840
|
+
await writer2.writeAttemptFor({
|
|
841
|
+
id: "q2",
|
|
842
|
+
experimentId: "e",
|
|
843
|
+
agent: "bub",
|
|
844
|
+
verdict: "passed",
|
|
845
|
+
attempt: 0,
|
|
846
|
+
startedAt: "2026-07-02T08:00:00.000Z",
|
|
847
|
+
durationMs: 1,
|
|
848
|
+
assertions: [],
|
|
849
|
+
});
|
|
850
|
+
const carried: EvalResult = {
|
|
851
|
+
...original.result,
|
|
852
|
+
experimentId: "e",
|
|
853
|
+
agent: "bub",
|
|
854
|
+
artifactBase: `${original.ref.snapshot}/${original.ref.attempt}`,
|
|
855
|
+
};
|
|
856
|
+
await writer2.writeAttemptFor(carried);
|
|
857
|
+
await writer2.finish();
|
|
858
|
+
|
|
859
|
+
const opened2 = await openResults(root);
|
|
860
|
+
const newest = opened2.experiments[0].latest;
|
|
861
|
+
const carriedAttempt = newest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
862
|
+
expect(carriedAttempt.locator).toBe(originalLocator);
|
|
863
|
+
expect(resolveLocator(opened2, originalLocator).evalId).toBe("q1");
|
|
864
|
+
|
|
865
|
+
// 反证:如果按「新快照的 startedAt」重算,会得到一个不同的字符串——证明确实是原样复制,不是重算。
|
|
866
|
+
const wronglyRecomputed = encodeAttemptLocator({
|
|
867
|
+
experimentId: "e",
|
|
868
|
+
snapshotStartedAt: newest.startedAt,
|
|
869
|
+
evalId: "q1",
|
|
870
|
+
attempt: 0,
|
|
662
871
|
});
|
|
872
|
+
expect(originalLocator).not.toBe(wronglyRecomputed);
|
|
873
|
+
});
|
|
663
874
|
|
|
664
|
-
|
|
665
|
-
const
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
"
|
|
686
|
-
"
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
}
|
|
737
|
-
{
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
875
|
+
it("resolveLocator:malformed 与 not-found 是两种可判别的错误", async () => {
|
|
876
|
+
const root = await makeRoot();
|
|
877
|
+
await writeSnapshot(root, "e", "2026-07-01T08-00-00-000Z-a", meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" }));
|
|
878
|
+
const results = await openResults(root);
|
|
879
|
+
|
|
880
|
+
expect(() => resolveLocator(results, "not-a-locator")).toThrow(MalformedLocatorError);
|
|
881
|
+
expect(() => resolveLocator(results, "@1nosuch1")).toThrow(LocatorNotFoundError); // 语法合法(7 位 body),索引里没有
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
it("两个不同身份的 attempt 撞出同一个 locator 字符串:openResults() 抛 LocatorCollisionError,不静默覆盖", async () => {
|
|
885
|
+
const root = await makeRoot();
|
|
886
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
887
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
888
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
889
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
890
|
+
await writer.finish();
|
|
891
|
+
|
|
892
|
+
// 人为制造撞车:把 q2 的 locator 改成和 q1 一样(真实哈希撞车不可复现,这里直接模拟其效果——
|
|
893
|
+
// 索引建立逻辑只关心「同一 locator 字符串映射到身份三元组不同的两个 attempt」)。
|
|
894
|
+
const q1Path = join(snap.dir, "q1/a0/result.json");
|
|
895
|
+
const q2Path = join(snap.dir, "q2/a0/result.json");
|
|
896
|
+
const q1Record = JSON.parse(await readFile(q1Path, "utf-8"));
|
|
897
|
+
const q2Record = JSON.parse(await readFile(q2Path, "utf-8"));
|
|
898
|
+
q2Record.locator = q1Record.locator;
|
|
899
|
+
await writeFile(q2Path, JSON.stringify(q2Record), "utf-8");
|
|
900
|
+
|
|
901
|
+
await expect(openResults(root)).rejects.toThrow(LocatorCollisionError);
|
|
902
|
+
});
|
|
903
|
+
|
|
904
|
+
it("多 experiment:同 evalId/attempt 在不同 experiment 下产出不同 locator,resolveLocator 精确定位到各自的 experiment", async () => {
|
|
905
|
+
const root = await makeRoot();
|
|
906
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
907
|
+
const snapA = await writer.snapshot({ experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
908
|
+
const snapB = await writer.snapshot({ experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
909
|
+
await snapA.writeAttempt({ id: "algebra/q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
910
|
+
await snapB.writeAttempt({ id: "algebra/q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
911
|
+
await writer.finish();
|
|
912
|
+
|
|
913
|
+
const results = await openResults(root);
|
|
914
|
+
const a = results.experiments.find((e) => e.id === "compare/bub")!.latest.evals[0]!.attempts[0]!;
|
|
915
|
+
const b = results.experiments.find((e) => e.id === "compare/codex")!.latest.evals[0]!.attempts[0]!;
|
|
916
|
+
// 同 evalId、同 attempt 序号,只有 experimentId 不同 → locator 必须不同(身份元组含 experimentId)。
|
|
917
|
+
expect(a.locator).not.toBe(b.locator);
|
|
918
|
+
|
|
919
|
+
expect(resolveLocator(results, a.locator!).experimentId).toBe("compare/bub");
|
|
920
|
+
expect(resolveLocator(results, b.locator!).experimentId).toBe("compare/codex");
|
|
921
|
+
});
|
|
922
|
+
|
|
923
|
+
it("同一 evalId 的不同 attempt 序号产出不同 locator,resolveLocator 各自精确定位到对应 attempt", async () => {
|
|
924
|
+
const root = await makeRoot();
|
|
925
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
926
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
927
|
+
await snap.writeAttempt({ id: "q1", verdict: "failed", attempt: 0, durationMs: 1, assertions: [] });
|
|
928
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 1, durationMs: 1, assertions: [] });
|
|
929
|
+
await writer.finish();
|
|
930
|
+
|
|
931
|
+
const results = await openResults(root);
|
|
932
|
+
const attempts = results.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts;
|
|
933
|
+
expect(attempts).toHaveLength(2);
|
|
934
|
+
const [a0, a1] = attempts;
|
|
935
|
+
expect(a0!.locator).not.toBe(a1!.locator);
|
|
936
|
+
expect(resolveLocator(results, a0!.locator!).result).toMatchObject({ attempt: 0, verdict: "failed" });
|
|
937
|
+
expect(resolveLocator(results, a1!.locator!).result).toMatchObject({ attempt: 1, verdict: "passed" });
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
it("历史快照(非 latest)的 attempt 依然被建进索引,resolveLocator 能定位到旧快照里的那份", async () => {
|
|
941
|
+
const root = await makeRoot();
|
|
942
|
+
const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
943
|
+
const monday = await writer1.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
944
|
+
await monday.writeAttempt({ id: "q1", verdict: "failed", attempt: 0, durationMs: 1, assertions: [] });
|
|
945
|
+
await writer1.finish();
|
|
946
|
+
|
|
947
|
+
const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
948
|
+
const friday = await writer2.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-05T08:00:00.000Z" });
|
|
949
|
+
await friday.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
950
|
+
await writer2.finish();
|
|
951
|
+
|
|
952
|
+
const results = await openResults(root);
|
|
953
|
+
const exp = results.experiments[0]!;
|
|
954
|
+
expect(exp.snapshots).toHaveLength(2); // 两次快照都在(忠实磁盘,不合并/不丢弃历史)
|
|
955
|
+
const oldAttempt = exp.snapshots.find((s) => s.startedAt === "2026-07-01T08:00:00.000Z")!.attempts[0]!;
|
|
956
|
+
const newAttempt = exp.snapshots.find((s) => s.startedAt === "2026-07-05T08:00:00.000Z")!.attempts[0]!;
|
|
957
|
+
expect(oldAttempt.locator).not.toBe(newAttempt.locator); // 不同 startedAt → 不同身份 → 不同 locator
|
|
958
|
+
|
|
959
|
+
expect(resolveLocator(results, oldAttempt.locator!).result.verdict).toBe("failed");
|
|
960
|
+
expect(resolveLocator(results, newAttempt.locator!).result.verdict).toBe("passed");
|
|
961
|
+
});
|
|
962
|
+
|
|
963
|
+
it("手工构造的 Results(未经 openResults())上调 resolveLocator:索引查不到,统一按 not-found 处理,不抛意外错误", () => {
|
|
964
|
+
const snapshot = fakeSnapshot({ experimentId: "e", startedAt: "2026-07-01T08:00:00.000Z", dir: "/tmp/e/s1" });
|
|
965
|
+
const attempt = fakeAttempt(snapshot, record({ id: "q1", attempt: 0 }));
|
|
966
|
+
snapshot.attempts = [attempt];
|
|
967
|
+
snapshot.evals = [{ id: "q1", attempts: [attempt] }];
|
|
968
|
+
const handMadeResults: Results = {
|
|
969
|
+
experiments: [{ id: "e", snapshots: [snapshot], latest: snapshot, evalIds: ["q1"] }],
|
|
970
|
+
skipped: [],
|
|
971
|
+
// filter() 本测试不调用,用不到,给个占位实现即可满足 Selection 接口。
|
|
972
|
+
latest: () => ({ snapshots: [snapshot], warnings: [], filter: () => { throw new Error("not implemented"); } }),
|
|
973
|
+
};
|
|
974
|
+
// 这份 locator 语法合法、甚至真的对应 handMadeResults 里那个 attempt 的身份,
|
|
975
|
+
// 但 handMadeResults 没经过 openResults(),locatorIndexByResults 里查不到它 —— 空索引,not-found。
|
|
976
|
+
const syntacticallyValidLocator = encodeAttemptLocator({
|
|
977
|
+
experimentId: "e",
|
|
978
|
+
snapshotStartedAt: "2026-07-01T08:00:00.000Z",
|
|
979
|
+
evalId: "q1",
|
|
980
|
+
attempt: 0,
|
|
981
|
+
});
|
|
982
|
+
expect(() => resolveLocator(handMadeResults, syntacticallyValidLocator)).toThrow(LocatorNotFoundError);
|
|
983
|
+
});
|
|
984
|
+
|
|
985
|
+
it("copySnapshots:普通(非 sources)attempt 的 locator 原样复制,目标结果根上 resolveLocator 依然命中", async () => {
|
|
986
|
+
const root = await makeRoot();
|
|
987
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
988
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
989
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
990
|
+
await snap.writeAttempt({ id: "q1", verdict: "failed", attempt: 1, durationMs: 1, assertions: [] });
|
|
991
|
+
await writer.finish();
|
|
992
|
+
|
|
993
|
+
const results = await openResults(root);
|
|
994
|
+
const [a0, a1] = results.experiments[0]!.latest.evals[0]!.attempts;
|
|
995
|
+
const locator0 = a0!.locator!;
|
|
996
|
+
const locator1 = a1!.locator!;
|
|
997
|
+
|
|
998
|
+
const dest = join(await makeRoot(), "published");
|
|
999
|
+
await copySnapshots(results.latest(), dest, { artifacts: [] });
|
|
1000
|
+
|
|
1001
|
+
const destResults = await openResults(dest);
|
|
1002
|
+
expect(resolveLocator(destResults, locator0).result.attempt).toBe(0);
|
|
1003
|
+
expect(resolveLocator(destResults, locator1).result.attempt).toBe(1);
|
|
1004
|
+
});
|
|
1005
|
+
});
|
|
1006
|
+
|
|
1007
|
+
// ───────────────────────── sources 去重仓库 ─────────────────────────
|
|
1008
|
+
|
|
1009
|
+
describe("sources · 快照级去重仓库", () => {
|
|
1010
|
+
it("两个 attempt 共享字节相同的 eval 源码:去重仓库只落一份 blob(文件数 = 1)", async () => {
|
|
1011
|
+
const root = await makeRoot();
|
|
1012
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1013
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1014
|
+
const content = "export default { test() {} };\n";
|
|
1015
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
|
|
1016
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: [{ path: "evals/shared.eval.ts", content }] });
|
|
1017
|
+
await writer.finish();
|
|
1018
|
+
|
|
1019
|
+
const storeFiles = await readdir(join(snap.dir, "sources"));
|
|
1020
|
+
expect(storeFiles).toHaveLength(1);
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
it("两个 attempt 的 eval 源码内容不同:各自落一份 blob(文件数 = 2)", async () => {
|
|
1024
|
+
const root = await makeRoot();
|
|
1025
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1026
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1027
|
+
await snap.writeAttempt(
|
|
1028
|
+
{ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
|
|
1029
|
+
{ sources: [{ path: "evals/a.eval.ts", content: "export default { test() {} };\n" }] },
|
|
1030
|
+
);
|
|
1031
|
+
await snap.writeAttempt(
|
|
1032
|
+
{ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] },
|
|
1033
|
+
{ sources: [{ path: "evals/b.eval.ts", content: "export default { test() { /* different */ } };\n" }] },
|
|
1034
|
+
);
|
|
1035
|
+
await writer.finish();
|
|
1036
|
+
|
|
1037
|
+
const storeFiles = await readdir(join(snap.dir, "sources"));
|
|
1038
|
+
expect(storeFiles).toHaveLength(2);
|
|
1039
|
+
});
|
|
1040
|
+
|
|
1041
|
+
it("经真实 --resume carry 流程(writeAttemptFor 的 artifactBase 分支)携带的 attempt,其 sources() 引用在新快照里依然能解到原快照内容", async () => {
|
|
1042
|
+
const root = await makeRoot();
|
|
1043
|
+
const content = "export default { test() {} };\n";
|
|
1044
|
+
const writer1 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1045
|
+
await writer1.writeAttemptFor({
|
|
1046
|
+
id: "q1",
|
|
1047
|
+
experimentId: "e",
|
|
1048
|
+
agent: "bub",
|
|
1049
|
+
verdict: "passed",
|
|
1050
|
+
attempt: 0,
|
|
1051
|
+
startedAt: "2026-07-01T08:00:00.000Z",
|
|
1052
|
+
durationMs: 1,
|
|
1053
|
+
assertions: [],
|
|
1054
|
+
sources: [{ path: "evals/q1.eval.ts", content }],
|
|
1055
|
+
});
|
|
1056
|
+
await writer1.finish();
|
|
1057
|
+
|
|
1058
|
+
const opened1 = await openResults(root);
|
|
1059
|
+
const original = opened1.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts[0]!;
|
|
1060
|
+
expect(await original.sources()).toEqual([{ path: "evals/q1.eval.ts", content }]);
|
|
1061
|
+
|
|
1062
|
+
// 第二轮:q2 是真正新跑的(锚定新快照的 startedAt 明确晚于原快照),q1 是 carry 合入 ——
|
|
1063
|
+
// artifactBase 指回第一轮的快照,与 locator carry 测试同一套构造手法。
|
|
1064
|
+
const writer2 = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1065
|
+
await writer2.writeAttemptFor({
|
|
1066
|
+
id: "q2",
|
|
1067
|
+
experimentId: "e",
|
|
1068
|
+
agent: "bub",
|
|
1069
|
+
verdict: "passed",
|
|
1070
|
+
attempt: 0,
|
|
1071
|
+
startedAt: "2026-07-02T08:00:00.000Z",
|
|
1072
|
+
durationMs: 1,
|
|
1073
|
+
assertions: [],
|
|
1074
|
+
});
|
|
1075
|
+
const carried: EvalResult = {
|
|
1076
|
+
...original.result,
|
|
1077
|
+
experimentId: "e",
|
|
1078
|
+
agent: "bub",
|
|
1079
|
+
artifactBase: `${original.ref.snapshot}/${original.ref.attempt}`,
|
|
1080
|
+
};
|
|
1081
|
+
await writer2.writeAttemptFor(carried);
|
|
1082
|
+
await writer2.finish();
|
|
1083
|
+
|
|
1084
|
+
const opened2 = await openResults(root);
|
|
1085
|
+
const carriedAttempt = opened2.experiments[0]!.latest.evals.find((e) => e.id === "q1")!.attempts[0]!;
|
|
1086
|
+
// 新快照下没有为携带条目重新写 sources.json/blob(carry 分支不写 artifact),
|
|
1087
|
+
// sources() 必须靠 artifactBase 回退到原快照的去重仓库才能解出内容。
|
|
1088
|
+
expect(await carriedAttempt.sources()).toEqual([{ path: "evals/q1.eval.ts", content }]);
|
|
1089
|
+
});
|
|
1090
|
+
|
|
1091
|
+
it("同一快照内相同内容只落一份 blob;不同内容各一份;attempt.sources() 各自读回正确内容", async () => {
|
|
1092
|
+
const root = await makeRoot();
|
|
1093
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1094
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1095
|
+
const shared = [{ path: "evals/shared.eval.ts", content: "export default { test() {} };\n" }];
|
|
1096
|
+
const other = [{ path: "evals/other.eval.ts", content: "export default { test() { /* different */ } };\n" }];
|
|
1097
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
|
|
1098
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
|
|
1099
|
+
await snap.writeAttempt({ id: "q3", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: other });
|
|
1100
|
+
await writer.finish();
|
|
1101
|
+
|
|
1102
|
+
const storeFiles = await readdir(join(snap.dir, "sources"));
|
|
1103
|
+
expect(storeFiles).toHaveLength(2); // 三份引用,内容只两种 → 两个 blob
|
|
1104
|
+
|
|
1105
|
+
// attempt 级 sources.json 只是引用(小,不含 content),不是全量内容。
|
|
1106
|
+
const q1Ref = JSON.parse(await readFile(join(snap.dir, "q1/a0/sources.json"), "utf-8"));
|
|
1107
|
+
expect(q1Ref).toEqual([{ path: "evals/shared.eval.ts", sha256: expect.any(String) }]);
|
|
1108
|
+
expect(JSON.stringify(q1Ref)).not.toContain("export default");
|
|
1109
|
+
|
|
1110
|
+
const results = await openResults(root);
|
|
1111
|
+
const evalById = (id: string) => results.experiments[0].latest.evals.find((e) => e.id === id)!.attempts[0];
|
|
1112
|
+
await expect(evalById("q1").sources()).resolves.toEqual(shared);
|
|
1113
|
+
await expect(evalById("q2").sources()).resolves.toEqual(shared);
|
|
1114
|
+
await expect(evalById("q3").sources()).resolves.toEqual(other);
|
|
1115
|
+
});
|
|
1116
|
+
|
|
1117
|
+
it("携带条目(artifactBase 回退)的 sources() 仍能解到原快照的去重仓库", async () => {
|
|
1118
|
+
const root = await makeRoot();
|
|
1119
|
+
const oldSnap = await writeSnapshot(
|
|
1120
|
+
root,
|
|
1121
|
+
"e",
|
|
1122
|
+
"2026-06-30T08-00-00-000Z-xxxx",
|
|
1123
|
+
meta({ experimentId: "e", agent: "bub", startedAt: "2026-06-30T08:00:00.000Z", completedAt: "2026-06-30T08:10:00.000Z" }),
|
|
1124
|
+
);
|
|
1125
|
+
await writeResultFile(oldSnap, "q1/a0", record({ id: "q1", attempt: 0, hasSources: true }));
|
|
1126
|
+
await mkdir(join(oldSnap, "sources"), { recursive: true });
|
|
1127
|
+
await writeFile(join(oldSnap, "sources", "abc123.json"), JSON.stringify({ content: "export default {};\n" }), "utf-8");
|
|
1128
|
+
await writeArtifactFile(oldSnap, "q1/a0", "sources.json", [{ path: "evals/q1.eval.ts", sha256: "abc123" }]);
|
|
1129
|
+
|
|
1130
|
+
const newSnap = await writeSnapshot(
|
|
1131
|
+
root,
|
|
1132
|
+
"e",
|
|
1133
|
+
"2026-07-01T08-00-00-000Z-yyyy",
|
|
1134
|
+
meta({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z", completedAt: "2026-07-01T08:10:00.000Z" }),
|
|
1135
|
+
);
|
|
1136
|
+
await writeResultFile(
|
|
1137
|
+
newSnap,
|
|
1138
|
+
"q1/a0",
|
|
1139
|
+
record({ id: "q1", attempt: 0, startedAt: "2026-06-30T08:01:00.000Z", artifactBase: "e/2026-06-30T08-00-00-000Z-xxxx/q1/a0", hasSources: true }),
|
|
1140
|
+
);
|
|
1141
|
+
|
|
1142
|
+
const results = await openResults(root);
|
|
1143
|
+
const carried = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
1144
|
+
expect(await carried.sources()).toEqual([{ path: "evals/q1.eval.ts", content: "export default {};\n" }]);
|
|
1145
|
+
});
|
|
1146
|
+
|
|
1147
|
+
it("copySnapshots:sources 引用与去重仓库一起复制,内容按目的地重新去重(同一份不重复落盘)", async () => {
|
|
1148
|
+
const root = await makeRoot();
|
|
1149
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1150
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1151
|
+
const shared = [{ path: "evals/shared.eval.ts", content: "export default { test() {} };\n" }];
|
|
1152
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
|
|
1153
|
+
await snap.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] }, { sources: shared });
|
|
1154
|
+
await writer.finish();
|
|
1155
|
+
|
|
1156
|
+
const originalLocator = JSON.parse(await readFile(join(snap.dir, "q1/a0/result.json"), "utf-8")).locator;
|
|
1157
|
+
|
|
1158
|
+
const results = await openResults(root);
|
|
1159
|
+
const dest = join(await makeRoot(), "published");
|
|
1160
|
+
await copySnapshots(results.latest(), dest, { artifacts: ["sources"] });
|
|
1161
|
+
|
|
1162
|
+
const destSnapDir = join(dest, "e", basename(snap.dir));
|
|
1163
|
+
const destStoreFiles = await readdir(join(destSnapDir, "sources"));
|
|
1164
|
+
expect(destStoreFiles).toHaveLength(1); // 复制后在目的地重新按内容去重,仍只一份
|
|
1165
|
+
|
|
1166
|
+
const destResults = await openResults(dest);
|
|
1167
|
+
const q1 = destResults.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
1168
|
+
expect(await q1.sources()).toEqual(shared);
|
|
1169
|
+
expect(q1.result.locator).toBe(originalLocator); // locator 随 result.json 原样复制,不重算
|
|
1170
|
+
});
|
|
1171
|
+
});
|
|
1172
|
+
|
|
1173
|
+
// ───────────────────────── loadAnnotatedEvalSource(端到端打通) ─────────────────────────
|
|
1174
|
+
|
|
1175
|
+
describe("loadAnnotatedEvalSource · discovery 捕获 → 去重存储 → 检索 → 标注 打通链路", () => {
|
|
1176
|
+
it("给一个真实落盘的 attempt,取回 sources() 内容并按 loc 标注断言", async () => {
|
|
1177
|
+
const root = await makeRoot();
|
|
1178
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1179
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1180
|
+
const content = "import { defineEval } from \"niceeval\";\nexport default defineEval({\n test() {},\n});\n";
|
|
1181
|
+
const assertions = [
|
|
1182
|
+
{ name: "check-1", passed: true, severity: "gate", score: 1, loc: { file: "evals/a.eval.ts", line: 3 } },
|
|
1183
|
+
{ name: "no-loc", passed: false, severity: "soft", score: 0 },
|
|
1184
|
+
] as unknown as EvalResult["assertions"];
|
|
1185
|
+
await snap.writeAttempt(
|
|
1186
|
+
{ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions },
|
|
1187
|
+
{ sources: [{ path: "evals/a.eval.ts", content }] },
|
|
1188
|
+
);
|
|
1189
|
+
await writer.finish();
|
|
1190
|
+
|
|
1191
|
+
const results = await openResults(root);
|
|
1192
|
+
const attempt = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
1193
|
+
const annotated = await loadAnnotatedEvalSource(attempt);
|
|
1194
|
+
expect(annotated).not.toBeNull();
|
|
1195
|
+
expect(annotated!.sourcePath).toBe("evals/a.eval.ts");
|
|
1196
|
+
expect(annotated!.lines[2]!.assertions.map((a) => a.name)).toEqual(["check-1"]);
|
|
1197
|
+
expect(annotated!.unmapped.map((a) => a.name)).toEqual(["no-loc"]);
|
|
1198
|
+
expect(annotated!.summary).toMatchObject({ totalAssertions: 2, mappedAssertions: 1, unmappedAssertions: 1 });
|
|
1199
|
+
});
|
|
1200
|
+
|
|
1201
|
+
it("没有 sources() 时返回 null,不伪造空文档", async () => {
|
|
1202
|
+
const root = await makeRoot();
|
|
1203
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
1204
|
+
const snap = await writer.snapshot({ experimentId: "e", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
1205
|
+
await snap.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1, assertions: [] });
|
|
1206
|
+
await writer.finish();
|
|
1207
|
+
|
|
1208
|
+
const results = await openResults(root);
|
|
1209
|
+
const attempt = results.experiments[0].latest.evals.find((e) => e.id === "q1")!.attempts[0];
|
|
1210
|
+
expect(await loadAnnotatedEvalSource(attempt)).toBeNull();
|
|
762
1211
|
});
|
|
763
1212
|
});
|