niceeval 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +23 -23
- package/README.zh.md +6 -6
- package/dist/agents/types.d.ts +69 -7
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +57 -3
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +32 -24
- package/dist/report/aggregate.js +158 -50
- package/dist/report/built-in/index.d.ts +2 -0
- package/dist/report/built-in/index.js +8 -0
- package/dist/report/components.d.ts +93 -160
- package/dist/report/components.js +377 -114
- package/dist/report/compute.d.ts +87 -81
- package/dist/report/compute.js +597 -417
- package/dist/report/flag.d.ts +32 -6
- package/dist/report/flag.js +92 -4
- package/dist/report/format.d.ts +19 -11
- package/dist/report/format.js +30 -13
- package/dist/report/index.d.ts +16 -16
- package/dist/report/index.js +20 -21
- package/dist/report/load.js +3 -2
- package/dist/report/locale.d.ts +57 -33
- package/dist/report/locale.js +122 -56
- package/dist/report/metrics.d.ts +23 -4
- package/dist/report/metrics.js +110 -25
- package/dist/report/primitives.d.ts +48 -15
- package/dist/report/primitives.js +135 -26
- package/dist/report/react/AttemptList.d.ts +9 -7
- package/dist/report/react/AttemptList.js +17 -10
- package/dist/report/react/DeltaTable.js +19 -18
- package/dist/report/react/EvalList.d.ts +4 -4
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +10 -0
- package/dist/report/react/ExperimentComparison.js +12 -0
- package/dist/report/react/ExperimentList.d.ts +4 -3
- package/dist/report/react/ExperimentList.js +17 -18
- package/dist/report/react/MetricBars.js +5 -4
- package/dist/report/react/MetricLine.js +12 -5
- package/dist/report/react/MetricMatrix.js +1 -1
- package/dist/report/react/MetricScatter.js +59 -28
- package/dist/report/react/MetricTable.js +2 -12
- package/dist/report/react/ScopeSummary.d.ts +10 -0
- package/dist/report/react/ScopeSummary.js +17 -0
- package/dist/report/react/Scoreboard.js +6 -6
- package/dist/report/react/cell.js +2 -2
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +5 -9
- package/dist/report/react/fixtures.js +110 -147
- package/dist/report/react/index.d.ts +15 -5
- package/dist/report/react/index.js +18 -7
- package/dist/report/report.d.ts +137 -16
- package/dist/report/report.js +259 -28
- package/dist/report/text/faces.d.ts +17 -19
- package/dist/report/text/faces.js +253 -184
- package/dist/report/text/plot.js +1 -1
- package/dist/report/text/table.js +38 -7
- package/dist/report/tree.d.ts +90 -40
- package/dist/report/tree.js +252 -94
- package/dist/report/types.d.ts +247 -284
- package/dist/report/types.js +4 -3
- package/dist/report/web.d.ts +21 -5
- package/dist/report/web.js +42 -16
- package/dist/results/select.d.ts +38 -16
- package/dist/results/select.js +73 -25
- package/dist/results/types.d.ts +49 -14
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +4 -2
- package/dist/shared/aggregate.js +8 -7
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/README.md +44 -0
- package/docs-site/zh/examples/ai-agent-application.mdx +63 -0
- package/docs-site/zh/examples/coding-agent-extensions.mdx +57 -0
- package/docs-site/zh/examples/index.mdx +50 -0
- package/docs-site/zh/{concepts → explanation}/adapter.mdx +31 -13
- package/docs-site/zh/{concepts → explanation}/assert.mdx +7 -7
- package/docs-site/zh/{concepts → explanation}/drive.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/evals.mdx +4 -4
- package/docs-site/zh/{concepts → explanation}/experiment.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/hitl.mdx +8 -8
- package/docs-site/zh/{concepts → explanation}/judge.mdx +5 -5
- package/docs-site/zh/{concepts → explanation}/overview.mdx +11 -11
- package/docs-site/zh/{guides → explanation}/runner.mdx +18 -8
- package/docs-site/zh/{concepts → explanation}/tier.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/agent-feedback-loop.mdx +35 -33
- package/docs-site/zh/{guides → how-to}/authoring.mdx +35 -2
- package/docs-site/zh/{guides → how-to}/ci-integration.mdx +23 -12
- package/docs-site/zh/{guides → how-to}/connect-otel.mdx +6 -6
- package/docs-site/zh/{guides → how-to}/connect-your-agent.mdx +47 -21
- package/docs-site/zh/{guides → how-to}/custom-reports.mdx +34 -39
- package/docs-site/zh/{guides → how-to}/dataset-fanout.mdx +25 -3
- package/docs-site/zh/{guides → how-to}/experiments.mdx +12 -5
- package/docs-site/zh/how-to/publish-report.mdx +105 -0
- package/docs-site/zh/{guides → how-to}/reporters.mdx +2 -2
- package/docs-site/zh/{guides → how-to}/sandbox-agent.mdx +56 -7
- package/docs-site/zh/how-to/sandbox-providers.mdx +350 -0
- package/docs-site/zh/{guides → how-to}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{guides → how-to}/viewing-results.mdx +82 -39
- package/docs-site/zh/{guides → how-to}/write-experiment.mdx +6 -4
- package/docs-site/zh/{guides → how-to}/write-send.mdx +30 -14
- package/docs-site/zh/index.mdx +24 -26
- package/docs-site/zh/introduction.mdx +8 -8
- package/docs-site/zh/reference/builtin-agents.mdx +32 -5
- package/docs-site/zh/reference/capabilities.mdx +8 -8
- package/docs-site/zh/reference/cli.mdx +40 -12
- package/docs-site/zh/reference/define-agent.mdx +58 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +3 -3
- package/docs-site/zh/reference/expect.mdx +26 -1
- package/docs-site/zh/{guides → reference}/official-adapters.mdx +32 -8
- package/docs-site/zh/{guides → reference}/report-components.mdx +45 -33
- package/docs-site/zh/{guides → reference}/results-data.mdx +21 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +57 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +212 -0
- package/docs-site/zh/{quickstart.mdx → tutorials/quickstart.mdx} +5 -17
- package/package.json +10 -2
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +13 -2
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/streaming.ts +2 -2
- package/src/agents/types.ts +71 -8
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +446 -124
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +162 -30
- package/src/context/session.test.ts +2 -1
- package/src/context/session.ts +115 -7
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +81 -17
- package/src/i18n/zh-CN.ts +80 -17
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +178 -61
- package/src/report/built-in/index.tsx +9 -0
- package/src/report/components.tsx +625 -279
- package/src/report/compute.ts +723 -491
- package/src/report/dual-render.test.tsx +741 -1024
- package/src/report/flag.ts +104 -12
- package/src/report/format.ts +32 -12
- package/src/report/index.ts +119 -46
- package/src/report/load.ts +3 -2
- package/src/report/locale.ts +136 -65
- package/src/report/metrics.ts +108 -25
- package/src/report/primitives.tsx +196 -45
- package/src/report/react/AttemptList.tsx +30 -43
- package/src/report/react/DeltaTable.tsx +63 -45
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +73 -0
- package/src/report/react/ExperimentList.tsx +50 -32
- package/src/report/react/MetricBars.tsx +5 -4
- package/src/report/react/MetricLine.tsx +13 -8
- package/src/report/react/MetricMatrix.tsx +2 -2
- package/src/report/react/MetricScatter.tsx +86 -34
- package/src/report/react/MetricTable.tsx +4 -76
- package/src/report/react/ScopeSummary.tsx +86 -0
- package/src/report/react/Scoreboard.tsx +28 -10
- package/src/report/react/cell.tsx +2 -2
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +89 -5
- package/src/report/react/fixtures.ts +114 -154
- package/src/report/react/index.tsx +24 -39
- package/src/report/react/render.test.tsx +138 -158
- package/src/report/react/styles.css +243 -82
- package/src/report/report.test.ts +779 -841
- package/src/report/report.ts +423 -41
- package/src/report/text/faces.ts +290 -193
- package/src/report/text/plot.ts +1 -1
- package/src/report/text/table.ts +44 -7
- package/src/report/tree.ts +362 -104
- package/src/report/types.ts +261 -271
- package/src/report/web.ts +63 -20
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +13 -11
- package/src/results/attempt-evidence.ts +20 -13
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +150 -60
- package/src/results/host-equivalence.test.ts +34 -20
- package/src/results/index.ts +12 -4
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +15 -5
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +89 -54
- package/src/results/select.ts +104 -34
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +43 -14
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +243 -37
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +8 -6
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +50 -67
- package/src/show/index.ts +127 -56
- package/src/show/render.ts +662 -131
- package/src/show/report-host.test.ts +188 -0
- package/src/show/report-host.ts +375 -0
- package/src/show/show.test.ts +320 -54
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/App.test.tsx +69 -0
- package/src/view/app/App.tsx +144 -48
- package/src/view/app/components/AttemptModal.tsx +423 -11
- package/src/view/app/components/CodeView.tsx +41 -14
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +37 -17
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/app/main.tsx +13 -8
- package/src/view/app/pages/{RunsPage.tsx → AttemptsPage.tsx} +6 -6
- package/src/view/app/types.ts +4 -1
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +10 -3
- package/src/view/data.ts +155 -49
- package/src/view/index.ts +56 -41
- package/src/view/server.ts +37 -15
- package/src/view/shared/types.ts +34 -5
- package/src/view/styles.css +227 -0
- package/src/view/view-report.test.ts +167 -62
- package/dist/report/built-ins/experiment-comparison.d.ts +0 -1
- package/dist/report/built-ins/experiment-comparison.js +0 -13
- package/dist/report/built-ins/index.d.ts +0 -1
- package/dist/report/built-ins/index.js +0 -2
- package/dist/report/react/GroupSummary.d.ts +0 -8
- package/dist/report/react/GroupSummary.js +0 -8
- package/dist/report/react/RunOverview.d.ts +0 -8
- package/dist/report/react/RunOverview.js +0 -12
- package/docs-site/zh/example/ai-agent-application.mdx +0 -152
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +0 -167
- package/docs-site/zh/example/claude-code-codex-skill.mdx +0 -152
- package/docs-site/zh/example/showcase.mdx +0 -39
- package/docs-site/zh/guides/publish-report.mdx +0 -91
- package/docs-site/zh/guides/sandbox-providers.mdx +0 -102
- package/src/report/built-in-user-parity.test.tsx +0 -640
- package/src/report/built-ins/experiment-comparison.tsx +0 -19
- package/src/report/built-ins/index.ts +0 -2
- package/src/report/react/GroupSummary.tsx +0 -66
- package/src/report/react/RunOverview.tsx +0 -109
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
- /package/docs-site/zh/{example/tier1-ai-sdk-v7.mdx → examples/integrations/ai-sdk-v7.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-claude-sdk.mdx → examples/integrations/claude-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-codex-sdk.mdx → examples/integrations/codex-sdk.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-langgraph.mdx → examples/integrations/langgraph.mdx} +0 -0
- /package/docs-site/zh/{example/tier1-pi-sdk.mdx → examples/integrations/pi-sdk.mdx} +0 -0
- /package/docs-site/zh/{guides → how-to}/fixtures.mdx +0 -0
package/src/results/copy.ts
CHANGED
|
@@ -7,20 +7,38 @@
|
|
|
7
7
|
// knownEvalIds(复制时刻该实验已知的 eval 并集),发布目录上重新 openResults().latest(),
|
|
8
8
|
// 残缺警告被同一套机制重新算出来,不靠发布者转述。
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
11
11
|
import { basename, dirname, join, resolve } from "node:path";
|
|
12
|
-
import type { EvalResult } from "../types.ts";
|
|
12
|
+
import type { EvalResult, StreamEvent, TraceSpan } from "../types.ts";
|
|
13
13
|
import { RESULTS_FORMAT } from "../types.ts";
|
|
14
14
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, experimentDirOf } from "./format.ts";
|
|
15
15
|
import { experimentOfSnapshot } from "./open.ts";
|
|
16
16
|
import { isNewerSnapshot } from "./select.ts";
|
|
17
17
|
import { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
18
|
-
import
|
|
19
|
-
|
|
18
|
+
import {
|
|
19
|
+
PUBLISH_FILE_MAX_BYTES,
|
|
20
|
+
redactEvents,
|
|
21
|
+
redactExperimentInfo,
|
|
22
|
+
redactJsonValue,
|
|
23
|
+
redactResultRecord,
|
|
24
|
+
redactSpans,
|
|
25
|
+
type Redactor,
|
|
26
|
+
} from "./publish.ts";
|
|
27
|
+
import type { ArtifactKind, AttemptHandle, Scope, Snapshot, SnapshotMeta } from "./types.ts";
|
|
28
|
+
|
|
29
|
+
/** 缺省携带的 artifact:events / trace / o11y / agentSetup / sources;diff 不截断、可达百 MB,缺省不带。 */
|
|
30
|
+
const DEFAULT_PUBLISH_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "sources"];
|
|
31
|
+
const VALID_ARTIFACTS: ArtifactKind[] = ["events", "trace", "o11y", "agentSetup", "diff", "sources"];
|
|
20
32
|
|
|
21
33
|
export interface CopySnapshotsOptions {
|
|
22
|
-
/** 要带上的 artifact
|
|
34
|
+
/** 要带上的 artifact 种类;缺省带 events / trace / o11y / agentSetup / sources,不带 diff。 */
|
|
23
35
|
artifacts?: ArtifactKind[];
|
|
36
|
+
/**
|
|
37
|
+
* 发布消毒(必填,没有隐式默认):传函数逐值消毒自由文本字段,或显式传 `false` 声明
|
|
38
|
+
* 「这批数据可以原文发布」——「不消毒」必须是写在代码里的选择,不是忘了传参数的副作用。
|
|
39
|
+
* 结构字段(格式、判定、身份、路径、哈希)永不经过 redactor。
|
|
40
|
+
*/
|
|
41
|
+
redact: Redactor | false;
|
|
24
42
|
}
|
|
25
43
|
|
|
26
44
|
export interface CopySnapshotsResult {
|
|
@@ -32,25 +50,31 @@ export interface CopySnapshotsResult {
|
|
|
32
50
|
|
|
33
51
|
/**
|
|
34
52
|
* 把选中快照复制成 `destDir` 下的一个标准结果根目录(`<experiment-dir>/<源快照目录名>/`,
|
|
35
|
-
* 快照目录名原样保留,身份不变)。输入收
|
|
53
|
+
* 快照目录名原样保留,身份不变)。输入收 Scope 或手工挑的 Snapshot[]
|
|
36
54
|
* (与 Reports 计算函数同一输入约定)。目标目录非空即报错,不静默覆盖、不合并。
|
|
37
55
|
*/
|
|
38
56
|
export async function copySnapshots(
|
|
39
|
-
|
|
57
|
+
scope: Scope | readonly Snapshot[],
|
|
40
58
|
destDir: string,
|
|
41
|
-
opts
|
|
59
|
+
opts: CopySnapshotsOptions,
|
|
42
60
|
): Promise<CopySnapshotsResult> {
|
|
43
|
-
const selected = Array.isArray(
|
|
61
|
+
const selected = Array.isArray(scope) ? (scope as readonly Snapshot[]) : (scope as Scope).snapshots;
|
|
44
62
|
if (selected.length === 0) {
|
|
45
63
|
throw new Error(
|
|
46
64
|
"copySnapshots got no snapshots to copy. Check the experiments filter, or pass snapshots from openResults().latest().",
|
|
47
65
|
);
|
|
48
66
|
}
|
|
49
|
-
|
|
67
|
+
if (opts?.redact === undefined) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
'copySnapshots requires an explicit "redact" option: pass a (text) => string sanitizer, or the literal false to declare this data safe to publish verbatim.',
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
const redactor: Redactor | undefined = opts.redact === false ? undefined : opts.redact;
|
|
73
|
+
const kinds = opts.artifacts ?? [...DEFAULT_PUBLISH_ARTIFACTS];
|
|
50
74
|
for (const kind of kinds) {
|
|
51
|
-
if (!
|
|
75
|
+
if (!VALID_ARTIFACTS.includes(kind)) {
|
|
52
76
|
throw new Error(
|
|
53
|
-
`Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${
|
|
77
|
+
`Unknown artifact kind "${String(kind)}" in copySnapshots options. Valid kinds: ${VALID_ARTIFACTS.join(", ")}.`,
|
|
54
78
|
);
|
|
55
79
|
}
|
|
56
80
|
}
|
|
@@ -68,101 +92,167 @@ export async function copySnapshots(
|
|
|
68
92
|
continue;
|
|
69
93
|
}
|
|
70
94
|
warnings.push(
|
|
71
|
-
`warning: multiple snapshots selected for experiment "${snapshot.experimentId}"; kept the newest one, dropped the rest. Dedupe with
|
|
95
|
+
`warning: multiple snapshots selected for experiment "${snapshot.experimentId}"; kept the newest one, dropped the rest. Dedupe with Scope.filter() or pick a single snapshot per experiment before copySnapshots to avoid this.`,
|
|
72
96
|
);
|
|
73
97
|
if (isNewerSnapshot(snapshot, existing)) byExperiment.set(snapshot.experimentId, snapshot);
|
|
74
98
|
}
|
|
75
99
|
|
|
76
|
-
|
|
100
|
+
// 发布前整文件预检:先规划并序列化全部目标文件(消毒发生在序列化后、写盘前),任一文件
|
|
101
|
+
// 超过 PUBLISH_FILE_MAX_BYTES 就整体失败,不留半成品目标目录。
|
|
102
|
+
const planned: PlannedFile[] = [];
|
|
77
103
|
for (const snapshot of byExperiment.values()) {
|
|
78
|
-
await
|
|
104
|
+
planned.push(...(await planOneSnapshot(snapshot, [...selected], dest, kinds, redactor)));
|
|
105
|
+
}
|
|
106
|
+
const oversized = planned.filter((f) => f.bytes.byteLength > PUBLISH_FILE_MAX_BYTES);
|
|
107
|
+
if (oversized.length > 0) {
|
|
108
|
+
const lines = oversized.map(
|
|
109
|
+
(f) =>
|
|
110
|
+
` ${f.source ?? f.path}: ${f.bytes.byteLength} bytes (limit ${PUBLISH_FILE_MAX_BYTES}). Exclude that artifact kind from "artifacts", or regenerate the history with the current writer.`,
|
|
111
|
+
);
|
|
112
|
+
throw new Error(`copySnapshots publish precheck failed; ${oversized.length} file(s) exceed the 50 MiB publish budget:\n${lines.join("\n")}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
await mkdir(dest, { recursive: true });
|
|
116
|
+
for (const file of planned) {
|
|
117
|
+
await mkdir(dirname(file.path), { recursive: true });
|
|
118
|
+
await writeFile(file.path, file.bytes);
|
|
79
119
|
}
|
|
80
120
|
|
|
81
121
|
return { dir: dest, warnings };
|
|
82
122
|
}
|
|
83
123
|
|
|
84
|
-
|
|
124
|
+
interface PlannedFile {
|
|
125
|
+
/** 目标绝对路径。 */
|
|
126
|
+
path: string;
|
|
127
|
+
/** 序列化(且已消毒)后的字节。 */
|
|
128
|
+
bytes: Buffer;
|
|
129
|
+
/** 源文件路径(预检报错定位用);重建的 json 没有单一源。 */
|
|
130
|
+
source?: string;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function planOneSnapshot(
|
|
134
|
+
snapshot: Snapshot,
|
|
135
|
+
selected: Snapshot[],
|
|
136
|
+
destRoot: string,
|
|
137
|
+
kinds: ArtifactKind[],
|
|
138
|
+
redactor: Redactor | undefined,
|
|
139
|
+
): Promise<PlannedFile[]> {
|
|
85
140
|
const destSnapDir = join(destRoot, experimentDirOf(snapshot.experimentId), basename(snapshot.dir));
|
|
86
|
-
|
|
141
|
+
const planned: PlannedFile[] = [];
|
|
87
142
|
|
|
88
143
|
// sources 的去重仓库(sources/<sha256>.json)是快照级的:同一份源码被多个 attempt 引用时,
|
|
89
|
-
// 复制到目的地也只应该有一份——这个 Set
|
|
90
|
-
const
|
|
144
|
+
// 复制到目的地也只应该有一份——这个 Set 记录本快照已经规划过的 hash,整快照的 attempt 共享。
|
|
145
|
+
const plannedSourceHashes = new Set<string>();
|
|
91
146
|
for (const attempt of snapshot.attempts) {
|
|
92
|
-
await
|
|
147
|
+
planned.push(...(await planOneAttempt(attempt, destSnapDir, kinds, plannedSourceHashes, redactor)));
|
|
93
148
|
}
|
|
94
149
|
|
|
95
150
|
const knownEvalIds = experimentOfSnapshot(snapshot)?.evalIds ?? fallbackUnion(selected, snapshot.experimentId);
|
|
151
|
+
const experiment =
|
|
152
|
+
snapshot.experiment !== undefined && redactor
|
|
153
|
+
? (redactExperimentInfo(snapshot.experiment as unknown as Record<string, unknown>, redactor) as unknown as SnapshotMeta["experiment"])
|
|
154
|
+
: snapshot.experiment;
|
|
96
155
|
const meta: SnapshotMeta = {
|
|
97
156
|
format: RESULTS_FORMAT,
|
|
98
157
|
schemaVersion: snapshot.schemaVersion,
|
|
99
158
|
producer: snapshot.producer,
|
|
100
159
|
experimentId: snapshot.experimentId,
|
|
101
|
-
...(
|
|
160
|
+
...(experiment !== undefined ? { experiment } : {}),
|
|
102
161
|
agent: snapshot.agent,
|
|
103
162
|
...(snapshot.model !== undefined ? { model: snapshot.model } : {}),
|
|
104
163
|
startedAt: snapshot.startedAt,
|
|
105
164
|
...(snapshot.completedAt !== undefined ? { completedAt: snapshot.completedAt } : {}),
|
|
106
165
|
...(knownEvalIds.length ? { knownEvalIds } : {}),
|
|
107
|
-
...(snapshot.name !== undefined
|
|
166
|
+
...(snapshot.name !== undefined
|
|
167
|
+
? { name: redactor && typeof snapshot.name === "string" ? redactor(snapshot.name) : snapshot.name }
|
|
168
|
+
: {}),
|
|
169
|
+
// 发布根标记只声明流程,不证明结果:applied = 消毒函数对全部自由文本字段跑过;
|
|
170
|
+
// none = 作者显式的原文发布声明。view --out 的防呆据此分级。
|
|
171
|
+
publish: { redaction: redactor ? "applied" : "none" },
|
|
108
172
|
};
|
|
109
|
-
|
|
173
|
+
planned.push({ path: join(destSnapDir, SNAPSHOT_FILE), bytes: Buffer.from(JSON.stringify(meta, null, 2), "utf-8") });
|
|
174
|
+
return planned;
|
|
110
175
|
}
|
|
111
176
|
|
|
112
|
-
async function
|
|
177
|
+
async function planOneAttempt(
|
|
113
178
|
attempt: AttemptHandle,
|
|
114
179
|
destSnapDir: string,
|
|
115
180
|
kinds: ArtifactKind[],
|
|
116
|
-
|
|
117
|
-
|
|
181
|
+
plannedSourceHashes: Set<string>,
|
|
182
|
+
redactor: Redactor | undefined,
|
|
183
|
+
): Promise<PlannedFile[]> {
|
|
118
184
|
const destAttemptDir = join(destSnapDir, attempt.ref.attempt);
|
|
119
|
-
|
|
185
|
+
const planned: PlannedFile[] = [];
|
|
120
186
|
|
|
121
187
|
// sources 是唯一「两层」的 artifact(attempt 级引用 + 快照级去重仓库),不能像其它四类那样
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
// 回退链(携带条目复制后,原快照可能不在目的地里,必须此刻把内容落到自己脚下)。
|
|
188
|
+
// 单文件原字节完事——原字节只是引用,不带内容。走读取面已经会解引用+回退的 attempt.sources()
|
|
189
|
+
// 拿到完整内容,消毒后按(消毒后)内容哈希重新去重落盘——发布根里引用与内容永远一致。
|
|
125
190
|
const genericKinds = kinds.filter((k) => k !== "sources");
|
|
126
191
|
const files = await findArtifactFiles(attempt, genericKinds);
|
|
127
|
-
await Promise.all(files.map(({ kind, source }) => copyFile(source, join(destAttemptDir, artifactFileOf(kind)))));
|
|
128
192
|
const copied = new Set(files.map((f) => f.kind));
|
|
193
|
+
for (const { kind, source } of files) {
|
|
194
|
+
const raw = await readFile(source);
|
|
195
|
+
const bytes = redactor ? redactArtifactBytes(kind, raw, redactor) : raw;
|
|
196
|
+
planned.push({ path: join(destAttemptDir, artifactFileOf(kind)), bytes, source });
|
|
197
|
+
}
|
|
129
198
|
|
|
130
199
|
if (kinds.includes("sources")) {
|
|
131
|
-
const
|
|
132
|
-
if (
|
|
200
|
+
const sources = await attempt.sources();
|
|
201
|
+
if (sources && sources.length > 0) {
|
|
202
|
+
copied.add("sources");
|
|
203
|
+
const destStoreDir = join(destSnapDir, "sources");
|
|
204
|
+
const refs: { path: string; sha256: string }[] = [];
|
|
205
|
+
for (const src of sources) {
|
|
206
|
+
const content = redactor ? redactor(src.content) : src.content;
|
|
207
|
+
const sha256 = hashEvalSource(normalizeEvalSource(content));
|
|
208
|
+
refs.push({ path: src.path, sha256 });
|
|
209
|
+
if (!plannedSourceHashes.has(sha256)) {
|
|
210
|
+
planned.push({
|
|
211
|
+
path: join(destStoreDir, `${sha256}.json`),
|
|
212
|
+
bytes: Buffer.from(JSON.stringify({ content }), "utf-8"),
|
|
213
|
+
});
|
|
214
|
+
plannedSourceHashes.add(sha256);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
planned.push({
|
|
218
|
+
path: join(destAttemptDir, artifactFileOf("sources")),
|
|
219
|
+
bytes: Buffer.from(JSON.stringify(refs), "utf-8"),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
133
222
|
}
|
|
134
223
|
|
|
135
|
-
|
|
136
|
-
|
|
224
|
+
let record = slimForCopy(attempt.result, copied);
|
|
225
|
+
if (redactor) record = redactResultRecord(record, redactor);
|
|
226
|
+
planned.push({ path: join(destAttemptDir, RESULT_FILE), bytes: Buffer.from(JSON.stringify(record, null, 2), "utf-8") });
|
|
227
|
+
return planned;
|
|
137
228
|
}
|
|
138
229
|
|
|
139
|
-
/**
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
await mkdir(destStoreDir, { recursive: true });
|
|
160
|
-
await writeFile(join(destStoreDir, `${sha256}.json`), JSON.stringify({ content: src.content }), "utf-8");
|
|
161
|
-
copiedSourceHashes.add(sha256);
|
|
230
|
+
/** 单个 artifact 文件的消毒:按种类解析 JSON、走各自的自由文本标注,重新序列化。 */
|
|
231
|
+
function redactArtifactBytes(kind: ArtifactKind, raw: Buffer, redactor: Redactor): Buffer {
|
|
232
|
+
try {
|
|
233
|
+
const parsed = JSON.parse(raw.toString("utf-8")) as unknown;
|
|
234
|
+
let next: unknown;
|
|
235
|
+
switch (kind) {
|
|
236
|
+
case "events":
|
|
237
|
+
next = redactEvents(parsed as StreamEvent[], redactor);
|
|
238
|
+
break;
|
|
239
|
+
case "trace":
|
|
240
|
+
next = redactSpans(parsed as TraceSpan[], redactor);
|
|
241
|
+
break;
|
|
242
|
+
// diff 的 before/after/内容与 o11y 的命令 / 错误 / URL 都是自由文本;路径键为结构字段。
|
|
243
|
+
case "diff":
|
|
244
|
+
case "o11y":
|
|
245
|
+
case "agentSetup":
|
|
246
|
+
next = redactJsonValue(parsed, redactor);
|
|
247
|
+
break;
|
|
248
|
+
default:
|
|
249
|
+
next = parsed;
|
|
162
250
|
}
|
|
251
|
+
return Buffer.from(JSON.stringify(next), "utf-8");
|
|
252
|
+
} catch {
|
|
253
|
+
// 解析不了的按原字节透传(malformed 数据不该在发布路径上被静默改写)。
|
|
254
|
+
return raw;
|
|
163
255
|
}
|
|
164
|
-
await writeFile(join(destAttemptDir, artifactFileOf("sources")), JSON.stringify(refs), "utf-8");
|
|
165
|
-
return true;
|
|
166
256
|
}
|
|
167
257
|
|
|
168
258
|
/** 目标目录非空即报错:盘上不该出现「我没写的东西被动过」的惊讶;发布脚本要幂等就自己先清目录。 */
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// show / view 宿主等价契约(设计契约:plan/show-view-equivalence.md 完成定义;
|
|
2
3
|
// docs/feature/reports/architecture.md「Selection 是计算入口」;docs/feature/results/library.md「选择快照」)。
|
|
3
4
|
//
|
|
@@ -23,7 +24,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
|
23
24
|
import { tmpdir } from "node:os";
|
|
24
25
|
import { join } from "node:path";
|
|
25
26
|
import { openResults } from "./index.ts";
|
|
26
|
-
import type {
|
|
27
|
+
import type { Scope, ScopeWarning } from "./index.ts";
|
|
27
28
|
import { selectCurrentResults, type ResultScope } from "./select.ts";
|
|
28
29
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
29
30
|
import { runShow, type ShowFlags } from "../show/index.ts";
|
|
@@ -131,7 +132,7 @@ interface NormSelection {
|
|
|
131
132
|
experiments: NormExperiment[];
|
|
132
133
|
}
|
|
133
134
|
|
|
134
|
-
function normalizeWarning(w:
|
|
135
|
+
function normalizeWarning(w: ScopeWarning): NormWarning {
|
|
135
136
|
switch (w.kind) {
|
|
136
137
|
case "partial-coverage":
|
|
137
138
|
return { kind: w.kind, experimentId: w.experimentId, covered: w.covered, total: w.total };
|
|
@@ -143,7 +144,7 @@ function normalizeWarning(w: SelectionWarning): NormWarning {
|
|
|
143
144
|
}
|
|
144
145
|
}
|
|
145
146
|
|
|
146
|
-
function normalizeSelection(selection:
|
|
147
|
+
function normalizeSelection(selection: Scope): NormSelection {
|
|
147
148
|
return {
|
|
148
149
|
warnings: selection.warnings.map(normalizeWarning),
|
|
149
150
|
experiments: selection.snapshots.map((snapshot) => ({
|
|
@@ -166,7 +167,7 @@ function normalizeSelection(selection: Selection): NormSelection {
|
|
|
166
167
|
async function showText(root: string, patterns: string[], flags: ShowFlags = {}): Promise<string> {
|
|
167
168
|
let out = "";
|
|
168
169
|
let err = "";
|
|
169
|
-
const code = await runShow(root, patterns, {
|
|
170
|
+
const code = await runShow(root, patterns, { results: root, ...flags }, {
|
|
170
171
|
out: (s) => (out += s),
|
|
171
172
|
err: (s) => (err += s),
|
|
172
173
|
width: 120,
|
|
@@ -178,8 +179,8 @@ async function showText(root: string, patterns: string[], flags: ShowFlags = {})
|
|
|
178
179
|
|
|
179
180
|
/** view 的 web 面:loadViewScan → 报告槽 HTML(en)。 */
|
|
180
181
|
async function viewHtml(root: string, opts: ViewScanOptions = {}): Promise<string> {
|
|
181
|
-
const {
|
|
182
|
-
return
|
|
182
|
+
const { reportPages } = await loadViewScan(root, opts);
|
|
183
|
+
return reportPages.map((page) => page.html.en).join("\n");
|
|
183
184
|
}
|
|
184
185
|
|
|
185
186
|
/** 两个宿主构造给选择器的 scope 完全同形:验证读源无误,避免"我以为它们一样"。 */
|
|
@@ -339,7 +340,7 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
|
|
|
339
340
|
const rootA = await makeRoot();
|
|
340
341
|
const rootB = await makeRoot();
|
|
341
342
|
// 失败 eval 让 id 现于两面(通过 eval 只进折叠子行,不便断言);qa / qb 各根独有。
|
|
342
|
-
const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0,
|
|
343
|
+
const boom = { assertions: [{ name: "succeeded()", severity: "gate" as const, score: 0, outcome: "failed" as const, detail: "boom" }] };
|
|
343
344
|
await writeSnapshot(rootA, "2026-07-01T08-00-00-000Z", { experimentId: "onlyA/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [res("qa", "failed", boom)]);
|
|
344
345
|
await writeSnapshot(rootB, "2026-07-02T08-00-00-000Z", { experimentId: "onlyB/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [res("qb", "failed", boom)]);
|
|
345
346
|
const normB = normalizeSelection(selectCurrentResults(await openResults(rootB)));
|
|
@@ -389,7 +390,7 @@ async function seedPartialRerun(): Promise<string> {
|
|
|
389
390
|
const root = await makeRoot();
|
|
390
391
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
391
392
|
res("q1", "passed"),
|
|
392
|
-
res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0,
|
|
393
|
+
res("q2", "failed", { assertions: [{ name: 'fileChanged("q2.tsx")', severity: "gate", score: 0, outcome: "failed" as const, detail: "file was not modified" }] }),
|
|
393
394
|
]);
|
|
394
395
|
await writeSnapshot(root, "2026-07-02T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-02T08:00:00.000Z" }, [
|
|
395
396
|
res("q1", "passed"),
|
|
@@ -404,7 +405,7 @@ async function seedPartialCoverage(): Promise<string> {
|
|
|
404
405
|
root,
|
|
405
406
|
"2026-07-01T08-00-00-000Z",
|
|
406
407
|
{ experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z", knownEvalIds: ["q1", "q2"] },
|
|
407
|
-
[res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
408
|
+
[res("q1", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] })],
|
|
408
409
|
);
|
|
409
410
|
return root;
|
|
410
411
|
}
|
|
@@ -415,9 +416,10 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
415
416
|
const text = await showText(root, []);
|
|
416
417
|
const html = await viewHtml(root);
|
|
417
418
|
for (const face of [text, html]) {
|
|
418
|
-
expect(face).toContain("compare/bub");
|
|
419
419
|
expect(face).toContain("q2"); // 周一补齐的失败 eval,若宿主回退 results.latest() 就会消失
|
|
420
420
|
}
|
|
421
|
+
expect(text).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
|
|
422
|
+
expect(html).toContain('data-sort-value="compare/bub"');
|
|
421
423
|
expect(text).toContain("1 passed / 1 failed");
|
|
422
424
|
expect(html).toContain("50%");
|
|
423
425
|
// 现刻水位覆盖齐全 → 两面都不出 partial-coverage(text: "verdicts cover"; html: data-kind)。
|
|
@@ -431,7 +433,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
431
433
|
it("位置前缀收窄:两扇门都只见范围内的 eval,范围外一致排除", async () => {
|
|
432
434
|
const root = await makeRoot();
|
|
433
435
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
434
|
-
res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
436
|
+
res("weather/brooklyn", "failed", { assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
|
|
435
437
|
res("weather/queens", "passed"),
|
|
436
438
|
res("algebra/quadratic", "passed"),
|
|
437
439
|
]);
|
|
@@ -451,20 +453,32 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
451
453
|
await writeFile(
|
|
452
454
|
reportPath,
|
|
453
455
|
[
|
|
456
|
+
"// 手搭新 defineReport 产物(kind + brand + 规范化页列表);组合组件从 ctx.scope 读注入范围。",
|
|
454
457
|
'const FACES = Symbol.for("niceeval.report.faces");',
|
|
455
|
-
|
|
456
|
-
"
|
|
458
|
+
'const COMPOSE = Symbol.for("niceeval.report.compose");',
|
|
459
|
+
"const Leaf = (props) => Leaf[FACES].web(props);",
|
|
460
|
+
"Leaf[FACES] = {",
|
|
457
461
|
' web: (props) => "EVALS[" + props.ids + "]",',
|
|
458
462
|
' text: (props) => "EVALS[" + props.ids + "]",',
|
|
459
463
|
"};",
|
|
464
|
+
"const Echo = () => { throw new Error(\"compose only\"); };",
|
|
465
|
+
"Echo[COMPOSE] = (_props, ctx) => ({",
|
|
466
|
+
' $$typeof: Symbol.for("react.transitional.element"),',
|
|
467
|
+
" type: Leaf,",
|
|
468
|
+
' props: { ids: ctx.scope.snapshots.flatMap((s) => s.evals.map((e) => e.id)).sort().join(",") },',
|
|
469
|
+
" key: null,",
|
|
470
|
+
"});",
|
|
460
471
|
"export default {",
|
|
472
|
+
' kind: "report",',
|
|
461
473
|
' [Symbol.for("niceeval.report.definition")]: true,',
|
|
462
|
-
"
|
|
463
|
-
|
|
464
|
-
"
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
"
|
|
474
|
+
" links: [],",
|
|
475
|
+
" scripts: [],",
|
|
476
|
+
" styles: [],",
|
|
477
|
+
" pages: [{",
|
|
478
|
+
' id: "report",',
|
|
479
|
+
' title: "Report",',
|
|
480
|
+
' content: { $$typeof: Symbol.for("react.transitional.element"), type: Echo, props: {}, key: null },',
|
|
481
|
+
" }],",
|
|
468
482
|
"};",
|
|
469
483
|
"",
|
|
470
484
|
].join("\n"),
|
|
@@ -497,7 +511,7 @@ function hasPartialCoverageHtml(html: string): boolean {
|
|
|
497
511
|
it("通过计数一致:同一 fixture 两面反映 3 过 1 败", async () => {
|
|
498
512
|
const root = await makeRoot();
|
|
499
513
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
500
|
-
res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0,
|
|
514
|
+
res("q1", "failed", { durationMs: 40_000, estimatedCostUSD: 0.04, assertions: [{ name: "succeeded()", severity: "gate", score: 0, outcome: "failed" as const, detail: "boom" }] }),
|
|
501
515
|
res("q2", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
|
502
516
|
res("q3", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
|
503
517
|
res("q4", "passed", { durationMs: 2_000, estimatedCostUSD: 0.02 }),
|
package/src/results/index.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// niceeval/results —— 实验结果数据的读写库(定稿见 docs/feature/results/library.md、docs/feature/results/architecture.md)。
|
|
2
2
|
//
|
|
3
|
-
// 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、latest()
|
|
3
|
+
// 读:openResults(实验 → 快照 → eval → attempt 分层、skipped、latest() / current() Scope);
|
|
4
4
|
// 写:createResultsWriter(快照声明 + attempt 增量落盘 + finish 补 completedAt);
|
|
5
5
|
// 发布:copySnapshots(格式感知复制 + knownEvalIds 补记);
|
|
6
6
|
// 身份:dedupeAttempts(跨快照聚合前按 (experimentId, evalId, attempt, startedAt) 去重)、isNewerSnapshot。
|
|
@@ -29,9 +29,11 @@ export {
|
|
|
29
29
|
} from "./locator.ts";
|
|
30
30
|
export {
|
|
31
31
|
buildAnnotatedEvalSource,
|
|
32
|
+
deriveSendAnnotations,
|
|
32
33
|
type AnnotatedEvalSource,
|
|
33
34
|
type AnnotatedEvalSourceSummary,
|
|
34
35
|
type AnnotatedSourceLine,
|
|
36
|
+
type SendAnnotation,
|
|
35
37
|
} from "./annotated-source.ts";
|
|
36
38
|
export { loadAnnotatedEvalSource } from "./attempt-source.ts";
|
|
37
39
|
export {
|
|
@@ -42,7 +44,13 @@ export {
|
|
|
42
44
|
} from "./attempt-evidence.ts";
|
|
43
45
|
export { hashEvalSource, normalizeEvalSource } from "./source-hash.ts";
|
|
44
46
|
export { groupIncompatibleVersionSkips, type SkippedVersionGroup } from "./skipped-notice.ts";
|
|
45
|
-
export {
|
|
47
|
+
export {
|
|
48
|
+
comparabilityConfigOf,
|
|
49
|
+
dedupeAttempts,
|
|
50
|
+
deepEqualJson,
|
|
51
|
+
isNewerSnapshot,
|
|
52
|
+
type ComparabilityConfig,
|
|
53
|
+
} from "./select.ts";
|
|
46
54
|
export { copySnapshots, type CopySnapshotsOptions, type CopySnapshotsResult } from "./copy.ts";
|
|
47
55
|
export {
|
|
48
56
|
createResultsWriter,
|
|
@@ -71,8 +79,8 @@ export {
|
|
|
71
79
|
type Eval,
|
|
72
80
|
type Experiment,
|
|
73
81
|
type Results,
|
|
74
|
-
type
|
|
75
|
-
type
|
|
82
|
+
type Scope,
|
|
83
|
+
type ScopeWarning,
|
|
76
84
|
type SkippedDir,
|
|
77
85
|
type Snapshot,
|
|
78
86
|
type SnapshotMeta,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/results/cases.md
|
|
1
2
|
// AttemptLocator 单测(定稿见 docs/feature/results/library.md「按 locator 寻址一个 attempt」、docs/concepts.md「Attempt 定位符」):
|
|
2
3
|
// 确定性编码、字段敏感(任一身份字段变化都换 locator)、@ 前缀与定长 base36 格式、decode 的语法
|
|
3
4
|
// 校验分支、大批量下的实际无撞车、buildLocatorIndex 的撞车检测(用可注入的 encode 强制制造撞车,
|
|
@@ -79,10 +80,6 @@ describe("encodeAttemptLocator · 确定性与格式", () => {
|
|
|
79
80
|
});
|
|
80
81
|
|
|
81
82
|
describe("decodeAttemptLocator · 语法校验", () => {
|
|
82
|
-
it("接受一个真实编码出来的 locator", () => {
|
|
83
|
-
expect(decodeAttemptLocator(encodeAttemptLocator(id()))).toEqual({ valid: true, scheme: 1 });
|
|
84
|
-
});
|
|
85
|
-
|
|
86
83
|
it.each([
|
|
87
84
|
["", "空字符串"],
|
|
88
85
|
["7k2m9qz", "缺 @ 前缀"],
|
|
@@ -149,24 +146,6 @@ describe("buildLocatorIndex", () => {
|
|
|
149
146
|
expect(err.message).toContain("collision");
|
|
150
147
|
});
|
|
151
148
|
|
|
152
|
-
it("撞车中若第三个身份与已记录身份相同,不重复抛错(只有真正不同的身份对才算撞车)", () => {
|
|
153
|
-
const sameEncode = (() => {
|
|
154
|
-
let calls = 0;
|
|
155
|
-
return (_identity: AttemptIdentity) => {
|
|
156
|
-
calls++;
|
|
157
|
-
return `@1samesame` as ReturnType<typeof encodeAttemptLocator>;
|
|
158
|
-
};
|
|
159
|
-
})();
|
|
160
|
-
const identity = id();
|
|
161
|
-
const index = buildLocatorIndex(
|
|
162
|
-
[
|
|
163
|
-
{ identity, handle: "first" },
|
|
164
|
-
{ identity, handle: "second" },
|
|
165
|
-
],
|
|
166
|
-
sameEncode,
|
|
167
|
-
);
|
|
168
|
-
expect(index.size).toBe(1);
|
|
169
|
-
});
|
|
170
149
|
});
|
|
171
150
|
|
|
172
151
|
describe("resolveAttemptLocator", () => {
|
package/src/results/open.ts
CHANGED
|
@@ -11,8 +11,9 @@ import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
|
11
11
|
import type { EvalResult } from "../types.ts";
|
|
12
12
|
import type { O11ySummary, StreamEvent, TraceSpan } from "../types.ts";
|
|
13
13
|
import type { AgentSetupManifest, DiffData, SourceArtifact } from "../types.ts";
|
|
14
|
+
import { deriveDiffData } from "../scoring/diff.ts";
|
|
14
15
|
import { RESULT_FILE, SNAPSHOT_FILE, artifactFileOf, classifySnapshot } from "./format.ts";
|
|
15
|
-
import { isNewerSnapshot, selectLatest } from "./select.ts";
|
|
16
|
+
import { isNewerSnapshot, selectCurrentResults, selectLatest } from "./select.ts";
|
|
16
17
|
import {
|
|
17
18
|
encodeAttemptLocator,
|
|
18
19
|
resolveAttemptLocator,
|
|
@@ -26,7 +27,7 @@ import type {
|
|
|
26
27
|
Eval,
|
|
27
28
|
Experiment,
|
|
28
29
|
Results,
|
|
29
|
-
|
|
30
|
+
Scope,
|
|
30
31
|
SkippedDir,
|
|
31
32
|
Snapshot,
|
|
32
33
|
SnapshotMeta,
|
|
@@ -170,13 +171,17 @@ export async function openResults(dir: string): Promise<Results> {
|
|
|
170
171
|
}
|
|
171
172
|
|
|
172
173
|
function makeResults(experiments: Experiment[], skipped: SkippedDir[]): Results {
|
|
173
|
-
|
|
174
|
+
const results: Results = {
|
|
174
175
|
experiments,
|
|
175
176
|
skipped,
|
|
176
|
-
latest(opts?: { experiments?: string | string[] }):
|
|
177
|
+
latest(opts?: { experiments?: string | string[] }): Scope {
|
|
177
178
|
return selectLatest(experiments, opts);
|
|
178
179
|
},
|
|
180
|
+
current(opts?: { experiments?: string | string[] }): Scope {
|
|
181
|
+
return selectCurrentResults(results, { experiment: opts?.experiments });
|
|
182
|
+
},
|
|
179
183
|
};
|
|
184
|
+
return results;
|
|
180
185
|
}
|
|
181
186
|
|
|
182
187
|
// ───────────────────────── 目录扫描 ─────────────────────────
|
|
@@ -360,7 +365,12 @@ function makeAttempt(snapshot: Snapshot, snapshotDir: string, attemptDir: string
|
|
|
360
365
|
trace: lazyArtifact<TraceSpan[]>(candidates, "trace", record.trace),
|
|
361
366
|
o11y: lazyArtifact<O11ySummary>(candidates, "o11y", record.o11y),
|
|
362
367
|
agentSetup: lazyArtifact<AgentSetupManifest>(candidates, "agentSetup", record.agentSetup),
|
|
363
|
-
diff:
|
|
368
|
+
diff: (() => {
|
|
369
|
+
// diff.json 落盘的是逐窗口 delta 序列(DiffArtifact);文件级视图在读取面派生。
|
|
370
|
+
const raw = lazyArtifact<import("../types.ts").DiffArtifact>(candidates, "diff", record.diff);
|
|
371
|
+
let memo: Promise<DiffData | null> | undefined;
|
|
372
|
+
return () => (memo ??= raw().then((windows) => (windows === null ? null : deriveDiffData(windows))));
|
|
373
|
+
})(),
|
|
364
374
|
sources: lazySources(candidates, candidateSnapshotRoots, record.sources),
|
|
365
375
|
};
|
|
366
376
|
}
|