niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
package/src/view/data.test.ts
CHANGED
|
@@ -14,10 +14,19 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
|
14
14
|
import { tmpdir } from "node:os";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { afterEach, describe, expect, it } from "vitest";
|
|
17
|
-
import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan } from "./data.ts";
|
|
17
|
+
import { IncompatibleResultsError, ViewInputError, loadLatestResultsPerEval, loadViewScan, type ViewScan } from "./data.ts";
|
|
18
|
+
import type { AttemptHandle } from "../results/index.ts";
|
|
18
19
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
19
20
|
import { encodeAttemptLocator } from "../results/locator.ts";
|
|
20
21
|
|
|
22
|
+
/** scan.attemptsByBase 按 base 建索引;测试按 eval id 找回单个 attempt(该 eval 在本 fixture 里唯一时用)。 */
|
|
23
|
+
function attemptByEvalId(scan: ViewScan, evalId: string): AttemptHandle {
|
|
24
|
+
for (const handle of scan.attemptsByBase.values()) {
|
|
25
|
+
if (handle.evalId === evalId) return handle;
|
|
26
|
+
}
|
|
27
|
+
throw new Error(`no attempt found for eval id "${evalId}"`);
|
|
28
|
+
}
|
|
29
|
+
|
|
21
30
|
const roots: string[] = [];
|
|
22
31
|
async function makeRoot(): Promise<string> {
|
|
23
32
|
const root = await mkdtemp(join(tmpdir(), "niceeval-viewdata-"));
|
|
@@ -132,9 +141,9 @@ describe("loadViewScan · skipped 三种原因进 viewData", () => {
|
|
|
132
141
|
const incomplete = [...byReason.values()].find((s) => s.reason === "incomplete")!;
|
|
133
142
|
expect(incomplete.dir).toContain("2026-07-05T08-00-00-000Z");
|
|
134
143
|
|
|
135
|
-
// 正常快照照常进报告槽(默认报告的榜单)
|
|
144
|
+
// 正常快照照常进报告槽(默认报告的榜单),坏快照不拖垮整页;证据室索引也只有 exp/a 一条。
|
|
136
145
|
expect(reportHtml.en).toContain("exp/a");
|
|
137
|
-
expect(
|
|
146
|
+
expect([...scan.attemptsByBase.values()].map((a) => a.experimentId)).toEqual(["exp/a"]);
|
|
138
147
|
});
|
|
139
148
|
|
|
140
149
|
it("单文件模式指向版本不同的报告:抛 IncompatibleResultsError(CLI 打印提示退出)", async () => {
|
|
@@ -182,24 +191,29 @@ describe("loadViewScan · 报告槽是现刻水位口径,裸跑与局部收窄
|
|
|
182
191
|
evalId: "q2",
|
|
183
192
|
attempt: 0,
|
|
184
193
|
});
|
|
185
|
-
expect(reportHtml.en).toContain(
|
|
186
|
-
expect(reportHtml.en).toContain(
|
|
194
|
+
expect(reportHtml.en).toContain(`attempt/${encodeURIComponent(q1Locator)}.html`); // q1 来自周二
|
|
195
|
+
expect(reportHtml.en).toContain(`attempt/${encodeURIComponent(q2Locator)}.html`); // q2 来自周一
|
|
187
196
|
|
|
188
197
|
// 两题都有真实判定,不是伪残缺:报告槽不再出 partial-coverage 警告。
|
|
189
198
|
expect(reportHtml.en).not.toContain('data-kind="partial-coverage"');
|
|
190
199
|
|
|
191
|
-
// 证据室的 latest 标记仍按 results.latest() 口径(周二快照),与报告槽 Selection 无关。
|
|
192
|
-
const latest = viewData.snapshots.filter((s) => s.latest);
|
|
193
|
-
expect(latest).toHaveLength(1);
|
|
194
|
-
expect(latest[0]!.startedAt).toBe("2026-07-02T08:00:00.000Z");
|
|
195
200
|
expect(viewData.lastRunAt).toBe("2026-07-02T08:00:00.000Z");
|
|
196
201
|
// 合成 Selection 的 attempts 来自周一、周二两个物理 run。
|
|
197
202
|
expect(viewData.composedRuns).toBe(2);
|
|
198
203
|
|
|
199
|
-
//
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
204
|
+
// 历史 attempt 仍各自可达(attempt/<locator>.html 的证据室索引,不随报告槽 Selection 收窄):
|
|
205
|
+
// 周二的 q1(现刻水位选中的那份)、周一的 q2(唯一一次,现刻水位也选它)都在;
|
|
206
|
+
// 周一自己的 q1(被周二顶替,不再是现刻水位选中的那份)locator 不同,dedup 也不吞它——
|
|
207
|
+
// 它是独立的历史事实,不是 --resume 字面携带的复印件。
|
|
208
|
+
expect(scan.attemptPages!.locators.has(q1Locator)).toBe(true);
|
|
209
|
+
expect(scan.attemptPages!.locators.has(q2Locator)).toBe(true);
|
|
210
|
+
const q1MondayLocator = encodeAttemptLocator({
|
|
211
|
+
experimentId: "exp/a",
|
|
212
|
+
snapshotStartedAt: "2026-07-01T08:00:00.000Z",
|
|
213
|
+
evalId: "q1",
|
|
214
|
+
attempt: 0,
|
|
215
|
+
});
|
|
216
|
+
expect(scan.attemptPages!.locators.has(q1MondayLocator)).toBe(true);
|
|
203
217
|
});
|
|
204
218
|
});
|
|
205
219
|
|
|
@@ -224,20 +238,19 @@ describe("loadViewScan · 跨快照去重(--resume 携带的复印件只算一
|
|
|
224
238
|
res("q2", "passed"),
|
|
225
239
|
]);
|
|
226
240
|
|
|
227
|
-
const
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
241
|
+
const scan = await loadViewScan(root);
|
|
242
|
+
const { artifactDirs, attemptsByBase } = scan;
|
|
243
|
+
// 全部落盘(证据室索引的数据面)里 q1 只出现一次:复印件不灌票。
|
|
244
|
+
const allAttempts = [...attemptsByBase.values()];
|
|
245
|
+
expect(allAttempts.filter((a) => a.evalId === "q1")).toHaveLength(1);
|
|
246
|
+
const q1 = allAttempts.find((a) => a.evalId === "q1")!;
|
|
232
247
|
// 证据身份(locator)跟着最新落盘走:身份元组里的 snapshotStartedAt 是新快照的 startedAt。
|
|
233
248
|
expect(q1.locator).toBe(
|
|
234
249
|
encodeAttemptLocator({ experimentId: "exp/a", snapshotStartedAt: "2026-07-02T08:00:00.000Z", evalId: "q1", attempt: 0 }),
|
|
235
250
|
);
|
|
236
|
-
expect(q1.artifactBase).toBe("exp_a/2026-07-01T08-00-00-000Z/q1/a0"); // artifact 仍指原快照
|
|
251
|
+
expect(q1.result.artifactBase).toBe("exp_a/2026-07-01T08-00-00-000Z/q1/a0"); // artifact 仍指原快照
|
|
237
252
|
// 静态导出(--out)能把携带条目的 artifact 一并带走。
|
|
238
253
|
expect(artifactDirs.get("exp_a/2026-07-01T08-00-00-000Z/q1/a0")).toBe(join(oldDir, "q1", "a0"));
|
|
239
|
-
// 条目全被吸走的旧快照不再出现。
|
|
240
|
-
expect(viewData.snapshots.filter((s) => !s.latest)).toHaveLength(0);
|
|
241
254
|
});
|
|
242
255
|
});
|
|
243
256
|
|
|
@@ -256,28 +269,27 @@ describe("loadViewScan · 新布局落盘直接可读(写入面 / 读取面同
|
|
|
256
269
|
);
|
|
257
270
|
|
|
258
271
|
const scan = await loadViewScan(root);
|
|
259
|
-
const { viewData } = scan;
|
|
260
272
|
const reportHtml = scan.reportPages[0]!.html;
|
|
261
273
|
// 报告槽(ExperimentComparison 的 web 面):Experiment 工作台行 + 官方通过率格子(1 过 1 败 = 50%)。
|
|
262
274
|
expect(reportHtml.en).toContain("compare/bub");
|
|
263
275
|
expect(reportHtml.en).toContain("50%");
|
|
264
|
-
const
|
|
265
|
-
expect(snapshot.
|
|
266
|
-
expect(snapshot.
|
|
267
|
-
expect(snapshot.run).toBe("compare_bub/2026-07-03T08-00-00-000Z");
|
|
276
|
+
const attempts = [...scan.attemptsByBase.values()];
|
|
277
|
+
expect(attempts.every((a) => a.snapshot.agent === "bub")).toBe(true);
|
|
278
|
+
expect(attempts.every((a) => a.snapshot.startedAt === "2026-07-03T08:00:00.000Z")).toBe(true);
|
|
268
279
|
// 每条结果的 locator 都能由身份元组(experimentId/快照 startedAt/evalId/attempt 下标)独立复算,
|
|
269
280
|
// 证明它不是随手塞的占位值,而是真从落盘产物(snapshot.json + result.json)算出来的。
|
|
270
|
-
expect(
|
|
271
|
-
experimentId:
|
|
272
|
-
snapshotStartedAt: snapshot.startedAt,
|
|
273
|
-
evalId:
|
|
274
|
-
attempt:
|
|
281
|
+
expect(attempts.every((a) => a.locator === encodeAttemptLocator({
|
|
282
|
+
experimentId: a.experimentId,
|
|
283
|
+
snapshotStartedAt: a.snapshot.startedAt,
|
|
284
|
+
evalId: a.evalId,
|
|
285
|
+
attempt: a.result.attempt,
|
|
275
286
|
}))).toBe(true);
|
|
276
|
-
// 本快照跑出的条目落盘没有 artifactBase 字段:读取面按 `${ref.snapshot}/${ref.attempt}`
|
|
277
|
-
//
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
expect(q1
|
|
287
|
+
// 本快照跑出的条目落盘没有 artifactBase 字段:读取面按 `${ref.snapshot}/${ref.attempt}` 现算——
|
|
288
|
+
// scan.attemptsByBase 正是按这份现算出的 base 建的索引,q1 应恰好落在这个 key 上。
|
|
289
|
+
const q1Base = "compare_bub/2026-07-03T08-00-00-000Z/q1/a0";
|
|
290
|
+
const q1 = scan.attemptsByBase.get(q1Base)!;
|
|
291
|
+
expect(q1).toBeTruthy();
|
|
292
|
+
expect(q1.result.hasEvents).toBe(true);
|
|
281
293
|
});
|
|
282
294
|
});
|
|
283
295
|
|
package/src/view/data.ts
CHANGED
|
@@ -7,29 +7,23 @@
|
|
|
7
7
|
// 见 docs/feature/reports/view.md「打开与收窄」。
|
|
8
8
|
|
|
9
9
|
import { readFileSync, statSync } from "node:fs";
|
|
10
|
-
import {
|
|
11
|
-
import { dedupeAttempts, openResults } from "../results/index.ts";
|
|
10
|
+
import { dirname, extname, join, relative, resolve } from "node:path";
|
|
11
|
+
import { dedupeAttempts, loadAttemptEvidence, openResults } from "../results/index.ts";
|
|
12
12
|
import type { AttemptHandle, Results, Scope, Snapshot, SkippedDir } from "../results/index.ts";
|
|
13
|
+
import type { AttemptLocator } from "../results/locator.ts";
|
|
13
14
|
import {
|
|
15
|
+
buildHostReportMeta,
|
|
14
16
|
loadHostReport,
|
|
15
17
|
renderHostPageHtml,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
type
|
|
19
|
-
type
|
|
20
|
-
|
|
21
|
-
} from "../show/report-host.ts";
|
|
18
|
+
type ReportAsset,
|
|
19
|
+
type ReportDefinition,
|
|
20
|
+
type ReportPage,
|
|
21
|
+
type HeadTag,
|
|
22
|
+
} from "../report/runtime/host.ts";
|
|
22
23
|
import { selectCurrentResults, filterExperiments } from "../results/select.ts";
|
|
23
24
|
import { evalPrefixPredicate } from "../shared/aggregate.ts";
|
|
24
25
|
import type { EvalResult } from "../types.ts";
|
|
25
|
-
import type {
|
|
26
|
-
SkippedRunNotice,
|
|
27
|
-
ViewData,
|
|
28
|
-
ViewEvalResult,
|
|
29
|
-
ViewReportMeta,
|
|
30
|
-
ViewReportPageHtml,
|
|
31
|
-
ViewSnapshot,
|
|
32
|
-
} from "./shared/types.ts";
|
|
26
|
+
import type { SkippedRunNotice, ViewData, ViewReportMeta, ViewReportPageHtml } from "./shared/types.ts";
|
|
33
27
|
import { t } from "../i18n/index.ts";
|
|
34
28
|
import { RESULTS_SCHEMA_VERSION } from "../types.ts";
|
|
35
29
|
|
|
@@ -58,8 +52,26 @@ export interface ViewScan {
|
|
|
58
52
|
reportPages: ViewReportPageHtml[];
|
|
59
53
|
/** 外壳注入资产(styles / scripts;{src} 已按路径纪律解析成 inline 内容),只进 web 面。 */
|
|
60
54
|
shellAssets: { styles: string[]; scripts: string[]; head: ResolvedHeadTag[] };
|
|
55
|
+
/**
|
|
56
|
+
* 报告声明了 attempt-input page 时才存在(architecture.md「Attempt 详情是一张参数化 page」)。
|
|
57
|
+
* `locators` 是收窄后有效根内可达的 locator → AttemptHandle(与 scope-input pages 同一份
|
|
58
|
+
* `scopedExperiments ∩ matchEval ∩ survivors` 口径——去重只吞掉 `--resume` 携带的字面重复,
|
|
59
|
+
* 不排除真实历史 attempt,见 view.md「打开与收窄」);站点管线(site.ts)据此为每个可达
|
|
60
|
+
* locator 生成一份 `attempt/<locator>.html`,不在这份索引里的 locator 不出站。
|
|
61
|
+
* `render` 装配一个 locator 的 `AttemptEvidence` 并渲染该 page 两种语言的内容 HTML(不含外层
|
|
62
|
+
* 文档 —— 独立 HTML 文档的组装是 site.ts 的事);pageFailure 语义与 scope pages 一致。
|
|
63
|
+
*/
|
|
64
|
+
attemptPages?: {
|
|
65
|
+
page: ReportPage;
|
|
66
|
+
locators: Map<AttemptLocator, AttemptHandle>;
|
|
67
|
+
render(locator: AttemptLocator, handle: AttemptHandle): Promise<{ en: string; "zh-CN": string }>;
|
|
68
|
+
};
|
|
61
69
|
}
|
|
62
70
|
|
|
71
|
+
/** attempt 页面内容(不是 index.html 的 scope-input page)链去同目录 attempt/ 下的兄弟文档,
|
|
72
|
+
* 不带 `attempt/` 目录前缀(site.ts「站点管线」;两处 href 的对应关系与拆分只住在那)。 */
|
|
73
|
+
const SIBLING_ATTEMPT_HREF = (locator: AttemptLocator): string => `${encodeURIComponent(locator)}.html`;
|
|
74
|
+
|
|
63
75
|
/** view 宿主输入的组合语义(与 show 对齐,docs/feature/reports/architecture.md「Scope 是计算入口」)。 */
|
|
64
76
|
export interface ViewScanOptions {
|
|
65
77
|
/** eval id 前缀(位置参数):把根滤成有效根,页面 Scope 与证据(快照明细、artifact 清单)一致收窄。 */
|
|
@@ -213,9 +225,6 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
213
225
|
|
|
214
226
|
// 报告槽 Selection:恒经现刻水位选择器合成,与 show 裸跑同口径(两扇门判定不分叉)。
|
|
215
227
|
const selection = selectCurrentResults(results, { experiment: opts.experiment, patterns });
|
|
216
|
-
// latestPerExperiment 只服务证据室 UI 的 latest 标记(ViewSnapshot.latest / viewData.snapshots),
|
|
217
|
-
// 与报告槽 Selection 完全无关,绝不复用为报告 Selection。
|
|
218
|
-
const latestPerExperiment = results.latest();
|
|
219
228
|
|
|
220
229
|
if (patterns.length > 0 && selection.snapshots.every((s) => s.evals.length === 0)) {
|
|
221
230
|
const known = [
|
|
@@ -238,44 +247,21 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
238
247
|
const matchEval = patterns.length > 0 ? evalPrefixPredicate(patterns) : () => true;
|
|
239
248
|
|
|
240
249
|
// 跨快照按身份键去重:--resume 携带的条目在多份落盘里重复,只保留最新快照里的那份
|
|
241
|
-
// (与官方计算函数的聚合口径一致,
|
|
250
|
+
// (与官方计算函数的聚合口径一致,attempt/<locator>.html 的计数因此不被复印件灌票)。
|
|
242
251
|
const artifactDirs = new Map<string, string>();
|
|
243
252
|
const attemptsByBase = new Map<string, AttemptHandle>();
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
const
|
|
253
|
+
// 报告没有 attempt-input page 时不建这份索引:没有 attempt/ 目录,站点管线不需要它
|
|
254
|
+
// (view.md「静态导出」)。
|
|
255
|
+
const attemptsByLocator = slot.attemptPage ? new Map<AttemptLocator, AttemptHandle>() : undefined;
|
|
247
256
|
const allAttempts: AttemptHandle[] = [];
|
|
248
257
|
for (const exp of scopedExperiments) {
|
|
249
258
|
for (const snap of exp.snapshots) allAttempts.push(...snap.attempts.filter((a) => matchEval(a.evalId)));
|
|
250
259
|
}
|
|
251
|
-
const
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
if (patterns.length > 0 && !exp.evalIds.some((id) => matchEval(id))) continue;
|
|
257
|
-
for (const snap of exp.snapshots) {
|
|
258
|
-
const kept = snap.attempts.filter((a) => survivors.has(a));
|
|
259
|
-
const latest = latestSet.has(snap);
|
|
260
|
-
// 条目全被去重吸走的历史快照不再携带(它的内容原样活在更新的落盘里)。
|
|
261
|
-
if (kept.length === 0 && !latest) continue;
|
|
262
|
-
snapshots.push({
|
|
263
|
-
experimentId: snap.experimentId,
|
|
264
|
-
agent: snap.agent,
|
|
265
|
-
...(snap.model !== undefined ? { model: snap.model } : {}),
|
|
266
|
-
startedAt: snap.startedAt,
|
|
267
|
-
// 与 reader 的 AttemptRef.snapshot 同一公式(祖父目录名/自身目录名),不依赖 root——
|
|
268
|
-
// 单文件模式的 root 是从目标文件上跳算出的,两条计算各自独立更不容易踩偏差。
|
|
269
|
-
run: `${basename(dirname(snap.dir))}/${basename(snap.dir)}`,
|
|
270
|
-
latest,
|
|
271
|
-
results: kept.map((a) => {
|
|
272
|
-
const { annotated, base, abs } = annotateResult(a, root);
|
|
273
|
-
artifactDirs.set(base, abs);
|
|
274
|
-
attemptsByBase.set(base, a);
|
|
275
|
-
return annotated;
|
|
276
|
-
}),
|
|
277
|
-
});
|
|
278
|
-
}
|
|
260
|
+
for (const attempt of dedupeAttempts(allAttempts).attempts) {
|
|
261
|
+
const { base, abs } = artifactLocation(attempt, root);
|
|
262
|
+
artifactDirs.set(base, abs);
|
|
263
|
+
attemptsByBase.set(base, attempt);
|
|
264
|
+
if (attemptsByLocator && attempt.locator !== undefined) attemptsByLocator.set(attempt.locator, attempt);
|
|
279
265
|
}
|
|
280
266
|
|
|
281
267
|
// 全局最新快照(跨有效根内全部实验):viewData.lastRunAt 从这里取。页内 hero 的「最后运行」
|
|
@@ -293,7 +279,6 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
293
279
|
// 反向引用取——每个 attempt 的 .snapshot 恒指向它真实所在的贡献快照(无论 Scope
|
|
294
280
|
// 是否合成),所以这条对裸跑与收窄一律成立,不需要分支。
|
|
295
281
|
composedRuns: new Set(selection.snapshots.flatMap((s) => s.attempts.map((a) => a.snapshot.dir))).size,
|
|
296
|
-
snapshots,
|
|
297
282
|
skippedRuns: results.skipped.map(toSkippedNotice),
|
|
298
283
|
report: slot.meta,
|
|
299
284
|
};
|
|
@@ -303,6 +288,9 @@ export async function loadViewScan(input?: string, opts: ViewScanOptions = {}):
|
|
|
303
288
|
attemptsByBase,
|
|
304
289
|
reportPages: slot.pages,
|
|
305
290
|
shellAssets: slot.shellAssets,
|
|
291
|
+
...(slot.attemptPage && attemptsByLocator
|
|
292
|
+
? { attemptPages: { page: slot.attemptPage, locators: attemptsByLocator, render: slot.renderAttemptPage } }
|
|
293
|
+
: {}),
|
|
306
294
|
};
|
|
307
295
|
}
|
|
308
296
|
|
|
@@ -345,7 +333,7 @@ export interface ResolvedHeadTag {
|
|
|
345
333
|
}
|
|
346
334
|
|
|
347
335
|
/** head 标签 attrs 的 src/href 解析:外链透传,本地路径按 `{src}` 同一路径纪律落成绝对路径并验存在。 */
|
|
348
|
-
function resolveShellHead(tags:
|
|
336
|
+
function resolveShellHead(tags: readonly HeadTag[], baseDir: string | undefined): ResolvedHeadTag[] {
|
|
349
337
|
const out: ResolvedHeadTag[] = [];
|
|
350
338
|
for (const entry of tags) {
|
|
351
339
|
const attrs = { ...(entry.attrs ?? {}) };
|
|
@@ -378,7 +366,7 @@ function resolveShellHead(tags: HostHeadTag[], baseDir: string | undefined): Res
|
|
|
378
366
|
}
|
|
379
367
|
|
|
380
368
|
/** 外壳 `{src}` 资产的路径纪律(shell.md「行为约束」):相对报告文件解析;拒绝 `..` 路径段、绝对路径与 `~`。 */
|
|
381
|
-
function resolveShellAssets(assets:
|
|
369
|
+
function resolveShellAssets(assets: readonly ReportAsset[], baseDir: string | undefined, kind: "styles" | "scripts"): string[] {
|
|
382
370
|
const out: string[] = [];
|
|
383
371
|
for (const asset of assets) {
|
|
384
372
|
if (asset.inline !== undefined) {
|
|
@@ -421,17 +409,32 @@ async function renderReportSlot(
|
|
|
421
409
|
meta: ViewReportMeta;
|
|
422
410
|
pages: ViewReportPageHtml[];
|
|
423
411
|
shellAssets: { styles: string[]; scripts: string[]; head: ResolvedHeadTag[] };
|
|
412
|
+
attemptPage: ReportPage | undefined;
|
|
413
|
+
renderAttemptPage: (locator: AttemptLocator, handle: AttemptHandle) => Promise<{ en: string; "zh-CN": string }>;
|
|
424
414
|
}> {
|
|
425
415
|
// 报告 runtime 走预编译产物(dist/report/**,`pnpm run build:report` 产出),不受 view
|
|
426
|
-
// 消费方 cwd/tsconfig 影响;装载与渲染统一经 ../
|
|
427
|
-
const hostReport:
|
|
416
|
+
// 消费方 cwd/tsconfig 影响;装载与渲染统一经 ../report/runtime/host.ts(两个宿主共用的中性联系面)。
|
|
417
|
+
const hostReport: ReportDefinition = await loadHostReport(report?.cwd ?? process.cwd(), report?.path, {
|
|
428
418
|
freshImport: true,
|
|
429
419
|
});
|
|
430
420
|
|
|
431
|
-
|
|
432
|
-
|
|
421
|
+
// scope-input pages 只有这些参与本函数的「全部烘进 index.html」渲染;attempt-input page(如果
|
|
422
|
+
// 报告声明了)没有 locator 就不能 resolve,它的每-locator 静态文档是独立机制,不在这里渲染
|
|
423
|
+
// (docs/feature/reports/architecture.md「Attempt 详情是一张参数化 page」)。
|
|
424
|
+
const scopePages = hostReport.pages.filter((p) => p.input !== "attempt");
|
|
425
|
+
const navigablePages = scopePages.filter((p) => p.navigation !== false);
|
|
426
|
+
|
|
427
|
+
const initialPageId = page ?? navigablePages[0]?.id ?? scopePages[0]?.id;
|
|
428
|
+
const initialPage = initialPageId !== undefined ? scopePages.find((p) => p.id === initialPageId) : undefined;
|
|
429
|
+
if (initialPageId === undefined || !initialPage) {
|
|
430
|
+
const requested = hostReport.pages.find((p) => p.id === page);
|
|
431
|
+
if (requested?.input === "attempt") {
|
|
432
|
+
throw new ViewInputError(
|
|
433
|
+
`error: page "${page}" in ${report?.path ?? "the built-in report"} is an attempt-input page and needs a locator — it cannot be opened as the initial page directly.`,
|
|
434
|
+
);
|
|
435
|
+
}
|
|
433
436
|
throw new ViewInputError(
|
|
434
|
-
`error: page "${page}" not found in ${report?.path ?? "the built-in report"}. Available pages: ${
|
|
437
|
+
`error: page "${page}" not found in ${report?.path ?? "the built-in report"}. Available pages: ${navigablePages.map((p) => p.id).join(", ")}`,
|
|
435
438
|
);
|
|
436
439
|
}
|
|
437
440
|
|
|
@@ -442,10 +445,10 @@ async function renderReportSlot(
|
|
|
442
445
|
head: resolveShellHead(hostReport.head, reportDir),
|
|
443
446
|
};
|
|
444
447
|
|
|
448
|
+
const hostMeta = await buildHostReportMeta(hostReport, selection);
|
|
445
449
|
const pages: ViewReportPageHtml[] = [];
|
|
446
|
-
for (const hostPage of
|
|
447
|
-
const
|
|
448
|
-
const ctx = { scope: selection, results, report: meta };
|
|
450
|
+
for (const hostPage of scopePages) {
|
|
451
|
+
const ctx = { scope: selection, results, report: hostMeta, page: { id: hostPage.id, input: "scope" as const } };
|
|
449
452
|
try {
|
|
450
453
|
pages.push({
|
|
451
454
|
id: hostPage.id,
|
|
@@ -464,13 +467,42 @@ async function renderReportSlot(
|
|
|
464
467
|
}
|
|
465
468
|
|
|
466
469
|
const meta: ViewReportMeta = {
|
|
467
|
-
title:
|
|
468
|
-
links: hostReport.links,
|
|
470
|
+
title: hostMeta.title,
|
|
471
|
+
links: [...hostReport.links],
|
|
469
472
|
...(hostReport.footer !== undefined ? { footer: hostReport.footer } : {}),
|
|
470
|
-
pages:
|
|
473
|
+
pages: scopePages.map((p) => ({ id: p.id, title: p.title })),
|
|
471
474
|
initialPageId,
|
|
472
475
|
};
|
|
473
|
-
|
|
476
|
+
|
|
477
|
+
const attemptPage = hostReport.pages.find((p) => p.input === "attempt");
|
|
478
|
+
// 装配一个 locator 的 AttemptEvidence 并渲染该 page 两种语言的内容 HTML(不含外层文档);
|
|
479
|
+
// pageFailure 语义与 scope pages 相同 —— 本地 server 折成该文档的完整错误反馈块,
|
|
480
|
+
// 静态导出直接抛出(writeSite 侧汇总成「整体失败,不留半套目录」)。
|
|
481
|
+
const renderAttemptPage = async (
|
|
482
|
+
locator: AttemptLocator,
|
|
483
|
+
handle: AttemptHandle,
|
|
484
|
+
): Promise<{ en: string; "zh-CN": string }> => {
|
|
485
|
+
const evidence = await loadAttemptEvidence(handle);
|
|
486
|
+
const ctx = {
|
|
487
|
+
scope: selection,
|
|
488
|
+
results,
|
|
489
|
+
report: hostMeta,
|
|
490
|
+
page: { id: attemptPage!.id, input: "attempt" as const, locator, evidence },
|
|
491
|
+
};
|
|
492
|
+
try {
|
|
493
|
+
return {
|
|
494
|
+
en: await renderHostPageHtml(attemptPage!, ctx, { locale: "en", attemptHref: SIBLING_ATTEMPT_HREF }),
|
|
495
|
+
"zh-CN": await renderHostPageHtml(attemptPage!, ctx, { locale: "zh-CN", attemptHref: SIBLING_ATTEMPT_HREF }),
|
|
496
|
+
};
|
|
497
|
+
} catch (e) {
|
|
498
|
+
if (pageFailure !== "embed") throw e;
|
|
499
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
500
|
+
const block = `<div class="nre nre-page-error"><pre>${escapeHtml(message)}</pre></div>`;
|
|
501
|
+
return { en: block, "zh-CN": block };
|
|
502
|
+
}
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
return { meta, pages, shellAssets, attemptPage, renderAttemptPage };
|
|
474
506
|
}
|
|
475
507
|
|
|
476
508
|
/**
|
|
@@ -504,24 +536,14 @@ function assertSingleFileReadable(results: Results, target: string): void {
|
|
|
504
536
|
}
|
|
505
537
|
|
|
506
538
|
/**
|
|
507
|
-
*
|
|
508
|
-
*
|
|
509
|
-
*
|
|
510
|
-
*
|
|
511
|
-
* 条目落盘没有这个字段,按 `${ref.snapshot}/${ref.attempt}` 现算;携带条目(--resume 合入)
|
|
512
|
-
* 落盘自带 artifactBase,指向原快照,原样沿用。
|
|
513
|
-
* 返回新对象,不 mutate 读入的结果;宿主机绝对路径只回给调用方写进 artifactDirs
|
|
514
|
-
* (server 端内存),不挂到 result 上,避免随 viewData 进静态 HTML。
|
|
539
|
+
* 一条 attempt 的 artifact 目录:相对 view 根的 base(前端据此 fetch trace.json 等)与宿主机
|
|
540
|
+
* 绝对路径。本快照跑出的条目落盘没有 artifactBase 字段,按 `${ref.snapshot}/${ref.attempt}`
|
|
541
|
+
* 现算;携带条目(--resume 合入)落盘自带 artifactBase,指向原快照,原样沿用。绝对路径只
|
|
542
|
+
* 回给调用方写进 artifactDirs(server 端内存),不进 viewData,避免随静态 HTML 泄漏宿主机路径。
|
|
515
543
|
*/
|
|
516
|
-
function
|
|
517
|
-
attempt
|
|
518
|
-
|
|
519
|
-
): { annotated: ViewEvalResult; base: string; abs: string } {
|
|
520
|
-
const r = attempt.result;
|
|
521
|
-
const base = r.artifactBase ?? `${attempt.ref.snapshot}/${attempt.ref.attempt}`;
|
|
522
|
-
const abs = join(root, base);
|
|
523
|
-
const annotated: ViewEvalResult = { ...r, locator: attempt.locator, artifactBase: base };
|
|
524
|
-
return { annotated, base, abs };
|
|
544
|
+
function artifactLocation(attempt: AttemptHandle, root: string): { base: string; abs: string } {
|
|
545
|
+
const base = attempt.result.artifactBase ?? `${attempt.ref.snapshot}/${attempt.ref.attempt}`;
|
|
546
|
+
return { base, abs: join(root, base) };
|
|
525
547
|
}
|
|
526
548
|
|
|
527
549
|
/**
|
package/src/view/index.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { planSite, writeSite } from "./site.ts";
|
|
|
10
10
|
import type { ViewOptions } from "./server.ts";
|
|
11
11
|
|
|
12
12
|
export { startViewServer, type ViewOptions, type ViewServer } from "./server.ts";
|
|
13
|
-
export { planSite, writeSite, renderHtml, type SitePlan, type SiteFile } from "./site.ts";
|
|
13
|
+
export { planSite, writeSite, renderHtml, renderStandaloneAttemptDocument, type SitePlan, type SiteFile } from "./site.ts";
|
|
14
14
|
export {
|
|
15
15
|
IncompatibleResultsError,
|
|
16
16
|
ViewInputError,
|
package/src/view/server.ts
CHANGED
|
@@ -5,9 +5,19 @@
|
|
|
5
5
|
// (pageFailure: "embed")。位置参数 / --exp 收窄是管线输入,不是宿主语义——两宿主同义。
|
|
6
6
|
|
|
7
7
|
import { createServer, type Server } from "node:http";
|
|
8
|
-
import { type ViewScanOptions } from "./data.ts";
|
|
9
|
-
import { planSite, readSiteFile, type SitePlan } from "./site.ts";
|
|
8
|
+
import { loadViewScan, type ViewScanOptions } from "./data.ts";
|
|
9
|
+
import { planSite, readSiteFile, renderStandaloneAttemptDocument, type SitePlan } from "./site.ts";
|
|
10
10
|
import { formatThrown } from "../util.ts";
|
|
11
|
+
import type { AttemptLocator } from "../results/locator.ts";
|
|
12
|
+
|
|
13
|
+
const HTML_TYPE = "text/html; charset=utf-8";
|
|
14
|
+
|
|
15
|
+
/** `attempt/<locator>.html` 站点路径 → 磁盘/清单键用的原始 locator(未编码,含字面 `@`;
|
|
16
|
+
* 见 site.ts「站点管线」对编码边界的说明)。不是这个形状返回 undefined。 */
|
|
17
|
+
function attemptLocatorFromSitePath(sitePath: string): AttemptLocator | undefined {
|
|
18
|
+
if (!sitePath.startsWith("attempt/") || !sitePath.endsWith(".html")) return undefined;
|
|
19
|
+
return sitePath.slice("attempt/".length, -".html".length) as AttemptLocator;
|
|
20
|
+
}
|
|
11
21
|
|
|
12
22
|
export interface ViewOptions {
|
|
13
23
|
input?: string;
|
|
@@ -70,6 +80,21 @@ export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServe
|
|
|
70
80
|
file = plan.files.get(sitePath);
|
|
71
81
|
}
|
|
72
82
|
if (!file) {
|
|
83
|
+
// 本地宿主的 attempt 详情路由越过收窄,对完整结果根解析(docs/engineering/unit-tests/
|
|
84
|
+
// reports/cases.md 第 198/220 行;与 `show @<locator>` 同一套「各自结果根语义寻址」,
|
|
85
|
+
// 不是 SitePlan 清单之外的旁路取数——这条路由本来就不该受 --exp/eval 前缀收窄限制,
|
|
86
|
+
// 与「server 不提供清单之外的路径」的奇偶保证不冲突,那条保证只约束收窄之内的路径)。
|
|
87
|
+
const locator = attemptLocatorFromSitePath(sitePath);
|
|
88
|
+
if (locator !== undefined) {
|
|
89
|
+
const fullScan = await loadViewScan(input, { ...scanOptions, experiment: undefined, patterns: [] }).catch(() => undefined);
|
|
90
|
+
const handle = fullScan?.attemptPages?.locators.get(locator);
|
|
91
|
+
if (fullScan && handle) {
|
|
92
|
+
const body = await renderStandaloneAttemptDocument(fullScan, locator, handle);
|
|
93
|
+
res.writeHead(200, { "content-type": HTML_TYPE, "cache-control": "no-store" });
|
|
94
|
+
res.end(body);
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
73
98
|
res.writeHead(404, { "content-type": "text/plain; charset=utf-8" });
|
|
74
99
|
res.end("not found");
|
|
75
100
|
return;
|
|
@@ -80,6 +105,11 @@ export async function startViewServer(opts: ViewOptions = {}): Promise<ViewServe
|
|
|
80
105
|
res.end("not found");
|
|
81
106
|
return;
|
|
82
107
|
}
|
|
108
|
+
// 同一路径同一 plan 生命周期内不重复求值(architecture.md「管线以 page 实例为单位执行」):
|
|
109
|
+
// lazy 产出器求值一次后把结果写回清单,下一次同路径请求(未触发 rebuild 之前)直接命中。
|
|
110
|
+
if (file.source.kind === "lazy") {
|
|
111
|
+
plan.files.set(sitePath, { ...file, source: { kind: "content", body: body as string } });
|
|
112
|
+
}
|
|
83
113
|
res.writeHead(200, { "content-type": file.contentType, "cache-control": "no-store" });
|
|
84
114
|
res.end(body);
|
|
85
115
|
} catch (e) {
|
package/src/view/shared/types.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// server(data.ts)与前端(app/)共用的 view 数据形状。
|
|
2
2
|
// viewData 会被序列化进静态 HTML,两边必须对同一份声明编程;只允许 type import。
|
|
3
3
|
//
|
|
4
|
-
// viewData
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// viewData 只携带壳需要的东西:skipped、项目名与 run 元信息。attempt 明细不在这里——
|
|
5
|
+
// 每份 attempt/<locator>.html 是独立静态文档(site.ts),不通过 viewData 这条通道下发;
|
|
6
|
+
// 统计口径(KPI / 榜单 / 挑选警告)整体住在报告槽的静态 HTML 里(ExperimentComparison 或
|
|
7
|
+
// --report 的报告自己算),壳与报告之间没有第二条数据通道。
|
|
7
8
|
|
|
8
|
-
import type {
|
|
9
|
-
import type { ReportLocale } from "../../report/locale.ts";
|
|
9
|
+
import type { LocalizedText } from "../../types.ts";
|
|
10
|
+
import type { ReportLocale } from "../../report/model/locale.ts";
|
|
10
11
|
import type { AttemptLocator } from "../../results/locator.ts";
|
|
11
12
|
|
|
12
13
|
export type { AttemptLocator };
|
|
@@ -47,30 +48,6 @@ export interface ViewReportMeta {
|
|
|
47
48
|
initialPageId: string;
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
/** view 侧的 attempt 结果 = 瘦身后的 EvalResult + loader 注入的深链身份(不透明 AttemptLocator,
|
|
51
|
-
* `#/attempt/@<locator>` 路由的参数,与 Reports 的 MetricCell.refs / `ctx.attemptHref` 同一身份契约)
|
|
52
|
-
* 与 artifact 基址。 */
|
|
53
|
-
export type ViewEvalResult = EvalResult & { locator?: AttemptLocator };
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* 快照 = 单次跑的实验(experiment × 一次运行),与 niceeval/results 的 Snapshot 同口径。
|
|
57
|
-
* 携带 attempt 明细供证据室(钻取 / AttemptModal / Runs / Traces)渲染;
|
|
58
|
-
* 榜单统计不从这里算,吃 ViewData.table / overview 的官方产物。
|
|
59
|
-
*/
|
|
60
|
-
export interface ViewSnapshot {
|
|
61
|
-
experimentId: string;
|
|
62
|
-
agent: string;
|
|
63
|
-
model?: string;
|
|
64
|
-
startedAt: string;
|
|
65
|
-
/** 快照的根相对路径(= niceeval/results 的 AttemptRef.snapshot,两段:`<experiment-dir>/<snapshot-dir>`)。 */
|
|
66
|
-
run: string;
|
|
67
|
-
/** 是否为该实验在 results.latest() 口径下的最新一次快照 —— 证据室的 latest 标记,与报告槽 Selection
|
|
68
|
-
(现刻水位,可能合成自更早快照)是两个独立概念,不要混用。 */
|
|
69
|
-
latest: boolean;
|
|
70
|
-
/** 该快照的 attempt 明细(跨快照去重后的幸存条目;locator / artifactBase 已注入)。 */
|
|
71
|
-
results: ViewEvalResult[];
|
|
72
|
-
}
|
|
73
|
-
|
|
74
51
|
/**
|
|
75
52
|
* 目录扫描里被跳过的 run 的结构化条目;三种原因与 niceeval/results 的 skipped 一致。
|
|
76
53
|
* 页面上的呈现不走它:不可读快照已形成 `unreadable-snapshot` Scope warning,由报告页内的
|
|
@@ -99,8 +76,6 @@ export interface ViewData {
|
|
|
99
76
|
lastRunAt?: string;
|
|
100
77
|
/** 报告槽 Selection 合成自几个物理 run。 */
|
|
101
78
|
composedRuns: number;
|
|
102
|
-
/** 全部历史快照(跨快照按身份键去重后);attempt 详情路由对这份完整集合解析,不随报告 Scope 收窄。 */
|
|
103
|
-
snapshots: ViewSnapshot[];
|
|
104
79
|
/** 读不了的落盘(三种原因);呈现走报告页内的 ScopeWarnings(unreadable-snapshot warning)。 */
|
|
105
80
|
skippedRuns?: SkippedRunNotice[];
|
|
106
81
|
/** 报告外壳与页导航的声明(规范化后);缺省时前端按单页 `report` 兜底。 */
|
|
@@ -35,7 +35,7 @@ async function seedSnapshot(root: string, experimentId: string, evalId: string):
|
|
|
35
35
|
{ type: "message", role: "user", text: "hi", loc: { file: "evals/a.eval.ts", line: 3 } },
|
|
36
36
|
{ type: "message", role: "assistant", text: "hello there" },
|
|
37
37
|
],
|
|
38
|
-
trace: [{ name: "turn", kind: "turn"
|
|
38
|
+
trace: [{ traceId: "t1", spanId: "s1", name: "turn", kind: "turn", startMs: 0, endMs: 100 }],
|
|
39
39
|
diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }] as never,
|
|
40
40
|
},
|
|
41
41
|
);
|
|
@@ -114,4 +114,27 @@ describe("站点管线奇偶:server 与 --out 是同一份产物", () => {
|
|
|
114
114
|
const direct = await (await fetch(new URL(`/${eventsPath}`, server.url))).text();
|
|
115
115
|
expect(await res.text()).toBe(direct);
|
|
116
116
|
});
|
|
117
|
+
|
|
118
|
+
it("本地宿主的 attempt 详情路由越过 --exp 收窄,对完整结果根解析(cases.md 第 198/220 行,与 show @<locator> 同一套「各自结果根语义寻址」)", async () => {
|
|
119
|
+
const root = await makeDir("niceeval-parity-unnarrowed-");
|
|
120
|
+
await seedSnapshot(root, "exp-a", "q1");
|
|
121
|
+
await seedSnapshot(root, "exp-b", "q2");
|
|
122
|
+
server = await startViewServer({ input: root, scan: { experiment: "exp-a" } });
|
|
123
|
+
|
|
124
|
+
const full = await planSite(root); // 不收窄,拿到两个实验各自的 attempt 路径
|
|
125
|
+
const attemptPaths = [...full.files.keys()].filter((p) => p.startsWith("attempt/"));
|
|
126
|
+
expect(attemptPaths).toHaveLength(2);
|
|
127
|
+
|
|
128
|
+
// 收窄后的 plan 只剩 exp-a 的那份(row 205 已覆盖导出侧的同一断言;这里确认 plan 本身也收窄)。
|
|
129
|
+
const narrowed = await planSite(root, { experiment: "exp-a" });
|
|
130
|
+
const narrowedPaths = new Set(narrowed.files.keys());
|
|
131
|
+
const outsideScopePath = attemptPaths.find((p) => !narrowedPaths.has(p))!;
|
|
132
|
+
expect(outsideScopePath).toBeTruthy();
|
|
133
|
+
|
|
134
|
+
// 但本地 server 仍能直接打开它:路由越过收窄,对完整结果根解析。
|
|
135
|
+
const res = await fetch(new URL(`/${outsideScopePath}`, server.url));
|
|
136
|
+
expect(res.status).toBe(200);
|
|
137
|
+
const body = await res.text();
|
|
138
|
+
expect(body).toContain("q2"); // exp-b 的证据内容真的解析出来了,不是空白/占位
|
|
139
|
+
});
|
|
117
140
|
});
|