niceeval 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +15 -7
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +254 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +235 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +527 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +174 -0
- package/dist/report/components/entity-lists/index.d.ts +51 -0
- package/dist/report/components/entity-lists/index.js +165 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
- package/dist/report/{tree.js → definition/tree.js} +10 -12
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
- package/dist/report/{format.js → model/format.js} +48 -4
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +12 -2
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +149 -23
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +15 -1
- package/dist/shared/aggregate.js +32 -1
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +11 -11
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +35 -32
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +193 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +103 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
- package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +8 -4
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +77 -64
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +30 -15
- package/src/i18n/zh-CN.ts +30 -15
- package/src/index.ts +2 -0
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/{react → assets}/styles.css +136 -112
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
- package/src/report/components/attempt-detail/compute.ts +287 -0
- package/src/report/components/attempt-detail/faces.ts +257 -0
- package/src/report/components/attempt-detail/index.tsx +583 -0
- package/src/report/components/attempt-detail/validate.test.ts +235 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +207 -0
- package/src/report/components/entity-lists/index.tsx +241 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +47 -17
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +58 -22
- package/src/report/index.ts +126 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +59 -5
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +143 -22
- package/src/report/react/index.tsx +52 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +54 -3
- package/src/results/select.ts +30 -6
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +158 -23
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b.ts +12 -13
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +30 -1
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +58 -22
- package/src/show/render.ts +16 -240
- package/src/show/show.test.ts +128 -35
- package/src/view/app/App.test.tsx +8 -7
- package/src/view/app/App.tsx +135 -51
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +122 -89
- package/src/view/index.ts +3 -13
- package/src/view/server.ts +36 -6
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +18 -806
- package/src/view/view-report.test.ts +182 -34
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
+
// 「外壳、页面与 Tabs」——宿主装载分流(文件 vs 内建)。
|
|
3
|
+
// 装载规范化(content/pages/extends、page id、input/navigation、标题回退、icon)全部是
|
|
4
|
+
// `defineReport` 自己的产物,由 src/report/runtime/dual-render.test.tsx 直接对着真实实现测试;
|
|
5
|
+
// 这里只测 host facade 仍然私有的编排:内建报告分流与 LocalizedText 回退。
|
|
6
|
+
|
|
7
|
+
import { describe, expect, it } from "vitest";
|
|
8
|
+
import { HostReportError, loadHostReport, localizeText } from "./host.ts";
|
|
9
|
+
// dist-sourced:裸宿主装载的就是这份预编译产物的默认导出(show 与 view 同一条路),
|
|
10
|
+
// raw-src import 会是另一份模块实例,引用等同断言必须对着 dist。
|
|
11
|
+
import distBuiltInReport from "../../../dist/report/built-in/index.js";
|
|
12
|
+
|
|
13
|
+
describe("裸宿主装载内建报告", () => {
|
|
14
|
+
it("缺省(无 --report)装载 niceeval/report/built-in 的默认导出:同引用,页与其 content 同引用", async () => {
|
|
15
|
+
const host = await loadHostReport(process.cwd(), undefined);
|
|
16
|
+
const builtIn = distBuiltInReport as { pages: readonly { id: string; content: unknown }[] };
|
|
17
|
+
expect(host).toBe(distBuiltInReport);
|
|
18
|
+
expect(host.pages.map((p: { id: string }) => p.id)).toEqual(builtIn.pages.map((p) => p.id));
|
|
19
|
+
for (let i = 0; i < host.pages.length; i++) {
|
|
20
|
+
expect(host.pages[i]!.content).toBe(builtIn.pages[i]!.content);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("--report <文件> 走文件装载;找不到文件时是 HostReportError 同族的可预期错误", async () => {
|
|
25
|
+
// loadHostReport 对文件路径委托 dist 里的 loadReportFile;这里只验证分流本身发生
|
|
26
|
+
// (文件装载错误的具体文案由 src/report/runtime/load.ts 自己的测试覆盖)。
|
|
27
|
+
await expect(loadHostReport(process.cwd(), "does/not/exist.tsx")).rejects.toThrow();
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
describe("LocalizedText 回退:locale → en → 键字典序第一个非空值", () => {
|
|
32
|
+
it("三级回退各自命中;undefined 输入原样返回 undefined", () => {
|
|
33
|
+
expect(localizeText({ "zh-CN": "中", en: "E" }, "zh-CN")).toBe("中");
|
|
34
|
+
expect(localizeText({ "zh-CN": "中", en: "E" }, "fr")).toBe("E");
|
|
35
|
+
expect(localizeText({ "zh-TW": "繁", ja: "日" }, "en")).toBe("日"); // ja < zh-TW 字典序
|
|
36
|
+
expect(localizeText("plain", "en")).toBe("plain");
|
|
37
|
+
expect(localizeText({}, "en")).toBeUndefined();
|
|
38
|
+
expect(localizeText(undefined, "en")).toBeUndefined();
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("HostReportError 是可预期用户错误(与 ReportLoadError 同待遇,不是内部异常)", () => {
|
|
43
|
+
expect(new HostReportError("x")).toBeInstanceOf(Error);
|
|
44
|
+
});
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
// 报告装载与逐页渲染的中性宿主 facade:show 与 view 共用(docs/feature/reports/architecture.md
|
|
2
|
+
// 「共享内核与两个宿主的代码边界」「单一 report runtime 身份」)。ReportDefinition / ReportPage /
|
|
3
|
+
// ReportMeta 的类型体系,以及装载规范化、resolve、text/web render 全部住在 niceeval/report 的
|
|
4
|
+
// 构建单元(src/report/**,经 `pnpm run build:report` 编译进 dist/report/**);这里只做两个
|
|
5
|
+
// 宿主都需要、但只属于宿主编排的一件事——文件 vs 内建报告的装载分流、逐页渲染——不重复声明
|
|
6
|
+
// 任何报告类型或规范化逻辑。终端专属的可复制命令拼装(`showCommand`)不在这里,只有 show 需要,
|
|
7
|
+
// 住在 src/show/command.ts。
|
|
8
|
+
//
|
|
9
|
+
// 本文件物理上住在 src/report/runtime/(host facade 的架构归属),但**不**参与
|
|
10
|
+
// tsconfig.report-build.json 的编译单元(见该文件的 exclude 注释):它是调用方经 tsx 直接执行的
|
|
11
|
+
// raw TypeScript,内部一律动态 import 兄弟 dist/report/** 产物——不是所有 show / view 命令都
|
|
12
|
+
// 渲染报告,静态 import 会让不需要报告的命令也背上 react / react-dom 依赖(web.ts 静态 import
|
|
13
|
+
// react-dom/server);同一进程也不能混用 raw src/report/** 与 dist/report/** 的同一状态模块,
|
|
14
|
+
// 所以值和类型都从同一批 dist 模块拿,不从本文件的兄弟源码拿。
|
|
15
|
+
|
|
16
|
+
import type { Results, Scope } from "../../results/index.ts";
|
|
17
|
+
import type { LocalizedText } from "../../types.ts";
|
|
18
|
+
import type { AttemptLocator } from "../../results/locator.ts";
|
|
19
|
+
import type { PageContext } from "../../../dist/report/definition/tree.js";
|
|
20
|
+
import type {
|
|
21
|
+
ReportDefinition,
|
|
22
|
+
ReportMeta,
|
|
23
|
+
ReportPage,
|
|
24
|
+
} from "../../../dist/report/definition/report.js";
|
|
25
|
+
|
|
26
|
+
export type { PageContext } from "../../../dist/report/definition/tree.js";
|
|
27
|
+
export type {
|
|
28
|
+
HeadTag,
|
|
29
|
+
ReportAsset,
|
|
30
|
+
ReportDefinition,
|
|
31
|
+
ReportMeta,
|
|
32
|
+
ReportMetaPage,
|
|
33
|
+
ReportPage,
|
|
34
|
+
} from "../../../dist/report/definition/report.js";
|
|
35
|
+
|
|
36
|
+
/** 可预期的装载用户错误(与 ReportLoadError 同待遇:打一句直说问题与下一步,不抛堆栈)。 */
|
|
37
|
+
export class HostReportError extends Error {}
|
|
38
|
+
|
|
39
|
+
// ───────────────────────── 装载 ─────────────────────────
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* 装载宿主报告:`--report <file>` 走 dist 里的 `loadReportFile`;缺省(裸 show / 裸 view)
|
|
43
|
+
* 装载内建 `standard`。两条路的产物都已经是 `defineReport` 规范化后的 `ReportDefinition`——
|
|
44
|
+
* 没有第二个规范化步骤,`defineReport` 本身就是唯一规范化点。
|
|
45
|
+
*/
|
|
46
|
+
export async function loadHostReport(
|
|
47
|
+
cwd: string,
|
|
48
|
+
reportPath: string | undefined,
|
|
49
|
+
options?: { freshImport?: boolean },
|
|
50
|
+
): Promise<ReportDefinition> {
|
|
51
|
+
if (reportPath !== undefined) {
|
|
52
|
+
const { loadReportFile } = await import("../../../dist/report/runtime/load.js");
|
|
53
|
+
return loadReportFile(cwd, reportPath, options) as Promise<ReportDefinition>;
|
|
54
|
+
}
|
|
55
|
+
const { standard } = await import("../../../dist/report/built-in/index.js");
|
|
56
|
+
return standard as ReportDefinition;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** ctx.report 的构建(不携带当前页——那是 HostRenderContext.page 的事)。 */
|
|
60
|
+
export async function buildHostReportMeta(definition: ReportDefinition, scope: Scope): Promise<ReportMeta> {
|
|
61
|
+
const { buildReportMeta } = await import("../../../dist/report/definition/report.js");
|
|
62
|
+
return buildReportMeta(definition, scope);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* LocalizedText 的确定回退(shell.md「行为约束」):当前 locale → en → 按 locale 键字典序的
|
|
67
|
+
* 第一个非空值。undefined 输入原样返回 undefined——两个宿主用它给"可能没声明"的字段
|
|
68
|
+
* (页标题、外壳字段)取显示字符串。算法与 niceeval/report 的 resolveLocalizedText 是同一份
|
|
69
|
+
* shell.md 文档契约的两处实现(这里没有报告类型或规范化状态,纯字符串函数,不经 dist 边界)。
|
|
70
|
+
*/
|
|
71
|
+
export function localizeText(text: LocalizedText | undefined, locale: string): string | undefined {
|
|
72
|
+
if (text === undefined) return undefined;
|
|
73
|
+
if (typeof text === "string") return text || undefined;
|
|
74
|
+
const exact = text[locale];
|
|
75
|
+
if (exact) return exact;
|
|
76
|
+
if (text.en) return text.en;
|
|
77
|
+
for (const key of Object.keys(text).sort()) {
|
|
78
|
+
if (text[key]) return text[key];
|
|
79
|
+
}
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ───────────────────────── 索引命令上下文 ─────────────────────────
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* 宿主索引命令的完整上下文(docs/feature/reports/show/reports.md「索引命令携带完整上下文」)。
|
|
87
|
+
* 只是数据形状;拼出实际可复制的 `niceeval show ...` 命令字符串是 show 自己的事
|
|
88
|
+
* (`src/show/command.ts` 的 `showCommand`)——view 走网页路由,不生成终端命令。
|
|
89
|
+
*/
|
|
90
|
+
export interface HostCommandContext {
|
|
91
|
+
patterns: string[];
|
|
92
|
+
results?: string;
|
|
93
|
+
experiment?: string;
|
|
94
|
+
report?: string;
|
|
95
|
+
page?: string;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ───────────────────────── 逐页渲染 ─────────────────────────
|
|
99
|
+
|
|
100
|
+
/** 逐页渲染的宿主上下文:官方口径的 Scope、结果根读取面、规范化声明与当前页判别。 */
|
|
101
|
+
export interface HostRenderContext {
|
|
102
|
+
scope: Scope;
|
|
103
|
+
results: Results;
|
|
104
|
+
report: ReportMeta;
|
|
105
|
+
page: PageContext;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export interface HostTextRenderOptions {
|
|
109
|
+
width?: number;
|
|
110
|
+
locale?: string;
|
|
111
|
+
/** 索引命令的完整上下文(docs/feature/reports/show/reports.md);逐页渲染时透传。 */
|
|
112
|
+
commandContext?: HostCommandContext;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** 渲染一页的 text 面。 */
|
|
116
|
+
export async function renderHostPageText(
|
|
117
|
+
page: ReportPage,
|
|
118
|
+
ctx: HostRenderContext,
|
|
119
|
+
options: HostTextRenderOptions,
|
|
120
|
+
): Promise<string> {
|
|
121
|
+
const { renderReportTreeToText } = await import("../../../dist/report/runtime/text.js");
|
|
122
|
+
return renderReportTreeToText(page.content, ctx, options);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 渲染一页的 web 面(静态 HTML)。attemptHref 缺省时报告有 attempt-input page 就用
|
|
127
|
+
* niceeval/report 的根相对默认值(`attempt/<encodeURIComponent(locator)>.html`,index.html
|
|
128
|
+
* 视角);从 attempt 页面自身内容渲染时(该 page 引用了其它 locator)view 显式传入同级
|
|
129
|
+
* 相对版本覆盖它——两种情形都不在这里判断,只透传。
|
|
130
|
+
*/
|
|
131
|
+
export async function renderHostPageHtml(
|
|
132
|
+
page: ReportPage,
|
|
133
|
+
ctx: HostRenderContext,
|
|
134
|
+
options: { locale: string; attemptHref?: (locator: AttemptLocator) => string },
|
|
135
|
+
): Promise<string> {
|
|
136
|
+
const { renderReportTreeToStaticHtml } = await import("../../../dist/report/runtime/web.js");
|
|
137
|
+
return renderReportTreeToStaticHtml(page.content, ctx, options);
|
|
138
|
+
}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import { existsSync, statSync } from "node:fs";
|
|
8
8
|
import { resolve } from "node:path";
|
|
9
9
|
import { pathToFileURL } from "node:url";
|
|
10
|
-
import { isReportDefinition, type ReportDefinition } from "
|
|
10
|
+
import { isReportDefinition, type ReportDefinition } from "../definition/report.ts";
|
|
11
11
|
|
|
12
12
|
/** 可预期的装载错误:宿主打一句英文直说问题与下一步,不抛堆栈。 */
|
|
13
13
|
export class ReportLoadError extends Error {}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// text 宿主(show)的渲染入口:装载好的 ReportDefinition/page → resolve(组合展开 + spec 取数)
|
|
2
|
+
// → validate(两面资格)→ render(纯同步字符输出)。web 宿主的对应入口在 ./web.ts(那一侧才
|
|
3
|
+
// import react-dom)。管线以页为单位执行;defineReport 本身与 ReportDefinition 的类型体系在
|
|
4
|
+
// ../definition/report.ts,这里只做渲染编排与宿主联系面(页选择、索引命令拼装)。
|
|
5
|
+
|
|
6
|
+
import type { Results, Scope } from "../../results/types.ts";
|
|
7
|
+
import type { AttemptLocator } from "../../results/locator.ts";
|
|
8
|
+
import {
|
|
9
|
+
createTextContext,
|
|
10
|
+
renderNodeToText,
|
|
11
|
+
resolveReportTree,
|
|
12
|
+
validateReportTree,
|
|
13
|
+
ResolveMemo,
|
|
14
|
+
type PageContext,
|
|
15
|
+
type ReportNode,
|
|
16
|
+
type TextRenderOptions,
|
|
17
|
+
} from "../definition/tree.ts";
|
|
18
|
+
import {
|
|
19
|
+
buildReportMeta,
|
|
20
|
+
resolveReportTitle,
|
|
21
|
+
type ReportDefinition,
|
|
22
|
+
type ReportMeta,
|
|
23
|
+
type ReportMetaPage,
|
|
24
|
+
type ReportPage,
|
|
25
|
+
} from "../definition/report.ts";
|
|
26
|
+
import { resolveLocalizedText, type ReportLocale } from "../model/locale.ts";
|
|
27
|
+
|
|
28
|
+
/** 默认下钻命令:`niceeval show <locator>` 是 show 已实现的真实 CLI 语法,不需要反查 eval id 再拼近似命令。 */
|
|
29
|
+
const DEFAULT_ATTEMPT_COMMAND = (locator: AttemptLocator): string => `niceeval show ${locator}`;
|
|
30
|
+
|
|
31
|
+
// ───────────────────────── 页选择与 text 宿主入口 ─────────────────────────
|
|
32
|
+
|
|
33
|
+
/** `--page` 未命中:宿主据此按用法错误退出并列出可用页 id(只列 navigation !== false 的)。 */
|
|
34
|
+
export class ReportPageNotFoundError extends Error {
|
|
35
|
+
readonly pageId: string;
|
|
36
|
+
readonly available: string[];
|
|
37
|
+
constructor(pageId: string, available: string[]) {
|
|
38
|
+
super(`page "${pageId}" not found. Available pages: ${available.join(", ")}`);
|
|
39
|
+
this.pageId = pageId;
|
|
40
|
+
this.available = available;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** 显式请求了 attempt-input page,但当前入口没有 locator 可注入 evidence。 */
|
|
45
|
+
export class ReportPageNeedsLocatorError extends Error {
|
|
46
|
+
readonly pageId: string;
|
|
47
|
+
constructor(pageId: string) {
|
|
48
|
+
super(
|
|
49
|
+
`Page "${pageId}" is an attempt-input page and needs a locator — it cannot be opened with --page or #/page/<id> directly. ` +
|
|
50
|
+
"Use the host's locator addressing instead (niceeval show @<locator>, or the view attempt route), which resolves this page with the matching AttemptEvidence.",
|
|
51
|
+
);
|
|
52
|
+
this.pageId = pageId;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* 挑选要渲染的 page:省略 pageId 时挑第一张 `navigation !== false` 的页(跳过参数化详情页,
|
|
58
|
+
* 它没有 locator 就不可打开);显式 pageId 命中 attempt-input page 时报
|
|
59
|
+
* ReportPageNeedsLocatorError——这个入口没有 locator,不能拿 Scope 强行 resolve。
|
|
60
|
+
*/
|
|
61
|
+
export function pickReportPage(definition: ReportDefinition, pageId?: string): ReportPage {
|
|
62
|
+
if (pageId === undefined) {
|
|
63
|
+
return definition.pages.find((p) => p.navigation !== false) ?? definition.pages[0];
|
|
64
|
+
}
|
|
65
|
+
const page = definition.pages.find((p) => p.id === pageId);
|
|
66
|
+
if (!page) {
|
|
67
|
+
throw new ReportPageNotFoundError(
|
|
68
|
+
pageId,
|
|
69
|
+
definition.pages.filter((p) => p.navigation !== false).map((p) => p.id),
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
if (page.input === "attempt") throw new ReportPageNeedsLocatorError(page.id);
|
|
73
|
+
return page;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** 宿主注入的渲染上下文:官方口径挑好的 Scope 与结果根完整读取面。 */
|
|
77
|
+
export interface ReportHostContext {
|
|
78
|
+
scope: Scope;
|
|
79
|
+
/** 组合组件 ctx.results 的来源;历史视图从这里自行挑 Snapshot[]。 */
|
|
80
|
+
results: Results;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface RenderReportTextOptions extends TextRenderOptions {
|
|
84
|
+
/** 渲染哪一页;缺省第一张可导航页。命中 attempt-input page 抛 ReportPageNeedsLocatorError,未命中抛 ReportPageNotFoundError。 */
|
|
85
|
+
pageId?: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* text 宿主的装载语义:选页(只能是 scope-input page,见 pickReportPage)→ resolve(组合展开 +
|
|
90
|
+
* spec 取数,唯一的 await 边界)→ 树校验 → 遍历渲染 text 面。不需要 react-dom。宿主不在报告树外
|
|
91
|
+
* 另设警告通道——挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告放不放
|
|
92
|
+
* 是作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
|
|
93
|
+
*/
|
|
94
|
+
export async function renderReportToText(
|
|
95
|
+
definition: ReportDefinition,
|
|
96
|
+
ctx: ReportHostContext,
|
|
97
|
+
options?: RenderReportTextOptions,
|
|
98
|
+
): Promise<string> {
|
|
99
|
+
const page = pickReportPage(definition, options?.pageId);
|
|
100
|
+
const meta = buildReportMeta(definition, ctx.scope);
|
|
101
|
+
const hasAttemptPage = definition.pages.some((p) => p.input === "attempt");
|
|
102
|
+
const resolved = await resolveReportTree(page.content, {
|
|
103
|
+
scope: ctx.scope,
|
|
104
|
+
results: ctx.results,
|
|
105
|
+
report: meta,
|
|
106
|
+
page: { id: page.id, input: "scope" },
|
|
107
|
+
memo: new ResolveMemo(),
|
|
108
|
+
});
|
|
109
|
+
validateReportTree(resolved);
|
|
110
|
+
return renderNodeToText(
|
|
111
|
+
resolved,
|
|
112
|
+
createTextContext({
|
|
113
|
+
...options,
|
|
114
|
+
attemptCommand: options?.attemptCommand ?? (hasAttemptPage ? DEFAULT_ATTEMPT_COMMAND : undefined),
|
|
115
|
+
}),
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** 页索引标题行(show 多页索引 / view 导航共用的解析结果):按 locale 解析的标题字符串。 */
|
|
120
|
+
export function reportTitleText(definition: ReportDefinition, scope: Scope, locale: ReportLocale): string {
|
|
121
|
+
return resolveLocalizedText(resolveReportTitle(definition, scope), locale);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ───────────────────────── 逐页(树)渲染入口:宿主联系面 ─────────────────────────
|
|
125
|
+
|
|
126
|
+
/** 宿主索引命令的完整上下文(docs/feature/reports/show/reports.md「索引命令携带完整上下文」)。 */
|
|
127
|
+
export interface HostCommandContext {
|
|
128
|
+
patterns: string[];
|
|
129
|
+
results?: string;
|
|
130
|
+
experiment?: string;
|
|
131
|
+
report?: string;
|
|
132
|
+
page?: string;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function quoteArg(value: string): string {
|
|
136
|
+
return /^[A-Za-z0-9._/@-]+$/.test(value) ? value : `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** 按上下文拼组索引的可复制命令:`niceeval show <patterns> --exp <id> [--results/--report/--page]`。 */
|
|
140
|
+
function experimentCommandFor(ctx: HostCommandContext): (experimentIdPrefix: string) => string {
|
|
141
|
+
return (prefix) => {
|
|
142
|
+
const parts = ["niceeval show", ...ctx.patterns.map(quoteArg), `--exp ${quoteArg(prefix)}`];
|
|
143
|
+
if (ctx.results !== undefined) parts.push(`--results ${quoteArg(ctx.results)}`);
|
|
144
|
+
if (ctx.report !== undefined) parts.push(`--report ${quoteArg(ctx.report)}`);
|
|
145
|
+
if (ctx.page !== undefined) parts.push(`--page ${quoteArg(ctx.page)}`);
|
|
146
|
+
return parts.join(" ");
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** 逐页渲染的宿主上下文:官方口径的 Scope、结果根读取面、规范化声明(ctx.report)与当前页判别。 */
|
|
151
|
+
export interface ReportTreeHostContext {
|
|
152
|
+
scope: Scope;
|
|
153
|
+
results: Results;
|
|
154
|
+
report: ReportMeta;
|
|
155
|
+
/** 当前渲染的页:scope 分支只有 id;attempt 分支带 locator + evidence(宿主已完成寻址与装配)。 */
|
|
156
|
+
page: PageContext;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export interface RenderTreeTextOptions extends TextRenderOptions {
|
|
160
|
+
/** 组索引命令的完整上下文;给了就按它拼命令,experimentCommand 显式注入时以后者为准。 */
|
|
161
|
+
commandContext?: HostCommandContext;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* 渲染一页报告树的 text 面(宿主逐页调用;页选择归宿主):
|
|
166
|
+
* resolve(组合展开 + spec 取数)→ validate → render。宿主不在报告树外另设警告通道,
|
|
167
|
+
* 挑选警告由页内的 `ScopeWarnings` 组件呈现(内建报告每页都放它)。当前 definition 没有
|
|
168
|
+
* attempt-input page 时不注入默认下钻命令,调用方也没显式给,`ctx.attemptCommand` 就不存在。
|
|
169
|
+
*/
|
|
170
|
+
export async function renderReportTreeToText(
|
|
171
|
+
tree: ReportNode,
|
|
172
|
+
ctx: ReportTreeHostContext,
|
|
173
|
+
options?: RenderTreeTextOptions,
|
|
174
|
+
): Promise<string> {
|
|
175
|
+
const resolved = await resolveReportTree(tree, {
|
|
176
|
+
scope: ctx.scope,
|
|
177
|
+
results: ctx.results,
|
|
178
|
+
report: ctx.report,
|
|
179
|
+
page: ctx.page,
|
|
180
|
+
memo: new ResolveMemo(),
|
|
181
|
+
});
|
|
182
|
+
validateReportTree(resolved);
|
|
183
|
+
const hasAttemptPage = ctx.report.pages.some((p) => p.input === "attempt");
|
|
184
|
+
const textCtx = createTextContext({
|
|
185
|
+
...options,
|
|
186
|
+
attemptCommand: options?.attemptCommand ?? (hasAttemptPage ? DEFAULT_ATTEMPT_COMMAND : undefined),
|
|
187
|
+
...(options?.experimentCommand === undefined && options?.commandContext !== undefined
|
|
188
|
+
? { experimentCommand: experimentCommandFor(options.commandContext) }
|
|
189
|
+
: {}),
|
|
190
|
+
});
|
|
191
|
+
return renderNodeToText(resolved, textCtx);
|
|
192
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// web 宿主(view --report)的装载入口:同一棵树走 web 面,renderToStaticMarkup 吐静态
|
|
2
|
+
// HTML 烘进查看器的报告槽。只有这一侧真正 import react-dom(import 边界即运行时边界),
|
|
3
|
+
// 所以本文件不从 niceeval/report 的入口 re-export —— 宿主与测试按源路径 import。
|
|
4
|
+
|
|
5
|
+
import type { ReactNode } from "react";
|
|
6
|
+
import { renderToStaticMarkup } from "react-dom/server";
|
|
7
|
+
import type { AttemptLocator } from "../../results/locator.ts";
|
|
8
|
+
import type { Scope } from "../../results/types.ts";
|
|
9
|
+
import {
|
|
10
|
+
resolveReportTree,
|
|
11
|
+
runWithWebContext,
|
|
12
|
+
validateReportTree,
|
|
13
|
+
ResolveMemo,
|
|
14
|
+
type PageContext,
|
|
15
|
+
type WebContext,
|
|
16
|
+
} from "../definition/tree.ts";
|
|
17
|
+
import { DEFAULT_REPORT_LOCALE, type ReportLocale } from "../model/locale.ts";
|
|
18
|
+
import { buildReportMeta, type ReportDefinition } from "../definition/report.ts";
|
|
19
|
+
import { pickReportPage, type ReportHostContext } from "./text.ts";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* 默认证据室深链:view 静态站的 attempt 文档路径,根相对(与 index.html 同级 —— scope-input
|
|
23
|
+
* page 恒在文档树的根)。文件名是 URL 编码后的 locator(architecture.md「Attempt 详情是一张
|
|
24
|
+
* 参数化 page」);磁盘/清单键用未编码的原始 locator,两者的对应关系与拆分只住在 view 的站点
|
|
25
|
+
* 管线(site.ts),这里只产出 href 字符串。从 attempt 页面自身内容渲染时(极少见:自定义
|
|
26
|
+
* attempt-input page 里嵌了别的证据引用组件)view 显式传入同级相对版本覆盖这个默认值,
|
|
27
|
+
* 不依赖这里的根相对形态。
|
|
28
|
+
*/
|
|
29
|
+
const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `attempt/${encodeURIComponent(locator)}.html`;
|
|
30
|
+
|
|
31
|
+
export interface StaticHtmlOptions {
|
|
32
|
+
/** 渲染哪一页;缺省第一张可导航页。命中 attempt-input page 抛 ReportPageNeedsLocatorError。 */
|
|
33
|
+
pageId?: string;
|
|
34
|
+
/** 证据室深链;当前 definition 没有 attempt-input page 时不注入默认值,除非显式传入。 */
|
|
35
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
36
|
+
/** 官方组件 chrome 文案的 locale;默认 "en"。 */
|
|
37
|
+
locale?: ReportLocale;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* web 宿主的装载语义:选页(只能是 scope-input page)→ resolve(组合展开 + spec 取数,
|
|
42
|
+
* 唯一的 await 边界)→ 树校验(与 text 宿主同一遍)→ 静态渲染 web 面。宿主不在报告树外
|
|
43
|
+
* 另设警告通道——挑选警告的呈现件是 `ScopeWarnings` 组件,内建报告每页都放它,自定义报告
|
|
44
|
+
* 放不放是作者义务(docs/feature/reports/architecture.md「Scope 是计算入口」)。
|
|
45
|
+
*/
|
|
46
|
+
export async function renderReportToStaticHtml(
|
|
47
|
+
definition: ReportDefinition,
|
|
48
|
+
ctx: ReportHostContext,
|
|
49
|
+
options?: StaticHtmlOptions,
|
|
50
|
+
): Promise<string> {
|
|
51
|
+
const page = pickReportPage(definition, options?.pageId);
|
|
52
|
+
const meta = buildReportMeta(definition, ctx.scope);
|
|
53
|
+
const hasAttemptPage = definition.pages.some((p) => p.input === "attempt");
|
|
54
|
+
const resolved = await resolveReportTree(page.content, {
|
|
55
|
+
scope: ctx.scope,
|
|
56
|
+
results: ctx.results,
|
|
57
|
+
report: meta,
|
|
58
|
+
page: { id: page.id, input: "scope" },
|
|
59
|
+
memo: new ResolveMemo(),
|
|
60
|
+
});
|
|
61
|
+
validateReportTree(resolved);
|
|
62
|
+
const webCtx: WebContext = {
|
|
63
|
+
...(options?.attemptHref !== undefined
|
|
64
|
+
? { attemptHref: options.attemptHref }
|
|
65
|
+
: hasAttemptPage
|
|
66
|
+
? { attemptHref: DEFAULT_ATTEMPT_HREF }
|
|
67
|
+
: {}),
|
|
68
|
+
locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
|
|
69
|
+
};
|
|
70
|
+
return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* 渲染一页报告树的 web 面(宿主逐页调用;页选择归宿主):resolve → validate → 静态渲染。
|
|
75
|
+
* 挑选警告由页内的 `ScopeWarnings` 组件呈现,宿主不前置任何树外块。
|
|
76
|
+
* ctx.report 是宿主规范化后的声明,ctx.page 是当前页判别(scope 或 attempt 分支)。
|
|
77
|
+
*/
|
|
78
|
+
export async function renderReportTreeToStaticHtml(
|
|
79
|
+
tree: import("../definition/tree.ts").ReportNode,
|
|
80
|
+
ctx: {
|
|
81
|
+
scope: Scope;
|
|
82
|
+
results: import("../../results/types.ts").Results;
|
|
83
|
+
report: import("../definition/report.ts").ReportMeta;
|
|
84
|
+
page: PageContext;
|
|
85
|
+
},
|
|
86
|
+
options?: { attemptHref?: (locator: AttemptLocator) => string; locale?: ReportLocale },
|
|
87
|
+
): Promise<string> {
|
|
88
|
+
const resolved = await resolveReportTree(tree, {
|
|
89
|
+
scope: ctx.scope,
|
|
90
|
+
results: ctx.results,
|
|
91
|
+
report: ctx.report,
|
|
92
|
+
page: ctx.page,
|
|
93
|
+
memo: new ResolveMemo(),
|
|
94
|
+
});
|
|
95
|
+
validateReportTree(resolved);
|
|
96
|
+
const hasAttemptPage = ctx.report.pages.some((p) => p.input === "attempt");
|
|
97
|
+
const webCtx: WebContext = {
|
|
98
|
+
...(options?.attemptHref !== undefined
|
|
99
|
+
? { attemptHref: options.attemptHref }
|
|
100
|
+
: hasAttemptPage
|
|
101
|
+
? { attemptHref: DEFAULT_ATTEMPT_HREF }
|
|
102
|
+
: {}),
|
|
103
|
+
locale: options?.locale ?? DEFAULT_REPORT_LOCALE,
|
|
104
|
+
};
|
|
105
|
+
return runWithWebContext(webCtx, () => renderToStaticMarkup(resolved as ReactNode));
|
|
106
|
+
}
|
|
@@ -33,7 +33,7 @@ import { type AttemptIdentity, type AttemptLocator, encodeAttemptLocator } from
|
|
|
33
33
|
import { loadAnnotatedEvalSource } from "./attempt-source.ts";
|
|
34
34
|
import { deriveSendAnnotations, type AnnotatedEvalSource } from "./annotated-source.ts";
|
|
35
35
|
import { buildExecutionTree, type ExecutionTree } from "../o11y/execution-tree.ts";
|
|
36
|
-
import type { DiffData, EvalResult } from "../types.ts";
|
|
36
|
+
import type { DiffData, EvalResult, StreamEvent } from "../types.ts";
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
39
|
* 一个 Attempt 的 artifact 落盘位置。单一目录足够:一个 attempt 的全部 artifact
|
|
@@ -79,6 +79,9 @@ export interface AttemptEvidence {
|
|
|
79
79
|
* 的调用方直接读结构化字段(如按 experimentId 分组、按时间排序)。 */
|
|
80
80
|
identity: AttemptIdentity;
|
|
81
81
|
result: EvalResult;
|
|
82
|
+
/** 标准事件流(`--execution` 切面与 AttemptConversation 分轮的唯一事实来源);没有非空
|
|
83
|
+
* events() 就是 null。装配一次,`show` / `view` / 报告不再各自调用 `attempt.events()`。 */
|
|
84
|
+
events: readonly StreamEvent[] | null;
|
|
82
85
|
/** 运行时保存的 Eval 源码标注(`--source` 切面);没有 sources() 就是 null,不伪造空文档。 */
|
|
83
86
|
evalSource: AnnotatedEvalSource | null;
|
|
84
87
|
/** 标准事件流 + OTel enrichment(`--execution` 切面);没有非空 events 就是 null,
|
|
@@ -138,6 +141,7 @@ export async function loadAttemptEvidence(attempt: AttemptHandle): Promise<Attem
|
|
|
138
141
|
locator,
|
|
139
142
|
identity,
|
|
140
143
|
result: attempt.result,
|
|
144
|
+
events: hasEvents ? events : null,
|
|
141
145
|
evalSource,
|
|
142
146
|
execution,
|
|
143
147
|
diff,
|
|
@@ -73,6 +73,8 @@ interface SnapshotOpts {
|
|
|
73
73
|
/** 缺省 = 已收尾(completedAt = startedAt);置 true 则不写 completedAt,模拟中断快照。 */
|
|
74
74
|
unfinished?: boolean;
|
|
75
75
|
knownEvalIds?: string[];
|
|
76
|
+
/** 声明这份快照实际选中的 eval id 全集;省略 = 第三方 harness 未实现该字段。 */
|
|
77
|
+
selectedEvalIds?: string[];
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
/** 写一份新布局快照:snapshot.json + 各 attempt 的 result.json。返回快照目录绝对路径。 */
|
|
@@ -94,6 +96,9 @@ async function writeSnapshot(
|
|
|
94
96
|
startedAt: opts.startedAt,
|
|
95
97
|
...(opts.unfinished ? {} : { completedAt: opts.startedAt }),
|
|
96
98
|
...(opts.knownEvalIds ? { knownEvalIds: opts.knownEvalIds } : {}),
|
|
99
|
+
...(opts.selectedEvalIds !== undefined
|
|
100
|
+
? { experiment: { runs: 1, earlyExit: true, selectedEvalIds: opts.selectedEvalIds } }
|
|
101
|
+
: {}),
|
|
97
102
|
};
|
|
98
103
|
await writeFile(join(dir, "snapshot.json"), JSON.stringify(meta, null, 2), "utf-8");
|
|
99
104
|
for (const r of results) {
|
|
@@ -234,6 +239,51 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
|
|
|
234
239
|
expect(results.latest().warnings.some((w) => w.kind === "partial-coverage")).toBe(true);
|
|
235
240
|
});
|
|
236
241
|
|
|
242
|
+
it("合成快照的 selectedEvalIds 重建为最终 picks(q1 新快照 + q2 旧快照补齐),不是照抄某一来源的局部选择", async () => {
|
|
243
|
+
const root = await makeRoot();
|
|
244
|
+
await writeSnapshot(
|
|
245
|
+
root,
|
|
246
|
+
"2026-07-01T08-00-00-000Z",
|
|
247
|
+
{ experimentId: "compare/carry", startedAt: "2026-07-01T08:00:00.000Z", selectedEvalIds: ["q1", "q2"] },
|
|
248
|
+
[res("q1", "passed"), res("q2", "failed")],
|
|
249
|
+
);
|
|
250
|
+
await writeSnapshot(
|
|
251
|
+
root,
|
|
252
|
+
"2026-07-02T08-00-00-000Z",
|
|
253
|
+
{ experimentId: "compare/carry", startedAt: "2026-07-02T08:00:00.000Z", selectedEvalIds: ["q1"] },
|
|
254
|
+
[res("q1", "passed")],
|
|
255
|
+
);
|
|
256
|
+
const results = await openResults(root);
|
|
257
|
+
const scope = selectCurrentResults(results);
|
|
258
|
+
expect(scope.snapshots[0]!.experiment!.selectedEvalIds).toEqual(["q1", "q2"]);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
it("来源快照声明 selectedEvalIds:[q1] 却夹带 q2 的历史 attempt,合成结果不含该 q2", async () => {
|
|
262
|
+
const root = await makeRoot();
|
|
263
|
+
await writeSnapshot(
|
|
264
|
+
root,
|
|
265
|
+
"2026-07-01T08-00-00-000Z",
|
|
266
|
+
{ experimentId: "compare/leaky", startedAt: "2026-07-01T08:00:00.000Z", selectedEvalIds: ["q1"] },
|
|
267
|
+
[res("q1", "passed"), res("q2", "passed")], // q2 落盘了,但没被这次实验选中
|
|
268
|
+
);
|
|
269
|
+
const results = await openResults(root);
|
|
270
|
+
const scope = selectCurrentResults(results);
|
|
271
|
+
expect(scope.snapshots[0]!.evals.map((ev) => ev.id)).toEqual(["q1"]);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it("第三方快照缺 experiment.selectedEvalIds 时按其实际 evals 退化,不整份排除;与本方快照混合时各自按自己口径收窄", async () => {
|
|
275
|
+
const root = await makeRoot();
|
|
276
|
+
await writeSnapshot(
|
|
277
|
+
root,
|
|
278
|
+
"2026-07-01T08-00-00-000Z",
|
|
279
|
+
{ experimentId: "third-party/harness", startedAt: "2026-07-01T08:00:00.000Z" }, // 无 selectedEvalIds
|
|
280
|
+
[res("q1", "passed"), res("q2", "passed")],
|
|
281
|
+
);
|
|
282
|
+
const results = await openResults(root);
|
|
283
|
+
const scope = selectCurrentResults(results);
|
|
284
|
+
expect(scope.snapshots[0]!.evals.map((ev) => ev.id).sort()).toEqual(["q1", "q2"]);
|
|
285
|
+
});
|
|
286
|
+
|
|
237
287
|
it("场景3 同一 eval 多 attempts:最新快照整批替换旧 attempts,不跨快照混装", async () => {
|
|
238
288
|
const root = await makeRoot();
|
|
239
289
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "retry/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [
|
|
@@ -329,7 +379,7 @@ describe("selectCurrentResults · 现刻水位结构化身份", () => {
|
|
|
329
379
|
expect(algebra.warnings).toEqual([]);
|
|
330
380
|
});
|
|
331
381
|
|
|
332
|
-
it("场景8 --
|
|
382
|
+
it("场景8 --exp 分段前缀过滤:只留匹配段,不误配同前缀实验", async () => {
|
|
333
383
|
const root = await makeRoot();
|
|
334
384
|
await writeSnapshot(root, "2026-07-01T08-00-00-000Z", { experimentId: "compare/bub", startedAt: "2026-07-01T08:00:00.000Z" }, [res("q1", "passed")]);
|
|
335
385
|
await writeSnapshot(root, "2026-07-01T09-00-00-000Z", { experimentId: "compare/codex", agent: "codex", startedAt: "2026-07-01T09:00:00.000Z" }, [res("q1", "passed")]);
|
|
@@ -424,7 +474,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
424
474
|
}
|
|
425
475
|
expect(text).toMatch(/\bbub\s+default\s+bub\s+1s\s+50%/);
|
|
426
476
|
expect(html).toContain('data-sort-value="compare/bub"');
|
|
427
|
-
expect(text).toContain("1 passed
|
|
477
|
+
expect(text).toContain("1 passed · 1 failed");
|
|
428
478
|
expect(html).toContain("50%");
|
|
429
479
|
// 现刻水位覆盖齐全 → 两面都不出 partial-coverage(text: "verdicts cover"; html: data-kind)。
|
|
430
480
|
expect(hasPartialCoverageText(text)).toBe(false);
|
|
@@ -476,6 +526,7 @@ describe("宿主接线 · show text 面与 view web 面反映同一批事实", (
|
|
|
476
526
|
' kind: "report",',
|
|
477
527
|
' [Symbol.for("niceeval.report.definition")]: true,',
|
|
478
528
|
" links: [],",
|
|
529
|
+
" head: [],",
|
|
479
530
|
" scripts: [],",
|
|
480
531
|
" styles: [],",
|
|
481
532
|
" pages: [{",
|
|
@@ -522,7 +573,7 @@ function hasPartialCoverageHtml(html: string): boolean {
|
|
|
522
573
|
]);
|
|
523
574
|
const text = await showText(root, []);
|
|
524
575
|
const html = await viewHtml(root);
|
|
525
|
-
expect(text).toContain("3 passed
|
|
576
|
+
expect(text).toContain("3 passed · 1 failed");
|
|
526
577
|
expect(html).toContain("75%");
|
|
527
578
|
for (const face of [text, html]) expect(face).toContain("compare/bub");
|
|
528
579
|
// 无残缺:两面都不出 partial-coverage,布尔一致(某面偷偷补/漏警告即失配)。
|