niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
import { renderToStaticMarkup } from "react-dom/server";
|
|
8
8
|
import { describe, expect, it } from "vitest";
|
|
9
9
|
|
|
10
|
-
import type { EvalResult, Verdict } from "
|
|
11
|
-
import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "
|
|
12
|
-
import { makeScope } from "
|
|
10
|
+
import type { EvalResult, Verdict } from "../../types.ts";
|
|
11
|
+
import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../../results/index.ts";
|
|
12
|
+
import { makeScope } from "../../results/select.ts";
|
|
13
13
|
import {
|
|
14
14
|
createTextContext,
|
|
15
15
|
defineComponent,
|
|
@@ -18,36 +18,31 @@ import {
|
|
|
18
18
|
validateReportTree,
|
|
19
19
|
ResolveMemo,
|
|
20
20
|
type ReportNode,
|
|
21
|
-
} from "
|
|
21
|
+
} from "../definition/tree.ts";
|
|
22
|
+
import { buildReportMeta, defineReport, FALLBACK_REPORT_TITLE, resolveReportTitle } from "../definition/report.ts";
|
|
22
23
|
import {
|
|
23
|
-
buildReportMeta,
|
|
24
|
-
defineReport,
|
|
25
|
-
FALLBACK_REPORT_TITLE,
|
|
26
24
|
pickReportPage,
|
|
27
25
|
renderReportToText,
|
|
26
|
+
renderReportTreeToText,
|
|
27
|
+
ReportPageNeedsLocatorError,
|
|
28
28
|
ReportPageNotFoundError,
|
|
29
|
-
|
|
30
|
-
} from "./
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
} from "./components.tsx";
|
|
47
|
-
import { Col, Row, Section, Style, Tab, Table, Tabs, Text } from "./primitives.tsx";
|
|
48
|
-
import { attemptListData, metricScatterData } from "./compute.ts";
|
|
49
|
-
import { costUSD, defineMetric, endToEndPassRate } from "./metrics.ts";
|
|
50
|
-
import builtInReport, { standard } from "./built-in/index.tsx";
|
|
29
|
+
} from "./text.ts";
|
|
30
|
+
import { renderReportToStaticHtml, renderReportTreeToStaticHtml } from "./web.ts";
|
|
31
|
+
import { AttemptList, ExperimentList, FailureList } from "../components/entity-lists/index.tsx";
|
|
32
|
+
import { CopyFixPrompt, Hero, ScopeWarnings, TraceWaterfall } from "../components/site-components/index.tsx";
|
|
33
|
+
import { ExperimentComparison, ScopeSummary } from "../components/summaries/index.tsx";
|
|
34
|
+
import { MetricBars, MetricMatrix, MetricScatter, MetricTable } from "../components/metric-views/index.tsx";
|
|
35
|
+
import { AttemptDetail } from "../components/attempt-detail/index.tsx";
|
|
36
|
+
import { Col, Grid, Row, Section, Stat, Style, Tab, Table, Tabs, Text } from "../definition/primitives.tsx";
|
|
37
|
+
import { stringWidth } from "../model/text-layout.ts";
|
|
38
|
+
import { attemptListData, experimentListData } from "../components/entity-lists/compute.ts";
|
|
39
|
+
import { metricScatterData } from "../components/metric-views/compute.ts";
|
|
40
|
+
import { scopeSummaryData } from "../components/summaries/compute.ts";
|
|
41
|
+
import { costUSD, defineMetric, endToEndPassRate } from "../model/metrics.ts";
|
|
42
|
+
import { label } from "../model/flag.ts";
|
|
43
|
+
import builtInReport, { standard, standardAttemptPage } from "../built-in/index.tsx";
|
|
44
|
+
import type { AttemptEvidence } from "../../results/attempt-evidence.ts";
|
|
45
|
+
import { encodeAttemptLocator } from "../../results/locator.ts";
|
|
51
46
|
|
|
52
47
|
// ───────────────────────── fake 数据 ─────────────────────────
|
|
53
48
|
|
|
@@ -133,13 +128,32 @@ function resultsOf(snapshots: Snapshot[]): Results {
|
|
|
133
128
|
} as unknown as Results;
|
|
134
129
|
}
|
|
135
130
|
|
|
131
|
+
/** 最小 AttemptEvidence fixture:只用于验证 standardAttemptPage 的渲染管线接线,不摆事实。 */
|
|
132
|
+
function attemptEvidenceOf(): AttemptEvidence {
|
|
133
|
+
const identity = { experimentId: "exp/a", snapshotStartedAt: "2026-07-01T00:00:00.000Z", evalId: "eval/one", attempt: 0 };
|
|
134
|
+
return {
|
|
135
|
+
locator: encodeAttemptLocator(identity),
|
|
136
|
+
identity,
|
|
137
|
+
result: { id: "eval/one", agent: "agent-x", verdict: "passed" as Verdict, attempt: 0, durationMs: 1000, assertions: [] },
|
|
138
|
+
events: null,
|
|
139
|
+
evalSource: null,
|
|
140
|
+
execution: null,
|
|
141
|
+
diff: null,
|
|
142
|
+
trace: null,
|
|
143
|
+
artifactPaths: { dir: "/results/exp/a/eval-one/a0" },
|
|
144
|
+
capabilities: { source: false, execution: false, timing: false, diff: false },
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
136
148
|
/** 管线便捷入口:resolve + validate + text 渲染,报告声明用最小 meta。 */
|
|
137
149
|
async function renderTreeText(node: ReportNode, scope: Scope, width = 100): Promise<string> {
|
|
138
150
|
const definition = defineReport(node);
|
|
139
|
-
const
|
|
151
|
+
const page = pickReportPage(definition);
|
|
152
|
+
const resolved = await resolveReportTree(page.content, {
|
|
140
153
|
scope,
|
|
141
154
|
results: resultsOf(scope.snapshots),
|
|
142
|
-
report: buildReportMeta(definition, scope
|
|
155
|
+
report: buildReportMeta(definition, scope),
|
|
156
|
+
page: { id: page.id, input: "scope" },
|
|
143
157
|
memo: new ResolveMemo(),
|
|
144
158
|
});
|
|
145
159
|
validateReportTree(resolved);
|
|
@@ -336,14 +350,14 @@ describe("组合组件(函数形态)", () => {
|
|
|
336
350
|
expect(composed).toBe(manual);
|
|
337
351
|
});
|
|
338
352
|
|
|
339
|
-
it("ctx.results 可自行挑 Snapshot[] 喂 input;ctx.report 携带走完回退链的 title
|
|
353
|
+
it("ctx.results 可自行挑 Snapshot[] 喂 input;ctx.report 携带走完回退链的 title,ctx.page 携带当前页 id", async () => {
|
|
340
354
|
const named = snap({ experimentId: "hist/a", name: "Memory Evals", results: [res("q", "passed")] });
|
|
341
355
|
const scope = scopeOf([named]);
|
|
342
356
|
const Meta = defineComponent((_props: Record<never, never>, ctx) => {
|
|
343
357
|
const history = ctx.results.experiments[0]!.snapshots;
|
|
344
358
|
return (
|
|
345
359
|
<Col>
|
|
346
|
-
<Text>{`title=${typeof ctx.report.title === "string" ? ctx.report.title : "?"} page=${ctx.
|
|
360
|
+
<Text>{`title=${typeof ctx.report.title === "string" ? ctx.report.title : "?"} page=${ctx.page.id}`}</Text>
|
|
347
361
|
<ScopeSummary input={history} />
|
|
348
362
|
</Col>
|
|
349
363
|
);
|
|
@@ -507,13 +521,13 @@ describe("text/web 双面同源", () => {
|
|
|
507
521
|
res("algebra/retry", "passed", { attempt: 1 }),
|
|
508
522
|
],
|
|
509
523
|
});
|
|
510
|
-
const text = await renderTreeText(<ExperimentList
|
|
524
|
+
const text = await renderTreeText(<ExperimentList />, scopeOf([s]), 160);
|
|
511
525
|
expect(text.match(/algebra\/retry/g)).toHaveLength(1); // Eval 父行只出现一次
|
|
512
526
|
expect(text).toContain("├─");
|
|
513
527
|
expect(text).toContain("└─");
|
|
514
528
|
// 失败摘要只在 Attempt 子行(父行不复述)
|
|
515
529
|
expect(text.match(/equals\(42\)/g)).toHaveLength(1);
|
|
516
|
-
//
|
|
530
|
+
// 行标签默认缩成最短唯一后缀
|
|
517
531
|
expect(text).toContain("exp-a");
|
|
518
532
|
});
|
|
519
533
|
});
|
|
@@ -659,6 +673,281 @@ describe("Table 与文本排版原语", () => {
|
|
|
659
673
|
});
|
|
660
674
|
});
|
|
661
675
|
|
|
676
|
+
// ───────────────────────── Grid / Stat / Section.meta ─────────────────────────
|
|
677
|
+
|
|
678
|
+
describe("Grid / Stat / Section.meta", () => {
|
|
679
|
+
const ctx = (width = 100) => createTextContext({ width });
|
|
680
|
+
|
|
681
|
+
it("展平数组 / Fragment,空分支不占格,任意 ReportNode 可作 cell,Col 内多个 Stat 保持同一格", () => {
|
|
682
|
+
const groups = ["a", "b"];
|
|
683
|
+
const tree = (
|
|
684
|
+
<Grid columns={4}>
|
|
685
|
+
{groups.map((g) => (
|
|
686
|
+
<Stat key={g} label={g} value={g} />
|
|
687
|
+
))}
|
|
688
|
+
{null}
|
|
689
|
+
{undefined}
|
|
690
|
+
{false}
|
|
691
|
+
<>
|
|
692
|
+
<Text>free block</Text>
|
|
693
|
+
</>
|
|
694
|
+
<Col>
|
|
695
|
+
<Stat label="col-stat-1" value="1" />
|
|
696
|
+
<Stat label="col-stat-2" value="2" />
|
|
697
|
+
</Col>
|
|
698
|
+
</Grid>
|
|
699
|
+
);
|
|
700
|
+
const text = renderNodeToText(tree, ctx());
|
|
701
|
+
const html = renderToStaticMarkup(tree as never);
|
|
702
|
+
// a, b, free block, Col(两个 Stat) = 4 格;空分支不占格
|
|
703
|
+
expect((html.match(/nre-grid-cell/g) ?? []).length).toBe(4);
|
|
704
|
+
for (const face of [text, html]) {
|
|
705
|
+
expect(face).toContain("free block");
|
|
706
|
+
expect(face).toContain("col-stat-1");
|
|
707
|
+
expect(face).toContain("col-stat-2");
|
|
708
|
+
}
|
|
709
|
+
// 两面 cell 顺序一致:a 在 b 之前,free block 在 Col 之前
|
|
710
|
+
expect(text.indexOf("a")).toBeLessThan(text.indexOf("b"));
|
|
711
|
+
expect(text.indexOf("free block")).toBeLessThan(text.indexOf("col-stat-1"));
|
|
712
|
+
});
|
|
713
|
+
|
|
714
|
+
it("全部子节点为空分支时 0 格,text 面输出空串", () => {
|
|
715
|
+
const tree = <Grid columns={3}>{[null, false, undefined]}</Grid>;
|
|
716
|
+
expect(renderNodeToText(tree, ctx())).toBe("");
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
for (const bad of [0, -1, 1.5, NaN, Infinity]) {
|
|
720
|
+
it(`columns={${bad}} 给完整用户反馈`, () => {
|
|
721
|
+
const tree = <Grid columns={bad}>{<Stat label="x" value="y" />}</Grid>;
|
|
722
|
+
expect(() => renderNodeToText(tree, ctx())).toThrow(/columns/);
|
|
723
|
+
expect(() => renderToStaticMarkup(tree as never)).toThrow(/columns/);
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
it("columns=1 与 columns 大于 cell 数都正常渲染;variant / density 默认 plain / regular", () => {
|
|
728
|
+
const single = <Grid columns={1}>{[<Stat key="a" label="a" value="1" />, <Stat key="b" label="b" value="2" />]}</Grid>;
|
|
729
|
+
expect(renderNodeToText(single, ctx())).toContain("a");
|
|
730
|
+
const roomy = (
|
|
731
|
+
<Grid columns={99}>
|
|
732
|
+
<Stat label="only" value="1" />
|
|
733
|
+
</Grid>
|
|
734
|
+
);
|
|
735
|
+
const html = renderToStaticMarkup(roomy as never);
|
|
736
|
+
expect(html).toContain("nre-grid--plain");
|
|
737
|
+
expect(html).toContain("nre-grid--regular");
|
|
738
|
+
expect(html).toContain('style="--nre-grid-max-columns:99"');
|
|
739
|
+
});
|
|
740
|
+
|
|
741
|
+
it("web 初始 HTML 含全部格、稳定 root/cell/variant/density class 与最大列数事实,无 JS 也完整可读", () => {
|
|
742
|
+
const tree = (
|
|
743
|
+
<Grid columns={3} variant="boxed" density="compact">
|
|
744
|
+
<Stat label="one" value="1" />
|
|
745
|
+
<Stat label="two" value="2" />
|
|
746
|
+
<Stat label="three" value="3" />
|
|
747
|
+
</Grid>
|
|
748
|
+
);
|
|
749
|
+
const html = renderToStaticMarkup(tree as never);
|
|
750
|
+
expect(html).toContain("nre-grid--boxed");
|
|
751
|
+
expect(html).toContain("nre-grid--compact");
|
|
752
|
+
expect(html).toContain('style="--nre-grid-max-columns:3"');
|
|
753
|
+
expect((html.match(/nre-grid-cell/g) ?? []).length).toBe(3);
|
|
754
|
+
expect(html).toContain("one");
|
|
755
|
+
expect(html).toContain("two");
|
|
756
|
+
expect(html).toContain("three");
|
|
757
|
+
expect(html).not.toMatch(/<script/);
|
|
758
|
+
expect(html).not.toContain("hydrat");
|
|
759
|
+
});
|
|
760
|
+
|
|
761
|
+
it("boxed 每个 cell 四边完整;plain 只去掉边框与内边距,不改变列数与内容宽", () => {
|
|
762
|
+
const tree = (variant: "plain" | "boxed") => (
|
|
763
|
+
<Grid columns={2} variant={variant}>
|
|
764
|
+
<Stat label="left" value="1" />
|
|
765
|
+
<Stat label="right" value="2" />
|
|
766
|
+
</Grid>
|
|
767
|
+
);
|
|
768
|
+
const boxed = renderNodeToText(tree("boxed"), ctx(80));
|
|
769
|
+
const plain = renderNodeToText(tree("plain"), ctx(80));
|
|
770
|
+
expect(boxed.match(/┌/g)?.length).toBe(2);
|
|
771
|
+
expect(boxed.match(/┐/g)?.length).toBe(2);
|
|
772
|
+
expect(boxed.match(/└/g)?.length).toBe(2);
|
|
773
|
+
expect(boxed.match(/┘/g)?.length).toBe(2);
|
|
774
|
+
expect(plain).not.toContain("┌");
|
|
775
|
+
expect(plain).toContain("left");
|
|
776
|
+
expect(plain).toContain("right");
|
|
777
|
+
for (const line of [...boxed.split("\n"), ...plain.split("\n")]) {
|
|
778
|
+
expect(stringWidth(line)).toBeLessThanOrEqual(80);
|
|
779
|
+
}
|
|
780
|
+
});
|
|
781
|
+
|
|
782
|
+
it("compact 不合并或丢弃字段,label/value/detail 三者仍全部可见", () => {
|
|
783
|
+
const text = renderNodeToText(
|
|
784
|
+
<Grid columns={2} variant="boxed" density="compact">
|
|
785
|
+
<Stat label="label-x" value="value-x" detail="detail-x" />
|
|
786
|
+
<Stat label="label-y" value="value-y" detail="detail-y" />
|
|
787
|
+
</Grid>,
|
|
788
|
+
ctx(80),
|
|
789
|
+
);
|
|
790
|
+
for (const token of ["label-x", "value-x", "detail-x", "label-y", "value-y", "detail-y"]) {
|
|
791
|
+
expect(text).toContain(token);
|
|
792
|
+
}
|
|
793
|
+
});
|
|
794
|
+
|
|
795
|
+
it("text 宽面使用声明列数,继续收窄降为一列,不丢任何 cell", () => {
|
|
796
|
+
const cells = Array.from({ length: 6 }, (_, i) => <Stat key={i} label={`s${i}`} value={i} />);
|
|
797
|
+
const wide = renderNodeToText(<Grid columns={6}>{cells}</Grid>, ctx(200));
|
|
798
|
+
for (let i = 0; i < 6; i++) expect(wide).toContain(`s${i}`);
|
|
799
|
+
const narrow = renderNodeToText(<Grid columns={6}>{cells}</Grid>, ctx(15));
|
|
800
|
+
for (let i = 0; i < 6; i++) expect(narrow).toContain(`s${i}`);
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
it("Stat:LocalizedText、number(按 locale 格式化)、null(—)、数字 0、detail 省略与四种 tone", () => {
|
|
804
|
+
const html0 = renderToStaticMarkup(<Stat label="zero" value={0} /> as never);
|
|
805
|
+
expect(html0).toContain(">0<");
|
|
806
|
+
expect(html0).not.toContain("—");
|
|
807
|
+
const htmlNull = renderToStaticMarkup(<Stat label="none" value={null} /> as never);
|
|
808
|
+
expect(htmlNull).toContain("—");
|
|
809
|
+
const htmlNumber = renderToStaticMarkup(<Stat label="n" value={1234} /> as never);
|
|
810
|
+
expect(htmlNumber).toContain("1,234"); // 默认 locale "en"
|
|
811
|
+
const htmlLocalized = renderToStaticMarkup(<Stat label={{ en: "Label", "zh-CN": "标签" }} value="v" /> as never);
|
|
812
|
+
expect(htmlLocalized).toContain("Label");
|
|
813
|
+
// detail 省略:text 面不留空行(恰好两行:label、value)
|
|
814
|
+
const textNoDetail = renderNodeToText(<Stat label="l" value="v" />, ctx());
|
|
815
|
+
expect(textNoDetail.split("\n")).toEqual(["l", "v"]);
|
|
816
|
+
const textWithDetail = renderNodeToText(<Stat label="l" value="v" detail="d" />, ctx());
|
|
817
|
+
expect(textWithDetail.split("\n")).toEqual(["l", "v", "d"]);
|
|
818
|
+
for (const tone of ["neutral", "positive", "negative", "warning"] as const) {
|
|
819
|
+
const html = renderToStaticMarkup(<Stat label="t" value="v" tone={tone} /> as never);
|
|
820
|
+
expect(html).toContain(`nre-stat--${tone}`);
|
|
821
|
+
}
|
|
822
|
+
// tone 不出现在 text 面(不新造 ANSI 颜色协议,不泄露内部词)
|
|
823
|
+
const textToned = renderNodeToText(<Stat label="t" value="v" tone="positive" />, ctx());
|
|
824
|
+
expect(textToned).not.toMatch(/positive|negative|warning|neutral/);
|
|
825
|
+
});
|
|
826
|
+
|
|
827
|
+
it("Section.meta:web 面同行(header 结构),text 面同行右对齐;省略 meta 时旧行为不变", () => {
|
|
828
|
+
const withMeta = (
|
|
829
|
+
<Section title="Title" meta="6/6 done">
|
|
830
|
+
<Text>body</Text>
|
|
831
|
+
</Section>
|
|
832
|
+
);
|
|
833
|
+
const html = renderToStaticMarkup(withMeta as never);
|
|
834
|
+
expect(html).toContain("nre-section-header");
|
|
835
|
+
expect(html).toContain("nre-section-meta");
|
|
836
|
+
expect(html).toContain("Title");
|
|
837
|
+
expect(html).toContain("6/6 done");
|
|
838
|
+
const text = renderNodeToText(withMeta, ctx());
|
|
839
|
+
const firstLine = text.split("\n")[0];
|
|
840
|
+
expect(firstLine).toContain("Title");
|
|
841
|
+
expect(firstLine).toContain("6/6 done");
|
|
842
|
+
expect(stringWidth(firstLine)).toBeLessThanOrEqual(100);
|
|
843
|
+
|
|
844
|
+
// 放不下时以两格缩进换行(meta 本身够长,createTextContext 最窄也钳在 20 列)
|
|
845
|
+
const longMeta = (
|
|
846
|
+
<Section title="Title" meta="this meta text is intentionally too long to fit on the title line">
|
|
847
|
+
<Text>body</Text>
|
|
848
|
+
</Section>
|
|
849
|
+
);
|
|
850
|
+
const narrowText = renderNodeToText(longMeta, ctx(20));
|
|
851
|
+
const lines = narrowText.split("\n");
|
|
852
|
+
expect(lines[0]).toBe("Title");
|
|
853
|
+
expect(lines[1]!.startsWith(" ")).toBe(true);
|
|
854
|
+
expect(narrowText).toContain("intentionally");
|
|
855
|
+
|
|
856
|
+
// 省略 meta:旧结构不变(裸 <h2>,无 header 包裹)
|
|
857
|
+
const withoutMeta = (
|
|
858
|
+
<Section title="Plain">
|
|
859
|
+
<Text>body</Text>
|
|
860
|
+
</Section>
|
|
861
|
+
);
|
|
862
|
+
const htmlPlain = renderToStaticMarkup(withoutMeta as never);
|
|
863
|
+
expect(htmlPlain).not.toContain("nre-section-header");
|
|
864
|
+
expect(htmlPlain).toContain('<h2 class="nre-section-title">Plain</h2>');
|
|
865
|
+
});
|
|
866
|
+
|
|
867
|
+
it("目标运行总览示例:恰好 100 显示列时两个 Grid 都降为三列,21 个 Stat 全部存在且索引递增,逐行不越界", () => {
|
|
868
|
+
const groupStats: Array<[string, string, string]> = [
|
|
869
|
+
["平均净 R / case", "+0.479 R", "累计 +2.877 R"],
|
|
870
|
+
["单笔期望", "+0.093 R", "已成交交易"],
|
|
871
|
+
["Episode 胜率", "66.7%", "4 / 6 cases"],
|
|
872
|
+
["MFE / MAE", "0.87 / 0.71", "捕获 4.3%"],
|
|
873
|
+
["交易胜率", "41.9%", "13 / 31 笔"],
|
|
874
|
+
["持有 / 回撤", "1.5 / 1.47 R", "bars / max DD"],
|
|
875
|
+
["方向命中", "66.7%", "cutoff → horizon"],
|
|
876
|
+
["完成率", "100.0%", "6 / 6"],
|
|
877
|
+
["Profit Factor", "1.29", "盈利 R / 亏损 R"],
|
|
878
|
+
["执行成本", "$1.09", "0 bps"],
|
|
879
|
+
["参与 / 成交", "100.0% / 100.0%", "6 个方向订单"],
|
|
880
|
+
["耗时 / 首次决策", "207.4 s / B0.8", "34.7 tools · 84927 tokens"],
|
|
881
|
+
];
|
|
882
|
+
const compactStats: Array<[string, string]> = [
|
|
883
|
+
["初始 1H", "0 bars"],
|
|
884
|
+
["初始日线", "250 bars"],
|
|
885
|
+
["初始周线", "104 bars"],
|
|
886
|
+
["回放窗口", "— sessions"],
|
|
887
|
+
["回放 1H", "20 bars"],
|
|
888
|
+
["首次决策", "B0 起自主决定"],
|
|
889
|
+
["待成交窗口", "— bars"],
|
|
890
|
+
["强平提醒", "T-5 → T-1"],
|
|
891
|
+
["长桥日 / 周回填", "0 / 0"],
|
|
892
|
+
];
|
|
893
|
+
const cols: ReportNode[] = [];
|
|
894
|
+
for (let i = 0; i < groupStats.length; i += 2) {
|
|
895
|
+
const [a, b] = [groupStats[i]!, groupStats[i + 1]!];
|
|
896
|
+
cols.push(
|
|
897
|
+
<Col key={i}>
|
|
898
|
+
<Stat label={a[0]} value={a[1]} detail={a[2]} tone="positive" />
|
|
899
|
+
<Stat label={b[0]} value={b[1]} detail={b[2]} tone="positive" />
|
|
900
|
+
</Col>,
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
const tree = (
|
|
904
|
+
<Section title="运行总览" meta="6/6 完成 · 31 笔完整交易">
|
|
905
|
+
<Grid columns={6} variant="boxed">
|
|
906
|
+
{cols}
|
|
907
|
+
</Grid>
|
|
908
|
+
<Grid columns={9} variant="boxed" density="compact">
|
|
909
|
+
{compactStats.map(([label, value]) => (
|
|
910
|
+
<Stat key={label} label={label} value={value} />
|
|
911
|
+
))}
|
|
912
|
+
</Grid>
|
|
913
|
+
</Section>
|
|
914
|
+
);
|
|
915
|
+
const text = renderNodeToText(tree, ctx(100));
|
|
916
|
+
for (const line of text.split("\n")) {
|
|
917
|
+
expect(stringWidth(line)).toBeLessThanOrEqual(100);
|
|
918
|
+
}
|
|
919
|
+
// 全部 21 个 Stat 都存在(声明序不丢格)
|
|
920
|
+
for (const [label] of [...groupStats, ...compactStats]) {
|
|
921
|
+
expect(text.indexOf(label), `缺少 ${label}`).toBeGreaterThan(-1);
|
|
922
|
+
}
|
|
923
|
+
// row-major 顺序:3 列的一整个物理行文本互相交织(同一 box 行内左右并排打印),
|
|
924
|
+
// 但物理行与物理行之间是严格顺序的(用 \n\n 隔开)——按行分组断言 min/max 递增,
|
|
925
|
+
// 比断言展平后的逐字段顺序更贴合实际排版,不会把「同一行内左右并排」误判成乱序。
|
|
926
|
+
// lastIndexOf:第二个 Grid 的「首次决策」是第一个 Grid「耗时 / 首次决策」的子串,
|
|
927
|
+
// indexOf 会先命中那个子串——两个 Grid 各自的真实位置都是各自标签的最后一次出现。
|
|
928
|
+
const rowIndexRange = (labels: string[]): { min: number; max: number } => {
|
|
929
|
+
const indices = labels.map((label) => text.lastIndexOf(label));
|
|
930
|
+
return { min: Math.min(...indices), max: Math.max(...indices) };
|
|
931
|
+
};
|
|
932
|
+
const grid1Rows = [
|
|
933
|
+
groupStats.slice(0, 6).map((s) => s[0]),
|
|
934
|
+
groupStats.slice(6, 12).map((s) => s[0]),
|
|
935
|
+
].map(rowIndexRange);
|
|
936
|
+
const grid2Rows = [
|
|
937
|
+
compactStats.slice(0, 3).map((s) => s[0]),
|
|
938
|
+
compactStats.slice(3, 6).map((s) => s[0]),
|
|
939
|
+
compactStats.slice(6, 9).map((s) => s[0]),
|
|
940
|
+
].map(rowIndexRange);
|
|
941
|
+
for (let i = 1; i < grid1Rows.length; i++) expect(grid1Rows[i]!.min).toBeGreaterThan(grid1Rows[i - 1]!.max);
|
|
942
|
+
for (let i = 1; i < grid2Rows.length; i++) expect(grid2Rows[i]!.min).toBeGreaterThan(grid2Rows[i - 1]!.max);
|
|
943
|
+
// Grid 1(6 列声明)整体先于 Grid 2(9 列声明,compact)
|
|
944
|
+
expect(grid2Rows[0]!.min).toBeGreaterThan(grid1Rows[grid1Rows.length - 1]!.max);
|
|
945
|
+
// 两个 Grid 都降到三列:每个 boxed cell 四边完整,6 + 9 = 15 个 cell 对应 5+3=... 实际按 3 列计数四角
|
|
946
|
+
expect(text.match(/┌/g)?.length).toBe(6 + 9);
|
|
947
|
+
expect(text.match(/└/g)?.length).toBe(6 + 9);
|
|
948
|
+
});
|
|
949
|
+
});
|
|
950
|
+
|
|
662
951
|
// ───────────────────────── Tabs ─────────────────────────
|
|
663
952
|
|
|
664
953
|
describe("Tabs", () => {
|
|
@@ -755,6 +1044,55 @@ describe("defineReport 装载规范化", () => {
|
|
|
755
1044
|
expect(() => defineReport({ title: {}, content: null })).toThrow(/no non-empty value/);
|
|
756
1045
|
});
|
|
757
1046
|
|
|
1047
|
+
it("page 省略 input 规范化为 scope + navigation:true;显式 attempt 必须 navigation:false", () => {
|
|
1048
|
+
const definition = defineReport({
|
|
1049
|
+
pages: [
|
|
1050
|
+
{ id: "report", title: "Report", content: null },
|
|
1051
|
+
{ id: "attempt", title: "Attempt", input: "attempt", navigation: false, content: null },
|
|
1052
|
+
],
|
|
1053
|
+
});
|
|
1054
|
+
expect(definition.pages[0]).toMatchObject({ input: "scope", navigation: true });
|
|
1055
|
+
expect(definition.pages[1]).toMatchObject({ input: "attempt", navigation: false });
|
|
1056
|
+
|
|
1057
|
+
expect(() =>
|
|
1058
|
+
defineReport({
|
|
1059
|
+
pages: [
|
|
1060
|
+
// @ts-expect-error input:"attempt" 必须显式 navigation:false;这里模拟无类型 JS 输入
|
|
1061
|
+
{ id: "a", title: "A", input: "attempt", content: null },
|
|
1062
|
+
],
|
|
1063
|
+
}),
|
|
1064
|
+
).toThrow(/navigation: false/);
|
|
1065
|
+
expect(() =>
|
|
1066
|
+
defineReport({
|
|
1067
|
+
pages: [
|
|
1068
|
+
// @ts-expect-error navigation:true 与 input:"attempt" 互斥;这里模拟无类型 JS 输入
|
|
1069
|
+
{ id: "a", title: "A", input: "attempt", navigation: true, content: null },
|
|
1070
|
+
],
|
|
1071
|
+
}),
|
|
1072
|
+
).toThrow(/navigation: false/);
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
it("navigation: false 的 scope-input page 不进导航但仍是普通 scope page", () => {
|
|
1076
|
+
const definition = defineReport({
|
|
1077
|
+
pages: [
|
|
1078
|
+
{ id: "report", title: "Report", content: null },
|
|
1079
|
+
{ id: "hidden", title: "Hidden", navigation: false, content: null },
|
|
1080
|
+
],
|
|
1081
|
+
});
|
|
1082
|
+
expect(definition.pages[1]).toMatchObject({ input: "scope", navigation: false });
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
it("一份 definition 最多一张 attempt-input page,第二张同类 page 装载报错", () => {
|
|
1086
|
+
expect(() =>
|
|
1087
|
+
defineReport({
|
|
1088
|
+
pages: [
|
|
1089
|
+
{ id: "a1", title: "A1", input: "attempt", navigation: false, content: null },
|
|
1090
|
+
{ id: "a2", title: "A2", input: "attempt", navigation: false, content: null },
|
|
1091
|
+
],
|
|
1092
|
+
}),
|
|
1093
|
+
).toThrow(/at most one input: "attempt" page/);
|
|
1094
|
+
});
|
|
1095
|
+
|
|
758
1096
|
it("{src} 资产拒绝 .. 路径段、绝对路径与 ~;inline/src 互斥", () => {
|
|
759
1097
|
expect(() => defineReport({ content: null, scripts: [{ src: "../x.js" }] })).toThrow(/not allowed/);
|
|
760
1098
|
expect(() => defineReport({ content: null, scripts: [{ src: "/abs.js" }] })).toThrow(/not allowed/);
|
|
@@ -786,6 +1124,34 @@ describe("defineReport 装载规范化", () => {
|
|
|
786
1124
|
expect(pickReportPage(single, "report").id).toBe("report");
|
|
787
1125
|
});
|
|
788
1126
|
|
|
1127
|
+
it("pickReportPage 缺省跳过 navigation:false 的页,只挑第一张可导航页;可用列表也只含可导航页", () => {
|
|
1128
|
+
const definition = defineReport({
|
|
1129
|
+
pages: [
|
|
1130
|
+
{ id: "hidden", title: "Hidden", navigation: false, content: null },
|
|
1131
|
+
{ id: "overview", title: "Overview", content: null },
|
|
1132
|
+
{ id: "attempt", title: "Attempt", input: "attempt", navigation: false, content: null },
|
|
1133
|
+
],
|
|
1134
|
+
});
|
|
1135
|
+
expect(pickReportPage(definition).id).toBe("overview");
|
|
1136
|
+
try {
|
|
1137
|
+
pickReportPage(definition, "typo");
|
|
1138
|
+
expect.unreachable();
|
|
1139
|
+
} catch (e) {
|
|
1140
|
+
expect(e).toBeInstanceOf(ReportPageNotFoundError);
|
|
1141
|
+
expect((e as ReportPageNotFoundError).available).toEqual(["overview"]);
|
|
1142
|
+
}
|
|
1143
|
+
});
|
|
1144
|
+
|
|
1145
|
+
it("显式选择 attempt-input page 但没有 locator:ReportPageNeedsLocatorError", () => {
|
|
1146
|
+
const definition = defineReport({
|
|
1147
|
+
pages: [
|
|
1148
|
+
{ id: "report", title: "Report", content: null },
|
|
1149
|
+
{ id: "attempt", title: "Attempt", input: "attempt", navigation: false, content: null },
|
|
1150
|
+
],
|
|
1151
|
+
});
|
|
1152
|
+
expect(() => pickReportPage(definition, "attempt")).toThrow(ReportPageNeedsLocatorError);
|
|
1153
|
+
});
|
|
1154
|
+
|
|
789
1155
|
it("标题回退链:def.title → 唯一且相同的快照 name → 内置文案「Eval 运行结果 / Eval Results」;en 相同 zh 不同也落内置文案", () => {
|
|
790
1156
|
const named = snap({ experimentId: "t/a", name: "Memory Evals", results: [res("q", "passed")] });
|
|
791
1157
|
const definition = defineReport(<ScopeSummary />);
|
|
@@ -823,13 +1189,14 @@ describe("defineReport 装载规范化", () => {
|
|
|
823
1189
|
// ───────────────────────── 内建报告 ─────────────────────────
|
|
824
1190
|
|
|
825
1191
|
describe("内建报告", () => {
|
|
826
|
-
it("
|
|
1192
|
+
it("四页普通 defineReport:页 id、页名与逐页组件构成和 built-in.md 全文一致,第四页是不进导航的 attempt-input page", () => {
|
|
827
1193
|
expect(builtInReport.kind).toBe("report");
|
|
828
|
-
expect(builtInReport.pages.map((p) => p.id)).toEqual(["report", "attempts", "traces"]);
|
|
1194
|
+
expect(builtInReport.pages.map((p) => p.id)).toEqual(["report", "attempts", "traces", "attempt"]);
|
|
829
1195
|
expect(builtInReport.pages.map((p) => p.title)).toEqual([
|
|
830
1196
|
{ en: "Report", "zh-CN": "报告" },
|
|
831
1197
|
"Attempts",
|
|
832
1198
|
{ en: "Traces", "zh-CN": "追踪" },
|
|
1199
|
+
"Attempt",
|
|
833
1200
|
]);
|
|
834
1201
|
// 逐页组件构成:每页一个 Col,children 按 built-in.md 全文的声明序,全部是公开组件。
|
|
835
1202
|
const childTypes = (content: unknown) => {
|
|
@@ -837,7 +1204,7 @@ describe("内建报告", () => {
|
|
|
837
1204
|
expect(col.type).toBe(Col);
|
|
838
1205
|
return col.props.children;
|
|
839
1206
|
};
|
|
840
|
-
const [reportPage, attemptsPage, tracesPage] = builtInReport.pages;
|
|
1207
|
+
const [reportPage, attemptsPage, tracesPage, attemptPage] = builtInReport.pages;
|
|
841
1208
|
expect(childTypes(reportPage!.content).map((c) => c.type)).toEqual([
|
|
842
1209
|
Hero,
|
|
843
1210
|
ScopeWarnings,
|
|
@@ -848,6 +1215,16 @@ describe("内建报告", () => {
|
|
|
848
1215
|
expect(attemptsChildren.map((c) => c.type)).toEqual([Hero, ScopeWarnings, AttemptList]);
|
|
849
1216
|
expect(attemptsChildren[2]!.props.filter).toBe(true);
|
|
850
1217
|
expect(childTypes(tracesPage!.content).map((c) => c.type)).toEqual([Hero, ScopeWarnings, TraceWaterfall]);
|
|
1218
|
+
// 第四页是参数化详情页:content 就是裸 AttemptDetail(不套 Col),input/navigation 与文档一致。
|
|
1219
|
+
// defineReport 规范化会重建页对象(id/title/content 逐字段拷贝),所以整页对象不可能与
|
|
1220
|
+
// 具名导出 standardAttemptPage 保持引用相等;但 content 字段是原样透传,同引用证明
|
|
1221
|
+
// standard.tsx 确实把 standardAttemptPage 整个复用进了 pages 数组,不是另抄一份同形的
|
|
1222
|
+
// <AttemptDetail /> JSX(两次书写同样的 JSX 字面量在运行时是两个不同的 element 对象)。
|
|
1223
|
+
expect(attemptPage!.content).toBe(standardAttemptPage.content);
|
|
1224
|
+
expect(attemptPage).toEqual(standardAttemptPage);
|
|
1225
|
+
expect(attemptPage!.input).toBe("attempt");
|
|
1226
|
+
expect(attemptPage!.navigation).toBe(false);
|
|
1227
|
+
expect((attemptPage!.content as { type: unknown }).type).toBe(AttemptDetail);
|
|
851
1228
|
});
|
|
852
1229
|
|
|
853
1230
|
it("与 --report 同内容文件完全等价:同一棵页树经同一条管线渲染出逐字节相同的两面", async () => {
|
|
@@ -876,6 +1253,7 @@ describe("内建报告", () => {
|
|
|
876
1253
|
title: { en: "Traces", "zh-CN": "追踪" },
|
|
877
1254
|
content: <Col><Hero /><ScopeWarnings /><TraceWaterfall /></Col>,
|
|
878
1255
|
},
|
|
1256
|
+
{ id: "attempt", title: "Attempt", input: "attempt" as const, navigation: false as const, content: <AttemptDetail /> },
|
|
879
1257
|
],
|
|
880
1258
|
});
|
|
881
1259
|
for (const pageId of ["report", "attempts", "traces"]) {
|
|
@@ -886,22 +1264,55 @@ describe("内建报告", () => {
|
|
|
886
1264
|
await renderReportToStaticHtml(user, ctx, { pageId }),
|
|
887
1265
|
);
|
|
888
1266
|
}
|
|
1267
|
+
// 第四页(attempt-input,不能经 pageId 挑选)另走 tree 级渲染入口逐字节比对。
|
|
1268
|
+
const evidence = attemptEvidenceOf();
|
|
1269
|
+
const page = { id: "attempt", input: "attempt" as const, locator: evidence.locator, evidence };
|
|
1270
|
+
const treeCtx = { scope, results: resultsOf([s1, s2]), report: buildReportMeta(builtInReport, scope), page };
|
|
1271
|
+
expect(await renderReportTreeToText(standardAttemptPage.content, treeCtx, { width: 120 })).toBe(
|
|
1272
|
+
await renderReportTreeToText(<AttemptDetail />, treeCtx, { width: 120 }),
|
|
1273
|
+
);
|
|
1274
|
+
expect(await renderReportTreeToStaticHtml(standardAttemptPage.content, treeCtx)).toBe(
|
|
1275
|
+
await renderReportTreeToStaticHtml(<AttemptDetail />, treeCtx),
|
|
1276
|
+
);
|
|
889
1277
|
});
|
|
890
1278
|
|
|
891
|
-
it("首页 text
|
|
892
|
-
const g1 = snap({ experimentId: "compare/
|
|
893
|
-
const g2 = snap({ experimentId: "dev/
|
|
894
|
-
const
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
{ width: 120 },
|
|
898
|
-
);
|
|
1279
|
+
it("首页 web/text 两面都展示完整 Scope,不同深度目录的 experiment 同屏且行标签缩成最短唯一后缀;无组选择器或组索引", async () => {
|
|
1280
|
+
const g1 = snap({ experimentId: "compare/alpha-exp", results: [res("q", "passed")] });
|
|
1281
|
+
const g2 = snap({ experimentId: "dev/beta-exp", results: [res("q", "failed")] });
|
|
1282
|
+
const ctx = { scope: scopeOf([g1, g2]), results: resultsOf([g1, g2]) };
|
|
1283
|
+
|
|
1284
|
+
const text = await renderReportToText(builtInReport, ctx, { width: 120 });
|
|
899
1285
|
// Hero text 面:标题行(回退链终点)+ 最后运行 meta 行在页首
|
|
900
|
-
expect(
|
|
901
|
-
expect(
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
expect(
|
|
1286
|
+
expect(text.split("\n")[0]).toBe("Eval Results");
|
|
1287
|
+
expect(text).toContain("Last run");
|
|
1288
|
+
// 完整 Scope 直接展示:两个不同路径的 experiment 都可见,不是组索引 + 单组查看命令;
|
|
1289
|
+
// 行标签是各自末段(两个 id 互不撞名,不必加长),不带父路径。
|
|
1290
|
+
expect(text).toContain("alpha-exp");
|
|
1291
|
+
expect(text).toContain("beta-exp");
|
|
1292
|
+
expect(text).not.toContain("compare/alpha-exp");
|
|
1293
|
+
expect(text).not.toContain("dev/beta-exp");
|
|
1294
|
+
expect(text).not.toContain("niceeval show --exp compare");
|
|
1295
|
+
expect(text).not.toContain("niceeval show --exp dev");
|
|
1296
|
+
expect(text).not.toContain("niceeval exp compare");
|
|
1297
|
+
expect(text).not.toContain("niceeval exp dev");
|
|
1298
|
+
|
|
1299
|
+
const html = await renderReportToStaticHtml(builtInReport, ctx);
|
|
1300
|
+
expect(html).not.toContain('role="tablist"');
|
|
1301
|
+
expect(html).not.toContain("data-nre-experiment-group");
|
|
1302
|
+
expect(html).toContain(">alpha-exp</b>");
|
|
1303
|
+
expect(html).toContain(">beta-exp</b>");
|
|
1304
|
+
// 完整 id 仍是排序键,只是不再重复出现在显示文字里
|
|
1305
|
+
expect(html).toContain('data-sort-value="compare/alpha-exp"');
|
|
1306
|
+
expect(html).toContain('data-sort-value="dev/beta-exp"');
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
it("0/1/多 experiment 均可渲染;0 个时三个叶子组件各自显示自己的空态", async () => {
|
|
1310
|
+
const empty = { scope: scopeOf([]), results: resultsOf([]) };
|
|
1311
|
+
const emptyText = await renderReportToText(builtInReport, empty, { width: 120 });
|
|
1312
|
+
expect(emptyText).toContain("No data to plot Cost × Pass rate"); // MetricScatter 零点空态
|
|
1313
|
+
expect(emptyText).toContain("No attempts"); // ExperimentList 零行空态
|
|
1314
|
+
const emptyHtml = await renderReportToStaticHtml(builtInReport, empty);
|
|
1315
|
+
expect(emptyHtml).toContain("No data to plot Cost × Pass rate");
|
|
905
1316
|
|
|
906
1317
|
const priced = snap({
|
|
907
1318
|
experimentId: "compare/priced",
|
|
@@ -912,12 +1323,149 @@ describe("内建报告", () => {
|
|
|
912
1323
|
{ scope: scopeOf([priced]), results: resultsOf([priced]) },
|
|
913
1324
|
{ width: 140 },
|
|
914
1325
|
);
|
|
915
|
-
expect(single).toContain("Eval / Attempt"); //
|
|
1326
|
+
expect(single).toContain("Eval / Attempt"); // 单 experiment 直接展示散点与实验明细
|
|
916
1327
|
// 成本轴(better: lower)反向渲染,「更好」恒指向右上;两轴都声明 better → 提示在场
|
|
917
1328
|
expect(single).toContain("better → upper right");
|
|
918
1329
|
});
|
|
919
1330
|
});
|
|
920
1331
|
|
|
1332
|
+
// ───────────────────────── ExperimentComparison(组合组件)─────────────────────────
|
|
1333
|
+
|
|
1334
|
+
describe("ExperimentComparison(组合组件)", () => {
|
|
1335
|
+
/** 展开树里 [ScopeSummary, MetricScatter, ExperimentList] 三个已解析元素。 */
|
|
1336
|
+
async function resolveComparisonChildren(
|
|
1337
|
+
node: ReportNode,
|
|
1338
|
+
snapshots: Snapshot[],
|
|
1339
|
+
): Promise<Array<{ props: { data: unknown } }>> {
|
|
1340
|
+
const scope = scopeOf(snapshots);
|
|
1341
|
+
const definition = defineReport(node);
|
|
1342
|
+
const page = pickReportPage(definition);
|
|
1343
|
+
const resolved = (await resolveReportTree(page.content, {
|
|
1344
|
+
scope,
|
|
1345
|
+
results: resultsOf(snapshots),
|
|
1346
|
+
report: buildReportMeta(definition, scope),
|
|
1347
|
+
page: { id: page.id, input: "scope" },
|
|
1348
|
+
memo: new ResolveMemo(),
|
|
1349
|
+
})) as unknown as { props: { children: Array<{ props: { data: unknown } }> } };
|
|
1350
|
+
return resolved.props.children;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
it("等价于手写的 Col<ScopeSummary/MetricScatter/ExperimentList>:同一份 input 下两棵树渲染逐字节相同(text 与 web)", async () => {
|
|
1354
|
+
const a = snap({ experimentId: "cmp/a", agent: "bub", results: [res("q", "passed")] });
|
|
1355
|
+
const b = snap({ experimentId: "cmp/b", agent: "codex", results: [res("q", "failed")] });
|
|
1356
|
+
const ctx = { scope: scopeOf([a, b]), results: resultsOf([a, b]) };
|
|
1357
|
+
|
|
1358
|
+
const viaCompose = defineReport(<ExperimentComparison />);
|
|
1359
|
+
const viaHandwritten = defineReport(
|
|
1360
|
+
<Col>
|
|
1361
|
+
<ScopeSummary />
|
|
1362
|
+
<MetricScatter points="experiment" series="agent" x={costUSD} y={endToEndPassRate} />
|
|
1363
|
+
<ExperimentList filter />
|
|
1364
|
+
</Col>,
|
|
1365
|
+
);
|
|
1366
|
+
expect(await renderReportToText(viaCompose, ctx, { width: 120 })).toBe(
|
|
1367
|
+
await renderReportToText(viaHandwritten, ctx, { width: 120 }),
|
|
1368
|
+
);
|
|
1369
|
+
expect(await renderReportToStaticHtml(viaCompose, ctx)).toBe(await renderReportToStaticHtml(viaHandwritten, ctx));
|
|
1370
|
+
});
|
|
1371
|
+
|
|
1372
|
+
it("不同深度目录的 experiments 一律进同一份 data;展开树里 ScopeSummary / MetricScatter / ExperimentList 的解析结果与直接调用三个函数深等", async () => {
|
|
1373
|
+
const g1a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
|
|
1374
|
+
const g1b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
|
|
1375
|
+
const g2 = snap({ experimentId: "bench/long/x", results: [res("q", "passed")] });
|
|
1376
|
+
const solo = snap({ experimentId: "standalone", results: [res("q", "failed")] });
|
|
1377
|
+
const all = [g1a, g1b, g2, solo];
|
|
1378
|
+
const [summaryEl, scatterEl, listEl] = await resolveComparisonChildren(<ExperimentComparison />, all);
|
|
1379
|
+
expect(listEl.props.data).toEqual(await experimentListData(all));
|
|
1380
|
+
expect(summaryEl.props.data).toEqual(await scopeSummaryData(all));
|
|
1381
|
+
expect(scatterEl.props.data).toEqual(
|
|
1382
|
+
await metricScatterData(all, { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate }),
|
|
1383
|
+
);
|
|
1384
|
+
});
|
|
1385
|
+
|
|
1386
|
+
it("series 缺省解析:Scope 内任一 experiment 声明 labels.line 时全图 line,完全无 line 时 agent;显式 series 覆盖缺省", async () => {
|
|
1387
|
+
const withCost = { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.1 } };
|
|
1388
|
+
const withLine = snap({ experimentId: "series/with-line", results: [res("q", "passed", withCost)] });
|
|
1389
|
+
withLine.experiment = { runs: 1, earlyExit: false, selectedEvalIds: ["q"], labels: { line: "codex" } };
|
|
1390
|
+
const withoutLine = snap({ experimentId: "series/plain", results: [res("q", "passed", withCost)] });
|
|
1391
|
+
|
|
1392
|
+
const textWithLine = await renderTreeText(<ExperimentComparison />, scopeOf([withLine, withoutLine]));
|
|
1393
|
+
expect(textWithLine).toContain("grouped by line");
|
|
1394
|
+
|
|
1395
|
+
const textNoLine = await renderTreeText(<ExperimentComparison />, scopeOf([withoutLine]));
|
|
1396
|
+
expect(textNoLine).toContain("grouped by agent");
|
|
1397
|
+
|
|
1398
|
+
const textExplicit = await renderTreeText(<ExperimentComparison series="agent" />, scopeOf([withLine]));
|
|
1399
|
+
expect(textExplicit).toContain("grouped by agent");
|
|
1400
|
+
});
|
|
1401
|
+
|
|
1402
|
+
it("connect 缺省跟随 series 解析:默认 line 时同 series 两点连线,默认 agent 时不连线", async () => {
|
|
1403
|
+
const withCost = { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.1 } };
|
|
1404
|
+
const lineA = snap({ experimentId: "connect/a", agent: "codex", results: [res("q", "passed", withCost)] });
|
|
1405
|
+
lineA.experiment = { runs: 1, earlyExit: false, selectedEvalIds: ["q"], labels: { line: "codex" } };
|
|
1406
|
+
const lineB = snap({ experimentId: "connect/b", agent: "codex", results: [res("q", "failed", withCost)] });
|
|
1407
|
+
lineB.experiment = { runs: 1, earlyExit: false, selectedEvalIds: ["q"], labels: { line: "codex" } };
|
|
1408
|
+
// 同一个 agent 分两个 experiment,声明 line 时归同一条线;位移摘要行(└ Pass rate …)只在
|
|
1409
|
+
// connect 开时出现(scatterText 的 connect 分支),是比裸 "→" 更可靠的标记
|
|
1410
|
+
// ("better → upper right" 提示行本身也含 "→",不能用来判连线)。
|
|
1411
|
+
const connected = await renderTreeText(<ExperimentComparison />, scopeOf([lineA, lineB]));
|
|
1412
|
+
expect(connected).toContain("grouped by line");
|
|
1413
|
+
expect(connected).toContain("└ Pass rate");
|
|
1414
|
+
|
|
1415
|
+
const plainA = snap({ experimentId: "connect/plain-a", agent: "codex", results: [res("q", "passed", withCost)] });
|
|
1416
|
+
const plainB = snap({ experimentId: "connect/plain-b", agent: "codex", results: [res("q", "failed", withCost)] });
|
|
1417
|
+
const unconnected = await renderTreeText(<ExperimentComparison />, scopeOf([plainA, plainB]));
|
|
1418
|
+
expect(unconnected).toContain("grouped by agent");
|
|
1419
|
+
expect(unconnected).not.toContain("└ Pass rate");
|
|
1420
|
+
});
|
|
1421
|
+
|
|
1422
|
+
it("line 缺省对整个 Scope 生效:混入一个声明 line 的实验后,没声明的实验落 (missing) 而非回退 agent;显式 series 覆盖全部", async () => {
|
|
1423
|
+
const lineA = snap({ experimentId: "mem/codex-baseline", agent: "codex", results: [res("q", "passed")] });
|
|
1424
|
+
lineA.experiment = { runs: 1, earlyExit: false, selectedEvalIds: ["q"], labels: { line: "codex", memory: "baseline" } };
|
|
1425
|
+
const lineB = snap({ experimentId: "mem/codex-mempal", agent: "codex", results: [res("q", "failed")] });
|
|
1426
|
+
lineB.experiment = { runs: 1, earlyExit: false, selectedEvalIds: ["q"], labels: { line: "codex", memory: "mempal" } };
|
|
1427
|
+
const plain = snap({ experimentId: "dev/one", agent: "codex", results: [res("q", "passed")] });
|
|
1428
|
+
const all = [lineA, lineB, plain];
|
|
1429
|
+
|
|
1430
|
+
const [, scatterEl] = await resolveComparisonChildren(<ExperimentComparison />, all);
|
|
1431
|
+
const scatterData = scatterEl.props.data as { seriesDimension?: string; rows: Array<{ key: string; series?: string }> };
|
|
1432
|
+
expect(scatterData.seriesDimension).toBe("line");
|
|
1433
|
+
const byKey = new Map(scatterData.rows.map((r) => [r.key, r.series]));
|
|
1434
|
+
expect(byKey.get("mem/codex-baseline")).toBe("codex");
|
|
1435
|
+
expect(byKey.get("dev/one")).toBe("(missing)");
|
|
1436
|
+
|
|
1437
|
+
const [, explicitScatterEl] = await resolveComparisonChildren(<ExperimentComparison series={label("memory")} />, all);
|
|
1438
|
+
expect((explicitScatterEl.props.data as { seriesDimension?: string }).seriesDimension).toBe("memory");
|
|
1439
|
+
});
|
|
1440
|
+
|
|
1441
|
+
it("ExperimentList 行标签默认缩成 Scope 内的最短唯一后缀,不改变 ScopeSummary / MetricScatter 输出或排序键", async () => {
|
|
1442
|
+
const a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
|
|
1443
|
+
const b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
|
|
1444
|
+
const ctx = { scope: scopeOf([a, b]), results: resultsOf([a, b]) };
|
|
1445
|
+
|
|
1446
|
+
const definition = defineReport(<ExperimentComparison />);
|
|
1447
|
+
const html = await renderReportToStaticHtml(definition, ctx);
|
|
1448
|
+
expect(html).toContain(">a</b>");
|
|
1449
|
+
expect(html).toContain(">b</b>");
|
|
1450
|
+
expect(html).toContain('data-sort-value="compare/a"');
|
|
1451
|
+
expect(html).toContain('data-sort-value="compare/b"');
|
|
1452
|
+
|
|
1453
|
+
// 换成末段撞名的一对(a/run、b/run 都叫 run)时两行都要加长到能区分为止,不是随口断言。
|
|
1454
|
+
const dupeA = snap({ experimentId: "compare/x/run", agent: "bub", results: [res("q", "passed")] });
|
|
1455
|
+
const dupeB = snap({ experimentId: "compare/y/run", agent: "codex", results: [res("q", "failed")] });
|
|
1456
|
+
const dupeHtml = await renderReportToStaticHtml(definition, { scope: scopeOf([dupeA, dupeB]), results: resultsOf([dupeA, dupeB]) });
|
|
1457
|
+
expect(dupeHtml).toContain(">x/run</b>");
|
|
1458
|
+
expect(dupeHtml).toContain(">y/run</b>");
|
|
1459
|
+
|
|
1460
|
+
// ScopeSummary / MetricScatter 完全不受行标签缩短影响
|
|
1461
|
+
const [summaryEl, scatterEl] = await resolveComparisonChildren(<ExperimentComparison />, [a, b]);
|
|
1462
|
+
expect(summaryEl.props.data).toEqual(await scopeSummaryData([a, b]));
|
|
1463
|
+
expect(scatterEl.props.data).toEqual(
|
|
1464
|
+
await metricScatterData([a, b], { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate }),
|
|
1465
|
+
);
|
|
1466
|
+
});
|
|
1467
|
+
});
|
|
1468
|
+
|
|
921
1469
|
// ───────────────────────── extends 与内建视图集合 ─────────────────────────
|
|
922
1470
|
|
|
923
1471
|
describe("extends 与内建视图集合", () => {
|
|
@@ -944,7 +1492,7 @@ describe("extends 与内建视图集合", () => {
|
|
|
944
1492
|
|
|
945
1493
|
// ctx.report.title 取 extends 上声明的 title
|
|
946
1494
|
const s = snap({ experimentId: "compare/a", results: [res("q", "passed")] });
|
|
947
|
-
expect(buildReportMeta(branded, scopeOf([s])
|
|
1495
|
+
expect(buildReportMeta(branded, scopeOf([s])).title).toBe("Memory Evals");
|
|
948
1496
|
});
|
|
949
1497
|
|
|
950
1498
|
it("无外壳字段的 extends 与内建逐页两面渲染逐字节相同", async () => {
|