niceeval 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +15 -7
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +254 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +235 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +527 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +174 -0
- package/dist/report/components/entity-lists/index.d.ts +51 -0
- package/dist/report/components/entity-lists/index.js +165 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
- package/dist/report/{tree.js → definition/tree.js} +10 -12
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
- package/dist/report/{format.js → model/format.js} +48 -4
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +12 -2
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +149 -23
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +15 -1
- package/dist/shared/aggregate.js +32 -1
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +11 -11
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +35 -32
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +193 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +103 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
- package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +8 -4
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +77 -64
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +30 -15
- package/src/i18n/zh-CN.ts +30 -15
- package/src/index.ts +2 -0
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/{react → assets}/styles.css +136 -112
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
- package/src/report/components/attempt-detail/compute.ts +287 -0
- package/src/report/components/attempt-detail/faces.ts +257 -0
- package/src/report/components/attempt-detail/index.tsx +583 -0
- package/src/report/components/attempt-detail/validate.test.ts +235 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +207 -0
- package/src/report/components/entity-lists/index.tsx +241 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +47 -17
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +58 -22
- package/src/report/index.ts +126 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +59 -5
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +143 -22
- package/src/report/react/index.tsx +52 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +54 -3
- package/src/results/select.ts +30 -6
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +158 -23
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b.ts +12 -13
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +30 -1
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +58 -22
- package/src/show/render.ts +16 -240
- package/src/show/show.test.ts +128 -35
- package/src/view/app/App.test.tsx +8 -7
- package/src/view/app/App.tsx +135 -51
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +122 -89
- package/src/view/index.ts +3 -13
- package/src/view/server.ts +36 -6
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +18 -806
- package/src/view/view-report.test.ts +182 -34
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -4,15 +4,15 @@
|
|
|
4
4
|
// skipped=null、null≠0、Scoreboard 固定分母(notRun/unscorable 分开)、权重最长前缀、
|
|
5
5
|
// 身份键去重、现刻水位、自定义指标 where/aggregate、evalGroup 完整父路径、verdict 权威、
|
|
6
6
|
// MetricCell 诚实、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary、
|
|
7
|
-
// scopeSummaryData 两级计票、
|
|
7
|
+
// scopeSummaryData 两级计票、experimentListData/scopeSummaryData 的 selectedEvalIds 投影、pairsByFlag、
|
|
8
8
|
// MetricLine 点身份、空数组反馈、metricTableData sort。
|
|
9
9
|
|
|
10
10
|
import { describe, expect, it } from "vitest";
|
|
11
11
|
|
|
12
|
-
import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "
|
|
13
|
-
import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "
|
|
14
|
-
import { makeScope, selectCurrentResults } from "
|
|
15
|
-
import type { Results } from "
|
|
12
|
+
import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../../types.ts";
|
|
13
|
+
import type { AttemptHandle, Scope, ScopeWarning, Snapshot } from "../../results/index.ts";
|
|
14
|
+
import { makeScope, selectCurrentResults } from "../../results/select.ts";
|
|
15
|
+
import type { Results } from "../../results/types.ts";
|
|
16
16
|
import {
|
|
17
17
|
assistantTurns,
|
|
18
18
|
costUSD,
|
|
@@ -23,23 +23,33 @@ import {
|
|
|
23
23
|
executionReliability,
|
|
24
24
|
repeatedFailedCommands,
|
|
25
25
|
taskPassRate,
|
|
26
|
-
} from "
|
|
27
|
-
import { flag, numericFlag } from "
|
|
26
|
+
} from "../model/metrics.ts";
|
|
27
|
+
import { flag, label, numericFlag, numericLabel } from "../model/flag.ts";
|
|
28
|
+
import { scatterText } from "./metric-views/faces.ts";
|
|
29
|
+
import { createTextContext } from "../definition/tree.ts";
|
|
30
|
+
import { colorIndexForKey, colorIndicesForKeys } from "../assets/colors.ts";
|
|
31
|
+
import { attemptListData, evalListData, experimentListData } from "./entity-lists/compute.ts";
|
|
32
|
+
import { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./entity-lists/index.tsx";
|
|
28
33
|
import {
|
|
29
|
-
attemptListData,
|
|
30
34
|
deltaTableData,
|
|
31
|
-
evalListData,
|
|
32
|
-
experimentComparisonData,
|
|
33
|
-
experimentListData,
|
|
34
35
|
metricLineData,
|
|
35
36
|
metricMatrixData,
|
|
36
37
|
metricScatterData,
|
|
37
38
|
metricTableData,
|
|
38
39
|
pairsByFlag,
|
|
39
|
-
scopeSummaryData,
|
|
40
40
|
scoreboardData,
|
|
41
|
-
} from "./compute.ts";
|
|
42
|
-
import {
|
|
41
|
+
} from "./metric-views/compute.ts";
|
|
42
|
+
import {
|
|
43
|
+
validateDeltaData,
|
|
44
|
+
validateLineData,
|
|
45
|
+
validateMatrixData,
|
|
46
|
+
validateScatterData,
|
|
47
|
+
validateScoreboardData,
|
|
48
|
+
validateTableData,
|
|
49
|
+
} from "./metric-views/index.tsx";
|
|
50
|
+
import { scopeSummaryData } from "./summaries/compute.ts";
|
|
51
|
+
import { validateScopeSummaryData } from "./summaries/index.tsx";
|
|
52
|
+
import { evalGroupOf } from "../model/aggregate.ts";
|
|
43
53
|
|
|
44
54
|
// ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
|
|
45
55
|
|
|
@@ -183,7 +193,7 @@ describe("两级聚合口径", () => {
|
|
|
183
193
|
expect(cells[executionReliability.name]!.value).toBeCloseTo(2 / 3);
|
|
184
194
|
});
|
|
185
195
|
|
|
186
|
-
it("2 passed + 5 errored
|
|
196
|
+
it("2 passed + 5 errored 的默认通过率是 2/7,不是 100%", async () => {
|
|
187
197
|
const s = snap({
|
|
188
198
|
experimentId: "exp/err",
|
|
189
199
|
results: [
|
|
@@ -595,7 +605,7 @@ describe("实体列表 data", () => {
|
|
|
595
605
|
expect(byEval.get("list/errored")!.costUSD).toBeNull();
|
|
596
606
|
});
|
|
597
607
|
|
|
598
|
-
it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens
|
|
608
|
+
it("experimentListData:evalVerdicts / endToEndPassRate / costUSD / durationMs / tokens 齐全,默认按端到端通过率降序", async () => {
|
|
599
609
|
const winner = snap({ experimentId: "exp/win", results: [res("a", "passed"), res("b", "passed")] });
|
|
600
610
|
const loser = snap({ experimentId: "exp/lose", results: [res("a", "failed"), res("b", "passed")] });
|
|
601
611
|
const items = await experimentListData([loser, winner]);
|
|
@@ -667,32 +677,59 @@ describe("scopeSummaryData", () => {
|
|
|
667
677
|
});
|
|
668
678
|
});
|
|
669
679
|
|
|
670
|
-
// ─────────────────────────
|
|
680
|
+
// ───────────────────────── experimentListData / scopeSummaryData 的 selectedEvalIds 投影 ─────────────────────────
|
|
681
|
+
// ExperimentComparison 已收为普通组合组件(把同一个 input 原样透传给 ScopeSummary /
|
|
682
|
+
// MetricScatter / ExperimentList,不再有自己的 data 形态);经它展开后与直接调用这三个
|
|
683
|
+
// 函数深等的验证挪到 dual-render.test.tsx(需要 resolve 管线)。这里只测三个函数自己的
|
|
684
|
+
// selectedEvalIds 投影契约。
|
|
671
685
|
|
|
672
|
-
describe("
|
|
673
|
-
it("
|
|
686
|
+
describe("experimentListData / scopeSummaryData 的 selectedEvalIds 投影", () => {
|
|
687
|
+
it("不同深度目录的 experiments 一律进同一份 data,不按父路径分组比较", async () => {
|
|
674
688
|
const g1a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
|
|
675
689
|
const g1b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
|
|
676
690
|
const g2 = snap({ experimentId: "bench/long/x", results: [res("q", "passed")] });
|
|
677
691
|
const solo = snap({ experimentId: "standalone", results: [res("q", "errored", { error: erroredWith("x") })] });
|
|
692
|
+
const items = await experimentListData([g1a, g1b, g2, solo]);
|
|
693
|
+
expect(items.map((e) => e.experimentId).sort()).toEqual([
|
|
694
|
+
"bench/long/x",
|
|
695
|
+
"compare/a",
|
|
696
|
+
"compare/b",
|
|
697
|
+
"standalone",
|
|
698
|
+
]);
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
it("每个 experiment 只保留自己选择的 eval:A 声明 selectedEvalIds:[q1] 但夹带 q2 attempt,B 只选 q2;q2 不污染 A", async () => {
|
|
702
|
+
const a = snap({
|
|
703
|
+
experimentId: "exp/a",
|
|
704
|
+
results: [res("q1", "passed"), res("q2", "failed")],
|
|
705
|
+
experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q1"] },
|
|
706
|
+
});
|
|
707
|
+
const b = snap({
|
|
708
|
+
experimentId: "exp/b",
|
|
709
|
+
results: [res("q2", "passed")],
|
|
710
|
+
experiment: { runs: 1, earlyExit: false, selectedEvalIds: ["q2"] },
|
|
711
|
+
});
|
|
678
712
|
|
|
679
|
-
const
|
|
680
|
-
|
|
713
|
+
const items = await experimentListData([a, b]);
|
|
714
|
+
const rowA = items.find((e) => e.experimentId === "exp/a")!;
|
|
715
|
+
const rowB = items.find((e) => e.experimentId === "exp/b")!;
|
|
716
|
+
expect(rowA.evals).toBe(1); // 只统计 q1,夹带的 q2 attempt 不计入
|
|
717
|
+
expect(rowA.evalRows.map((r) => r.evalId)).toEqual(["q1"]);
|
|
718
|
+
expect(rowB.evals).toBe(1);
|
|
719
|
+
expect(rowB.evalRows.map((r) => r.evalId)).toEqual(["q2"]);
|
|
720
|
+
|
|
721
|
+
// 汇总口径同样不被污染:全 Scope 只有 2 个 eval(exp/a·q1、exp/b·q2),不是 3 个。
|
|
722
|
+
const summary = await scopeSummaryData([a, b]);
|
|
723
|
+
expect(summary.evals).toBe(2);
|
|
724
|
+
});
|
|
681
725
|
|
|
682
|
-
|
|
683
|
-
const
|
|
684
|
-
|
|
685
|
-
metricScatterData([g1a, g1b], { points: "experiment", series: "agent", x: costUSD, y: endToEndPassRate }),
|
|
686
|
-
experimentListData([g1a, g1b]),
|
|
687
|
-
]);
|
|
688
|
-
expect(compare.summary).toEqual(summary);
|
|
689
|
-
expect(compare.scatter).toEqual(scatter);
|
|
690
|
-
expect(compare.experiments).toEqual(experiments);
|
|
726
|
+
it("第三方快照缺 experiment 信息时仍可见,按其实际 evals 参与(selectedEvalIdsOf 退化)", async () => {
|
|
727
|
+
const thirdParty = snap({ experimentId: "third-party/exp", results: [res("q", "passed")] });
|
|
728
|
+
expect(thirdParty.experiment).toBeUndefined();
|
|
691
729
|
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
expect(compareRefs.some((ref) => soloRefs.includes(ref))).toBe(false);
|
|
730
|
+
const items = await experimentListData([thirdParty]);
|
|
731
|
+
expect(items.map((e) => e.experimentId)).toEqual(["third-party/exp"]);
|
|
732
|
+
expect(items[0]!.evals).toBe(1);
|
|
696
733
|
});
|
|
697
734
|
});
|
|
698
735
|
|
|
@@ -918,22 +955,79 @@ describe("deltaTableData", () => {
|
|
|
918
955
|
];
|
|
919
956
|
};
|
|
920
957
|
|
|
921
|
-
it("
|
|
958
|
+
it("input Scope 内删除该 flag 后配置深相等才配对;a 取 baseline(缺省=未声明),label 用完整 a experiment id,按 (a 末段, 显示键) 字典序", async () => {
|
|
922
959
|
const data = await deltaTableData(matrixSnaps(), {
|
|
923
960
|
by: "experiment",
|
|
924
961
|
pairs: pairsByFlag("memory"),
|
|
925
962
|
metrics: [endToEndPassRate],
|
|
926
963
|
});
|
|
927
964
|
expect(data.rows.map((r) => r.label)).toEqual([
|
|
928
|
-
"bub · memory=agents-md",
|
|
929
|
-
"codex · memory=agents-md",
|
|
930
|
-
"codex · memory=mempal",
|
|
931
|
-
"gemini · memory=agents-md",
|
|
932
|
-
"gemini · memory=mempal",
|
|
965
|
+
"mem/bub · memory=agents-md",
|
|
966
|
+
"mem/codex · memory=agents-md",
|
|
967
|
+
"mem/codex · memory=mempal",
|
|
968
|
+
"mem/gemini · memory=agents-md",
|
|
969
|
+
"mem/gemini · memory=mempal",
|
|
933
970
|
]);
|
|
934
971
|
expect(data.experiments).toBe(8);
|
|
935
972
|
});
|
|
936
973
|
|
|
974
|
+
it("配对边界只是 input Scope,不额外按 experiment id 的目录前缀分组:不同前缀但可比性配置相同就配对", async () => {
|
|
975
|
+
const mk = (id: string, memory?: string) =>
|
|
976
|
+
snap({
|
|
977
|
+
experimentId: id,
|
|
978
|
+
agent: "bub",
|
|
979
|
+
results: [res("q", "passed")],
|
|
980
|
+
experiment: {
|
|
981
|
+
runs: 1,
|
|
982
|
+
earlyExit: false,
|
|
983
|
+
selectedEvalIds: [],
|
|
984
|
+
...(memory !== undefined ? { flags: { memory } } : {}),
|
|
985
|
+
},
|
|
986
|
+
});
|
|
987
|
+
const data = await deltaTableData([mk("compare/codex"), mk("bench/codex", "agents-md")], {
|
|
988
|
+
by: "experiment",
|
|
989
|
+
pairs: pairsByFlag("memory"),
|
|
990
|
+
metrics: [endToEndPassRate],
|
|
991
|
+
});
|
|
992
|
+
expect(data.rows.map((r) => r.label)).toEqual(["compare/codex · memory=agents-md"]);
|
|
993
|
+
expect(data.rows[0]!.a.key).toBe("compare/codex");
|
|
994
|
+
expect(data.rows[0]!.b.key).toBe("bench/codex");
|
|
995
|
+
});
|
|
996
|
+
|
|
997
|
+
it("排序只看 a 末段与 flag 显示键,不看完整 label 字符串:a 末段相同、完整 id 不同的两个 bucket 不因目录前缀打乱顺序", async () => {
|
|
998
|
+
const mk = (id: string, model: string, memory?: string) =>
|
|
999
|
+
snap({
|
|
1000
|
+
experimentId: id,
|
|
1001
|
+
model,
|
|
1002
|
+
agent: "bub",
|
|
1003
|
+
results: [res("q", "passed")],
|
|
1004
|
+
experiment: {
|
|
1005
|
+
runs: 1,
|
|
1006
|
+
earlyExit: false,
|
|
1007
|
+
selectedEvalIds: [],
|
|
1008
|
+
...(memory !== undefined ? { flags: { memory } } : {}),
|
|
1009
|
+
},
|
|
1010
|
+
});
|
|
1011
|
+
// 两个 bucket 因 model 不同而彼此不可比(不会互相配对),但都各自派生一对;
|
|
1012
|
+
// 两对的 a 末段都是 "codex",flagKey 分别是 "z-value" 与 "a-value"。
|
|
1013
|
+
const data = await deltaTableData(
|
|
1014
|
+
[
|
|
1015
|
+
mk("alpha/codex", "m1"), // bucket 1 baseline
|
|
1016
|
+
mk("alpha/codex-variant", "m1", "z-value"), // bucket 1 b 侧
|
|
1017
|
+
mk("zeta/codex", "m2"), // bucket 2 baseline
|
|
1018
|
+
mk("zeta/codex-variant", "m2", "a-value"), // bucket 2 b 侧
|
|
1019
|
+
],
|
|
1020
|
+
{ by: "experiment", pairs: pairsByFlag("memory"), metrics: [endToEndPassRate] },
|
|
1021
|
+
);
|
|
1022
|
+
// 只看 a 末段("codex",两对相同)与 flag 显示键("a-value" < "z-value")排序:
|
|
1023
|
+
// zeta 那对(flagKey "a-value")排在 alpha 那对(flagKey "z-value")之前——若误按完整
|
|
1024
|
+
// label 字符串排序,"alpha/codex..." 会先于 "zeta/codex...",顺序会相反。
|
|
1025
|
+
expect(data.rows.map((r) => r.label)).toEqual([
|
|
1026
|
+
"zeta/codex · memory=a-value",
|
|
1027
|
+
"alpha/codex · memory=z-value",
|
|
1028
|
+
]);
|
|
1029
|
+
});
|
|
1030
|
+
|
|
937
1031
|
it("可比性配置不同(model 不同)的两实验不配对", async () => {
|
|
938
1032
|
const base = snap({
|
|
939
1033
|
experimentId: "mm/a",
|
|
@@ -977,13 +1071,13 @@ describe("deltaTableData", () => {
|
|
|
977
1071
|
pairs: pairsByFlag("memory", { baseline: "agents-md" }),
|
|
978
1072
|
metrics: [endToEndPassRate],
|
|
979
1073
|
});
|
|
980
|
-
// a = *--agents-md;b = 未声明(显示键 (missing))与 mempal
|
|
1074
|
+
// a = *--agents-md;b = 未声明(显示键 (missing))与 mempal;label 用完整 a experiment id。
|
|
981
1075
|
expect(data.rows.map((r) => r.label)).toEqual([
|
|
982
|
-
"bub--agents-md · memory=(missing)",
|
|
983
|
-
"codex--agents-md · memory=(missing)",
|
|
984
|
-
"codex--agents-md · memory=mempal",
|
|
985
|
-
"gemini--agents-md · memory=(missing)",
|
|
986
|
-
"gemini--agents-md · memory=mempal",
|
|
1076
|
+
"mem/bub--agents-md · memory=(missing)",
|
|
1077
|
+
"mem/codex--agents-md · memory=(missing)",
|
|
1078
|
+
"mem/codex--agents-md · memory=mempal",
|
|
1079
|
+
"mem/gemini--agents-md · memory=(missing)",
|
|
1080
|
+
"mem/gemini--agents-md · memory=mempal",
|
|
987
1081
|
]);
|
|
988
1082
|
});
|
|
989
1083
|
});
|
|
@@ -1009,3 +1103,179 @@ describe("examScore", () => {
|
|
|
1009
1103
|
expect(cellOf("crashed").samples).toBe(1); // 0 分是测得的事实,不是缺数据
|
|
1010
1104
|
});
|
|
1011
1105
|
});
|
|
1106
|
+
|
|
1107
|
+
// ───────────────────────── labels 维度、series 归类与 connect ─────────────────────────
|
|
1108
|
+
|
|
1109
|
+
describe("labels 维度、series 归类与 connect", () => {
|
|
1110
|
+
const labeled = (
|
|
1111
|
+
experimentId: string,
|
|
1112
|
+
labels: Record<string, string | number> | undefined,
|
|
1113
|
+
verdicts: Verdict[],
|
|
1114
|
+
agent = "agent-x",
|
|
1115
|
+
) =>
|
|
1116
|
+
snap({
|
|
1117
|
+
experimentId,
|
|
1118
|
+
agent,
|
|
1119
|
+
results: verdicts.map((v, i) => res(`q${i}`, v, { agent })),
|
|
1120
|
+
experiment: {
|
|
1121
|
+
runs: 1,
|
|
1122
|
+
earlyExit: false,
|
|
1123
|
+
selectedEvalIds: verdicts.map((_, i) => `q${i}`),
|
|
1124
|
+
...(labels ? { labels } : {}),
|
|
1125
|
+
},
|
|
1126
|
+
});
|
|
1127
|
+
|
|
1128
|
+
it("label() 按声明值分组、未声明归 (missing);numericLabel 对字符串值返回 null 不猜序", async () => {
|
|
1129
|
+
const table = await metricTableData(
|
|
1130
|
+
[labeled("m/one", { memory: "mempal" }, ["passed"]), labeled("m/two", undefined, ["failed"])],
|
|
1131
|
+
{ rows: label("memory"), columns: [endToEndPassRate] },
|
|
1132
|
+
);
|
|
1133
|
+
expect(table.rows.map((r) => r.key).sort()).toEqual(["(missing)", "mempal"]);
|
|
1134
|
+
|
|
1135
|
+
const line = await metricLineData(
|
|
1136
|
+
[labeled("m/num", { contextK: 32 }, ["passed"]), labeled("m/str", { contextK: "big" }, ["passed"])],
|
|
1137
|
+
{ x: numericLabel("contextK"), y: endToEndPassRate },
|
|
1138
|
+
);
|
|
1139
|
+
expect(line.rows.filter((r) => r.x === null)).toHaveLength(1);
|
|
1140
|
+
expect(line.rows.some((r) => r.x === 32)).toBe(true);
|
|
1141
|
+
expect(line.rows.some((r) => r.x === 0)).toBe(false);
|
|
1142
|
+
});
|
|
1143
|
+
|
|
1144
|
+
it("series 数组解析为复合维度:name 以 × 连接、值以 · 连接,缺失成员沿用 (missing)", async () => {
|
|
1145
|
+
const a = labeled("c/one", { memory: "mempal" }, ["passed"], "codex");
|
|
1146
|
+
const b = labeled("c/two", undefined, ["failed"], "codex");
|
|
1147
|
+
const data = await metricScatterData([a, b], {
|
|
1148
|
+
points: "experiment",
|
|
1149
|
+
series: ["agent", label("memory")],
|
|
1150
|
+
x: costUSD,
|
|
1151
|
+
y: endToEndPassRate,
|
|
1152
|
+
});
|
|
1153
|
+
expect(data.seriesDimension).toBe("agent × memory");
|
|
1154
|
+
const byKey = new Map(data.rows.map((r) => [r.key, r.series]));
|
|
1155
|
+
expect(byKey.get("c/one")).toBe("codex · mempal");
|
|
1156
|
+
expect(byKey.get("c/two")).toBe("codex · (missing)");
|
|
1157
|
+
// 单成员数组等价于单维度
|
|
1158
|
+
const single = await metricScatterData([a], { points: "experiment", series: ["agent"], x: costUSD, y: endToEndPassRate });
|
|
1159
|
+
expect(single.seriesDimension).toBe("agent");
|
|
1160
|
+
expect(single.rows[0]!.series).toBe("codex");
|
|
1161
|
+
});
|
|
1162
|
+
|
|
1163
|
+
const cell = (value: number, display: string): import("../model/types.ts").MetricCell => ({
|
|
1164
|
+
value,
|
|
1165
|
+
display,
|
|
1166
|
+
samples: 1,
|
|
1167
|
+
total: 1,
|
|
1168
|
+
refs: [],
|
|
1169
|
+
});
|
|
1170
|
+
const lineScatter = (): import("../model/types.ts").ScatterData => ({
|
|
1171
|
+
pointDimension: "experiment",
|
|
1172
|
+
seriesDimension: "line",
|
|
1173
|
+
x: { key: "costUSD", label: "Cost", unit: "$", better: "lower" },
|
|
1174
|
+
y: { key: "endToEndPassRate", label: "Pass rate", unit: "%", better: "higher" },
|
|
1175
|
+
rows: [
|
|
1176
|
+
{ key: "mem/claude-mempal", series: "claude", x: cell(0.55, "$0.55"), y: cell(0.75, "75%") },
|
|
1177
|
+
{ key: "mem/claude-baseline", series: "claude", x: cell(0.35, "$0.35"), y: cell(0.5, "50%") },
|
|
1178
|
+
{ key: "mem/bub", series: "bub", x: cell(0.09, "$0.09"), y: cell(0.875, "87.5%") },
|
|
1179
|
+
],
|
|
1180
|
+
});
|
|
1181
|
+
|
|
1182
|
+
it("scatterText:标记按图例顺序分配(series 字典序、series 内 x 升序),标题标注归类维度;默认无箭头", () => {
|
|
1183
|
+
const text = scatterText(lineScatter(), createTextContext({ width: 80 }));
|
|
1184
|
+
expect(text).toContain("grouped by line");
|
|
1185
|
+
// bub < claude:A 给 bub;claude 内按成本升序 baseline(B) → mempal(C)
|
|
1186
|
+
expect(text).toContain("bub A mem/bub");
|
|
1187
|
+
expect(text).toContain("B mem/claude-baseline");
|
|
1188
|
+
expect(text).toContain("C mem/claude-mempal");
|
|
1189
|
+
expect(text).not.toContain("→ C mem/claude-mempal");
|
|
1190
|
+
expect(text).not.toContain("└ Pass rate");
|
|
1191
|
+
});
|
|
1192
|
+
|
|
1193
|
+
it("scatterText connect:图例按 x 升序 → 串联并给逐段位移摘要;单点 series 无箭头无摘要;坐标图内不画折线", () => {
|
|
1194
|
+
const text = scatterText(lineScatter(), createTextContext({ width: 80 }), { connect: true });
|
|
1195
|
+
expect(text).toContain("B mem/claude-baseline → C mem/claude-mempal");
|
|
1196
|
+
expect(text).toContain("└ Pass rate +25pt · Cost +$0.20");
|
|
1197
|
+
expect(text).toContain("A mem/bub");
|
|
1198
|
+
expect(text).not.toContain("A mem/bub →");
|
|
1199
|
+
// 坐标图折线用 · 描画;connect 的 text 投影是图例的位移摘要,网格行(│ 开头)不出现折线点
|
|
1200
|
+
const plotRows = text.split("\n").filter((l) => l.includes("│"));
|
|
1201
|
+
expect(plotRows.length).toBeGreaterThan(0);
|
|
1202
|
+
expect(plotRows.every((l) => !l.includes("·"))).toBe(true);
|
|
1203
|
+
});
|
|
1204
|
+
|
|
1205
|
+
it("同图撞色按图例顺序线性探测空格;无冲突键仍取散列格;超过色板才复用", () => {
|
|
1206
|
+
// 真实回归现场:bub / claude-code / codex 在同一张图里必须三色互异
|
|
1207
|
+
const real = colorIndicesForKeys(["bub", "claude-code", "codex"]);
|
|
1208
|
+
expect(new Set(real.values()).size).toBe(3);
|
|
1209
|
+
// 构造性冲突:找两个散列同格的键,后到者让位,先到者保持散列格(跨图稳定)
|
|
1210
|
+
const pool = Array.from({ length: 40 }, (_, i) => `k${i}`);
|
|
1211
|
+
const byIdx = new Map<number, string[]>();
|
|
1212
|
+
for (const k of pool) {
|
|
1213
|
+
const idx = colorIndexForKey(k);
|
|
1214
|
+
byIdx.set(idx, [...(byIdx.get(idx) ?? []), k]);
|
|
1215
|
+
}
|
|
1216
|
+
const [first, second] = [...byIdx.values()].find((keys) => keys.length >= 2)!;
|
|
1217
|
+
const resolved = colorIndicesForKeys([first, second]);
|
|
1218
|
+
expect(resolved.get(first)).toBe(colorIndexForKey(first));
|
|
1219
|
+
expect(resolved.get(second)).not.toBe(resolved.get(first));
|
|
1220
|
+
// 前 6 个键占满 6 色;第 7 个开始只能复用
|
|
1221
|
+
const seven = colorIndicesForKeys(["s0", "s1", "s2", "s3", "s4", "s5", "s6"]);
|
|
1222
|
+
expect(new Set([...seven.values()].slice(0, 6)).size).toBe(6);
|
|
1223
|
+
});
|
|
1224
|
+
});
|
|
1225
|
+
|
|
1226
|
+
describe("validate*Data 接受真实计算产物(不是只接受手写 literal)", () => {
|
|
1227
|
+
// 手写 literal 通过校验只证明 validator 认得自己设想的形状;真正的把关是喂真实计算产物——
|
|
1228
|
+
// 尤其是省略可选字段(如自定义指标不声明 label)的真实路径,不是校验作者假想的边界。
|
|
1229
|
+
const a = snap({ experimentId: "compare/a", results: [res("q1", "passed"), res("q2", "failed")] });
|
|
1230
|
+
const b = snap({ experimentId: "compare/b", results: [res("q1", "failed"), res("q2", "passed")] });
|
|
1231
|
+
const scope = scopeOf([a, b]);
|
|
1232
|
+
const noLabelMetric = defineMetric({ name: "custom-no-label", value: (attempt) => (attempt.result.verdict === "passed" ? 1 : 0) });
|
|
1233
|
+
|
|
1234
|
+
it("metricTableData:含未声明 label 的自定义指标", async () => {
|
|
1235
|
+
const table = await metricTableData(scope, { rows: "agent", columns: [costUSD, noLabelMetric] });
|
|
1236
|
+
expect(validateTableData(table)).toBeNull();
|
|
1237
|
+
});
|
|
1238
|
+
|
|
1239
|
+
it("metricMatrixData", async () => {
|
|
1240
|
+
const matrix = await metricMatrixData(scope, { rows: "experiment", columns: "eval", cell: endToEndPassRate });
|
|
1241
|
+
expect(validateMatrixData(matrix)).toBeNull();
|
|
1242
|
+
});
|
|
1243
|
+
|
|
1244
|
+
it("metricScatterData:含缺 y 的点(null MetricCell.value)", async () => {
|
|
1245
|
+
const scatter = await metricScatterData(scope, { points: "experiment", x: costUSD, y: noLabelMetric });
|
|
1246
|
+
expect(validateScatterData(scatter)).toBeNull();
|
|
1247
|
+
});
|
|
1248
|
+
|
|
1249
|
+
it("metricLineData", async () => {
|
|
1250
|
+
const line = await metricLineData(scope, { x: numericFlag("budget"), y: endToEndPassRate });
|
|
1251
|
+
expect(validateLineData(line)).toBeNull();
|
|
1252
|
+
});
|
|
1253
|
+
|
|
1254
|
+
it("scoreboardData", async () => {
|
|
1255
|
+
const board = await scoreboardData(scope, { rows: "agent", questions: ["q1", "q2"] });
|
|
1256
|
+
expect(validateScoreboardData(board)).toBeNull();
|
|
1257
|
+
});
|
|
1258
|
+
|
|
1259
|
+
it("deltaTableData(含 pairsByFlag 派生 pair)", async () => {
|
|
1260
|
+
const withFlag = snap({
|
|
1261
|
+
experimentId: "compare/withflag",
|
|
1262
|
+
results: [res("q1", "passed")],
|
|
1263
|
+
experiment: { runs: 1, earlyExit: false, selectedEvalIds: [], flags: { memory: "on" } },
|
|
1264
|
+
});
|
|
1265
|
+
const withoutFlag = snap({ experimentId: "compare/noflag", results: [res("q1", "failed")] });
|
|
1266
|
+
const deltaScope = scopeOf([withFlag, withoutFlag]);
|
|
1267
|
+
const delta = await deltaTableData(deltaScope, { by: "experiment", pairs: pairsByFlag("memory"), metrics: [costUSD] });
|
|
1268
|
+
expect(validateDeltaData(delta)).toBeNull();
|
|
1269
|
+
});
|
|
1270
|
+
|
|
1271
|
+
it("scopeSummaryData", async () => {
|
|
1272
|
+
const summary = await scopeSummaryData(scope);
|
|
1273
|
+
expect(validateScopeSummaryData(summary)).toBeNull();
|
|
1274
|
+
});
|
|
1275
|
+
|
|
1276
|
+
it("experimentListData / evalListData / attemptListData(同一 Scope 三种粒度)", async () => {
|
|
1277
|
+
expect(validateExperimentListData(await experimentListData(scope))).toBeNull();
|
|
1278
|
+
expect(validateEvalListData(await evalListData(scope))).toBeNull();
|
|
1279
|
+
expect(validateAttemptListData(await attemptListData(scope))).toBeNull();
|
|
1280
|
+
});
|
|
1281
|
+
});
|
|
@@ -6,30 +6,38 @@
|
|
|
6
6
|
// 也是它们展开区里"逐条 attempt"那一层的唯一实现(通过 AttemptRow 导出复用,不重写一遍)。
|
|
7
7
|
|
|
8
8
|
import type { ReactElement } from "react";
|
|
9
|
-
import type { AttemptListItem } from "
|
|
10
|
-
import type { AttemptLocator } from "
|
|
11
|
-
import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "
|
|
12
|
-
import { colorClassForKey } from "
|
|
13
|
-
import {
|
|
9
|
+
import type { AttemptListItem } from "../../model/types.ts";
|
|
10
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
11
|
+
import { DEFAULT_REPORT_LOCALE, countText, localeText, type ReportLocale } from "../../model/locale.ts";
|
|
12
|
+
import { colorClassForKey } from "../../assets/colors.ts";
|
|
13
|
+
import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
|
|
14
|
+
import { cx } from "../shared.ts";
|
|
14
15
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* locator + 判定符,AttemptList/EvalList/ExperimentList 共用。没有 target(当前报告没有
|
|
18
|
+
* declare attempt-input page,也没有显式 attemptHref)时是纯文本,不生成空 href 或假链接
|
|
19
|
+
* (docs/feature/reports/architecture.md「Attempt 详情是一张参数化 page」)。
|
|
20
|
+
*/
|
|
18
21
|
export function AttemptLocatorBadge({
|
|
19
22
|
item,
|
|
20
|
-
attemptHref
|
|
23
|
+
attemptHref,
|
|
21
24
|
}: {
|
|
22
25
|
item: Pick<AttemptListItem, "locator" | "verdict">;
|
|
23
26
|
attemptHref?: (locator: AttemptLocator) => string;
|
|
24
27
|
}): ReactElement {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
href={attemptHref(item.locator)}
|
|
29
|
-
>
|
|
28
|
+
const className = cx("nre-locator", `nre-verdict-${item.verdict}`);
|
|
29
|
+
const content = (
|
|
30
|
+
<>
|
|
30
31
|
{item.locator}
|
|
31
32
|
<span className="nre-locator-mark">{verdictMark(item.verdict)}</span>
|
|
33
|
+
</>
|
|
34
|
+
);
|
|
35
|
+
return attemptHref ? (
|
|
36
|
+
<a className={className} href={attemptHref(item.locator)}>
|
|
37
|
+
{content}
|
|
32
38
|
</a>
|
|
39
|
+
) : (
|
|
40
|
+
<span className={className}>{content}</span>
|
|
33
41
|
);
|
|
34
42
|
}
|
|
35
43
|
|
|
@@ -44,7 +52,7 @@ export function failureSummaryText(item: Pick<AttemptListItem, "failureSummary"
|
|
|
44
52
|
/** 一条 Attempt 的比较卡片;完整 assertions 通过 locator 下钻,不在列表内展开。 */
|
|
45
53
|
export function AttemptRow({
|
|
46
54
|
item,
|
|
47
|
-
attemptHref
|
|
55
|
+
attemptHref,
|
|
48
56
|
locale = DEFAULT_REPORT_LOCALE,
|
|
49
57
|
}: {
|
|
50
58
|
item: AttemptListItem;
|
|
@@ -74,7 +82,7 @@ export function AttemptList({
|
|
|
74
82
|
data,
|
|
75
83
|
total,
|
|
76
84
|
filter = false,
|
|
77
|
-
attemptHref
|
|
85
|
+
attemptHref,
|
|
78
86
|
className,
|
|
79
87
|
locale = DEFAULT_REPORT_LOCALE,
|
|
80
88
|
}: {
|
|
Binary file
|
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
// ExperimentList:实体列表的第一级。web 面是一行一个 experiment 的固定八列比较表
|
|
2
|
-
// (Experiment / Model / Agent / Avg
|
|
2
|
+
// (Experiment / Model / Agent / Avg. time / Pass rate / Tokens / Cost / Results),
|
|
3
3
|
// 每行用原生 <details> 展开到 Eval 与 Attempt locator。数据完全来自 experimentListData(),
|
|
4
4
|
// 组件不重算、不推断组边界。
|
|
5
5
|
|
|
6
6
|
import type { ReactElement } from "react";
|
|
7
|
-
import type { AttemptLocator } from "
|
|
8
|
-
import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "
|
|
9
|
-
import { experimentDisplayName } from "
|
|
10
|
-
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "
|
|
7
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
8
|
+
import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../../model/types.ts";
|
|
9
|
+
import { experimentDisplayName } from "../../model/format.ts";
|
|
10
|
+
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
|
|
11
11
|
import { AttemptLocatorBadge, failureSummaryText } from "./AttemptList.tsx";
|
|
12
|
-
import { MetricCellView } from "
|
|
13
|
-
import { colorClassForKey } from "
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
|
|
12
|
+
import { MetricCellView } from "../cell.tsx";
|
|
13
|
+
import { colorClassForKey } from "../../assets/colors.ts";
|
|
14
|
+
import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
|
|
15
|
+
import { cx } from "../shared.ts";
|
|
17
16
|
|
|
18
17
|
const verdictOrder = ["passed", "failed", "errored", "skipped"] as const;
|
|
19
18
|
|
|
@@ -39,7 +38,7 @@ function VerdictSummary({ item, locale }: { item: ExperimentListItem; locale: Re
|
|
|
39
38
|
const parts = verdictOrder
|
|
40
39
|
.filter((verdict) => item.evalVerdicts[verdict] > 0)
|
|
41
40
|
.map((verdict) => `${item.evalVerdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
|
|
42
|
-
return <span className="nre-experiment-pill">{parts.join("
|
|
41
|
+
return <span className="nre-experiment-pill">{parts.join(" · ") || "—"}</span>;
|
|
43
42
|
}
|
|
44
43
|
|
|
45
44
|
function ExperimentAttemptRow({
|
|
@@ -50,7 +49,7 @@ function ExperimentAttemptRow({
|
|
|
50
49
|
}: {
|
|
51
50
|
attempt: AttemptListItem;
|
|
52
51
|
last: boolean;
|
|
53
|
-
attemptHref
|
|
52
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
54
53
|
locale: ReportLocale;
|
|
55
54
|
}): ReactElement {
|
|
56
55
|
const reason = failureSummaryText(attempt, locale);
|
|
@@ -76,7 +75,7 @@ function EvalAttempts({
|
|
|
76
75
|
locale,
|
|
77
76
|
}: {
|
|
78
77
|
row: ExperimentListEvalRow;
|
|
79
|
-
attemptHref
|
|
78
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
80
79
|
locale: ReportLocale;
|
|
81
80
|
}): ReactElement {
|
|
82
81
|
const duration = row.durationMs.value === null ? localeText(locale, "cell.missing") : formatDurationMs(row.durationMs.value);
|
|
@@ -128,16 +127,16 @@ function ExperimentRow({
|
|
|
128
127
|
relativeTo,
|
|
129
128
|
}: {
|
|
130
129
|
item: ExperimentListItem;
|
|
131
|
-
attemptHref
|
|
130
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
132
131
|
locale: ReportLocale;
|
|
133
132
|
relativeTo?: string;
|
|
134
133
|
}): ReactElement {
|
|
135
134
|
return (
|
|
136
135
|
<details className="nre-experiment-entry">
|
|
137
136
|
<summary className="nre-experiment-summary">
|
|
138
|
-
{/* relativeTo 只影响显示末段;完整 id 仍是排序 /
|
|
137
|
+
{/* relativeTo 只影响显示末段;完整 id 仍是排序 / 过滤 / 折叠的键,系列色跟随 agent */}
|
|
139
138
|
<span className="nre-experiment-name" data-sort-value={item.experimentId}>
|
|
140
|
-
<b className={cx("nre-experiment-id", "nre-key"
|
|
139
|
+
<b className={cx("nre-experiment-id", "nre-key")}>
|
|
141
140
|
{experimentDisplayName(item.experimentId, relativeTo)}
|
|
142
141
|
</b>
|
|
143
142
|
<small>
|
|
@@ -147,7 +146,9 @@ function ExperimentRow({
|
|
|
147
146
|
</small>
|
|
148
147
|
</span>
|
|
149
148
|
<span data-sort-value={item.model ?? ""}>{item.model ?? localeText(locale, "experimentList.defaultModel")}</span>
|
|
150
|
-
<span data-sort-value={item.agent}>
|
|
149
|
+
<span data-sort-value={item.agent}>
|
|
150
|
+
<span className={cx("nre-experiment-agent", "nre-key", colorClassForKey(item.agent))}>{item.agent}</span>
|
|
151
|
+
</span>
|
|
151
152
|
<span className="nre-num" data-sort-value={item.durationMs.value ?? ""}>
|
|
152
153
|
<MetricCellView cell={item.durationMs} locale={locale} />
|
|
153
154
|
</span>
|
|
@@ -179,7 +180,7 @@ function ExperimentRow({
|
|
|
179
180
|
|
|
180
181
|
export function ExperimentList({
|
|
181
182
|
data,
|
|
182
|
-
attemptHref
|
|
183
|
+
attemptHref,
|
|
183
184
|
filter = false,
|
|
184
185
|
className,
|
|
185
186
|
locale = DEFAULT_REPORT_LOCALE,
|
|
@@ -209,10 +210,12 @@ export function ExperimentList({
|
|
|
209
210
|
<button
|
|
210
211
|
type="button"
|
|
211
212
|
data-nre-experiment-sort={index}
|
|
212
|
-
className={index === 4
|
|
213
|
+
className={cx(index >= 3 && index <= 6 && "nre-num-head", index === 4 && "nre-sort-desc")}
|
|
213
214
|
key={label}
|
|
215
|
+
title={index === 4 ? localeText(locale, "experimentList.passRateDescription") : undefined}
|
|
214
216
|
>
|
|
215
|
-
{label}
|
|
217
|
+
<span className="nre-sort-label">{label}</span>
|
|
218
|
+
<span className="nre-sort-icon" aria-hidden="true" />
|
|
216
219
|
</button>
|
|
217
220
|
))}
|
|
218
221
|
</div>
|