niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
// 官方双面组件的装配点:web 面(./ExperimentList.tsx / EvalList.tsx / AttemptList.tsx 的纯
|
|
2
|
+
// React 组件)+ text 面(./faces.ts)+ resolve 解析面(spec 形态由管线代调配套 ./compute.ts)。
|
|
3
|
+
// FailureList 是组合组件,内部就是 attemptListData → 过滤 → AttemptList data 形态,不产生
|
|
4
|
+
// 自己的 data。
|
|
5
|
+
|
|
6
|
+
import { defineComponent, type ReportComponent } from "../../definition/tree.ts";
|
|
7
|
+
import type { AttemptListItem, EvalListItem, ExperimentListItem, ReportInput } from "../../model/types.ts";
|
|
8
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
9
|
+
import { collectItems, locatorOf, resolveInput } from "../../model/aggregate.ts";
|
|
10
|
+
import type { ReportLocale } from "../../model/locale.ts";
|
|
11
|
+
import {
|
|
12
|
+
arrayProblem,
|
|
13
|
+
cellProblem,
|
|
14
|
+
isObject,
|
|
15
|
+
makeDataComponent,
|
|
16
|
+
hrefOf,
|
|
17
|
+
tallyProblem,
|
|
18
|
+
type ChromeProps,
|
|
19
|
+
type DataProps,
|
|
20
|
+
type Validator,
|
|
21
|
+
} from "../shared.ts";
|
|
22
|
+
import { attemptListData, evalListData, experimentListData } from "./compute.ts";
|
|
23
|
+
import { attemptListText, evalListText, experimentListText } from "./faces.ts";
|
|
24
|
+
import { AttemptList as AttemptListWeb } from "./AttemptList.tsx";
|
|
25
|
+
import { EvalList as EvalListWeb } from "./EvalList.tsx";
|
|
26
|
+
import { ExperimentList as ExperimentListWeb } from "./ExperimentList.tsx";
|
|
27
|
+
|
|
28
|
+
/** AttemptListItem(src/report/model/types.ts):三个组件族共用的叶子形状(独立 data 或嵌套在 evalRows/attempts 里)。 */
|
|
29
|
+
function attemptListItemProblem(value: unknown, path: string): string | null {
|
|
30
|
+
if (!isObject(value)) return `"${path}" must be an object`;
|
|
31
|
+
if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
32
|
+
if (typeof value.evalId !== "string") return `"${path}.evalId" must be a string`;
|
|
33
|
+
if (typeof value.attempt !== "number") return `"${path}.attempt" must be a number`;
|
|
34
|
+
if (typeof value.agent !== "string") return `"${path}.agent" must be a string`;
|
|
35
|
+
if (typeof value.verdict !== "string") return `"${path}.verdict" must be a string`;
|
|
36
|
+
if (!(value.failureSummary === null || typeof value.failureSummary === "string")) {
|
|
37
|
+
return `"${path}.failureSummary" must be a string or null`;
|
|
38
|
+
}
|
|
39
|
+
if (typeof value.moreFailures !== "number") return `"${path}.moreFailures" must be a number`;
|
|
40
|
+
const examScoreProblem = cellProblem(value.examScore, `${path}.examScore`);
|
|
41
|
+
if (examScoreProblem !== null) return examScoreProblem;
|
|
42
|
+
if (typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
43
|
+
if (!(value.costUSD === null || typeof value.costUSD === "number")) return `"${path}.costUSD" must be a number or null`;
|
|
44
|
+
if (typeof value.locator !== "string") return `"${path}.locator" must be a string`;
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export const validateExperimentListData: Validator = (data) =>
|
|
49
|
+
arrayProblem(data, "data", (item, path) => {
|
|
50
|
+
if (!isObject(item)) return `"${path}" must be an object`;
|
|
51
|
+
if (typeof item.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
52
|
+
if (typeof item.agent !== "string") return `"${path}.agent" must be a string`;
|
|
53
|
+
const verdictsProblem = tallyProblem(item.evalVerdicts, `${path}.evalVerdicts`);
|
|
54
|
+
if (verdictsProblem !== null) return verdictsProblem;
|
|
55
|
+
const passRateProblem = cellProblem(item.endToEndPassRate, `${path}.endToEndPassRate`);
|
|
56
|
+
if (passRateProblem !== null) return passRateProblem;
|
|
57
|
+
const costProblem = cellProblem(item.costUSD, `${path}.costUSD`);
|
|
58
|
+
if (costProblem !== null) return costProblem;
|
|
59
|
+
const durationProblem = cellProblem(item.durationMs, `${path}.durationMs`);
|
|
60
|
+
if (durationProblem !== null) return durationProblem;
|
|
61
|
+
const tokensProblem = cellProblem(item.tokens, `${path}.tokens`);
|
|
62
|
+
if (tokensProblem !== null) return tokensProblem;
|
|
63
|
+
if (typeof item.evals !== "number") return `"${path}.evals" must be a number`;
|
|
64
|
+
if (typeof item.attempts !== "number") return `"${path}.attempts" must be a number`;
|
|
65
|
+
if (typeof item.lastRunAt !== "string") return `"${path}.lastRunAt" must be a string`;
|
|
66
|
+
return arrayProblem(item.evalRows, `${path}.evalRows`, (row, rowPath) => {
|
|
67
|
+
if (!isObject(row) || typeof row.evalId !== "string") {
|
|
68
|
+
return `"${rowPath}" must be an object with a string "evalId"`;
|
|
69
|
+
}
|
|
70
|
+
const rowDurationProblem = cellProblem(row.durationMs, `${rowPath}.durationMs`);
|
|
71
|
+
if (rowDurationProblem !== null) return rowDurationProblem;
|
|
72
|
+
const rowCostProblem = cellProblem(row.costUSD, `${rowPath}.costUSD`);
|
|
73
|
+
if (rowCostProblem !== null) return rowCostProblem;
|
|
74
|
+
return arrayProblem(row.attempts, `${rowPath}.attempts`, attemptListItemProblem);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
export const validateEvalListData: Validator = (data) =>
|
|
78
|
+
arrayProblem(data, "data", (item, path) => {
|
|
79
|
+
if (!isObject(item)) return `"${path}" must be an object`;
|
|
80
|
+
if (typeof item.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
81
|
+
if (typeof item.evalId !== "string") return `"${path}.evalId" must be a string`;
|
|
82
|
+
if (typeof item.verdict !== "string") return `"${path}.verdict" must be a string`;
|
|
83
|
+
const examScoreProblem = cellProblem(item.examScore, `${path}.examScore`);
|
|
84
|
+
if (examScoreProblem !== null) return examScoreProblem;
|
|
85
|
+
const durationProblem = cellProblem(item.durationMs, `${path}.durationMs`);
|
|
86
|
+
if (durationProblem !== null) return durationProblem;
|
|
87
|
+
const costProblem = cellProblem(item.costUSD, `${path}.costUSD`);
|
|
88
|
+
if (costProblem !== null) return costProblem;
|
|
89
|
+
return arrayProblem(item.attempts, `${path}.attempts`, attemptListItemProblem);
|
|
90
|
+
});
|
|
91
|
+
export const validateAttemptListData: Validator = (data) => arrayProblem(data, "data", attemptListItemProblem);
|
|
92
|
+
|
|
93
|
+
// ───────────────────────── 实体列表 ─────────────────────────
|
|
94
|
+
|
|
95
|
+
interface EntityListChrome extends ChromeProps {
|
|
96
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export type ExperimentListProps = DataProps<
|
|
100
|
+
readonly ExperimentListItem[],
|
|
101
|
+
Record<never, never>,
|
|
102
|
+
EntityListChrome & {
|
|
103
|
+
/** web 面在比较表前显示实验过滤框;text 面忽略。 */
|
|
104
|
+
filter?: boolean;
|
|
105
|
+
}
|
|
106
|
+
>;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。行标签是
|
|
110
|
+
* experiment id 在当前列表里的最短唯一后缀(与 MetricScatter 点标签同一算法,重名逐步
|
|
111
|
+
* 加长到能区分为止);完整 id 不受影响,仍是排序 / 过滤 / 折叠的键。
|
|
112
|
+
*/
|
|
113
|
+
export const ExperimentList = makeDataComponent<
|
|
114
|
+
readonly ExperimentListItem[],
|
|
115
|
+
Record<never, never>,
|
|
116
|
+
EntityListChrome & { filter?: boolean }
|
|
117
|
+
>({
|
|
118
|
+
name: "ExperimentList",
|
|
119
|
+
dataFnName: "experimentListData",
|
|
120
|
+
shapeName: "ExperimentListItem[]",
|
|
121
|
+
dataFn: (input) => experimentListData(input),
|
|
122
|
+
specKeys: [],
|
|
123
|
+
validate: validateExperimentListData,
|
|
124
|
+
web: (props, ctx) => (
|
|
125
|
+
<ExperimentListWeb
|
|
126
|
+
data={props.data}
|
|
127
|
+
filter={props.filter}
|
|
128
|
+
locale={props.locale ?? ctx.locale}
|
|
129
|
+
attemptHref={hrefOf(props, ctx)}
|
|
130
|
+
className={props.className}
|
|
131
|
+
/>
|
|
132
|
+
),
|
|
133
|
+
text: (props, ctx) => experimentListText(props.data, ctx),
|
|
134
|
+
}) as unknown as ReportComponent<ExperimentListProps>;
|
|
135
|
+
|
|
136
|
+
export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
|
|
137
|
+
|
|
138
|
+
/** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
|
|
139
|
+
export const EvalList = makeDataComponent<readonly EvalListItem[], Record<never, never>, EntityListChrome>({
|
|
140
|
+
name: "EvalList",
|
|
141
|
+
dataFnName: "evalListData",
|
|
142
|
+
shapeName: "EvalListItem[]",
|
|
143
|
+
dataFn: (input) => evalListData(input),
|
|
144
|
+
specKeys: [],
|
|
145
|
+
validate: validateEvalListData,
|
|
146
|
+
web: (props, ctx) => (
|
|
147
|
+
<EvalListWeb
|
|
148
|
+
data={props.data}
|
|
149
|
+
locale={props.locale ?? ctx.locale}
|
|
150
|
+
attemptHref={hrefOf(props, ctx)}
|
|
151
|
+
className={props.className}
|
|
152
|
+
/>
|
|
153
|
+
),
|
|
154
|
+
text: (props, ctx) => evalListText(props.data, ctx),
|
|
155
|
+
}) as unknown as ReportComponent<EvalListProps>;
|
|
156
|
+
|
|
157
|
+
export type AttemptListProps = DataProps<
|
|
158
|
+
readonly AttemptListItem[],
|
|
159
|
+
Record<never, never>,
|
|
160
|
+
EntityListChrome & {
|
|
161
|
+
/** 过滤 / 截断前的总数;省略时等于 data 长度。 */
|
|
162
|
+
total?: number;
|
|
163
|
+
/** web 面加过滤输入框(按 experiment、eval、agent、verdict 或摘要文本收窄行);渐进增强,不改变数据与 text 面。 */
|
|
164
|
+
filter?: boolean;
|
|
165
|
+
}
|
|
166
|
+
>;
|
|
167
|
+
|
|
168
|
+
/** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
|
|
169
|
+
export const AttemptList = makeDataComponent<
|
|
170
|
+
readonly AttemptListItem[],
|
|
171
|
+
Record<never, never>,
|
|
172
|
+
EntityListChrome & { total?: number; filter?: boolean }
|
|
173
|
+
>({
|
|
174
|
+
name: "AttemptList",
|
|
175
|
+
dataFnName: "attemptListData",
|
|
176
|
+
shapeName: "AttemptListItem[]",
|
|
177
|
+
dataFn: (input) => attemptListData(input),
|
|
178
|
+
specKeys: [],
|
|
179
|
+
validate: validateAttemptListData,
|
|
180
|
+
web: (props, ctx) => (
|
|
181
|
+
<AttemptListWeb
|
|
182
|
+
data={props.data}
|
|
183
|
+
total={props.total}
|
|
184
|
+
filter={props.filter}
|
|
185
|
+
locale={props.locale ?? ctx.locale}
|
|
186
|
+
attemptHref={hrefOf(props, ctx)}
|
|
187
|
+
className={props.className}
|
|
188
|
+
/>
|
|
189
|
+
),
|
|
190
|
+
text: (props, ctx) => attemptListText(props.data, props.total, ctx),
|
|
191
|
+
}) as unknown as ReportComponent<AttemptListProps>;
|
|
192
|
+
|
|
193
|
+
// ───────────────────────── FailureList(官方组合件)─────────────────────────
|
|
194
|
+
|
|
195
|
+
export interface FailureListProps {
|
|
196
|
+
/** 显示的最大条数;默认 20。 */
|
|
197
|
+
limit?: number;
|
|
198
|
+
/** 默认宿主注入的 Scope。 */
|
|
199
|
+
input?: ReportInput;
|
|
200
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
201
|
+
locale?: ReportLocale;
|
|
202
|
+
className?: string;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* 「现在有哪些失败要处理」的成品组合件:内部就是 attemptListData → 过滤 → AttemptList
|
|
207
|
+
* data 形态,与手写组合严格等价、没有私有能力(docs/feature/reports/library/entity-lists.md)。
|
|
208
|
+
* verdict ∈ failed / errored,按 attempt 开始时间降序(同刻按 locator 字典序),
|
|
209
|
+
* 截断到 limit(默认 20),total 报告截断前总数。
|
|
210
|
+
*/
|
|
211
|
+
export const FailureList = defineComponent<FailureListProps>(async (props, ctx) => {
|
|
212
|
+
const input = props.input ?? ctx.scope;
|
|
213
|
+
const all = await attemptListData(input);
|
|
214
|
+
// attempt 开始时间不在列表条目里(它不是列表展示字段);从同一 input 的读取面按 locator 对回。
|
|
215
|
+
const startedAtByLocator = new Map<string, string>();
|
|
216
|
+
for (const item of collectItems(resolveInput(input).snapshots)) {
|
|
217
|
+
startedAtByLocator.set(locatorOf(item), item.attempt.result.startedAt ?? "");
|
|
218
|
+
}
|
|
219
|
+
const failures = all
|
|
220
|
+
.filter((item) => item.verdict === "failed" || item.verdict === "errored")
|
|
221
|
+
.sort((a, b) => {
|
|
222
|
+
const ta = startedAtByLocator.get(a.locator) ?? "";
|
|
223
|
+
const tb = startedAtByLocator.get(b.locator) ?? "";
|
|
224
|
+
if (ta !== tb) return ta < tb ? 1 : -1; // 最近的失败在前
|
|
225
|
+
return a.locator < b.locator ? -1 : a.locator > b.locator ? 1 : 0;
|
|
226
|
+
});
|
|
227
|
+
const limit = props.limit ?? 20;
|
|
228
|
+
return (
|
|
229
|
+
<AttemptList
|
|
230
|
+
data={failures.slice(0, limit)}
|
|
231
|
+
total={failures.length}
|
|
232
|
+
attemptHref={props.attemptHref}
|
|
233
|
+
locale={props.locale}
|
|
234
|
+
className={props.className}
|
|
235
|
+
/>
|
|
236
|
+
);
|
|
237
|
+
});
|
|
238
|
+
FailureList.displayName = "FailureList";
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
|
+
// "validate*Data 递归覆盖到嵌套字段" 行:entity-lists 三个 validate*Data 的表驱动字段突变覆盖,AttemptListItem 的形状在三个组件族
|
|
3
|
+
// 共用(独立 data 或嵌套在 evalRows/attempts 里),重点覆盖嵌套 MetricCell/tally 字段与
|
|
4
|
+
// 深层嵌套(ExperimentListItem.evalRows[i].attempts[j])的路径定位。
|
|
5
|
+
|
|
6
|
+
import { describe, expect, it } from "vitest";
|
|
7
|
+
import { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./index.tsx";
|
|
8
|
+
|
|
9
|
+
const validCell = { value: 1, display: "1", samples: 1, total: 1, refs: [] };
|
|
10
|
+
const validTally = { passed: 1, failed: 0, errored: 0, skipped: 0 };
|
|
11
|
+
|
|
12
|
+
const validAttemptItem = {
|
|
13
|
+
experimentId: "compare/codex",
|
|
14
|
+
evalId: "q1",
|
|
15
|
+
attempt: 0,
|
|
16
|
+
agent: "codex",
|
|
17
|
+
verdict: "passed",
|
|
18
|
+
failureSummary: null,
|
|
19
|
+
moreFailures: 0,
|
|
20
|
+
examScore: validCell,
|
|
21
|
+
durationMs: 1000,
|
|
22
|
+
costUSD: 0.01,
|
|
23
|
+
locator: "@1abcdef2",
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
describe("validateAttemptListData", () => {
|
|
27
|
+
it("合规 literal 通过,costUSD 为 null 与 failureSummary 为 null 合法", () => {
|
|
28
|
+
expect(validateAttemptListData([{ ...validAttemptItem, costUSD: null }])).toBeNull();
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it("非数组整体报错", () => {
|
|
32
|
+
expect(validateAttemptListData({})).toMatch(/"data" must be an array/);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("[i].examScore 结构错误定位到嵌套 MetricCell 字段", () => {
|
|
36
|
+
const bad = [{ ...validAttemptItem, examScore: { value: 1 } }];
|
|
37
|
+
expect(validateAttemptListData(bad)).toMatch(/"data\[0\]\.examScore\.display"/);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("[i].verdict 非字符串报错", () => {
|
|
41
|
+
const bad = [{ ...validAttemptItem, verdict: 1 }];
|
|
42
|
+
expect(validateAttemptListData(bad)).toMatch(/"data\[0\]\.verdict"/);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("validateEvalListData", () => {
|
|
47
|
+
const valid = [
|
|
48
|
+
{
|
|
49
|
+
experimentId: "compare/codex",
|
|
50
|
+
evalId: "q1",
|
|
51
|
+
verdict: "passed",
|
|
52
|
+
examScore: validCell,
|
|
53
|
+
durationMs: validCell,
|
|
54
|
+
costUSD: validCell,
|
|
55
|
+
attempts: [validAttemptItem],
|
|
56
|
+
},
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
it("合规 literal 通过", () => {
|
|
60
|
+
expect(validateEvalListData(valid)).toBeNull();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("[i].durationMs(MetricCell)结构错误报错", () => {
|
|
64
|
+
const bad = [{ ...valid[0], durationMs: { value: 1, display: "1", samples: 1, total: 1 } }];
|
|
65
|
+
expect(validateEvalListData(bad)).toMatch(/"data\[0\]\.durationMs\.refs"/);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it("[i].attempts[j] 嵌套 AttemptListItem 结构错误定位到深层下标", () => {
|
|
69
|
+
const bad = [{ ...valid[0], attempts: [{ ...validAttemptItem, locator: 1 }] }];
|
|
70
|
+
expect(validateEvalListData(bad)).toMatch(/"data\[0\]\.attempts\[0\]\.locator"/);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("validateExperimentListData", () => {
|
|
75
|
+
const validEvalRow = { evalId: "q1", verdict: "passed", durationMs: validCell, costUSD: validCell, attempts: [validAttemptItem] };
|
|
76
|
+
const valid = [
|
|
77
|
+
{
|
|
78
|
+
experimentId: "compare/codex",
|
|
79
|
+
agent: "codex",
|
|
80
|
+
evalVerdicts: validTally,
|
|
81
|
+
endToEndPassRate: validCell,
|
|
82
|
+
costUSD: validCell,
|
|
83
|
+
durationMs: validCell,
|
|
84
|
+
tokens: validCell,
|
|
85
|
+
evals: 1,
|
|
86
|
+
attempts: 1,
|
|
87
|
+
lastRunAt: "2026-07-01T00:00:00Z",
|
|
88
|
+
evalRows: [validEvalRow],
|
|
89
|
+
},
|
|
90
|
+
];
|
|
91
|
+
|
|
92
|
+
it("合规 literal 通过", () => {
|
|
93
|
+
expect(validateExperimentListData(valid)).toBeNull();
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("[i].evalVerdicts(四态 tally)缺字段报错", () => {
|
|
97
|
+
const bad = [{ ...valid[0], evalVerdicts: { passed: 1, failed: 0, errored: 0 } }];
|
|
98
|
+
expect(validateExperimentListData(bad)).toMatch(/"data\[0\]\.evalVerdicts\.skipped"/);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("[i].evalRows[j].attempts[k] 三层嵌套报错精确定位", () => {
|
|
102
|
+
const bad = [
|
|
103
|
+
{
|
|
104
|
+
...valid[0],
|
|
105
|
+
evalRows: [{ ...validEvalRow, attempts: [{ ...validAttemptItem, examScore: null }] }],
|
|
106
|
+
},
|
|
107
|
+
];
|
|
108
|
+
expect(validateExperimentListData(bad)).toMatch(/"data\[0\]\.evalRows\[0\]\.attempts\[0\]\.examScore"/);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("model / flags 可选字段省略仍合法", () => {
|
|
112
|
+
expect(validateExperimentListData(valid)).toBeNull();
|
|
113
|
+
});
|
|
114
|
+
});
|
|
@@ -15,7 +15,7 @@ import type {
|
|
|
15
15
|
ScopeSummaryData,
|
|
16
16
|
ScoreboardData,
|
|
17
17
|
TableData,
|
|
18
|
-
} from "../types.ts";
|
|
18
|
+
} from "../model/types.ts";
|
|
19
19
|
import type { AttemptLocator } from "../../results/locator.ts";
|
|
20
20
|
|
|
21
21
|
const locator = (s: string): AttemptLocator => s as AttemptLocator;
|
|
@@ -24,7 +24,7 @@ export const passRateColumn: MetricColumn = { key: "task-pass-rate", label: "pas
|
|
|
24
24
|
export const codeLinesColumn: MetricColumn = { key: "code-lines", label: "code lines", unit: "lines", better: "lower" };
|
|
25
25
|
export const costColumn: MetricColumn = { key: "cost", label: "cost", unit: "$", better: "lower" };
|
|
26
26
|
|
|
27
|
-
/** scopeSummaryData
|
|
27
|
+
/** scopeSummaryData 的产物形态:两级计票恒随行,通过率与总成本是官方 MetricCell。 */
|
|
28
28
|
export const scopeSummaryData: ScopeSummaryData = {
|
|
29
29
|
range: { earliestStartedAt: "2026-07-01T10:00:00Z", latestStartedAt: "2026-07-01T11:30:00Z" },
|
|
30
30
|
experiments: 2,
|
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
// 0 对不是错误:派生形态(pairsByFlag)配不出对时显示明确空态并报告配对域实验数。
|
|
5
5
|
|
|
6
6
|
import type { ReactElement } from "react";
|
|
7
|
-
import type { DeltaData } from "
|
|
8
|
-
import { DEFAULT_REPORT_LOCALE, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "
|
|
9
|
-
import { MetricCellView } from "
|
|
10
|
-
import { colorClassForKey } from "
|
|
11
|
-
import { cx } from "
|
|
7
|
+
import type { DeltaData } from "../../model/types.ts";
|
|
8
|
+
import { DEFAULT_REPORT_LOCALE, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "../../model/locale.ts";
|
|
9
|
+
import { MetricCellView } from "../cell.tsx";
|
|
10
|
+
import { colorClassForKey } from "../../assets/colors.ts";
|
|
11
|
+
import { cx } from "../shared.ts";
|
|
12
12
|
|
|
13
13
|
const OUTCOME_CLASS: Record<string, string> = {
|
|
14
14
|
improved: "nre-delta-good",
|
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
// 组内按值排序,方向随 better;缺数据的系列不画柱、不编 0(与 text 面的 — 同口径)。
|
|
5
5
|
|
|
6
6
|
import type { ReactElement } from "react";
|
|
7
|
-
import type { MatrixData, MetricCell } from "
|
|
8
|
-
import type { AttemptLocator } from "
|
|
9
|
-
import { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "
|
|
10
|
-
import { colorClassForKey, seriesClassForKey } from "
|
|
11
|
-
import { cx } from "
|
|
7
|
+
import type { MatrixData, MetricCell } from "../../model/types.ts";
|
|
8
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
9
|
+
import { DEFAULT_REPORT_LOCALE, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "../../model/locale.ts";
|
|
10
|
+
import { colorClassForKey, seriesClassForKey } from "../../assets/colors.ts";
|
|
11
|
+
import { cx } from "../shared.ts";
|
|
12
12
|
|
|
13
13
|
const WIDTH = 640;
|
|
14
14
|
const HEIGHT = 320;
|
|
@@ -6,10 +6,10 @@
|
|
|
6
6
|
// 上色,深色主题跟随,不留内联 hex。
|
|
7
7
|
|
|
8
8
|
import type { ReactElement } from "react";
|
|
9
|
-
import type { LineData } from "
|
|
10
|
-
import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "
|
|
11
|
-
import {
|
|
12
|
-
import { cx } from "
|
|
9
|
+
import type { LineData } from "../../model/types.ts";
|
|
10
|
+
import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "../../model/locale.ts";
|
|
11
|
+
import { colorIndicesForKeys } from "../../assets/colors.ts";
|
|
12
|
+
import { cx } from "../shared.ts";
|
|
13
13
|
|
|
14
14
|
const WIDTH = 640;
|
|
15
15
|
const HEIGHT = 360;
|
|
@@ -101,6 +101,9 @@ export function MetricLine({
|
|
|
101
101
|
bySeries.get(key)!.push(p);
|
|
102
102
|
}
|
|
103
103
|
for (const list of bySeries.values()) list.sort((a, b) => a.xValue - b.xValue);
|
|
104
|
+
// 同图撞色消解:散列格作起点,按图例顺序线性探测空格(colors.ts 的契约注释)
|
|
105
|
+
const colorIdx = colorIndicesForKeys(seriesOrder.filter((s) => s !== ""));
|
|
106
|
+
const seriesClassOf = (series: string) => (series === "" ? "nre-series-none" : `nre-series-c${colorIdx.get(series) ?? 0}`);
|
|
104
107
|
|
|
105
108
|
const xTicks = xScale.lo === xScale.hi ? [xScale.lo] : [xScale.lo, xScale.hi];
|
|
106
109
|
const yTicks = yScale.lo === yScale.hi ? [yScale.lo] : [yScale.lo, yScale.hi];
|
|
@@ -157,7 +160,7 @@ export function MetricLine({
|
|
|
157
160
|
|
|
158
161
|
{seriesOrder.map((series) => {
|
|
159
162
|
const list = bySeries.get(series)!;
|
|
160
|
-
const seriesClass =
|
|
163
|
+
const seriesClass = seriesClassOf(series);
|
|
161
164
|
const labelAt = list.reduce((a, b) => (b.px > a.px ? b : a));
|
|
162
165
|
return (
|
|
163
166
|
<g key={series || "(single)"} className={cx("nre-line-series", seriesClass)} data-series={series || undefined}>
|
|
@@ -180,7 +183,7 @@ export function MetricLine({
|
|
|
180
183
|
{points.map((p, i) => {
|
|
181
184
|
const circle = (
|
|
182
185
|
<circle
|
|
183
|
-
className={cx("nre-line-point", p.series !== undefined ?
|
|
186
|
+
className={cx("nre-line-point", p.series !== undefined ? seriesClassOf(p.series) : "nre-series-none")}
|
|
184
187
|
data-key={p.key}
|
|
185
188
|
cx={p.px}
|
|
186
189
|
cy={p.py}
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
// 是格子里的普通 <a>,不 hydrate 也能点。
|
|
6
6
|
|
|
7
7
|
import type { ReactElement } from "react";
|
|
8
|
-
import type { MatrixData } from "
|
|
9
|
-
import type { AttemptLocator } from "
|
|
10
|
-
import { DEFAULT_REPORT_LOCALE, resolveMetricLabel, type ReportLocale } from "
|
|
11
|
-
import { MetricCellView } from "
|
|
12
|
-
import { colorClassForKey } from "
|
|
13
|
-
import { cx } from "
|
|
8
|
+
import type { MatrixData } from "../../model/types.ts";
|
|
9
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
10
|
+
import { DEFAULT_REPORT_LOCALE, resolveMetricLabel, type ReportLocale } from "../../model/locale.ts";
|
|
11
|
+
import { MetricCellView } from "../cell.tsx";
|
|
12
|
+
import { colorClassForKey } from "../../assets/colors.ts";
|
|
13
|
+
import { cx } from "../shared.ts";
|
|
14
14
|
|
|
15
15
|
export function MetricMatrix({
|
|
16
16
|
data,
|
|
@@ -3,22 +3,24 @@
|
|
|
3
3
|
// 「越靠右上越好」且仅当两轴都声明 better 时显示;刻度显示真实值。
|
|
4
4
|
// niceTicks 刻度 + 网格线;每个点直接标注(placePointLabels 候选位择优:避开其它标签、
|
|
5
5
|
// 数据点与画布边界,离开左右紧邻位时补 leader line);
|
|
6
|
-
//
|
|
6
|
+
// 默认不连线,`connect` 显式开启时同系列的点按 x 升序连折线(lineage series),
|
|
7
|
+
// 系列图例按显示键字典序列在图下。x 或 y 为 null 的点不画,
|
|
7
8
|
// 底部注脚如实报「n 个点缺数据」;hover 信息退化为 SVG <title>,不 hydrate 也在
|
|
8
9
|
// (enhance.js 在场时升级为样式化 tooltip)。配色走类名(nre-series-cN)由 CSS 上色,
|
|
9
|
-
// 深色主题下图表随令牌切换,不留内联 hex
|
|
10
|
+
// 深色主题下图表随令牌切换,不留内联 hex;同图 series 撞色按图例顺序线性探测消解
|
|
11
|
+
// (colorIndicesForKeys),跨图稳定让位给图内可辨。
|
|
10
12
|
|
|
11
13
|
import type { ReactElement } from "react";
|
|
12
|
-
import type { MetricColumn, ScatterData } from "
|
|
13
|
-
import { formatMetricValue } from "
|
|
14
|
-
import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "
|
|
14
|
+
import type { MetricColumn, ScatterData } from "../../model/types.ts";
|
|
15
|
+
import { formatMetricValue, shortestUniqueLabels } from "../../model/format.ts";
|
|
16
|
+
import { DEFAULT_REPORT_LOCALE, countText, localeText, resolveLocalizedText, resolveMetricLabel, type ReportLocale } from "../../model/locale.ts";
|
|
15
17
|
import { niceTicks, placePointLabels } from "./chart-math.ts";
|
|
16
|
-
import {
|
|
17
|
-
import { cx } from "
|
|
18
|
+
import { colorIndicesForKeys } from "../../assets/colors.ts";
|
|
19
|
+
import { cx } from "../shared.ts";
|
|
18
20
|
|
|
19
|
-
const WIDTH =
|
|
20
|
-
const HEIGHT =
|
|
21
|
-
const MARGIN = { top:
|
|
21
|
+
const WIDTH = 760;
|
|
22
|
+
const HEIGHT = 400;
|
|
23
|
+
const MARGIN = { top: 28, right: 32, bottom: 48, left: 64 };
|
|
22
24
|
const PLOT_W = WIDTH - MARGIN.left - MARGIN.right;
|
|
23
25
|
const PLOT_H = HEIGHT - MARGIN.top - MARGIN.bottom;
|
|
24
26
|
|
|
@@ -34,42 +36,6 @@ interface DrawablePoint {
|
|
|
34
36
|
py: number;
|
|
35
37
|
}
|
|
36
38
|
|
|
37
|
-
/**
|
|
38
|
-
* 点的直接标签:末段在当前 data 中唯一才缩成末段;重名时逐步加长为能区分它们的最短
|
|
39
|
-
* 路径后缀(完整 id 与两轴值仍进 <title>)。
|
|
40
|
-
*/
|
|
41
|
-
function pointLabels(keys: readonly string[]): Map<string, string> {
|
|
42
|
-
const segsOf = (key: string) => key.split("/").filter(Boolean);
|
|
43
|
-
const depth = new Map<string, number>(keys.map((key) => [key, 1]));
|
|
44
|
-
for (;;) {
|
|
45
|
-
const byLabel = new Map<string, string[]>();
|
|
46
|
-
for (const key of keys) {
|
|
47
|
-
const segs = segsOf(key);
|
|
48
|
-
const label = segs.slice(-Math.min(depth.get(key)!, segs.length)).join("/") || key;
|
|
49
|
-
byLabel.set(label, [...(byLabel.get(label) ?? []), key]);
|
|
50
|
-
}
|
|
51
|
-
let grew = false;
|
|
52
|
-
for (const group of byLabel.values()) {
|
|
53
|
-
if (group.length < 2) continue;
|
|
54
|
-
for (const key of group) {
|
|
55
|
-
const segs = segsOf(key);
|
|
56
|
-
if (depth.get(key)! < segs.length) {
|
|
57
|
-
depth.set(key, depth.get(key)! + 1);
|
|
58
|
-
grew = true;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
if (!grew) {
|
|
63
|
-
const out = new Map<string, string>();
|
|
64
|
-
for (const key of keys) {
|
|
65
|
-
const segs = segsOf(key);
|
|
66
|
-
out.set(key, segs.slice(-Math.min(depth.get(key)!, segs.length)).join("/") || key);
|
|
67
|
-
}
|
|
68
|
-
return out;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
39
|
/**
|
|
74
40
|
* 一根轴:niceTicks 撑出整齐的值域,值 → 像素做线性映射。轴方向跟随指标的 `better`:
|
|
75
41
|
* `better: "lower"` 的轴反向渲染(值大的一端在左 / 下),「更好」因此恒指向右与上;
|
|
@@ -89,11 +55,14 @@ function axisScale(values: number[], pixelLo: number, pixelHi: number, invert: b
|
|
|
89
55
|
|
|
90
56
|
export function MetricScatter({
|
|
91
57
|
data,
|
|
58
|
+
connect = false,
|
|
92
59
|
pointHref,
|
|
93
60
|
className,
|
|
94
61
|
locale = DEFAULT_REPORT_LOCALE,
|
|
95
62
|
}: {
|
|
96
63
|
data: ScatterData;
|
|
64
|
+
/** series 内按 x 升序连折线;默认 false,只给「线 = 同族变体」的 lineage series 用。 */
|
|
65
|
+
connect?: boolean;
|
|
97
66
|
pointHref?: (row: ScatterData["rows"][number]) => string;
|
|
98
67
|
className?: string;
|
|
99
68
|
locale?: ReportLocale;
|
|
@@ -138,7 +107,7 @@ export function MetricScatter({
|
|
|
138
107
|
MARGIN.top,
|
|
139
108
|
data.y.better === "lower",
|
|
140
109
|
);
|
|
141
|
-
const labelByKey =
|
|
110
|
+
const labelByKey = shortestUniqueLabels(drawableRows.map((r) => r.key));
|
|
142
111
|
// 方向提示恒为「越靠右上越好」,仅当两轴都声明 better 时显示——任一轴未声明,
|
|
143
112
|
// 组件不猜「更好」朝哪边,整图无提示。
|
|
144
113
|
const showBetterHint = data.x.better !== undefined && data.y.better !== undefined;
|
|
@@ -159,18 +128,20 @@ export function MetricScatter({
|
|
|
159
128
|
};
|
|
160
129
|
});
|
|
161
130
|
|
|
162
|
-
//
|
|
163
|
-
const seriesOrder: string[] = [];
|
|
131
|
+
// 系列分组:图例与配色按显示键字典序;connect 时 series 内按 x 升序连折线
|
|
164
132
|
const bySeries = new Map<string, DrawablePoint[]>();
|
|
165
133
|
for (const p of points) {
|
|
166
134
|
if (p.series === undefined) continue;
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
}
|
|
171
|
-
bySeries.get(p.series)!.push(p);
|
|
135
|
+
const list = bySeries.get(p.series) ?? [];
|
|
136
|
+
list.push(p);
|
|
137
|
+
bySeries.set(p.series, list);
|
|
172
138
|
}
|
|
139
|
+
const seriesOrder = [...bySeries.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
173
140
|
for (const list of bySeries.values()) list.sort((a, b) => a.xValue - b.xValue);
|
|
141
|
+
// 同图撞色消解:散列格作起点,按图例顺序线性探测空格(colors.ts 的契约注释)
|
|
142
|
+
const colorIdx = colorIndicesForKeys(seriesOrder);
|
|
143
|
+
const seriesClassOf = (series: string) => `nre-series-c${colorIdx.get(series) ?? 0}`;
|
|
144
|
+
const keyClassOf = (series: string) => `nre-c${colorIdx.get(series) ?? 0}`;
|
|
174
145
|
|
|
175
146
|
// 直接标签的候选位择优布局:锚向、避让方向都由布局按空间决定,画布边界含边距
|
|
176
147
|
const labels = placePointLabels(
|
|
@@ -238,20 +209,21 @@ export function MetricScatter({
|
|
|
238
209
|
{axisLabel(yLabel, data.y)}
|
|
239
210
|
</text>
|
|
240
211
|
|
|
241
|
-
{/*
|
|
242
|
-
{
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
212
|
+
{/* 系列折线:仅 connect 显式开启时画(lineage series 的位移),按 x 升序;类名上色,深色主题跟随 */}
|
|
213
|
+
{connect &&
|
|
214
|
+
seriesOrder.map((series) => {
|
|
215
|
+
const list = bySeries.get(series)!;
|
|
216
|
+
if (list.length < 2) return null;
|
|
217
|
+
return (
|
|
218
|
+
<polyline
|
|
219
|
+
key={series}
|
|
220
|
+
className={cx("nre-scatter-line", seriesClassOf(series))}
|
|
221
|
+
data-series={series}
|
|
222
|
+
points={list.map((p) => `${p.px},${p.py}`).join(" ")}
|
|
223
|
+
fill="none"
|
|
224
|
+
/>
|
|
225
|
+
);
|
|
226
|
+
})}
|
|
255
227
|
|
|
256
228
|
{/* 点:g 内带 <title>(无 JS 的原生 hover)、直接标签与 leader line;
|
|
257
229
|
pointHref 时包普通 <a>,静态导出也能下钻 */}
|
|
@@ -259,7 +231,7 @@ export function MetricScatter({
|
|
|
259
231
|
const label = labels[i];
|
|
260
232
|
const group = (
|
|
261
233
|
<g
|
|
262
|
-
className={cx("nre-scatter-point", p.series !== undefined ?
|
|
234
|
+
className={cx("nre-scatter-point", p.series !== undefined ? seriesClassOf(p.series) : "nre-series-none")}
|
|
263
235
|
data-key={p.key}
|
|
264
236
|
>
|
|
265
237
|
<title>{p.title}</title>
|
|
@@ -287,7 +259,7 @@ export function MetricScatter({
|
|
|
287
259
|
{seriesOrder.length > 0 && (
|
|
288
260
|
<figcaption className="nre-scatter-legend">
|
|
289
261
|
{seriesOrder.map((series) => (
|
|
290
|
-
<span key={series} className={cx("nre-legend-key", "nre-key",
|
|
262
|
+
<span key={series} className={cx("nre-legend-key", "nre-key", keyClassOf(series))}>
|
|
291
263
|
{series}
|
|
292
264
|
</span>
|
|
293
265
|
))}
|