niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
// ExperimentList:实体列表的第一级。web 面是一行一个 experiment 的固定八列比较表
|
|
2
|
-
// (Experiment / Model / Agent / Avg
|
|
2
|
+
// (Experiment / Model / Agent / Avg. time / Pass rate / Tokens / Cost / Results),
|
|
3
3
|
// 每行用原生 <details> 展开到 Eval 与 Attempt locator。数据完全来自 experimentListData(),
|
|
4
4
|
// 组件不重算、不推断组边界。
|
|
5
5
|
|
|
6
6
|
import type { ReactElement } from "react";
|
|
7
|
-
import type { AttemptLocator } from "
|
|
8
|
-
import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "
|
|
9
|
-
import {
|
|
10
|
-
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "
|
|
7
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
8
|
+
import type { AttemptListItem, ExperimentListEvalRow, ExperimentListItem } from "../../model/types.ts";
|
|
9
|
+
import { shortestUniqueLabels } from "../../model/format.ts";
|
|
10
|
+
import { DEFAULT_REPORT_LOCALE, localeText, type ReportLocale } from "../../model/locale.ts";
|
|
11
11
|
import { AttemptLocatorBadge, failureSummaryText } from "./AttemptList.tsx";
|
|
12
|
-
import { MetricCellView } from "
|
|
13
|
-
import { colorClassForKey } from "
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
const DEFAULT_ATTEMPT_HREF = (locator: AttemptLocator): string => `#/attempt/${locator}`;
|
|
12
|
+
import { MetricCellView } from "../cell.tsx";
|
|
13
|
+
import { colorClassForKey } from "../../assets/colors.ts";
|
|
14
|
+
import { formatDurationMs, formatUSD, verdictMark } from "../../model/format.ts";
|
|
15
|
+
import { cx } from "../shared.ts";
|
|
17
16
|
|
|
18
17
|
const verdictOrder = ["passed", "failed", "errored", "skipped"] as const;
|
|
19
18
|
|
|
@@ -39,7 +38,7 @@ function VerdictSummary({ item, locale }: { item: ExperimentListItem; locale: Re
|
|
|
39
38
|
const parts = verdictOrder
|
|
40
39
|
.filter((verdict) => item.evalVerdicts[verdict] > 0)
|
|
41
40
|
.map((verdict) => `${item.evalVerdicts[verdict]} ${localeText(locale, `verdict.${verdict}`)}`);
|
|
42
|
-
return <span className="nre-experiment-pill">{parts.join("
|
|
41
|
+
return <span className="nre-experiment-pill">{parts.join(" · ") || "—"}</span>;
|
|
43
42
|
}
|
|
44
43
|
|
|
45
44
|
function ExperimentAttemptRow({
|
|
@@ -50,7 +49,7 @@ function ExperimentAttemptRow({
|
|
|
50
49
|
}: {
|
|
51
50
|
attempt: AttemptListItem;
|
|
52
51
|
last: boolean;
|
|
53
|
-
attemptHref
|
|
52
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
54
53
|
locale: ReportLocale;
|
|
55
54
|
}): ReactElement {
|
|
56
55
|
const reason = failureSummaryText(attempt, locale);
|
|
@@ -76,7 +75,7 @@ function EvalAttempts({
|
|
|
76
75
|
locale,
|
|
77
76
|
}: {
|
|
78
77
|
row: ExperimentListEvalRow;
|
|
79
|
-
attemptHref
|
|
78
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
80
79
|
locale: ReportLocale;
|
|
81
80
|
}): ReactElement {
|
|
82
81
|
const duration = row.durationMs.value === null ? localeText(locale, "cell.missing") : formatDurationMs(row.durationMs.value);
|
|
@@ -123,23 +122,21 @@ function Flags({ flags, locale }: { flags: Record<string, unknown> | undefined;
|
|
|
123
122
|
|
|
124
123
|
function ExperimentRow({
|
|
125
124
|
item,
|
|
125
|
+
label,
|
|
126
126
|
attemptHref,
|
|
127
127
|
locale,
|
|
128
|
-
relativeTo,
|
|
129
128
|
}: {
|
|
130
129
|
item: ExperimentListItem;
|
|
131
|
-
|
|
130
|
+
label: string;
|
|
131
|
+
attemptHref?: (locator: AttemptLocator) => string;
|
|
132
132
|
locale: ReportLocale;
|
|
133
|
-
relativeTo?: string;
|
|
134
133
|
}): ReactElement {
|
|
135
134
|
return (
|
|
136
135
|
<details className="nre-experiment-entry">
|
|
137
136
|
<summary className="nre-experiment-summary">
|
|
138
|
-
{/*
|
|
137
|
+
{/* label 是当前列表里的最短唯一后缀;完整 id 仍是排序 / 过滤 / 折叠的键,系列色跟随 agent */}
|
|
139
138
|
<span className="nre-experiment-name" data-sort-value={item.experimentId}>
|
|
140
|
-
<b className={cx("nre-experiment-id", "nre-key"
|
|
141
|
-
{experimentDisplayName(item.experimentId, relativeTo)}
|
|
142
|
-
</b>
|
|
139
|
+
<b className={cx("nre-experiment-id", "nre-key")}>{label}</b>
|
|
143
140
|
<small>
|
|
144
141
|
{localeText(locale, "overview.evalsCount", { n: item.evals })}
|
|
145
142
|
{item.attempts > item.evals ? ` · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })}` : ""}
|
|
@@ -147,7 +144,9 @@ function ExperimentRow({
|
|
|
147
144
|
</small>
|
|
148
145
|
</span>
|
|
149
146
|
<span data-sort-value={item.model ?? ""}>{item.model ?? localeText(locale, "experimentList.defaultModel")}</span>
|
|
150
|
-
<span data-sort-value={item.agent}>
|
|
147
|
+
<span data-sort-value={item.agent}>
|
|
148
|
+
<span className={cx("nre-experiment-agent", "nre-key", colorClassForKey(item.agent))}>{item.agent}</span>
|
|
149
|
+
</span>
|
|
151
150
|
<span className="nre-num" data-sort-value={item.durationMs.value ?? ""}>
|
|
152
151
|
<MetricCellView cell={item.durationMs} locale={locale} />
|
|
153
152
|
</span>
|
|
@@ -179,19 +178,18 @@ function ExperimentRow({
|
|
|
179
178
|
|
|
180
179
|
export function ExperimentList({
|
|
181
180
|
data,
|
|
182
|
-
attemptHref
|
|
181
|
+
attemptHref,
|
|
183
182
|
filter = false,
|
|
184
183
|
className,
|
|
185
184
|
locale = DEFAULT_REPORT_LOCALE,
|
|
186
|
-
relativeTo,
|
|
187
185
|
}: {
|
|
188
186
|
data: readonly ExperimentListItem[];
|
|
189
187
|
attemptHref?: (locator: AttemptLocator) => string;
|
|
190
188
|
filter?: boolean;
|
|
191
189
|
className?: string;
|
|
192
190
|
locale?: ReportLocale;
|
|
193
|
-
relativeTo?: string;
|
|
194
191
|
}): ReactElement {
|
|
192
|
+
const experimentLabels = shortestUniqueLabels(data.map((item) => item.experimentId));
|
|
195
193
|
const labels = [
|
|
196
194
|
localeText(locale, "experimentList.experiment"),
|
|
197
195
|
localeText(locale, "table.model"),
|
|
@@ -209,16 +207,24 @@ export function ExperimentList({
|
|
|
209
207
|
<button
|
|
210
208
|
type="button"
|
|
211
209
|
data-nre-experiment-sort={index}
|
|
212
|
-
className={index === 4
|
|
210
|
+
className={cx(index >= 3 && index <= 6 && "nre-num-head", index === 4 && "nre-sort-desc")}
|
|
213
211
|
key={label}
|
|
212
|
+
title={index === 4 ? localeText(locale, "experimentList.passRateDescription") : undefined}
|
|
214
213
|
>
|
|
215
|
-
{label}
|
|
214
|
+
<span className="nre-sort-label">{label}</span>
|
|
215
|
+
<span className="nre-sort-icon" aria-hidden="true" />
|
|
216
216
|
</button>
|
|
217
217
|
))}
|
|
218
218
|
</div>
|
|
219
219
|
{data.length === 0 && <p className="nre-experiment-list-empty">{localeText(locale, "attemptList.empty")}</p>}
|
|
220
220
|
{data.map((item) => (
|
|
221
|
-
<ExperimentRow
|
|
221
|
+
<ExperimentRow
|
|
222
|
+
key={item.experimentId}
|
|
223
|
+
item={item}
|
|
224
|
+
label={experimentLabels.get(item.experimentId) ?? item.experimentId}
|
|
225
|
+
attemptHref={attemptHref}
|
|
226
|
+
locale={locale}
|
|
227
|
+
/>
|
|
222
228
|
))}
|
|
223
229
|
</div>
|
|
224
230
|
);
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// 计算函数(*Data):ReportInput → 一份组件数据。实体列表族(ExperimentList / EvalList /
|
|
2
|
+
// AttemptList / FailureList)的 *Data 都住在这里(docs/feature/reports/library/entity-lists.md)。
|
|
3
|
+
//
|
|
4
|
+
// 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
|
|
5
|
+
// - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
|
|
6
|
+
// - 聚合前按身份键去重(dedupeAttempts;missing-startedAt 不去重、如实保留、不透出警告);
|
|
7
|
+
// - null ≠ 0:缺数据不编数,覆盖率经 samples/total 如实暴露;
|
|
8
|
+
// - core 中立:只认 Metric / Dimension 接口,不出现具体 agent 名的分支。
|
|
9
|
+
|
|
10
|
+
import type {
|
|
11
|
+
AttemptListItem,
|
|
12
|
+
EvalListItem,
|
|
13
|
+
ExperimentListEvalRow,
|
|
14
|
+
ExperimentListItem,
|
|
15
|
+
ReportInput,
|
|
16
|
+
} from "../../model/types.ts";
|
|
17
|
+
import type { EvalResult } from "../../../types.ts";
|
|
18
|
+
import type { Snapshot } from "../../../results/types.ts";
|
|
19
|
+
import { comparabilityConfigOf, deepEqualJson } from "../../../results/select.ts";
|
|
20
|
+
import { foldEvalVerdict } from "../../../shared/verdict.ts";
|
|
21
|
+
import {
|
|
22
|
+
collectItems,
|
|
23
|
+
computeCell,
|
|
24
|
+
evalIdOf,
|
|
25
|
+
experimentIdOf,
|
|
26
|
+
fullEvalKey,
|
|
27
|
+
groupItems,
|
|
28
|
+
locatorOf,
|
|
29
|
+
resolveInput,
|
|
30
|
+
type Item,
|
|
31
|
+
} from "../../model/aggregate.ts";
|
|
32
|
+
import { attemptCostUSD, costUSD, durationMs, endToEndPassRate, examScore, tokens } from "../../model/metrics.ts";
|
|
33
|
+
import { compactAssertionSummary, primaryAssertionSummary, summaryText } from "../../../scoring/display.ts";
|
|
34
|
+
import { selectedEvalsOnly, summarizeItems } from "../shared-compute.ts";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* 一次 attempt 的单行结果摘要(Scoring display 契约):failed 取主失败断言摘要(不含
|
|
38
|
+
* "+N more",N 单独进 moreFailures),errored 取结构化 error 的一层摘要
|
|
39
|
+
* (phase · code · message),passed / skipped 为 null。
|
|
40
|
+
*/
|
|
41
|
+
export function failureSummaryOf(result: EvalResult): { summary: string | null; more: number } {
|
|
42
|
+
if (result.verdict === "errored" && result.error !== undefined) {
|
|
43
|
+
const parts = [result.error.phase, result.error.code, result.error.message].filter(
|
|
44
|
+
(part): part is string => typeof part === "string" && part.length > 0,
|
|
45
|
+
);
|
|
46
|
+
return { summary: summaryText(parts.join(" · ")), more: 0 };
|
|
47
|
+
}
|
|
48
|
+
if (result.verdict === "failed" || result.verdict === "errored") {
|
|
49
|
+
const primary = primaryAssertionSummary(result.assertions, result.verdict);
|
|
50
|
+
if (primary !== undefined) {
|
|
51
|
+
return {
|
|
52
|
+
summary: compactAssertionSummary({ ...primary, additionalFailures: 0 }),
|
|
53
|
+
more: primary.additionalFailures,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
if (result.verdict === "errored" && result.skipReason !== undefined) {
|
|
57
|
+
return { summary: summaryText(result.skipReason), more: 0 };
|
|
58
|
+
}
|
|
59
|
+
return { summary: null, more: 0 };
|
|
60
|
+
}
|
|
61
|
+
return { summary: null, more: 0 };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** AttemptList / ExperimentList / EvalList 共用的叶子构造:一个 Item → 一个 AttemptListItem。 */
|
|
65
|
+
async function attemptListItemOf(item: Item): Promise<AttemptListItem> {
|
|
66
|
+
const result = item.attempt.result;
|
|
67
|
+
const { summary, more } = failureSummaryOf(result);
|
|
68
|
+
return {
|
|
69
|
+
experimentId: experimentIdOf(item),
|
|
70
|
+
evalId: evalIdOf(item),
|
|
71
|
+
attempt: result.attempt,
|
|
72
|
+
agent: result.agent,
|
|
73
|
+
verdict: result.verdict,
|
|
74
|
+
failureSummary: summary,
|
|
75
|
+
moreFailures: more,
|
|
76
|
+
examScore: await computeCell(examScore, [item]),
|
|
77
|
+
durationMs: result.durationMs,
|
|
78
|
+
costUSD: attemptCostUSD(result),
|
|
79
|
+
locator: locatorOf(item),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** `attemptListData(input)`:每个 Attempt 一项,顺序取自 Scope 展平顺序(不重排)。 */
|
|
84
|
+
export async function attemptListData(input: ReportInput): Promise<AttemptListItem[]> {
|
|
85
|
+
const { snapshots } = resolveInput(input);
|
|
86
|
+
const items = collectItems(snapshots);
|
|
87
|
+
return Promise.all(items.map((item) => attemptListItemOf(item)));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** `evalListData(input)`:每个 `experimentId + evalId` 一项,按 evalId 再按 experimentId 升序。 */
|
|
91
|
+
export async function evalListData(input: ReportInput): Promise<EvalListItem[]> {
|
|
92
|
+
const { snapshots } = resolveInput(input);
|
|
93
|
+
const items = collectItems(snapshots);
|
|
94
|
+
const groups = new Map<string, Item[]>();
|
|
95
|
+
for (const item of items) {
|
|
96
|
+
const key = fullEvalKey(item);
|
|
97
|
+
const list = groups.get(key);
|
|
98
|
+
if (list) list.push(item);
|
|
99
|
+
else groups.set(key, [item]);
|
|
100
|
+
}
|
|
101
|
+
const out: EvalListItem[] = [];
|
|
102
|
+
for (const group of groups.values()) {
|
|
103
|
+
const sorted = [...group].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
104
|
+
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
105
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
106
|
+
out.push({
|
|
107
|
+
experimentId: experimentIdOf(sorted[0]!),
|
|
108
|
+
evalId: evalIdOf(sorted[0]!),
|
|
109
|
+
verdict,
|
|
110
|
+
examScore: await computeCell(examScore, sorted),
|
|
111
|
+
durationMs: await computeCell(durationMs, sorted),
|
|
112
|
+
costUSD: await computeCell(costUSD, sorted),
|
|
113
|
+
attempts,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
out.sort((a, b) => a.evalId.localeCompare(b.evalId) || a.experimentId.localeCompare(b.experimentId));
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* `experimentListData(input)`:每个 experiment 一项,展开到每道 Eval;初始按端到端通过率
|
|
122
|
+
* 从高到低(缺数据沉底,同分按 id)。一行只有一套 agent / model / flags 是输入约束:
|
|
123
|
+
* 宿主注入的 current() Scope 保证每个 experiment 只由可比性配置一致的快照拼成;作者自选
|
|
124
|
+
* Snapshot[] 时若同一 experiment 混入不一致的可比性配置,按完整用户反馈失败并指引——
|
|
125
|
+
* 看跨配置演化用 snapshot 维度或 MetricLine,不把两套配置拼成一行冒充单一配置。
|
|
126
|
+
*/
|
|
127
|
+
export async function experimentListData(input: ReportInput): Promise<ExperimentListItem[]> {
|
|
128
|
+
const { snapshots: rawSnapshots } = resolveInput(input);
|
|
129
|
+
const snapshots = selectedEvalsOnly(rawSnapshots);
|
|
130
|
+
|
|
131
|
+
// 可比性配置单义检查:同一 experiment 的输入快照必须共享一套可比性配置。
|
|
132
|
+
const configByExperiment = new Map<string, { snapshot: Snapshot; config: unknown }>();
|
|
133
|
+
for (const snapshot of snapshots) {
|
|
134
|
+
const config = comparabilityConfigOf(snapshot);
|
|
135
|
+
const existing = configByExperiment.get(snapshot.experimentId);
|
|
136
|
+
if (existing === undefined) {
|
|
137
|
+
configByExperiment.set(snapshot.experimentId, { snapshot, config });
|
|
138
|
+
} else if (!deepEqualJson(existing.config, config)) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`experimentListData got inconsistent comparability configs for experiment "${snapshot.experimentId}" ` +
|
|
141
|
+
`(snapshots ${existing.snapshot.startedAt} and ${snapshot.startedAt} differ in agent/model/reasoningEffort/flags/budget/timeoutMs/sandbox). ` +
|
|
142
|
+
"One row shows one configuration — it cannot honestly merge two. To chart evolution across configs, " +
|
|
143
|
+
'use the "snapshot" dimension or MetricLine; to show the current level, pass results.current() which selects a single config per experiment.',
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const items = collectItems(snapshots);
|
|
149
|
+
const groups = groupItems(items, "experiment");
|
|
150
|
+
const out: ExperimentListItem[] = [];
|
|
151
|
+
for (const [experimentId, group] of groups) {
|
|
152
|
+
const stats = summarizeItems(group);
|
|
153
|
+
const newest = [...group].sort((a, b) => b.snapshot.startedAt.localeCompare(a.snapshot.startedAt))[0]!;
|
|
154
|
+
const evalGroups = groupItems(group, "eval");
|
|
155
|
+
const evalRows: ExperimentListEvalRow[] = [];
|
|
156
|
+
for (const [evalId, evalItems] of evalGroups) {
|
|
157
|
+
const sorted = [...evalItems].sort((a, b) => a.attempt.result.attempt - b.attempt.result.attempt);
|
|
158
|
+
const verdict = foldEvalVerdict(sorted.map((item) => item.attempt.result));
|
|
159
|
+
const attempts = await Promise.all(sorted.map((item) => attemptListItemOf(item)));
|
|
160
|
+
evalRows.push({
|
|
161
|
+
evalId,
|
|
162
|
+
verdict,
|
|
163
|
+
durationMs: await computeCell(durationMs, sorted),
|
|
164
|
+
costUSD: await computeCell(costUSD, sorted),
|
|
165
|
+
attempts,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
const experiment = newest.snapshot.experiment ?? newest.attempt.result.experiment;
|
|
169
|
+
const model = newest.attempt.result.model ?? newest.snapshot.model;
|
|
170
|
+
out.push({
|
|
171
|
+
experimentId,
|
|
172
|
+
agent: newest.snapshot.agent || newest.attempt.result.agent,
|
|
173
|
+
...(model !== undefined ? { model } : {}),
|
|
174
|
+
...(experiment?.flags ? { flags: experiment.flags } : {}),
|
|
175
|
+
evalVerdicts: stats.verdicts,
|
|
176
|
+
endToEndPassRate: await computeCell(endToEndPassRate, group),
|
|
177
|
+
costUSD: await computeCell(costUSD, group),
|
|
178
|
+
durationMs: await computeCell(durationMs, group),
|
|
179
|
+
tokens: await computeCell(tokens, group),
|
|
180
|
+
evals: stats.evals,
|
|
181
|
+
attempts: stats.attempts,
|
|
182
|
+
lastRunAt: stats.lastRunAt!,
|
|
183
|
+
evalRows,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
// 初始态按端到端通过率(endToEndPassRate)从高到低,缺数据沉底;同分按 experiment id 稳定排序。
|
|
187
|
+
out.sort((a, b) => {
|
|
188
|
+
const va = a.endToEndPassRate.value;
|
|
189
|
+
const vb = b.endToEndPassRate.value;
|
|
190
|
+
if (va === null && vb === null) return a.experimentId.localeCompare(b.experimentId);
|
|
191
|
+
if (va === null) return 1;
|
|
192
|
+
if (vb === null) return -1;
|
|
193
|
+
return vb - va || a.experimentId.localeCompare(b.experimentId);
|
|
194
|
+
});
|
|
195
|
+
return out;
|
|
196
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// 实体列表族(ExperimentList / EvalList / AttemptList)的 text 面:同一份算好的数据,
|
|
2
|
+
// 渲染成终端字符(niceeval show 的形态)。三面共用的紧凑标记:`locator✓`(判定符紧跟
|
|
3
|
+
// locator,中间不留空格)。ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的
|
|
4
|
+
// 摘要,不重复整段 niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼
|
|
5
|
+
// `niceeval show <locator>`。零 react、零 IO、纯同步。
|
|
6
|
+
|
|
7
|
+
import type { AttemptListItem, EvalListItem, ExperimentListItem } from "../../model/types.ts";
|
|
8
|
+
import type { TextContext } from "../../definition/tree.ts";
|
|
9
|
+
import type { TableColumn, TableRow } from "../../definition/primitives.tsx";
|
|
10
|
+
import {
|
|
11
|
+
fitFailureSummary,
|
|
12
|
+
formatDurationMs,
|
|
13
|
+
formatUSD,
|
|
14
|
+
shortestUniqueLabels,
|
|
15
|
+
verdictMark,
|
|
16
|
+
} from "../../model/format.ts";
|
|
17
|
+
import { countText, localeText, type ReportLocale } from "../../model/locale.ts";
|
|
18
|
+
import { stringWidth, wrapDisplay } from "../../model/text-layout.ts";
|
|
19
|
+
import { renderTableText } from "../../definition/table-text.ts";
|
|
20
|
+
import { cellText, missingText, verdictTallyText, MISSING_MARK } from "../shared-faces.ts";
|
|
21
|
+
|
|
22
|
+
// ───────────────────────── 实体列表(ExperimentList / EvalList / AttemptList)─────────────────────────
|
|
23
|
+
//
|
|
24
|
+
// 三面共用的紧凑标记:`locator✓`(判定符紧跟 locator,中间不留空格)。
|
|
25
|
+
// ExperimentList / EvalList 逐 attempt 只列这一个标记 + 各自的摘要,不重复整段
|
|
26
|
+
// niceeval show 命令;要看某个 attempt 的完整证据,agent 自己拼 `niceeval show <locator>`。
|
|
27
|
+
|
|
28
|
+
function locatorBadge(item: { locator: string; verdict: AttemptListItem["verdict"] }): string {
|
|
29
|
+
return `${item.locator}${verdictMark(item.verdict)}`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* failureSummary + moreFailures 的展示形态:摘要在计算侧已按 Scoring display 契约折好,
|
|
34
|
+
* 这里只加 "+N more failures" 计数与宽度收口,不重算摘要。
|
|
35
|
+
*/
|
|
36
|
+
function attemptReasonText(item: AttemptListItem, locale: ReportLocale, maxChars: number): string | undefined {
|
|
37
|
+
if (item.failureSummary === null) return undefined;
|
|
38
|
+
const withMore =
|
|
39
|
+
item.moreFailures > 0
|
|
40
|
+
? `${item.failureSummary} · ${countText(locale, "entityList.moreFailures", item.moreFailures)}`
|
|
41
|
+
: item.failureSummary;
|
|
42
|
+
return fitFailureSummary(withMore, Math.max(24, maxChars));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ── ExperimentList ──
|
|
46
|
+
|
|
47
|
+
function experimentSummaryTable(
|
|
48
|
+
items: readonly ExperimentListItem[],
|
|
49
|
+
ctx: TextContext,
|
|
50
|
+
labels: Map<string, string>,
|
|
51
|
+
): string {
|
|
52
|
+
const locale = ctx.locale;
|
|
53
|
+
const compact = ctx.width < 100;
|
|
54
|
+
const columns: TableColumn[] = [
|
|
55
|
+
{ key: "experiment", header: compact && locale === "en" ? "Exp." : localeText(locale, "experimentList.experiment") },
|
|
56
|
+
{ key: "model", header: localeText(locale, "table.model") },
|
|
57
|
+
{ key: "agent", header: localeText(locale, "table.agent") },
|
|
58
|
+
{ key: "duration", header: compact && locale === "en" ? "Avg" : localeText(locale, "experimentList.avgDuration"), align: "right" },
|
|
59
|
+
{ key: "passRate", header: compact && locale === "en" ? "Pass" : localeText(locale, "experimentList.passRate"), align: "right" },
|
|
60
|
+
{ key: "result", header: localeText(locale, "experimentList.result") },
|
|
61
|
+
{ key: "tokens", header: localeText(locale, "experimentList.tokens"), align: "right" },
|
|
62
|
+
{ key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
|
|
63
|
+
];
|
|
64
|
+
const rows: TableRow[] = items.map((item) => ({
|
|
65
|
+
key: item.experimentId,
|
|
66
|
+
cells: {
|
|
67
|
+
experiment: labels.get(item.experimentId) ?? item.experimentId,
|
|
68
|
+
model: item.model ?? localeText(locale, "experimentList.defaultModel"),
|
|
69
|
+
agent: item.agent,
|
|
70
|
+
duration: cellText(item.durationMs, locale),
|
|
71
|
+
passRate: cellText(item.endToEndPassRate, locale),
|
|
72
|
+
result: verdictTallyText(item.evalVerdicts, locale),
|
|
73
|
+
tokens: cellText(item.tokens, locale),
|
|
74
|
+
cost: cellText(item.costUSD, locale),
|
|
75
|
+
},
|
|
76
|
+
}));
|
|
77
|
+
const metadata = items.flatMap((item) =>
|
|
78
|
+
wrapDisplay(
|
|
79
|
+
`${labels.get(item.experimentId) ?? item.experimentId}: ${localeText(locale, "overview.evalsCount", { n: item.evals })} · ${localeText(locale, "overview.attemptsCount", { n: item.attempts })} · ${item.lastRunAt}`,
|
|
80
|
+
Math.max(8, ctx.width - 2),
|
|
81
|
+
).map((line) => ` ${line}`),
|
|
82
|
+
);
|
|
83
|
+
return [renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx), metadata.join("\n")].join("\n");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function experimentDetailTable(item: ExperimentListItem, ctx: TextContext, label: string): string {
|
|
87
|
+
const locale = ctx.locale;
|
|
88
|
+
const columns: TableColumn[] = [
|
|
89
|
+
{ key: "status", header: localeText(locale, "experimentList.status") },
|
|
90
|
+
{ key: "entity", header: localeText(locale, "experimentList.evalAttempt") },
|
|
91
|
+
// Result 是可扫读的失败预览,不是证据面:两行放不下的以 … 收口,完整值走 locator 下钻。
|
|
92
|
+
{ key: "result", header: localeText(locale, "experimentList.result"), maxLines: 2 },
|
|
93
|
+
{ key: "duration", header: localeText(locale, "experimentList.duration"), align: "right" },
|
|
94
|
+
{ key: "cost", header: localeText(locale, "experimentList.cost"), align: "right" },
|
|
95
|
+
];
|
|
96
|
+
// Result 的字符预算 ≈ 两行 × 它能分到的列宽(总宽减其它列的自然宽与列距)。这里只做
|
|
97
|
+
// 粗预算;精确的按宽度收口由列的 maxLines 兜底。
|
|
98
|
+
const statusWidth = Math.max(
|
|
99
|
+
stringWidth(localeText(locale, "experimentList.status")),
|
|
100
|
+
...item.evalRows.map((row) => stringWidth(`${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`)),
|
|
101
|
+
);
|
|
102
|
+
const entityWidth = Math.max(
|
|
103
|
+
stringWidth(localeText(locale, "experimentList.evalAttempt")),
|
|
104
|
+
...item.evalRows.flatMap((row) => [stringWidth(row.evalId), ...row.attempts.map((a) => stringWidth(a.locator) + 3)]),
|
|
105
|
+
);
|
|
106
|
+
const fixedWidth = statusWidth + entityWidth + 8 /* duration */ + 6 /* cost */ + 3 * 4; /* 4 段列距 */
|
|
107
|
+
const resultBudget = Math.max(24, (ctx.width - fixedWidth) * 2);
|
|
108
|
+
const rows: TableRow[] = item.evalRows.flatMap((row) => {
|
|
109
|
+
// Eval 父行只承载折叠判定与题级聚合;失败摘要只在 Attempt 子行出现。
|
|
110
|
+
const parent: TableRow = {
|
|
111
|
+
key: row.evalId,
|
|
112
|
+
cells: {
|
|
113
|
+
status: `${verdictMark(row.verdict)} ${localeText(locale, `verdict.${row.verdict}`)}`,
|
|
114
|
+
entity: row.evalId,
|
|
115
|
+
result: "",
|
|
116
|
+
duration: localeText(locale, "entityList.average", { value: cellText(row.durationMs, locale) }),
|
|
117
|
+
cost: localeText(locale, "entityList.average", { value: cellText(row.costUSD, locale) }),
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
const attempts: TableRow[] = row.attempts.map((attempt, index) => ({
|
|
121
|
+
key: attempt.locator,
|
|
122
|
+
cells: {
|
|
123
|
+
status: ` ${verdictMark(attempt.verdict)}`,
|
|
124
|
+
entity: `${index === row.attempts.length - 1 ? "└─" : "├─"} ${attempt.locator}`,
|
|
125
|
+
result: attemptReasonText(attempt, locale, resultBudget) ?? MISSING_MARK,
|
|
126
|
+
duration: attempt.verdict === "skipped" && attempt.durationMs === 0 ? null : formatDurationMs(attempt.durationMs),
|
|
127
|
+
cost: attempt.costUSD === null ? null : formatUSD(attempt.costUSD),
|
|
128
|
+
},
|
|
129
|
+
}));
|
|
130
|
+
return [parent, ...attempts];
|
|
131
|
+
});
|
|
132
|
+
const flags = item.flags && Object.keys(item.flags).length > 0
|
|
133
|
+
? `${localeText(locale, "experimentList.flags")} ${Object.entries(item.flags)
|
|
134
|
+
.map(([key, value]) => `${key}=${typeof value === "string" ? value : JSON.stringify(value)}`)
|
|
135
|
+
.join(" · ")}`
|
|
136
|
+
: undefined;
|
|
137
|
+
return [
|
|
138
|
+
label,
|
|
139
|
+
flags,
|
|
140
|
+
renderTableText({ columns: columns as unknown as [TableColumn, ...TableColumn[]], rows, locale }, ctx),
|
|
141
|
+
]
|
|
142
|
+
.filter(Boolean)
|
|
143
|
+
.join("\n");
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function experimentListText(items: readonly ExperimentListItem[], ctx: TextContext): string {
|
|
147
|
+
if (items.length === 0) return localeText(ctx.locale, "attemptList.empty");
|
|
148
|
+
const labels = shortestUniqueLabels(items.map((item) => item.experimentId));
|
|
149
|
+
return [
|
|
150
|
+
experimentSummaryTable(items, ctx, labels),
|
|
151
|
+
...items.map((item) => experimentDetailTable(item, ctx, labels.get(item.experimentId) ?? item.experimentId)),
|
|
152
|
+
].join("\n\n");
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ── EvalList ──
|
|
156
|
+
|
|
157
|
+
function evalListAttemptLine(item: AttemptListItem, ctx: TextContext): string {
|
|
158
|
+
// 行式列表同守「Result 最多两行」:预算 = 两行终端宽,超出按尾截收口。
|
|
159
|
+
const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - stringWidth(locatorBadge(item)) - 6);
|
|
160
|
+
return ` ${locatorBadge(item)}${reason ? ` · ${reason}` : ""}`;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function evalListText(items: readonly EvalListItem[], ctx: TextContext): string {
|
|
164
|
+
const locale = ctx.locale;
|
|
165
|
+
if (items.length === 0) return localeText(locale, "attemptList.empty");
|
|
166
|
+
const blocks = items.map((item) => {
|
|
167
|
+
const identity = `${item.evalId} · ${item.experimentId} · ${localeText(locale, `verdict.${item.verdict}`)}`;
|
|
168
|
+
const summary = [
|
|
169
|
+
localeText(locale, "attemptList.score", { score: cellText(item.examScore, locale) }),
|
|
170
|
+
localeText(locale, "overview.attemptsCount", { n: item.attempts.length }),
|
|
171
|
+
localeText(locale, "entityList.average", {
|
|
172
|
+
value: item.durationMs.value === null ? missingText(locale) : formatDurationMs(item.durationMs.value),
|
|
173
|
+
}),
|
|
174
|
+
localeText(locale, "entityList.average", {
|
|
175
|
+
value: item.costUSD.value === null ? missingText(locale) : formatUSD(item.costUSD.value),
|
|
176
|
+
}),
|
|
177
|
+
].join(" · ");
|
|
178
|
+
const attemptLines = item.attempts.map((attempt) => evalListAttemptLine(attempt, ctx));
|
|
179
|
+
return [identity, ` ${summary}`, ...attemptLines].join("\n");
|
|
180
|
+
});
|
|
181
|
+
return blocks.join("\n\n");
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ── AttemptList ──
|
|
185
|
+
|
|
186
|
+
/** Attempt 比较卡片:只显示一条主失败摘要(至多两行终端宽);完整 assertions 走 locator 下钻。 */
|
|
187
|
+
function attemptListItemText(item: AttemptListItem, ctx: TextContext): string {
|
|
188
|
+
const head = [
|
|
189
|
+
`${verdictMark(item.verdict)} ${item.locator}`,
|
|
190
|
+
item.evalId,
|
|
191
|
+
item.experimentId,
|
|
192
|
+
formatDurationMs(item.durationMs),
|
|
193
|
+
...(item.costUSD !== null ? [formatUSD(item.costUSD)] : []),
|
|
194
|
+
].join(" · ");
|
|
195
|
+
const lines = [head];
|
|
196
|
+
const reason = attemptReasonText(item, ctx.locale, ctx.width * 2 - 4);
|
|
197
|
+
if (reason) lines.push(` ${reason}`);
|
|
198
|
+
return lines.join("\n");
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export function attemptListText(items: readonly AttemptListItem[], total: number | undefined, ctx: TextContext): string {
|
|
202
|
+
const locale = ctx.locale;
|
|
203
|
+
if (items.length === 0) return localeText(locale, "attemptList.empty");
|
|
204
|
+
const blocks = items.map((item) => attemptListItemText(item, ctx));
|
|
205
|
+
const remaining = (total ?? items.length) - items.length;
|
|
206
|
+
if (remaining > 0) blocks.push(localeText(locale, "attemptList.truncatedText", { n: remaining }));
|
|
207
|
+
return blocks.join("\n\n");
|
|
208
|
+
}
|