niceeval 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +254 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +235 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +527 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +174 -0
- package/dist/report/components/entity-lists/index.d.ts +51 -0
- package/dist/report/components/entity-lists/index.js +165 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
- package/dist/report/{format.js → model/format.js} +48 -4
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +149 -23
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +11 -11
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +35 -32
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +193 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +103 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +49 -49
- package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +8 -4
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +71 -57
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/{react → assets}/styles.css +124 -106
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
- package/src/report/components/attempt-detail/compute.ts +287 -0
- package/src/report/components/attempt-detail/faces.ts +257 -0
- package/src/report/components/attempt-detail/index.tsx +583 -0
- package/src/report/components/attempt-detail/validate.test.ts +235 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +207 -0
- package/src/report/components/entity-lists/index.tsx +241 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +47 -17
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +126 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +59 -5
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +143 -22
- package/src/report/react/index.tsx +52 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +158 -23
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b.ts +12 -13
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +141 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// cases: docs/engineering/unit-tests/reports/cases.md
|
|
2
2
|
// niceeval view 的报告槽与宿主组合语义(docs/feature/reports/architecture.md「Selection 是计算入口」
|
|
3
|
-
// 与裁决记录 6;公开行为准绳 docs-site/zh/
|
|
3
|
+
// 与裁决记录 6;公开行为准绳 docs-site/zh/tutorials/viewing-results.mdx / custom-reports.mdx)。
|
|
4
4
|
// 覆盖:
|
|
5
5
|
// - 组合语义与 show 对齐:位置前缀收窄报告槽 Scope、--exp 过滤、匹配不到直说;
|
|
6
6
|
// - 输入语义:位置参数只表示 eval id 前缀(不随文件系统状态改变),结果根走 --results,
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// fixture 直接写新布局(<expDir>/<snapDir>/snapshot.json + <evalId>/a<n>/result.json),
|
|
14
14
|
// 依据是 docs/feature/results/architecture.md 的稳定磁盘契约,不经 writer 运行时 API。
|
|
15
15
|
|
|
16
|
-
import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
|
16
|
+
import { mkdir, mkdtemp, readdir, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
|
17
17
|
import { existsSync } from "node:fs";
|
|
18
18
|
import { tmpdir } from "node:os";
|
|
19
19
|
import { join, resolve } from "node:path";
|
|
@@ -21,11 +21,12 @@ import { afterEach, describe, expect, it } from "vitest";
|
|
|
21
21
|
// dist-sourced: this must be the exact class loadViewScan()/data.ts's loadReportFile() throws
|
|
22
22
|
// (see src/view/data.ts's comment) — a raw-src import would be a structurally-identical but
|
|
23
23
|
// `instanceof`-incompatible class.
|
|
24
|
-
import { ReportLoadError } from "../../dist/report/load.js";
|
|
24
|
+
import { ReportLoadError } from "../../dist/report/runtime/load.js";
|
|
25
25
|
import { ViewInputError, loadViewScan, type ViewScan } from "./data.ts";
|
|
26
26
|
import { renderHtml } from "./site.ts";
|
|
27
27
|
import { buildView, resolveViewInput } from "./index.ts";
|
|
28
28
|
import { runShow } from "../show/index.ts";
|
|
29
|
+
import { createResultsWriter } from "../results/index.ts";
|
|
29
30
|
import { RESULTS_FORMAT, RESULTS_SCHEMA_VERSION, type EvalResult, type Verdict } from "../types.ts";
|
|
30
31
|
|
|
31
32
|
const EXAM_REPORT = resolve(__dirname, "../../test/fixtures/report/exam-report.tsx");
|
|
@@ -165,7 +166,6 @@ describe("loadViewScan · 组合语义", () => {
|
|
|
165
166
|
it("位置前缀收窄作用在有效根上:全部页与证据室一致缩小,被滤掉的 attempt 不烘进页面数据", async () => {
|
|
166
167
|
const root = await seedRoot();
|
|
167
168
|
const scan = await loadViewScan(root, { patterns: ["weather"] });
|
|
168
|
-
const { viewData } = scan;
|
|
169
169
|
const reportHtml = bareReportHtml(scan);
|
|
170
170
|
// 报告页:两实验都只剩 weather/brooklyn 一题,范围外的失败不再出现。
|
|
171
171
|
expect(reportHtml.en).toContain("compare/bub");
|
|
@@ -176,19 +176,17 @@ describe("loadViewScan · 组合语义", () => {
|
|
|
176
176
|
expect(pageHtml(scan, "traces").en).not.toContain("fixtures/button");
|
|
177
177
|
const bare = await loadViewScan(root);
|
|
178
178
|
expect(pageHtml(bare, "attempts").en).toContain("fixtures/button"); // 不收窄时在场,对照
|
|
179
|
-
// 有效根即证据室:被滤掉的 attempt 不在
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
.find((r) => r.id === "fixtures/button");
|
|
179
|
+
// 有效根即证据室:被滤掉的 attempt 不在 attemptsByBase / attemptPages.locators 里,
|
|
180
|
+
// 站点管线(site.ts)不会为它生成 attempt/<locator>.html。
|
|
181
|
+
const filteredOut = [...scan.attemptsByBase.values()].find((a) => a.evalId === "fixtures/button");
|
|
183
182
|
expect(filteredOut).toBeUndefined();
|
|
184
183
|
expect(scan.artifactDirs.size).toBe(
|
|
185
184
|
[...scan.artifactDirs.keys()].filter((base) => base.includes("weather/brooklyn")).length,
|
|
186
185
|
);
|
|
187
186
|
// 收窄之内、不在现刻水位的口径差异由 --exp 深链测试覆盖;不收窄时深链照常可达,对照:
|
|
188
|
-
const inBare = bare.
|
|
187
|
+
const inBare = [...bare.attemptsByBase.values()].find((a) => a.evalId === "fixtures/button");
|
|
189
188
|
expect(inBare?.locator).toBeTruthy();
|
|
190
|
-
|
|
191
|
-
expect(resolveAttemptLocator(bare.viewData.snapshots, inBare!.locator!)).toBe(inBare);
|
|
189
|
+
expect(bare.attemptPages!.locators.get(inBare!.locator!)).toBe(inBare);
|
|
192
190
|
});
|
|
193
191
|
|
|
194
192
|
it("--exp 过滤:报告槽 Selection 只留该实验", async () => {
|
|
@@ -242,13 +240,13 @@ describe("loadViewScan · 默认报告槽(裸跑)", () => {
|
|
|
242
240
|
it("报告槽双语渲染:同一棵树按 locale 渲染两遍,chrome 文案分语言、数据不分语言", async () => {
|
|
243
241
|
const root = await seedRoot();
|
|
244
242
|
const reportHtml = bareReportHtml(await loadViewScan(root));
|
|
245
|
-
expect(reportHtml.en).toContain("
|
|
246
|
-
expect(reportHtml["zh-CN"]).toContain("
|
|
243
|
+
expect(reportHtml.en).toContain("Pass rate"); // ExperimentList 主行(en)
|
|
244
|
+
expect(reportHtml["zh-CN"]).toContain("通过率"); // ExperimentList 主行(zh-CN)
|
|
247
245
|
for (const html of [reportHtml.en, reportHtml["zh-CN"]]) {
|
|
248
246
|
expect(html).toContain("compare/bub");
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
expect(html).toMatch(/href="
|
|
247
|
+
// 失败案例深链进独立 attempt 文档:不透明 AttemptLocator 编码进文件名
|
|
248
|
+
// (`attempt/<encodeURIComponent(locator)>.html`),不再是旧的 hash 路由。
|
|
249
|
+
expect(html).toMatch(/href="attempt\/%40[0-9a-z]+\.html"/);
|
|
252
250
|
expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
|
|
253
251
|
}
|
|
254
252
|
});
|
|
@@ -274,20 +272,24 @@ describe("loadViewScan · 默认报告槽(裸跑)", () => {
|
|
|
274
272
|
// ───────────────────────── 报告槽整槽替换 ─────────────────────────
|
|
275
273
|
|
|
276
274
|
describe("loadViewScan · --report 报告槽", () => {
|
|
277
|
-
it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style>
|
|
275
|
+
it("报告树渲染为静态 HTML:官方水位 + 自定义摆法 + <Style> 产物,零 <script>", async () => {
|
|
278
276
|
const root = await seedRoot();
|
|
279
277
|
const scan = await loadViewScan(root, { report: { path: EXAM_REPORT, cwd: root } });
|
|
280
278
|
const html = slotHtml(scan).en;
|
|
281
279
|
expect(html).toContain("考试成绩单"); // 自定义 Section
|
|
282
280
|
expect(html).toContain("nre-"); // 官方组件的稳定类名
|
|
283
281
|
expect(html).toContain("<style>.exam-note { color: #4a7; }</style>"); // <Style> 随树带走
|
|
284
|
-
|
|
282
|
+
// exam-report.tsx 是树形态 defineReport,没有声明 attempt-input page:locator 因此只是
|
|
283
|
+
// 纯文本,不生成假 href(architecture.md「Attempt 详情是一张参数化 page」)。
|
|
284
|
+
expect(html).toMatch(/<span class="nre-locator[^"]*">@[0-9a-z]+/);
|
|
285
|
+
expect(html).not.toMatch(/href="#\/attempt\//);
|
|
285
286
|
expect(html).not.toContain("<script"); // 报告槽产物零客户端 JS,不 hydrate
|
|
286
287
|
// 用户报告同样双语渲染两遍(壳按界面语言摆放)。
|
|
287
288
|
expect(slotHtml(scan)["zh-CN"]).toContain("考试成绩单");
|
|
288
|
-
//
|
|
289
|
-
|
|
290
|
-
expect(scan.
|
|
289
|
+
// 证据室索引(attemptsByBase,供 artifact/ 与 attempt/<locator>.html 两条出口共用)原样保留,
|
|
290
|
+
// 不受这份没有 attempt-input page 的自定义报告影响;每条都有 locator。
|
|
291
|
+
expect(scan.attemptsByBase.size).toBeGreaterThan(0);
|
|
292
|
+
expect([...scan.attemptsByBase.values()].every((a) => a.locator)).toBe(true);
|
|
291
293
|
expect(JSON.stringify(scan.viewData)).not.toContain("考试成绩单"); // 报告块不进 viewData
|
|
292
294
|
});
|
|
293
295
|
|
|
@@ -320,6 +322,27 @@ describe("loadViewScan · --report 报告槽", () => {
|
|
|
320
322
|
expect(text).toContain(needle);
|
|
321
323
|
expect(html.en).toContain(needle);
|
|
322
324
|
}
|
|
325
|
+
// Grid/Stat/Section.meta:两面共享同一批 label/value/detail/meta 终值,不要求布局逐字一致。
|
|
326
|
+
for (const needle of ["速览", "人工摘要,非计算结果", "及格线", "60 分", "最高分", "97 分", "compare/codex"]) {
|
|
327
|
+
expect(text).toContain(needle);
|
|
328
|
+
expect(html.en).toContain(needle);
|
|
329
|
+
}
|
|
330
|
+
// web 面是 Grid 结构(稳定 nre-grid/nre-grid-cell class),text 面无 JS 概念、只有内容。
|
|
331
|
+
expect(html.en).toContain("nre-grid");
|
|
332
|
+
expect(html.en).toContain("nre-grid-cell");
|
|
333
|
+
expect(html.en).toContain("nre-stat");
|
|
334
|
+
|
|
335
|
+
// text 面按显示宽度减列但不丢 Stat:极窄终端下两个 Stat 仍都完整可读。
|
|
336
|
+
let narrowText = "";
|
|
337
|
+
const narrowCode = await runShow(root, [], { results: root, report: EXAM_REPORT }, {
|
|
338
|
+
out: (s) => (narrowText += s),
|
|
339
|
+
err: () => {},
|
|
340
|
+
width: 30,
|
|
341
|
+
});
|
|
342
|
+
expect(narrowCode).toBe(0);
|
|
343
|
+
for (const needle of ["及格线", "60 分", "最高分", "97 分"]) {
|
|
344
|
+
expect(narrowText).toContain(needle);
|
|
345
|
+
}
|
|
323
346
|
});
|
|
324
347
|
|
|
325
348
|
it("报告文件缺失 / 默认导出不是 defineReport 产物:ReportLoadError 直说", async () => {
|
|
@@ -352,6 +375,7 @@ describe("loadViewScan · 外壳标题与 ReportLink.icon", () => {
|
|
|
352
375
|
' kind: "report",',
|
|
353
376
|
' title: { en: "Memory Evals", "zh-CN": "记忆能力评测" },',
|
|
354
377
|
' links: [{ label: "GitHub", href: "https://example.com", icon: { svg: "<svg data-mark></svg>" } }],',
|
|
378
|
+
" head: [],",
|
|
355
379
|
" scripts: [],",
|
|
356
380
|
" styles: [],",
|
|
357
381
|
' pages: [{ id: "report", title: "Report", content: { $$typeof: Symbol.for("react.transitional.element"), type: Block, props: {}, key: null } }],',
|
|
@@ -402,6 +426,7 @@ describe("loadViewScan · 报告文件变更整页重算", () => {
|
|
|
402
426
|
"const definition = {",
|
|
403
427
|
' kind: "report",',
|
|
404
428
|
" links: [],",
|
|
429
|
+
" head: [],",
|
|
405
430
|
" scripts: [],",
|
|
406
431
|
" styles: [],",
|
|
407
432
|
' pages: [{ id: "report", title: "Report", content: { $$typeof: Symbol.for("react.transitional.element"), type: Block, props: {}, key: null } }],',
|
|
@@ -463,7 +488,7 @@ describe("buildView · --out 与 --report", () => {
|
|
|
463
488
|
const artifactDir = join(root, "compare_codex", "2026-07-09T10-00-00-000Z", "weather", "brooklyn", "a0");
|
|
464
489
|
await mkdir(artifactDir, { recursive: true });
|
|
465
490
|
await writeFile(join(artifactDir, "events.json"), "[]", "utf-8");
|
|
466
|
-
await writeFile(join(artifactDir, "diff.json"),
|
|
491
|
+
await writeFile(join(artifactDir, "diff.json"), "[]", "utf-8");
|
|
467
492
|
await writeFile(join(artifactDir, "o11y.json"), "{}", "utf-8");
|
|
468
493
|
|
|
469
494
|
const out = join(root, "site");
|
|
@@ -525,3 +550,97 @@ describe("buildView · --out 与 --report", () => {
|
|
|
525
550
|
expect(block).not.toContain("<script");
|
|
526
551
|
});
|
|
527
552
|
});
|
|
553
|
+
|
|
554
|
+
// ───────────────────── attempt/<locator>.html:独立静态详情文档 ─────────────────────
|
|
555
|
+
|
|
556
|
+
describe("buildView · attempt/<locator>.html", () => {
|
|
557
|
+
it("收窄后有效根内每个可达 locator 各一份文档,收窄外 locator 不生成文件(cases.md 第 205 行)", async () => {
|
|
558
|
+
const root = await makeRoot();
|
|
559
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
560
|
+
const snapA = await writer.snapshot({ experimentId: "exp/a", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
561
|
+
await snapA.writeAttempt({ id: "q1", verdict: "passed", attempt: 0, durationMs: 1000, assertions: [] });
|
|
562
|
+
const snapB = await writer.snapshot({ experimentId: "exp/b", agent: "bub", startedAt: "2026-07-01T09:00:00.000Z" });
|
|
563
|
+
await snapB.writeAttempt({ id: "q2", verdict: "passed", attempt: 0, durationMs: 1000, assertions: [] });
|
|
564
|
+
await writer.finish();
|
|
565
|
+
|
|
566
|
+
const scanFull = await loadViewScan(root);
|
|
567
|
+
const scanNarrowed = await loadViewScan(root, { experiment: "exp/a" });
|
|
568
|
+
expect(scanFull.attemptPages!.locators.size).toBe(2);
|
|
569
|
+
expect(scanNarrowed.attemptPages!.locators.size).toBe(1);
|
|
570
|
+
const [keptLocator] = [...scanNarrowed.attemptPages!.locators.keys()];
|
|
571
|
+
const droppedLocator = [...scanFull.attemptPages!.locators.keys()].find((l) => l !== keptLocator)!;
|
|
572
|
+
|
|
573
|
+
const out = join(root, "site");
|
|
574
|
+
await buildView({ input: root, out, scan: { experiment: "exp/a" } });
|
|
575
|
+
const files = await readdir(join(out, "attempt"));
|
|
576
|
+
expect(files).toEqual([`${keptLocator}.html`]);
|
|
577
|
+
expect(existsSync(join(out, "attempt", `${droppedLocator}.html`))).toBe(false);
|
|
578
|
+
});
|
|
579
|
+
|
|
580
|
+
it("直接读取该文档(无 JavaScript 场景)即可见完整内容:身份/verdict/断言/时间树/diagnostics/usage/对话/trace/diff 都在可见的 en 区块里,不藏在 hidden 或 <template> 里(cases.md 第 206 行)", async () => {
|
|
581
|
+
const root = await makeRoot();
|
|
582
|
+
const writer = createResultsWriter(root, { producer: { name: "niceeval", version: "1.0.0" } });
|
|
583
|
+
const snap = await writer.snapshot({ experimentId: "compare/bub", agent: "bub", startedAt: "2026-07-01T08:00:00.000Z" });
|
|
584
|
+
await snap.writeAttempt(
|
|
585
|
+
{
|
|
586
|
+
id: "weather/brooklyn",
|
|
587
|
+
verdict: "failed",
|
|
588
|
+
attempt: 0,
|
|
589
|
+
durationMs: 4200,
|
|
590
|
+
assertions: [
|
|
591
|
+
{
|
|
592
|
+
name: "temperature matches forecast",
|
|
593
|
+
severity: "gate",
|
|
594
|
+
outcome: "failed",
|
|
595
|
+
score: 0,
|
|
596
|
+
expected: "72",
|
|
597
|
+
received: "71",
|
|
598
|
+
loc: { file: "evals/weather.eval.ts", line: 1 },
|
|
599
|
+
},
|
|
600
|
+
],
|
|
601
|
+
phases: [{ name: "eval.run", durationMs: 4000 }],
|
|
602
|
+
diagnostics: [{ code: "teardown-failed", level: "warning", phase: "sandbox.teardown", message: "container stop timed out" }],
|
|
603
|
+
usage: { inputTokens: 100, outputTokens: 50 },
|
|
604
|
+
},
|
|
605
|
+
{
|
|
606
|
+
sources: [{ path: "evals/weather.eval.ts", content: "export default 1;\n" }],
|
|
607
|
+
events: [
|
|
608
|
+
{ type: "message", role: "user", text: "what is the forecast for brooklyn" },
|
|
609
|
+
{ type: "message", role: "assistant", text: "72 degrees and sunny" },
|
|
610
|
+
],
|
|
611
|
+
trace: [{ traceId: "t1", spanId: "s1", name: "turn", kind: "turn", startMs: 0, endMs: 100 }],
|
|
612
|
+
diff: [{ window: "s1/t1", changes: { "a.txt": { status: "added", after: "1" } } }] as never,
|
|
613
|
+
},
|
|
614
|
+
);
|
|
615
|
+
await writer.finish();
|
|
616
|
+
|
|
617
|
+
const out = join(root, "site");
|
|
618
|
+
await buildView({ input: root, out });
|
|
619
|
+
const [locator] = [...(await loadViewScan(root)).attemptPages!.locators.keys()];
|
|
620
|
+
const html = await readFile(join(out, "attempt", `${locator}.html`), "utf-8");
|
|
621
|
+
|
|
622
|
+
// 结构:en 可见(无 hidden 属性),zh-CN 带 hidden——两者都不需要 JS 才能被浏览器正确
|
|
623
|
+
// 处理(hidden 是浏览器原生渲染指令,不是 JS 行为);没有 <template> 把内容整个锁起来。
|
|
624
|
+
expect(html).toMatch(/<div data-nre-locale="en">/);
|
|
625
|
+
expect(html).not.toMatch(/<div data-nre-locale="en"[^>]*\shidden/);
|
|
626
|
+
expect(html).toMatch(/<div data-nre-locale="zh-CN" hidden>/);
|
|
627
|
+
const body = html.split("<body>")[1]!.split("</body>")[0]!;
|
|
628
|
+
expect(body).not.toContain("<template"); // 不像 index.html 那样把内容锁进 <template>
|
|
629
|
+
const enBlock = html.split('<div data-nre-locale="en">')[1]!.split('<div data-nre-locale="zh-CN"')[0]!;
|
|
630
|
+
for (const needle of [
|
|
631
|
+
"weather/brooklyn", // 身份
|
|
632
|
+
"compare/bub",
|
|
633
|
+
"evals/weather.eval.ts", // source
|
|
634
|
+
"temperature matches forecast", // 断言
|
|
635
|
+
"71", // received(fix prompt 里的摘要行)
|
|
636
|
+
"4.0s", // 时间树(eval.run 4000ms)
|
|
637
|
+
"container stop timed out", // diagnostics
|
|
638
|
+
"output tokens", // usage
|
|
639
|
+
"what is the forecast", // 对话
|
|
640
|
+
"nre-attempt-trace", // trace
|
|
641
|
+
"a.txt", // diff 摘要
|
|
642
|
+
]) {
|
|
643
|
+
expect(enBlock, needle).toContain(needle);
|
|
644
|
+
}
|
|
645
|
+
});
|
|
646
|
+
});
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
import { type ReportComponent } from "./tree.ts";
|
|
2
|
-
import type { LocalizedText, ReportLocale } from "./locale.ts";
|
|
3
|
-
import type { AttemptLocator } from "../results/locator.ts";
|
|
4
|
-
import type { AttemptListItem, CopyFixPromptData, DeltaData, EvalListItem, ExperimentComparisonData, ExperimentListItem, HeroData, LineData, MatrixData, ReportInput, ScatterData, ScopeSummaryData, ScopeWarning, ScoreboardData, TableData, TraceWaterfallRow } from "./types.ts";
|
|
5
|
-
import { type DeltaTableOptions, type MetricLineOptions, type MetricMatrixOptions, type MetricScatterOptions, type MetricTableOptions, type ScoreboardOptions } from "./compute.ts";
|
|
6
|
-
type Never<T> = {
|
|
7
|
-
[K in keyof T]?: never;
|
|
8
|
-
};
|
|
9
|
-
/**
|
|
10
|
-
* 官方数据组件的统一 props 组合(docs/feature/reports/library/metric-views.md):
|
|
11
|
-
* data 形态(接收配套 *Data 的产物)或 spec 形态(Options 平铺 + 可选 input)。
|
|
12
|
-
*/
|
|
13
|
-
export type DataProps<Data, Options, Presentation> = ({
|
|
14
|
-
data: Data;
|
|
15
|
-
input?: never;
|
|
16
|
-
} & Never<Options> & Presentation) | ({
|
|
17
|
-
data?: never;
|
|
18
|
-
input?: ReportInput;
|
|
19
|
-
} & Options & Presentation);
|
|
20
|
-
interface ChromeProps {
|
|
21
|
-
/** chrome 文案 locale;省略时随宿主上下文(宿主外默认 "en")。 */
|
|
22
|
-
locale?: ReportLocale;
|
|
23
|
-
className?: string;
|
|
24
|
-
}
|
|
25
|
-
export type ScopeSummaryProps = DataProps<ScopeSummaryData, Record<never, never>, ChromeProps & {
|
|
26
|
-
/** 显示哪一级计票;默认 "eval"。data 恒携带两级,votes 只选择呈现。 */
|
|
27
|
-
votes?: "eval" | "attempt";
|
|
28
|
-
}>;
|
|
29
|
-
/** 范围摘要卡:时间窗、数量、两级计票、端到端成功率与总成本。 */
|
|
30
|
-
export declare const ScopeSummary: ReportComponent<ScopeSummaryProps>;
|
|
31
|
-
export type ExperimentComparisonProps = DataProps<ExperimentComparisonData, Record<never, never>, ChromeProps>;
|
|
32
|
-
/**
|
|
33
|
-
* 内建报告的默认组合件:先把 input 按可比组分区,再为每组分别计算 ScopeSummary、
|
|
34
|
-
* 成本 × 端到端成功率散点和 ExperimentList。web 面持有完整组索引并一次聚焦一组;
|
|
35
|
-
* text 面命中多个组时只显示组索引与可执行的单组查看命令,命中单组时才输出完整散点与列表。
|
|
36
|
-
*/
|
|
37
|
-
export declare const ExperimentComparison: ReportComponent<ExperimentComparisonProps>;
|
|
38
|
-
interface EntityListChrome extends ChromeProps {
|
|
39
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
40
|
-
}
|
|
41
|
-
export type ExperimentListProps = DataProps<readonly ExperimentListItem[], Record<never, never>, EntityListChrome & {
|
|
42
|
-
/** web 面在比较表前显示实验过滤框;text 面忽略。 */
|
|
43
|
-
filter?: boolean;
|
|
44
|
-
/**
|
|
45
|
-
* 可选父路径:两面的行标签去掉与它相同的前缀,只显示 experiment id 末段(默认
|
|
46
|
-
* `ExperimentComparison` 给每组传组键)。完整 id 仍是排序 / 着色 / 过滤 / 折叠的键。
|
|
47
|
-
*/
|
|
48
|
-
relativeTo?: string;
|
|
49
|
-
}>;
|
|
50
|
-
/** 实验列表:每项一个 experiment,固定八列比较表 + 展开到 Eval / Attempt。 */
|
|
51
|
-
export declare const ExperimentList: ReportComponent<ExperimentListProps>;
|
|
52
|
-
export type EvalListProps = DataProps<readonly EvalListItem[], Record<never, never>, EntityListChrome>;
|
|
53
|
-
/** Eval 列表:每项一个 experimentId + evalId,展开到这道题的 Attempt。 */
|
|
54
|
-
export declare const EvalList: ReportComponent<EvalListProps>;
|
|
55
|
-
export type AttemptListProps = DataProps<readonly AttemptListItem[], Record<never, never>, EntityListChrome & {
|
|
56
|
-
/** 过滤 / 截断前的总数;省略时等于 data 长度。 */
|
|
57
|
-
total?: number;
|
|
58
|
-
/** web 面加过滤输入框(按 experiment、eval、agent、verdict 或摘要文本收窄行);渐进增强,不改变数据与 text 面。 */
|
|
59
|
-
filter?: boolean;
|
|
60
|
-
}>;
|
|
61
|
-
/** Attempt 列表:实体列表的叶子层,每项一次 attempt 的判定、单行摘要与 locator。 */
|
|
62
|
-
export declare const AttemptList: ReportComponent<AttemptListProps>;
|
|
63
|
-
export interface FailureListProps {
|
|
64
|
-
/** 显示的最大条数;默认 20。 */
|
|
65
|
-
limit?: number;
|
|
66
|
-
/** 默认宿主注入的 Scope。 */
|
|
67
|
-
input?: ReportInput;
|
|
68
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
69
|
-
locale?: ReportLocale;
|
|
70
|
-
className?: string;
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* 「现在有哪些失败要处理」的成品组合件:内部就是 attemptListData → 过滤 → AttemptList
|
|
74
|
-
* data 形态,与手写组合严格等价、没有私有能力(docs/feature/reports/library/entity-lists.md)。
|
|
75
|
-
* verdict ∈ failed / errored,按 attempt 开始时间降序(同刻按 locator 字典序),
|
|
76
|
-
* 截断到 limit(默认 20),total 报告截断前总数。
|
|
77
|
-
*/
|
|
78
|
-
export declare const FailureList: ReportComponent<FailureListProps>;
|
|
79
|
-
/** `Hero` 的 props:标题缺省取 `ctx.report.title`(回退链后的站点标题)。 */
|
|
80
|
-
export interface HeroProps {
|
|
81
|
-
/** 覆盖标题;省略时取 ctx.report.title(回退链后的站点标题)。 */
|
|
82
|
-
title?: LocalizedText;
|
|
83
|
-
className?: string;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* `HeroCard`:Hero 的渲染件,双面组件,只收 data 形态——标题输入是站点声明与 Scope 的
|
|
87
|
-
* 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
|
|
88
|
-
* meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
|
|
89
|
-
* 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
|
|
90
|
-
* (docs/feature/reports/library/site-components.md「HeroCard」)。
|
|
91
|
-
*/
|
|
92
|
-
export declare const HeroCard: ReportComponent<HeroCardProps>;
|
|
93
|
-
/** `HeroCard` 的 props:标题 + `heroData()` 的产物,只有 data 形态。 */
|
|
94
|
-
export interface HeroCardProps {
|
|
95
|
-
title: LocalizedText;
|
|
96
|
-
data: HeroData;
|
|
97
|
-
className?: string;
|
|
98
|
-
}
|
|
99
|
-
/**
|
|
100
|
-
* `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
|
|
101
|
-
* 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
|
|
102
|
-
* 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
|
|
103
|
-
* `HeroCard` 显式传值(docs/feature/reports/library/site-components.md「Hero」)。
|
|
104
|
-
*/
|
|
105
|
-
export declare const Hero: ReportComponent<HeroProps>;
|
|
106
|
-
/**
|
|
107
|
-
* `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
|
|
108
|
-
* 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
|
|
109
|
-
* text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
|
|
110
|
-
* 这些组件、自己写替代组件(docs/feature/reports/library/site-components.md「PoweredBy」)。
|
|
111
|
-
*/
|
|
112
|
-
export declare const PoweredBy: ReportComponent<Record<never, never>>;
|
|
113
|
-
/** `ScopeWarnings` 的 props:spec 形态取宿主 Scope 的 warnings,data 形态收 `ScopeWarning[]`。 */
|
|
114
|
-
export type ScopeWarningsProps = DataProps<readonly ScopeWarning[], Record<never, never>, ChromeProps>;
|
|
115
|
-
/**
|
|
116
|
-
* `ScopeWarnings`:选择警告区,警告的唯一呈现组件。把 Scope 携带的 `ScopeWarning[]`
|
|
117
|
-
* 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
|
|
118
|
-
* integrity 组在前);web 面组头带去重后的可复制命令、明细收原生 `<details>`(总条数 ≤ 3
|
|
119
|
-
* 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入两面零输出
|
|
120
|
-
* (docs/feature/reports/library/site-components.md「ScopeWarnings」)。
|
|
121
|
-
*/
|
|
122
|
-
export declare const ScopeWarnings: ReportComponent<ScopeWarningsProps>;
|
|
123
|
-
/** `CopyFixPrompt` 的 props:spec 形态无选项,data 形态收 `copyFixPromptData()` 的产物。 */
|
|
124
|
-
export type CopyFixPromptProps = DataProps<CopyFixPromptData, Record<never, never>, ChromeProps>;
|
|
125
|
-
/**
|
|
126
|
-
* `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
|
|
127
|
-
* prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
|
|
128
|
-
* 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
|
|
129
|
-
* attempt 下钻命令本身(docs/feature/reports/library/site-components.md「CopyFixPrompt」)。
|
|
130
|
-
*/
|
|
131
|
-
export declare const CopyFixPrompt: ReportComponent<CopyFixPromptProps>;
|
|
132
|
-
/** `TraceWaterfall` 的 props:spec 形态无选项,data 形态收 `traceWaterfallData()` 的产物。 */
|
|
133
|
-
export type TraceWaterfallProps = DataProps<readonly TraceWaterfallRow[], Record<never, never>, ChromeProps & {
|
|
134
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
135
|
-
}>;
|
|
136
|
-
/**
|
|
137
|
-
* `TraceWaterfall`:每个 attempt 一行的执行时间瀑布,用 canonical OTel 字段显示被测 agent
|
|
138
|
-
* 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
|
|
139
|
-
* 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
|
|
140
|
-
* 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
|
|
141
|
-
* 生命周期节点不进瀑布(docs/feature/reports/library/site-components.md「TraceWaterfall」)。
|
|
142
|
-
*/
|
|
143
|
-
export declare const TraceWaterfall: ReportComponent<TraceWaterfallProps>;
|
|
144
|
-
export type MetricTableProps = DataProps<TableData, MetricTableOptions, ChromeProps & {
|
|
145
|
-
/** web 面在表格前渲染过滤输入框(enhance.js 接管);无 JS 时表格内容依旧完整。 */
|
|
146
|
-
filter?: boolean;
|
|
147
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
148
|
-
}>;
|
|
149
|
-
/** 榜单:一行一个维度值、一列一个指标,回答「谁整体更好」。 */
|
|
150
|
-
export declare const MetricTable: ReportComponent<MetricTableProps>;
|
|
151
|
-
export type MetricMatrixProps = DataProps<MatrixData, MetricMatrixOptions, ChromeProps & {
|
|
152
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
153
|
-
}>;
|
|
154
|
-
export type MetricBarsProps = MetricMatrixProps;
|
|
155
|
-
/** 逐题格子:行 × 列两个维度、格子里一个指标,回答「哪道题谁挂了」。 */
|
|
156
|
-
export declare const MetricMatrix: ReportComponent<MetricMatrixProps>;
|
|
157
|
-
/** 分组条形:同一份矩阵数据的另一种摆法;与 MetricMatrix 写同一份 spec 时只计算一次。 */
|
|
158
|
-
export declare const MetricBars: ReportComponent<MetricBarsProps>;
|
|
159
|
-
export type ScoreboardProps = DataProps<ScoreboardData, ScoreboardOptions, ChromeProps & {
|
|
160
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
161
|
-
}>;
|
|
162
|
-
/** 考试成绩单:总分 + 分科小计,固定分母、notRun / unscorable 分开如实报。 */
|
|
163
|
-
export declare const Scoreboard: ReportComponent<ScoreboardProps>;
|
|
164
|
-
export type MetricScatterProps = DataProps<ScatterData, MetricScatterOptions, ChromeProps & {
|
|
165
|
-
pointHref?: (row: ScatterData["rows"][number]) => string;
|
|
166
|
-
}>;
|
|
167
|
-
/** 两个指标之间的取舍:每个点一个维度值,x / y 各一个指标,series 只决定颜色和分组。 */
|
|
168
|
-
export declare const MetricScatter: ReportComponent<MetricScatterProps>;
|
|
169
|
-
export type MetricLineProps = DataProps<LineData, MetricLineOptions, ChromeProps & {
|
|
170
|
-
pointHref?: (row: LineData["rows"][number]) => string;
|
|
171
|
-
}>;
|
|
172
|
-
/** 趋势线:x 是 NumericAxis(参数轴),点身份 = (series, x)。 */
|
|
173
|
-
export declare const MetricLine: ReportComponent<MetricLineProps>;
|
|
174
|
-
export type DeltaTableProps = DataProps<DeltaData, DeltaTableOptions, ChromeProps & {
|
|
175
|
-
attemptHref?: (locator: AttemptLocator) => string;
|
|
176
|
-
}>;
|
|
177
|
-
/** 成对对比:每行一对(字面声明或 pairsByFlag 派生),格子里 A、B、Δ 三个值。 */
|
|
178
|
-
export declare const DeltaTable: ReportComponent<DeltaTableProps>;
|
|
179
|
-
export {};
|