niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// 计算函数(*Data):ReportInput → 一份组件数据。站点组件族(Hero / ScopeWarnings /
|
|
2
|
+
// CopyFixPrompt / TraceWaterfall)的 *Data 都住在这里
|
|
3
|
+
// (docs/feature/reports/library/site-components.md)。
|
|
4
|
+
//
|
|
5
|
+
// 共同约定(docs/feature/reports/architecture.md「指标聚合不变量」):
|
|
6
|
+
// - 第一参收 ReportInput = Scope | readonly Snapshot[];warnings 不进组件数据(宿主统一显示);
|
|
7
|
+
// - null ≠ 0:缺数据不编数。
|
|
8
|
+
|
|
9
|
+
import type { CopyFixPromptData, HeroData, ReportInput, ScopeWarning, TraceSpanSummary, TraceWaterfallRow } from "../../model/types.ts";
|
|
10
|
+
import type { TraceSpan } from "../../../types.ts";
|
|
11
|
+
import { collectItems, evalIdOf, experimentIdOf, locatorOf, resolveInput } from "../../model/aggregate.ts";
|
|
12
|
+
import { attemptListData } from "../entity-lists/compute.ts";
|
|
13
|
+
|
|
14
|
+
// ───────────────────────── 站点组件的计算函数(hero / warnings / fix prompt / trace)─────────────────────────
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* `heroData(input)`:站点标题区的运行 meta——`latestStartedAt` 取范围内最新快照的开始时间
|
|
18
|
+
* (空范围为 null,不编造当前时间),`snapshots` 计贡献当前水位的快照数
|
|
19
|
+
* (docs/feature/reports/library/site-components.md「HeroCard」)。
|
|
20
|
+
*/
|
|
21
|
+
export async function heroData(input: ReportInput): Promise<HeroData> {
|
|
22
|
+
const { snapshots } = resolveInput(input);
|
|
23
|
+
let latest: string | null = null;
|
|
24
|
+
for (const snapshot of snapshots) {
|
|
25
|
+
if (latest === null || snapshot.startedAt > latest) latest = snapshot.startedAt;
|
|
26
|
+
}
|
|
27
|
+
return { latestStartedAt: latest, snapshots: snapshots.length };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* `scopeWarningsData(input)`:Scope 携带的挑选警告原样透出;`input` 是裸 `Snapshot[]` 时
|
|
32
|
+
* 没有挑选过程、没有警告,返回空数组,也如实
|
|
33
|
+
* (docs/feature/reports/library/site-components.md「ScopeWarnings」)。
|
|
34
|
+
*/
|
|
35
|
+
export async function scopeWarningsData(input: ReportInput): Promise<readonly ScopeWarning[]> {
|
|
36
|
+
return resolveInput(input).warnings;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `copyFixPromptData(input)`:把范围内全部失败(verdict 为 failed / errored 的 attempt)
|
|
41
|
+
* 整理成一段可交给 coding agent 的修复 prompt——逐失败含 eval id、主失败摘要与 attempt
|
|
42
|
+
* 下钻命令(`niceeval show @<locator>`)。prompt 面向 agent,固定英文
|
|
43
|
+
* (docs/feature/reports/library/site-components.md「CopyFixPrompt」)。
|
|
44
|
+
*/
|
|
45
|
+
export async function copyFixPromptData(input: ReportInput): Promise<CopyFixPromptData> {
|
|
46
|
+
const items = await attemptListData(input);
|
|
47
|
+
const failures = items.filter((item) => item.verdict === "failed" || item.verdict === "errored");
|
|
48
|
+
if (failures.length === 0) return { prompt: "", failures: 0 };
|
|
49
|
+
const lines = failures
|
|
50
|
+
.map((item, i) => {
|
|
51
|
+
const reason =
|
|
52
|
+
item.failureSummary === null
|
|
53
|
+
? null
|
|
54
|
+
: item.moreFailures > 0
|
|
55
|
+
? `${item.failureSummary} (+${item.moreFailures} more failures)`
|
|
56
|
+
: item.failureSummary;
|
|
57
|
+
return [
|
|
58
|
+
`${i + 1}. eval "${item.evalId}" [experiment ${item.experimentId}] — ${item.verdict}`,
|
|
59
|
+
reason ? ` reason: ${reason}` : null,
|
|
60
|
+
` inspect: niceeval show ${item.locator}`,
|
|
61
|
+
]
|
|
62
|
+
.filter(Boolean)
|
|
63
|
+
.join("\n");
|
|
64
|
+
})
|
|
65
|
+
.join("\n");
|
|
66
|
+
const experiments = [...new Set(failures.map((item) => item.experimentId))].join(" / ");
|
|
67
|
+
const prompt = [
|
|
68
|
+
"Fix the failing evals from this niceeval run.",
|
|
69
|
+
"",
|
|
70
|
+
"## Failures",
|
|
71
|
+
lines,
|
|
72
|
+
"",
|
|
73
|
+
"## Steps",
|
|
74
|
+
"1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
|
|
75
|
+
"2. For each failure, run its inspect command above to see the verdict and assertions; add `--execution` for the full agent transcript (tool calls included), `--timing` for the execution timeline, and `--diff` for the workspace diff.",
|
|
76
|
+
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
77
|
+
`4. Re-run: \`npx niceeval exp ${experiments || "<experiment>"} <eval-id-prefix>\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
78
|
+
"5. Run `npx niceeval show` and confirm these failures are gone.",
|
|
79
|
+
].join("\n");
|
|
80
|
+
return { prompt, failures: failures.length };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** TraceSpan 的语义角色 → 瀑布摘要的 kind:turn 归入 agent(一轮就是一次 agent 调用),未识别落 other。 */
|
|
84
|
+
function waterfallKindOf(kind: TraceSpan["kind"]): TraceSpanSummary["kind"] {
|
|
85
|
+
switch (kind) {
|
|
86
|
+
case "agent":
|
|
87
|
+
case "turn":
|
|
88
|
+
return "agent";
|
|
89
|
+
case "model":
|
|
90
|
+
return "model";
|
|
91
|
+
case "tool":
|
|
92
|
+
return "tool";
|
|
93
|
+
default:
|
|
94
|
+
return "other";
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* `traceWaterfallData(input)`:每个 attempt 一行的执行时间瀑布摘要。span 事实只来自
|
|
100
|
+
* trace artifact(经 AttemptHandle 懒加载的 canonical OTel span);runner 生命周期节点
|
|
101
|
+
* (`result.phases`)不进瀑布。行内只汇总顶层 span(parentSpanId 缺失或不在本 trace 内),
|
|
102
|
+
* 按 startOffsetMs 升序;trace 缺失或为空时 `durationMs` 为 null、行照常出现
|
|
103
|
+
* (docs/feature/reports/library/site-components.md「TraceWaterfall」)。
|
|
104
|
+
*/
|
|
105
|
+
export async function traceWaterfallData(input: ReportInput): Promise<readonly TraceWaterfallRow[]> {
|
|
106
|
+
const { snapshots } = resolveInput(input);
|
|
107
|
+
const items = collectItems(snapshots);
|
|
108
|
+
return Promise.all(
|
|
109
|
+
items.map(async (item): Promise<TraceWaterfallRow> => {
|
|
110
|
+
const spans = await item.attempt.trace();
|
|
111
|
+
if (spans === null || spans.length === 0) {
|
|
112
|
+
return {
|
|
113
|
+
experimentId: experimentIdOf(item),
|
|
114
|
+
evalId: evalIdOf(item),
|
|
115
|
+
locator: locatorOf(item),
|
|
116
|
+
durationMs: null,
|
|
117
|
+
spans: [],
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const t0 = Math.min(...spans.map((s) => s.startMs));
|
|
121
|
+
const t1 = Math.max(...spans.map((s) => s.endMs));
|
|
122
|
+
const ids = new Set(spans.map((s) => s.spanId));
|
|
123
|
+
const topLevel = spans.filter((s) => s.parentSpanId === undefined || !ids.has(s.parentSpanId));
|
|
124
|
+
const summaries = topLevel
|
|
125
|
+
.map(
|
|
126
|
+
(s): TraceSpanSummary => ({
|
|
127
|
+
name: s.name,
|
|
128
|
+
kind: waterfallKindOf(s.kind),
|
|
129
|
+
startOffsetMs: s.startMs - t0,
|
|
130
|
+
durationMs: s.endMs - s.startMs,
|
|
131
|
+
failed: s.status === "error",
|
|
132
|
+
}),
|
|
133
|
+
)
|
|
134
|
+
.sort((a, b) => a.startOffsetMs - b.startOffsetMs);
|
|
135
|
+
return {
|
|
136
|
+
experimentId: experimentIdOf(item),
|
|
137
|
+
evalId: evalIdOf(item),
|
|
138
|
+
locator: locatorOf(item),
|
|
139
|
+
durationMs: Math.max(0, t1 - t0),
|
|
140
|
+
spans: summaries,
|
|
141
|
+
};
|
|
142
|
+
}),
|
|
143
|
+
);
|
|
144
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// 站点组件族(HeroCard / ScopeWarnings / TraceWaterfall)的 text 面:同一份算好的数据,
|
|
2
|
+
// 渲染成终端字符(niceeval show 的形态)。零 react、零 IO、纯同步。
|
|
3
|
+
|
|
4
|
+
import type { HeroData, ScopeWarning, TraceWaterfallRow } from "../../model/types.ts";
|
|
5
|
+
import type { LocalizedText } from "../../model/locale.ts";
|
|
6
|
+
import type { TextContext } from "../../definition/tree.ts";
|
|
7
|
+
import { countText, localeText, resolveLocalizedText } from "../../model/locale.ts";
|
|
8
|
+
import { formatDurationMs } from "../../model/format.ts";
|
|
9
|
+
import { groupScopeWarnings } from "./scope-warnings.ts";
|
|
10
|
+
|
|
11
|
+
/** ISO 时间 → "YYYY-MM-DD HH:mm"(本地时区);不可解析原样返回。 */
|
|
12
|
+
function formatDateTimeMinute(iso: string): string {
|
|
13
|
+
const date = new Date(iso);
|
|
14
|
+
if (Number.isNaN(date.valueOf())) return iso;
|
|
15
|
+
const pad = (n: number) => String(n).padStart(2, "0");
|
|
16
|
+
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}`;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// ───────────────────────── 站点组件(HeroCard / ScopeWarnings / TraceWaterfall)─────────────────────────
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* HeroCard 的 text 面:标题行 + meta 行(最后运行时间;空范围为内置「暂无运行」文案;
|
|
23
|
+
* 多快照时标注合成来源),不含品牌行(品牌行是纯 web 件,text 面零输出)。
|
|
24
|
+
*/
|
|
25
|
+
export function heroCardText(title: LocalizedText, data: HeroData, ctx: TextContext): string {
|
|
26
|
+
const locale = ctx.locale;
|
|
27
|
+
const meta =
|
|
28
|
+
data.latestStartedAt === null
|
|
29
|
+
? localeText(locale, "hero.noRuns")
|
|
30
|
+
: [
|
|
31
|
+
localeText(locale, "hero.lastRun", { time: formatDateTimeMinute(data.latestStartedAt) }),
|
|
32
|
+
...(data.snapshots > 1 ? [localeText(locale, "hero.composedSnapshots", { n: data.snapshots })] : []),
|
|
33
|
+
].join(" · ");
|
|
34
|
+
return `${resolveLocalizedText(title, locale)}\n${meta}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* ScopeWarnings 的 text 面:按动作聚合(../scope-warnings.ts,与 web 面共用),同构但不折叠——
|
|
39
|
+
* 多组时首行 "! <分类计数汇总>";每组一行组头 "! <标题> — <徽标> → <组头命令>",其下缩进
|
|
40
|
+
* 逐条原样打印 message(已以下一步收尾,不截断掉尾段)。空警告集零输出。
|
|
41
|
+
*/
|
|
42
|
+
export function scopeWarningsText(warnings: readonly ScopeWarning[], ctx: TextContext): string {
|
|
43
|
+
if (warnings.length === 0) return "";
|
|
44
|
+
const { summary, groups } = groupScopeWarnings(warnings, ctx.locale);
|
|
45
|
+
const lines: string[] = [];
|
|
46
|
+
// 汇总行只在多组时打印;单组时组头即汇总,不另起一行(web 面则恒以汇总行作外层 <summary>)。
|
|
47
|
+
if (groups.length > 1) lines.push(`! ${summary}`);
|
|
48
|
+
for (const group of groups) {
|
|
49
|
+
const badges = group.badges.length > 0 ? ` — ${group.badges.map((b) => b.text).join(" · ")}` : "";
|
|
50
|
+
const command = group.headCommand !== null ? ` → ${group.headCommand}` : "";
|
|
51
|
+
lines.push(`! ${group.title}${badges}${command}`);
|
|
52
|
+
for (const w of group.warnings) lines.push(`! ${w.message}`);
|
|
53
|
+
}
|
|
54
|
+
return lines.join("\n");
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* TraceWaterfall 的 text 面:每 attempt 一行——locator、总耗时(缺 trace 如实显示缺失)、
|
|
59
|
+
* 顶层 span 计数与失败标记,行尾是可复制的 `--timing` 下钻命令(经宿主注入的 attemptCommand
|
|
60
|
+
* 通道拼出,携带宿主上下文)。当前报告没有 attempt-input page 时 `ctx.attemptCommand`
|
|
61
|
+
* 不存在,行退化为纯文本,不生成假命令(architecture.md「Attempt 详情是一张参数化 page」)。
|
|
62
|
+
*/
|
|
63
|
+
export function traceWaterfallText(rows: readonly TraceWaterfallRow[], ctx: TextContext): string {
|
|
64
|
+
const locale = ctx.locale;
|
|
65
|
+
if (rows.length === 0) return localeText(locale, "traceWaterfall.empty");
|
|
66
|
+
return rows
|
|
67
|
+
.map((row) => {
|
|
68
|
+
const failedSpans = row.spans.filter((span) => span.failed).length;
|
|
69
|
+
const parts = [
|
|
70
|
+
row.locator,
|
|
71
|
+
row.evalId,
|
|
72
|
+
row.durationMs === null ? localeText(locale, "traceWaterfall.noTrace") : formatDurationMs(row.durationMs),
|
|
73
|
+
countText(locale, "traceWaterfall.spans", row.spans.length),
|
|
74
|
+
...(failedSpans > 0 ? [`✗ ${countText(locale, "traceWaterfall.failedSpans", failedSpans)}`] : []),
|
|
75
|
+
];
|
|
76
|
+
const line = parts.join(" · ");
|
|
77
|
+
return ctx.attemptCommand ? `${line} ${ctx.attemptCommand(row.locator)} --timing` : line;
|
|
78
|
+
})
|
|
79
|
+
.join("\n");
|
|
80
|
+
}
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
// 官方双面组件的装配点:web 面(./HeroCard.tsx 等纯 React 组件)+ text 面(./faces.ts)+
|
|
2
|
+
// resolve 解析面(spec 形态由管线代调配套 ./compute.ts)。Hero 是组合组件(装配 HeroCard),
|
|
3
|
+
// 不产生自己的 data;PoweredBy 无 props,没有对应的计算函数或 validate。
|
|
4
|
+
|
|
5
|
+
import { defineComponent, type ReportComponent } from "../../definition/tree.ts";
|
|
6
|
+
import type { CopyFixPromptData, HeroData, ScopeWarning, TraceWaterfallRow } from "../../model/types.ts";
|
|
7
|
+
import type { LocalizedText } from "../../model/locale.ts";
|
|
8
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
9
|
+
import {
|
|
10
|
+
arrayProblem,
|
|
11
|
+
dataShapeError,
|
|
12
|
+
isObject,
|
|
13
|
+
makeDataComponent,
|
|
14
|
+
hrefOf,
|
|
15
|
+
type ChromeProps,
|
|
16
|
+
type DataProps,
|
|
17
|
+
type Validator,
|
|
18
|
+
} from "../shared.ts";
|
|
19
|
+
import { copyFixPromptData, heroData, scopeWarningsData, traceWaterfallData } from "./compute.ts";
|
|
20
|
+
import { heroCardText, scopeWarningsText, traceWaterfallText } from "./faces.ts";
|
|
21
|
+
import { HeroCard as HeroCardWeb } from "./HeroCard.tsx";
|
|
22
|
+
import { PoweredBy as PoweredByWeb } from "./PoweredBy.tsx";
|
|
23
|
+
import { ScopeWarnings as ScopeWarningsWeb } from "./ScopeWarnings.tsx";
|
|
24
|
+
import { CopyFixPrompt as CopyFixPromptWeb } from "./CopyFixPrompt.tsx";
|
|
25
|
+
import { TraceWaterfall as TraceWaterfallWeb } from "./TraceWaterfall.tsx";
|
|
26
|
+
|
|
27
|
+
export const validateHeroData: Validator = (data) => {
|
|
28
|
+
if (!isObject(data)) return "expected an object";
|
|
29
|
+
if (!(data.latestStartedAt === null || typeof data.latestStartedAt === "string")) {
|
|
30
|
+
return '"latestStartedAt" must be a string or null';
|
|
31
|
+
}
|
|
32
|
+
if (typeof data.snapshots !== "number") return '"snapshots" must be a number';
|
|
33
|
+
return null;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const UNREADABLE_SNAPSHOT_REASONS = ["incompatible-version", "malformed", "incomplete"];
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* ScopeWarning(src/results/types.ts「警告 kind 全集」):按 `kind` 判别的联合,已登记 kind
|
|
40
|
+
* 各自的必填字段单独校验——`unreadable-snapshot` 没有 experimentId(非实验作用域)、
|
|
41
|
+
* `command` 恒可选。未登记的 `kind`(如未来版本新增的 warning)只要求 `kind` / `message`
|
|
42
|
+
* 这份两族共用的最小形状,不拒绝——`ScopeWarnings` 的分组渲染对未识别 kind 有专门的
|
|
43
|
+
* 单独成组回退(message 原样、按 integrity 归位),这条前向兼容路径本身就是契约的一部分,
|
|
44
|
+
* 结构校验不能比渲染逻辑更严。
|
|
45
|
+
*/
|
|
46
|
+
function scopeWarningProblem(value: unknown, path: string): string | null {
|
|
47
|
+
if (!isObject(value)) return `"${path}" must be a ScopeWarning object`;
|
|
48
|
+
if (typeof value.kind !== "string") return `"${path}.kind" must be a string`;
|
|
49
|
+
if (typeof value.message !== "string") return `"${path}.message" must be a string`;
|
|
50
|
+
switch (value.kind) {
|
|
51
|
+
case "partial-coverage":
|
|
52
|
+
if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
53
|
+
if (typeof value.covered !== "number") return `"${path}.covered" must be a number`;
|
|
54
|
+
if (typeof value.total !== "number") return `"${path}.total" must be a number`;
|
|
55
|
+
if (typeof value.command !== "string") return `"${path}.command" must be a string`;
|
|
56
|
+
return null;
|
|
57
|
+
case "stale-snapshot":
|
|
58
|
+
if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
59
|
+
if (typeof value.startedAt !== "string") return `"${path}.startedAt" must be a string`;
|
|
60
|
+
if (typeof value.latestStartedAt !== "string") return `"${path}.latestStartedAt" must be a string`;
|
|
61
|
+
if (typeof value.command !== "string") return `"${path}.command" must be a string`;
|
|
62
|
+
return null;
|
|
63
|
+
case "unfinished-snapshot":
|
|
64
|
+
if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
65
|
+
if (typeof value.startedAt !== "string") return `"${path}.startedAt" must be a string`;
|
|
66
|
+
if (typeof value.dir !== "string") return `"${path}.dir" must be a string`;
|
|
67
|
+
if (typeof value.command !== "string") return `"${path}.command" must be a string`;
|
|
68
|
+
return null;
|
|
69
|
+
case "unreadable-snapshot":
|
|
70
|
+
if (typeof value.dir !== "string") return `"${path}.dir" must be a string`;
|
|
71
|
+
if (typeof value.reason !== "string" || !UNREADABLE_SNAPSHOT_REASONS.includes(value.reason)) {
|
|
72
|
+
return `"${path}.reason" must be one of ${JSON.stringify(UNREADABLE_SNAPSHOT_REASONS)}`;
|
|
73
|
+
}
|
|
74
|
+
if (value.command !== undefined && typeof value.command !== "string") {
|
|
75
|
+
return `"${path}.command" must be a string when present`;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
default:
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export const validateScopeWarningsData: Validator = (data) => arrayProblem(data, "data", scopeWarningProblem);
|
|
84
|
+
|
|
85
|
+
export const validateCopyFixPromptData: Validator = (data) => {
|
|
86
|
+
if (!isObject(data)) return "expected an object";
|
|
87
|
+
if (typeof data.prompt !== "string") return '"prompt" must be a string';
|
|
88
|
+
if (typeof data.failures !== "number") return '"failures" must be a number';
|
|
89
|
+
return null;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const TRACE_SPAN_KINDS = ["agent", "model", "tool", "other"];
|
|
93
|
+
|
|
94
|
+
/** TraceSpanSummary(src/report/model/types.ts):瀑布行里的一个顶层 span 摘要。 */
|
|
95
|
+
function traceSpanSummaryProblem(value: unknown, path: string): string | null {
|
|
96
|
+
if (!isObject(value)) return `"${path}" must be a TraceSpanSummary { name, kind, startOffsetMs, durationMs, failed }`;
|
|
97
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
98
|
+
if (typeof value.kind !== "string" || !TRACE_SPAN_KINDS.includes(value.kind)) {
|
|
99
|
+
return `"${path}.kind" must be one of ${JSON.stringify(TRACE_SPAN_KINDS)}`;
|
|
100
|
+
}
|
|
101
|
+
if (typeof value.startOffsetMs !== "number") return `"${path}.startOffsetMs" must be a number`;
|
|
102
|
+
if (typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
103
|
+
if (typeof value.failed !== "boolean") return `"${path}.failed" must be a boolean`;
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export const validateTraceWaterfallData: Validator = (data) =>
|
|
108
|
+
arrayProblem(data, "data", (row, path) => {
|
|
109
|
+
if (!isObject(row)) return `"${path}" must be an object`;
|
|
110
|
+
if (typeof row.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
111
|
+
if (typeof row.evalId !== "string") return `"${path}.evalId" must be a string`;
|
|
112
|
+
if (typeof row.locator !== "string") return `"${path}.locator" must be a string`;
|
|
113
|
+
if (!(row.durationMs === null || typeof row.durationMs === "number")) {
|
|
114
|
+
return `"${path}.durationMs" must be a number or null`;
|
|
115
|
+
}
|
|
116
|
+
return arrayProblem(row.spans, `${path}.spans`, traceSpanSummaryProblem);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// ───────────────────────── 站点组件(Hero / PoweredBy / ScopeWarnings / CopyFixPrompt / TraceWaterfall)─────────────────────────
|
|
120
|
+
|
|
121
|
+
/** `Hero` 的 props:标题缺省取 `ctx.report.title`(回退链后的站点标题)。 */
|
|
122
|
+
export interface HeroProps {
|
|
123
|
+
/** 覆盖标题;省略时取 ctx.report.title(回退链后的站点标题)。 */
|
|
124
|
+
title?: LocalizedText;
|
|
125
|
+
className?: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** HeroCard 的 data 校验入口(它不经 makeDataComponent,数据形态是唯一形态)。 */
|
|
129
|
+
const assertHeroData = (data: unknown): HeroData => {
|
|
130
|
+
const problem = validateHeroData(data);
|
|
131
|
+
if (problem !== null) throw dataShapeError("HeroCard", "heroData", "HeroData", problem);
|
|
132
|
+
return data as HeroData;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `HeroCard`:Hero 的渲染件,双面组件,只收 data 形态——标题输入是站点声明与 Scope 的
|
|
137
|
+
* 合成物,没有单独的 spec 等价形。web 面渲染 hero 标题(h1)、按渲染 locale 格式化的运行
|
|
138
|
+
* meta(latestStartedAt 为 null 时内置「暂无运行」文案)与品牌行(等同 PoweredBy,恒含、
|
|
139
|
+
* 无拆除 prop);text 面输出标题行与 meta 行,不含品牌行
|
|
140
|
+
* (docs/feature/reports/library/site-components.md「HeroCard」)。
|
|
141
|
+
*/
|
|
142
|
+
export const HeroCard = defineComponent<HeroCardProps>({
|
|
143
|
+
web: (props, ctx) => {
|
|
144
|
+
assertHeroData(props.data);
|
|
145
|
+
return <HeroCardWeb title={props.title} data={props.data} className={props.className} locale={ctx.locale} />;
|
|
146
|
+
},
|
|
147
|
+
text: (props, ctx) => {
|
|
148
|
+
assertHeroData(props.data);
|
|
149
|
+
return heroCardText(props.title, props.data, ctx);
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
HeroCard.displayName = "HeroCard";
|
|
153
|
+
|
|
154
|
+
/** `HeroCard` 的 props:标题 + `heroData()` 的产物,只有 data 形态。 */
|
|
155
|
+
export interface HeroCardProps {
|
|
156
|
+
title: LocalizedText;
|
|
157
|
+
data: HeroData;
|
|
158
|
+
className?: string;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* `Hero`:页首的站点标题区——标题、最后运行时间、快照合成来源,恒含品牌行。官方组合组件,
|
|
163
|
+
* 与手写 `<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} />`
|
|
164
|
+
* 严格等价、没有私有能力;读 `ctx.report` 意味着输出跟随站点,要站点无关的标题区直接用
|
|
165
|
+
* `HeroCard` 显式传值(docs/feature/reports/library/site-components.md「Hero」)。
|
|
166
|
+
*/
|
|
167
|
+
export const Hero = defineComponent<HeroProps>(async ({ title, className }, ctx) => (
|
|
168
|
+
<HeroCard title={title ?? ctx.report.title} data={await heroData(ctx.scope)} className={className} />
|
|
169
|
+
));
|
|
170
|
+
Hero.displayName = "Hero";
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* `PoweredBy`:唯一的品牌件,无 props 双面组件。web 面渲染指向 niceeval 官网的一行品牌色
|
|
174
|
+
* 小字(`utm_source=report&utm_medium=powered-by`,`rel` 仅 `noopener` 以保留 Referer);
|
|
175
|
+
* text 面零输出。没有任何配置——品牌契约是「提供一个组件,不给开关」:不想要品牌就不用
|
|
176
|
+
* 这些组件、自己写替代组件(docs/feature/reports/library/site-components.md「PoweredBy」)。
|
|
177
|
+
*/
|
|
178
|
+
export const PoweredBy = defineComponent<Record<never, never>>({
|
|
179
|
+
web: () => <PoweredByWeb />,
|
|
180
|
+
text: () => "",
|
|
181
|
+
});
|
|
182
|
+
PoweredBy.displayName = "PoweredBy";
|
|
183
|
+
|
|
184
|
+
/** `ScopeWarnings` 的 props:spec 形态取宿主 Scope 的 warnings,data 形态收 `ScopeWarning[]`。 */
|
|
185
|
+
export type ScopeWarningsProps = DataProps<readonly ScopeWarning[], Record<never, never>, ChromeProps>;
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* `ScopeWarnings`:选择警告区,警告的唯一呈现组件。把 Scope 携带的 `ScopeWarning[]`
|
|
189
|
+
* 按「下一步动作」聚合渲染(带 experimentId 的按实验聚合、非实验作用域按 kind 聚合;
|
|
190
|
+
* integrity 组在前);web 面组头带去重后的可复制命令、明细收原生 `<details>`(总条数 ≤ 3
|
|
191
|
+
* 默认展开),text 面同构但不折叠。空警告集与裸 `Snapshot[]` 输入两面零输出
|
|
192
|
+
* (docs/feature/reports/library/site-components.md「ScopeWarnings」)。
|
|
193
|
+
*/
|
|
194
|
+
export const ScopeWarnings = makeDataComponent<readonly ScopeWarning[], Record<never, never>, ChromeProps>({
|
|
195
|
+
name: "ScopeWarnings",
|
|
196
|
+
dataFnName: "scopeWarningsData",
|
|
197
|
+
shapeName: "ScopeWarning[]",
|
|
198
|
+
dataFn: (input) => scopeWarningsData(input),
|
|
199
|
+
specKeys: [],
|
|
200
|
+
validate: validateScopeWarningsData,
|
|
201
|
+
web: (props, ctx) =>
|
|
202
|
+
props.data.length === 0 ? null : (
|
|
203
|
+
<ScopeWarningsWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />
|
|
204
|
+
),
|
|
205
|
+
text: (props, ctx) => scopeWarningsText(props.data, ctx),
|
|
206
|
+
}) as unknown as ReportComponent<ScopeWarningsProps>;
|
|
207
|
+
|
|
208
|
+
/** `CopyFixPrompt` 的 props:spec 形态无选项,data 形态收 `copyFixPromptData()` 的产物。 */
|
|
209
|
+
export type CopyFixPromptProps = DataProps<CopyFixPromptData, Record<never, never>, ChromeProps>;
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* `CopyFixPrompt`:把当前范围的全部失败整理成一段可交给 coding agent 的修复 prompt。
|
|
213
|
+
* prompt 在 resolve 阶段算好、烘进静态 HTML,无 JS 时在折叠块里完整可读,「复制」是增强层
|
|
214
|
+
* 行为;`failures` 为 0 时两面零输出;text 面恒零输出——终端里的等价能力是 `show` 的
|
|
215
|
+
* attempt 下钻命令本身(docs/feature/reports/library/site-components.md「CopyFixPrompt」)。
|
|
216
|
+
*/
|
|
217
|
+
export const CopyFixPrompt = makeDataComponent<CopyFixPromptData, Record<never, never>, ChromeProps>({
|
|
218
|
+
name: "CopyFixPrompt",
|
|
219
|
+
dataFnName: "copyFixPromptData",
|
|
220
|
+
shapeName: "CopyFixPromptData",
|
|
221
|
+
dataFn: (input) => copyFixPromptData(input),
|
|
222
|
+
specKeys: [],
|
|
223
|
+
validate: validateCopyFixPromptData,
|
|
224
|
+
web: (props, ctx) =>
|
|
225
|
+
props.data.failures === 0 ? null : (
|
|
226
|
+
<CopyFixPromptWeb data={props.data} locale={props.locale ?? ctx.locale} className={props.className} />
|
|
227
|
+
),
|
|
228
|
+
text: () => "",
|
|
229
|
+
}) as unknown as ReportComponent<CopyFixPromptProps>;
|
|
230
|
+
|
|
231
|
+
/** `TraceWaterfall` 的 props:spec 形态无选项,data 形态收 `traceWaterfallData()` 的产物。 */
|
|
232
|
+
export type TraceWaterfallProps = DataProps<
|
|
233
|
+
readonly TraceWaterfallRow[],
|
|
234
|
+
Record<never, never>,
|
|
235
|
+
ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
|
|
236
|
+
>;
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* `TraceWaterfall`:每个 attempt 一行的执行时间瀑布,用 canonical OTel 字段显示被测 agent
|
|
240
|
+
* 的原始 span(agent / model / tool)。web 面静态渲染顶层 span 分解条(失败 span 带失败
|
|
241
|
+
* 标记),行链接 attempt 详情;text 面每 attempt 一行(locator、总耗时、span 计数与失败
|
|
242
|
+
* 标记)+ 可复制的 `--timing` 下钻命令。trace 缺失的行照常出现并如实显示缺失;runner
|
|
243
|
+
* 生命周期节点不进瀑布(docs/feature/reports/library/site-components.md「TraceWaterfall」)。
|
|
244
|
+
*/
|
|
245
|
+
export const TraceWaterfall = makeDataComponent<
|
|
246
|
+
readonly TraceWaterfallRow[],
|
|
247
|
+
Record<never, never>,
|
|
248
|
+
ChromeProps & { attemptHref?: (locator: AttemptLocator) => string }
|
|
249
|
+
>({
|
|
250
|
+
name: "TraceWaterfall",
|
|
251
|
+
dataFnName: "traceWaterfallData",
|
|
252
|
+
shapeName: "TraceWaterfallRow[]",
|
|
253
|
+
dataFn: (input) => traceWaterfallData(input),
|
|
254
|
+
specKeys: [],
|
|
255
|
+
validate: validateTraceWaterfallData,
|
|
256
|
+
web: (props, ctx) => (
|
|
257
|
+
<TraceWaterfallWeb
|
|
258
|
+
data={props.data}
|
|
259
|
+
attemptHref={hrefOf(props, ctx)}
|
|
260
|
+
locale={props.locale ?? ctx.locale}
|
|
261
|
+
className={props.className}
|
|
262
|
+
/>
|
|
263
|
+
),
|
|
264
|
+
text: (props, ctx) => traceWaterfallText(props.data, ctx),
|
|
265
|
+
}) as unknown as ReportComponent<TraceWaterfallProps>;
|
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
// (docs/feature/results/library.md「警告 kind 全集」)经 locale 词典渲染,未登记的 kind
|
|
5
5
|
// 回退为单独成组、逐条 message 原样。
|
|
6
6
|
|
|
7
|
-
import type { ScopeWarning } from "
|
|
8
|
-
import { gapParts } from "
|
|
9
|
-
import { localeText, type ReportLocale, type ReportMessageKey } from "
|
|
7
|
+
import type { ScopeWarning } from "../../../results/types.ts";
|
|
8
|
+
import { gapParts } from "../../../results/select.ts";
|
|
9
|
+
import { localeText, type ReportLocale, type ReportMessageKey } from "../../model/locale.ts";
|
|
10
10
|
|
|
11
11
|
/** kind 表登记的类别:integrity(选中集合的分母可能不对)组排在 freshness(可能过期)之前。 */
|
|
12
12
|
export type WarningCategory = "integrity" | "freshness";
|
package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx}
RENAMED
|
@@ -7,9 +7,9 @@
|
|
|
7
7
|
|
|
8
8
|
import { describe, expect, it } from "vitest";
|
|
9
9
|
|
|
10
|
-
import type { AssertionResult, EvalResult, TraceSpan, Verdict } from "
|
|
11
|
-
import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "
|
|
12
|
-
import { makeScope } from "
|
|
10
|
+
import type { AssertionResult, EvalResult, TraceSpan, Verdict } from "../../../types.ts";
|
|
11
|
+
import type { AttemptHandle, Results, Scope, ScopeWarning, Snapshot } from "../../../results/index.ts";
|
|
12
|
+
import { makeScope } from "../../../results/select.ts";
|
|
13
13
|
import {
|
|
14
14
|
createTextContext,
|
|
15
15
|
defineComponent,
|
|
@@ -18,20 +18,15 @@ import {
|
|
|
18
18
|
validateReportTree,
|
|
19
19
|
ResolveMemo,
|
|
20
20
|
type ReportNode,
|
|
21
|
-
} from "
|
|
22
|
-
import { buildReportMeta, defineReport
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
ScopeWarnings,
|
|
31
|
-
TraceWaterfall,
|
|
32
|
-
} from "./components.tsx";
|
|
33
|
-
import { Text } from "./primitives.tsx";
|
|
34
|
-
import { attemptListData, copyFixPromptData, heroData, traceWaterfallData } from "./compute.ts";
|
|
21
|
+
} from "../../definition/tree.ts";
|
|
22
|
+
import { buildReportMeta, defineReport } from "../../definition/report.ts";
|
|
23
|
+
import { pickReportPage, renderReportToText, reportTitleText } from "../../runtime/text.ts";
|
|
24
|
+
import { renderReportToStaticHtml } from "../../runtime/web.ts";
|
|
25
|
+
import { AttemptList } from "../entity-lists/index.tsx";
|
|
26
|
+
import { CopyFixPrompt, Hero, HeroCard, PoweredBy, ScopeWarnings, TraceWaterfall } from "./index.tsx";
|
|
27
|
+
import { Text } from "../../definition/primitives.tsx";
|
|
28
|
+
import { attemptListData } from "../entity-lists/compute.ts";
|
|
29
|
+
import { copyFixPromptData, heroData, traceWaterfallData } from "./compute.ts";
|
|
35
30
|
|
|
36
31
|
// ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
|
|
37
32
|
|
|
@@ -120,22 +115,29 @@ function hostCtx(scope: Scope) {
|
|
|
120
115
|
return { scope, results: resultsOf(scope.snapshots) };
|
|
121
116
|
}
|
|
122
117
|
|
|
118
|
+
/** 站点组件测试固定"宿主已声明 attempt-input page"这条件,验证 attemptCommand/attemptHref 通道本身接得对;
|
|
119
|
+
* 「没有 attempt page 时退化成纯文本」由 report.test.ts / show.test.ts 的报告级测试单独覆盖。 */
|
|
120
|
+
const DEFAULT_TEST_ATTEMPT_COMMAND = (locator: string) => `niceeval show ${locator}`;
|
|
121
|
+
const DEFAULT_TEST_ATTEMPT_HREF = (locator: string) => `#/attempt/${locator}`;
|
|
122
|
+
|
|
123
123
|
/** 管线便捷入口:装载 → resolve → validate → text 渲染(与 show 同一条管线,不带宿主前置块)。 */
|
|
124
124
|
async function treeText(node: ReportNode, scope: Scope, width = 120): Promise<string> {
|
|
125
125
|
const definition = defineReport(node);
|
|
126
|
-
const
|
|
126
|
+
const page = pickReportPage(definition);
|
|
127
|
+
const resolved = await resolveReportTree(page.content, {
|
|
127
128
|
scope,
|
|
128
129
|
results: resultsOf(scope.snapshots),
|
|
129
|
-
report: buildReportMeta(definition, scope
|
|
130
|
+
report: buildReportMeta(definition, scope),
|
|
131
|
+
page: { id: page.id, input: "scope" },
|
|
130
132
|
memo: new ResolveMemo(),
|
|
131
133
|
});
|
|
132
134
|
validateReportTree(resolved);
|
|
133
|
-
return renderNodeToText(resolved, createTextContext({ width }));
|
|
135
|
+
return renderNodeToText(resolved, createTextContext({ width, attemptCommand: DEFAULT_TEST_ATTEMPT_COMMAND }));
|
|
134
136
|
}
|
|
135
137
|
|
|
136
138
|
/** 管线便捷入口:web 面静态 HTML(经 renderReportToStaticHtml;测试用无警告 Scope,免宿主前置块)。 */
|
|
137
139
|
async function treeHtml(node: ReportNode, scope: Scope): Promise<string> {
|
|
138
|
-
return renderReportToStaticHtml(defineReport(node), hostCtx(scope));
|
|
140
|
+
return renderReportToStaticHtml(defineReport(node), hostCtx(scope), { attemptHref: DEFAULT_TEST_ATTEMPT_HREF });
|
|
139
141
|
}
|
|
140
142
|
|
|
141
143
|
// ───────────────────────── 警告 fixture(按 ScopeWarning 联合造)─────────────────────────
|