niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "编写评估用例: 单轮、多轮和数据集模式"
|
|
3
|
+
sidebarTitle: "评估用例"
|
|
4
|
+
description: "用 defineEval 编写评估用例,包括单轮对话、多轮对话、数据驱动测试、Sandbox Workspace 和评估的生命周期与 Fixture。"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## `defineEval` 的结构
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { defineEval } from "niceeval";
|
|
11
|
+
|
|
12
|
+
export default defineEval({
|
|
13
|
+
description?: string;
|
|
14
|
+
tags?: string[];
|
|
15
|
+
judge?: JudgeConfig;
|
|
16
|
+
reporters?: Reporter[];
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
environment?: string;
|
|
19
|
+
metadata?: Record<string, unknown>;
|
|
20
|
+
async setup(sandbox, ctx) { /* task fixture + progress/diagnostic */ },
|
|
21
|
+
async test(t) { /* interactions + assertions */ },
|
|
22
|
+
});
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## `tags` 与 `environment`
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
// evals/coding/fix-button.eval.ts → id: coding/fix-button
|
|
29
|
+
export default defineEval({
|
|
30
|
+
description: "修复 Button 组件",
|
|
31
|
+
tags: ["coding", "frontend"],
|
|
32
|
+
environment: "node-22",
|
|
33
|
+
async test(t) { /* coding task */ },
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// evals/research/gpu-literature.eval.ts → id: research/gpu-literature
|
|
37
|
+
export default defineEval({
|
|
38
|
+
description: "在 GPU 环境检索论文",
|
|
39
|
+
tags: ["research"],
|
|
40
|
+
environment: "gpu",
|
|
41
|
+
async test(t) { /* research task */ },
|
|
42
|
+
});
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`tags` 可供 `--tag` 和 experiment 的 `evals` 谓词读取。`environment` 只声明环境 profile;具体 image / template 由 experiment 使用的 Sandbox 配置映射。
|
|
46
|
+
|
|
47
|
+
## 单轮评估
|
|
48
|
+
|
|
49
|
+
```ts
|
|
50
|
+
import { defineEval } from "niceeval";
|
|
51
|
+
import { includes } from "niceeval/expect";
|
|
52
|
+
|
|
53
|
+
export default defineEval({
|
|
54
|
+
description: "Brooklyn weather query",
|
|
55
|
+
async test(t) {
|
|
56
|
+
await t.send("What's the weather like in Brooklyn today?");
|
|
57
|
+
t.succeeded();
|
|
58
|
+
t.calledTool("get_weather", { input: { city: "Brooklyn" }, count: 1 });
|
|
59
|
+
t.check(t.reply, includes("sunny"));
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`t.send()` 驱动一次交互,`t.succeeded()` 和 `t.calledTool()` 是作用域断言,`t.check()` 是立即记录的值断言。
|
|
65
|
+
|
|
66
|
+
### `Turn` 对象
|
|
67
|
+
|
|
68
|
+
| 属性 | 说明 |
|
|
69
|
+
|---|---|
|
|
70
|
+
| `turn.events` | 标准事件流,主要事实来源 |
|
|
71
|
+
| `turn.data` | 结构化输出 |
|
|
72
|
+
| `turn.status` | `"completed"`、`"failed"` 或 `"waiting"` |
|
|
73
|
+
| `turn.usage` | token / cost 等 usage |
|
|
74
|
+
| `turn.message` | assistant 文本回复 |
|
|
75
|
+
| `turn.toolCalls` | 本轮工具调用 |
|
|
76
|
+
|
|
77
|
+
## 多轮评估
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
export default defineEval({
|
|
81
|
+
description: "Draft an email, then send it on confirmation",
|
|
82
|
+
async test(t) {
|
|
83
|
+
const draft = await t.send("Draft a follow-up email.");
|
|
84
|
+
draft.succeeded();
|
|
85
|
+
t.check(draft.message, includes("Best"));
|
|
86
|
+
|
|
87
|
+
await t.send("Looks good, send it.");
|
|
88
|
+
t.calledTool("send_email");
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
需要并行独立会话时,用 `t.newSession()`。
|
|
94
|
+
|
|
95
|
+
## 数据驱动测试(dataset fan-out)
|
|
96
|
+
|
|
97
|
+
一个文件可以导出评估用例数组:
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
export default rows.map((row) =>
|
|
101
|
+
defineEval({
|
|
102
|
+
description: row.task,
|
|
103
|
+
async test(t) {
|
|
104
|
+
await t.send(row.prompt);
|
|
105
|
+
t.check(t.reply, equals(row.expected));
|
|
106
|
+
},
|
|
107
|
+
}),
|
|
108
|
+
);
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
生成 ID 为 `sql/0000`、`sql/0001` 等。详见 [数据驱动测试](/zh/tutorials/dataset-fanout)。
|
|
112
|
+
|
|
113
|
+
## Sandbox workspace
|
|
114
|
+
|
|
115
|
+
Coding-agent 评估用例仍然是普通 `.eval.ts` 文件,只是 test 里会准备 Sandbox workspace、发送任务并检查文件结果:
|
|
116
|
+
|
|
117
|
+
```ts
|
|
118
|
+
import { defineEval } from "niceeval";
|
|
119
|
+
|
|
120
|
+
export default defineEval({
|
|
121
|
+
description: "Create a Button component",
|
|
122
|
+
async test(t) {
|
|
123
|
+
await t.sandbox.uploadDirectory("../workspaces/ts-starter");
|
|
124
|
+
await t.send("Create src/components/Button.tsx with label and onClick props.").then((turn) => turn.succeeded());
|
|
125
|
+
t.sandbox.fileChanged("src/components/Button.tsx");
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
详见 [Fixtures](/zh/tutorials/fixtures)。
|
|
131
|
+
|
|
132
|
+
## 输出信息
|
|
133
|
+
|
|
134
|
+
`setup` 用于这条评估用例的 Fixture。第二个参数绑定到评估用例 setup 阶段;`test(t)` 里的反馈绑定到评估用例 run 阶段:
|
|
135
|
+
|
|
136
|
+
```ts
|
|
137
|
+
export default defineEval({
|
|
138
|
+
async setup(sandbox, ctx) {
|
|
139
|
+
ctx.progress({ message: "安装 fixture 依赖" });
|
|
140
|
+
await sandbox.runCommand("npm", ["install"]);
|
|
141
|
+
},
|
|
142
|
+
|
|
143
|
+
async test(t) {
|
|
144
|
+
t.progress({ message: "上传隐藏测试", current: 1, total: 2 });
|
|
145
|
+
await t.sandbox.uploadDirectory("../fixtures/project");
|
|
146
|
+
|
|
147
|
+
const preflight = await inspectFixture();
|
|
148
|
+
if (preflight.usedFallback) {
|
|
149
|
+
t.diagnostic({
|
|
150
|
+
code: "fixture-check-degraded",
|
|
151
|
+
level: "warning",
|
|
152
|
+
message: "Fixture 预检使用了备用检查器",
|
|
153
|
+
data: { checker: preflight.checker },
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
await t.send("完成任务");
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`progress` 只更新运行中的短期状态,不进入结果。`diagnostic` 会写进当前 Attempt 的 `result.json`,但不会代替断言或自动改变判定:业务结论仍用 `t.check` / `t.require` / gate;基础设施无法继续时抛出异常。
|
|
163
|
+
|
|
164
|
+
## 评估的生命周期
|
|
165
|
+
|
|
166
|
+
`setup(sandbox, ctx)` 准备 Fixture;配一个 `teardown(sandbox, ctx)` 收尾,两者合起来是这条评估用例的 Fixture,每个 Attempt 各执行一次。执行顺序:`setup` 在 Sandbox 生命周期 Hook 和 git 基线锚点之后、`test(t)` 之前跑;`teardown` 是 Attempt 收尾链的第一段(先评估用例 teardown,再 agent teardown,最后 Sandbox teardown),这时 Sandbox 还活着,收尾代码可以照常读 Sandbox。
|
|
167
|
+
|
|
168
|
+
大多数 Fixture 不需要 `teardown`——写进 Sandbox 的起始文件、装的依赖随 Sandbox 销毁自动没了。需要 `teardown` 的是**Sandbox 外**的 Fixture:在共享外部服务里为这个 Attempt 建的临时资源(临时 repo、bucket、队列 topic),不收就泄漏。
|
|
169
|
+
|
|
170
|
+
同一条评估用例的多个 Attempt(`runs` 大于 1、或同批多个实验跑同一条评估用例)并发执行且共享同一个模块,`setup` 的句柄不能放进普通模块变量——会被后一个并发 Attempt 覆写。以 `sandbox` 实例作键存取(`WeakMap`):sandbox 与 Attempt 一一对应,天然是 per-attempt 键:
|
|
171
|
+
|
|
172
|
+
```ts
|
|
173
|
+
// evals/pr-review/close-stale.eval.ts
|
|
174
|
+
import { defineEval } from "niceeval";
|
|
175
|
+
import type { Sandbox } from "niceeval/sandbox";
|
|
176
|
+
|
|
177
|
+
// 并发 Attempt 共享本模块:句柄按 sandbox 键控,不用普通模块变量
|
|
178
|
+
const fixtures = new WeakMap<Sandbox, { repoUrl: string; destroy(): Promise<void> }>();
|
|
179
|
+
|
|
180
|
+
export default defineEval({
|
|
181
|
+
async setup(sandbox, ctx) {
|
|
182
|
+
ctx.progress({ message: "seeding fixture repo" });
|
|
183
|
+
const fixture = await createFixtureRepo("pr-review/close-stale"); // 沙箱外的临时资源
|
|
184
|
+
fixtures.set(sandbox, fixture);
|
|
185
|
+
await sandbox.runCommand("git", ["clone", fixture.repoUrl, "workspace"]);
|
|
186
|
+
},
|
|
187
|
+
async teardown(sandbox) {
|
|
188
|
+
await fixtures.get(sandbox)?.destroy(); // setup 抛错也会进来:没建成就跳过
|
|
189
|
+
},
|
|
190
|
+
async test(t) { /* 驱动 agent 清理 stale PR,断言 */ },
|
|
191
|
+
});
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
`teardown` 当且仅当这条 Attempt 走到过 `setup` 的时点才执行——`setup` 抛错不豁免,收尾代码要对可能没建成的资源做防御。`teardown` 抛错或超过 30 秒清理上限只记 `teardown-failed` 诊断,不改变这个 Attempt 已经产出的判定;要让某个收尾动作影响结论,在 `setup` / `test` 里抛错,不要指望 `teardown` 能改判。
|
|
195
|
+
|
|
196
|
+
## 命名约定
|
|
197
|
+
|
|
198
|
+
<CardGroup cols={2}>
|
|
199
|
+
<Card title="文件名" icon="file">
|
|
200
|
+
只有 `.eval.ts` 会被 runner 发现。
|
|
201
|
+
</Card>
|
|
202
|
+
<Card title="ID 命名空间" icon="folder">
|
|
203
|
+
`evals/billing/refund.eval.ts` 的 ID 是 `billing/refund`。
|
|
204
|
+
</Card>
|
|
205
|
+
<Card title="数据集" icon="database">
|
|
206
|
+
适合大量结构相同、输入不同的 case。
|
|
207
|
+
</Card>
|
|
208
|
+
<Card title="Sandbox workspace" icon="box">
|
|
209
|
+
适合 coding agent,需要真实文件系统、命令和 diff。
|
|
210
|
+
</Card>
|
|
211
|
+
</CardGroup>
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "在 GitHub Actions 和 CI 中运行 NiceEval"
|
|
3
3
|
sidebarTitle: "CI 集成"
|
|
4
|
-
description: "把 NiceEval 接入 GitHub Actions 或任意 CI
|
|
4
|
+
description: "把 NiceEval 接入 GitHub Actions 或任意 CI。评估用例失败时非零退出,输出 JUnit XML,并通过缓存加速重复运行。"
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
评估用例应该和测试一样进入 CI。它们能在 PR 阶段发现 agent 行为回退,也能在 nightly job 中跟踪模型、成本和延迟变化。
|
|
8
8
|
|
|
9
9
|
## 退出码
|
|
10
10
|
|
|
@@ -55,10 +55,10 @@ jobs:
|
|
|
55
55
|
把 provider token 放进 GitHub Actions secrets。
|
|
56
56
|
</Step>
|
|
57
57
|
<Step title="在 workflow 中传入 env">
|
|
58
|
-
|
|
58
|
+
只在运行评估用例的 step 暴露必要环境变量。
|
|
59
59
|
</Step>
|
|
60
60
|
<Step title="验证变量存在">
|
|
61
|
-
|
|
61
|
+
用最小评估用例或 `list` / dry run 先验证配置。
|
|
62
62
|
</Step>
|
|
63
63
|
</Steps>
|
|
64
64
|
|
|
@@ -74,7 +74,7 @@ niceeval: junit=.niceeval/junit.xml
|
|
|
74
74
|
|
|
75
75
|
退出码是第一层红绿信号;JSON、JUnit 和结果快照是完整机器接口,日志行只用于搜索和 annotation。`errored` 行带 locator、eval/experiment 身份、已知时的正式 phase,以及一层 `reason` 摘要。详细 cause、stack 和 diagnostics 保存在 Attempt 的 `result.json`,可在保留 artifact 后运行 `niceeval show @<locator>` 回顾。
|
|
76
76
|
|
|
77
|
-
CLI 显式要求的 JSON/JUnit 和默认 results artifact 都是 required 输出:写入失败必须让 job 判红,不能只留 warning 后退出 0
|
|
77
|
+
CLI 显式要求的 JSON/JUnit 和默认 results artifact 都是 required 输出:写入失败必须让 job 判红,不能只留 warning 后退出 0。把结果同时上报到 Braintrust 等实验平台的配置见 [Reporter 上报](./reporters)。
|
|
78
78
|
|
|
79
79
|
## 只检查发现
|
|
80
80
|
|
|
@@ -82,7 +82,7 @@ CLI 显式要求的 JSON/JUnit 和默认 results artifact 都是 required 输出
|
|
|
82
82
|
npx niceeval list
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
适合快速验证评估用例文件能加载、配置没有明显错误。
|
|
86
86
|
|
|
87
87
|
## 缓存 `.niceeval/`
|
|
88
88
|
|
|
@@ -94,13 +94,13 @@ npx niceeval list
|
|
|
94
94
|
npx niceeval exp ci --output ci --max-concurrency 2
|
|
95
95
|
```
|
|
96
96
|
|
|
97
|
-
标准 GitHub-hosted runner 上,
|
|
97
|
+
标准 GitHub-hosted runner 上,Sandbox 评估用例并发不宜过高。远程 HTTP 评估用例可以按服务限流能力调高。
|
|
98
98
|
|
|
99
99
|
## 推荐模式
|
|
100
100
|
|
|
101
101
|
<CardGroup cols={2}>
|
|
102
102
|
<Card title="PR runs" icon="code-pull-request">
|
|
103
|
-
|
|
103
|
+
只跑关键评估用例和高风险路径,保持反馈快。
|
|
104
104
|
</Card>
|
|
105
105
|
<Card title="Nightly 全量矩阵运行" icon="moon">
|
|
106
106
|
运行完整 experiment,记录 pass rate、成本和延迟趋势。
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: "OTel 接入"
|
|
3
3
|
sidebarTitle: "OTel 接入"
|
|
4
|
-
description: "
|
|
4
|
+
description: "把应用已有的 OTel span 发送给 NiceEval,在 niceeval view 中查看每轮调用瀑布图;评估用例断言仍以 Send 返回的事件和用量为准。"
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
OTel 接入**不改变断言的数据来源**。`t.calledTool`、`t.maxTokens` 和耗时判定都读取 Adapter 在 `send` 中返回的 `Turn`(`events` 与 `usage`),见[接入你的 Agent](/zh/tutorials/connect-your-agent)。
|
|
8
8
|
|
|
9
|
-
OTel
|
|
9
|
+
OTel span 用于生成 **`niceeval view` 中的调用瀑布图**。瀑布图按轮显示模型调用、工具执行、耗时和 token,帮助定位评估用例失败或轮次变慢的具体步骤。
|
|
10
10
|
|
|
11
11
|
如果你的应用已经在发 OTel trace——AI SDK 的 telemetry、LangGraph 的 LangSmith 导出、OpenLLMetry / OpenInference 自动埋点,或自己按 GenAI 语义埋的点——那瀑布图的数据你已经在生产了:让应用把 span 也发给 [NiceEval](https://niceeval.com/) 一份即可,应用代码一行不改,仍是无侵入(见 [Tier](/zh/explanation/tier))。
|
|
12
12
|
|
|
13
13
|
## 原理(一段话)
|
|
14
14
|
|
|
15
|
-
[NiceEval](https://niceeval.com/)
|
|
15
|
+
[NiceEval](https://niceeval.com/) 运行时启动本机 OTLP 接收器。应用发送的 span 会归属到对应 `send` 轮次,归一成 GenAI 语义后写入 `EvalResult.trace`,并由 `npx niceeval view` 显示为瀑布图。**span 只用于瀑布图,不进入事件流,也不参与断言**。埋点缺失、span 迟到或丢批只影响瀑布图完整性,不影响判定。
|
|
16
16
|
|
|
17
17
|
## 接法
|
|
18
18
|
|
|
@@ -44,7 +44,7 @@ export default defineAgent({
|
|
|
44
44
|
|
|
45
45
|
内置件不用做这件事:`uiMessageStreamAgent` 总会自动把 `ctx.telemetry.headers` 并入请求头。
|
|
46
46
|
|
|
47
|
-
**2.
|
|
47
|
+
**2. 向应用提供端点。** [NiceEval](https://niceeval.com/) 的接收端点属于**启动期配置,不从 `send` 传递**。标准 OTel SDK 只在进程启动时读取一次 `OTEL_*` 环境变量。按部署形态选择配置方式:
|
|
48
48
|
|
|
49
49
|
- **你自己长驻的服务(最常见)**:用**固定端口模式**,在 `niceeval.config.ts` 里钉住接收端口——写了这个配置就等于打开了 OTel 接入:
|
|
50
50
|
|
|
@@ -55,7 +55,7 @@ export default defineAgent({
|
|
|
55
55
|
});
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
-
服务启动时一次性配 `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces
|
|
58
|
+
服务启动时一次性配 `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces`,之后跑多少次评估用例都不用改。代价:端口共享意味着同一台机器同时只能跑一个 niceeval 进程;OTel Collector 扇出场景同理指向这个固定端点。`port` 被其它进程占用时 [NiceEval](https://niceeval.com/) 会直接报错提示换一个空闲端口,不会静默失败。
|
|
59
59
|
|
|
60
60
|
报给应用的接收端 hostname 默认是 `127.0.0.1`;docker 型 sandbox tracing 需要 `host.docker.internal`、或配了隧道的远程接入需要别的 hostname 时,加 `host` 覆盖:`telemetry: { host: "host.docker.internal", port: 4318 }`。这两个字段是 [NiceEval](https://niceeval.com/) 里配置 OTLP 接收的唯一入口,不读环境变量。
|
|
61
61
|
|
|
@@ -86,10 +86,10 @@ export default defineAgent({
|
|
|
86
86
|
LANGSMITH_OTEL_ENABLED=true \
|
|
87
87
|
LANGSMITH_OTEL_ONLY=true \
|
|
88
88
|
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces \
|
|
89
|
-
node server.js #
|
|
89
|
+
node server.js # 端点值按「向应用提供端点」一节选择:注入的 env 或固定端口
|
|
90
90
|
```
|
|
91
91
|
|
|
92
|
-
`LANGSMITH_OTEL_ONLY` 表示只发 OTLP、不发 LangSmith
|
|
92
|
+
`LANGSMITH_OTEL_ONLY` 表示只发 OTLP、不发 LangSmith 云端;需要双发时移除该变量。**Python** 版 `langsmith` SDK 会在 import 时自动注册 OTel hook。**JS** 版(`langsmith@0.7.x`)还需要显式调用 `langsmith/experimental/otel/setup` 导出的 `initializeOTEL()`,否则只输出警告,不产生 span。三个环境变量保持不变。Python 后端示例见 [`examples/zh/origin/langgraph`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/origin/langgraph),完整评测项目见 [`examples/zh/tier1/langgraph`](https://github.com/CorrectRoadH/niceeval/tree/main/examples/zh/tier1/langgraph)。
|
|
93
93
|
</Tab>
|
|
94
94
|
<Tab title="OpenLLMetry">
|
|
95
95
|
```ts
|
|
@@ -113,16 +113,16 @@ export default defineAgent({
|
|
|
113
113
|
</Tab>
|
|
114
114
|
</Tabs>
|
|
115
115
|
|
|
116
|
-
## span
|
|
116
|
+
## 把 span 归属到 Turn
|
|
117
117
|
|
|
118
|
-
|
|
118
|
+
并行跑多条评估用例时,同一个接收器会同时收到多条会话的 span,[NiceEval](https://niceeval.com/) 按两条路把它们归到各自的轮:
|
|
119
119
|
|
|
120
120
|
- **traceparent(推荐,并发安全)**:`send` 发请求时把 `ctx.telemetry.headers`(W3C trace context,每轮一个新 `traceparent`)spread 进请求头。应用的埋点支持 context 传播的话(标准 OTel HTTP 服务端埋点都支持),本轮 span 自动挂到 [NiceEval](https://niceeval.com/) 给的 trace 下,按 traceId 精确归属。
|
|
121
121
|
- **时间窗口(兜底)**:应用不传播 trace context 时,按 send 前后的时间窗归属。窗口只在串行下可靠,所以这种情况 [NiceEval](https://niceeval.com/) 会把这个 agent 的轮次串行执行并在日志里提示,不会静默混流;一旦确认 traceparent 生效,自动恢复并发。
|
|
122
122
|
|
|
123
123
|
应用侧要及时导出:瀑布图在意的是"这一轮的 span 及时到齐",用 `SimpleSpanProcessor`(或每轮 flush)——`BatchSpanProcessor` 的缓冲会让 span 跨轮迟到,瀑布图偶发缺尾巴多半是它。
|
|
124
124
|
|
|
125
|
-
##
|
|
125
|
+
## 保留现有 OTel 后端并双发
|
|
126
126
|
|
|
127
127
|
应用多半已经把 trace 发给自己的观测后端(Langfuse / SigNoz / 生产 collector)。接 [NiceEval](https://niceeval.com/) **不需要换后端、也不需要第二套埋点**:TracerProvider 支持挂多个 SpanProcessor——同一批 span,两个出口:
|
|
128
128
|
|
|
@@ -146,11 +146,11 @@ provider.register();
|
|
|
146
146
|
|
|
147
147
|
不方便改应用代码的话,OTel Collector 扇出——应用只发给 collector,collector 配两个 exporter(你的后端 + [NiceEval](https://niceeval.com/) 的固定端点)。代价是多运维一个组件。
|
|
148
148
|
|
|
149
|
-
##
|
|
149
|
+
## 用语义映射控制瀑布图内容
|
|
150
150
|
|
|
151
|
-
[NiceEval](https://niceeval.com/)
|
|
151
|
+
[NiceEval](https://niceeval.com/) 在绘制瀑布图前把每条 span 归一到 GenAI 语义。映射读取 `gen_ai.operation.name`(标准操作名)和归一后的 `kind`(语义角色):
|
|
152
152
|
|
|
153
|
-
|
|
|
153
|
+
| Span 语义 | 瀑布图内容 |
|
|
154
154
|
|---|---|
|
|
155
155
|
| `gen_ai.operation.name: "chat"`(或 `text_completion` / `embeddings`) | 按**模型调用**着色分类 |
|
|
156
156
|
| `gen_ai.operation.name: "execute_tool"`,或属性里有 `tool_name` | 按**工具执行**着色分类 |
|
|
@@ -178,7 +178,7 @@ function mapMySpans(spans: TraceSpan[]): TraceSpan[] {
|
|
|
178
178
|
// 私有命名 → 标准语义:op 写进 gen_ai.operation.name,kind 定着色
|
|
179
179
|
if (span.name === "my.llm.request") return tagSpan(span, { op: "chat", kind: "model" });
|
|
180
180
|
if (span.name === "my.tool.exec") {
|
|
181
|
-
//
|
|
181
|
+
// 把私有调用 id 映射成 call_id,使工具 I/O 能与 send 事件关联
|
|
182
182
|
const attributes = { ...span.attributes, call_id: String(span.attributes?.["my.callId"] ?? "") };
|
|
183
183
|
return tagSpan({ ...span, attributes }, { op: "execute_tool", kind: "tool" });
|
|
184
184
|
}
|
|
@@ -187,22 +187,22 @@ function mapMySpans(spans: TraceSpan[]): TraceSpan[] {
|
|
|
187
187
|
}
|
|
188
188
|
|
|
189
189
|
export default defineAgent({
|
|
190
|
-
name: "my-
|
|
190
|
+
name: "my-agent",
|
|
191
191
|
spanMapper: mapMySpans,
|
|
192
192
|
async send(input, ctx) { /* 同上文接法 */ },
|
|
193
193
|
});
|
|
194
194
|
```
|
|
195
195
|
|
|
196
|
-
|
|
196
|
+
`niceeval/adapter` 导出的 `mapCodexSpans` 是一个可参考的 `spanMapper` 实现。`spanMapper` 只影响观测展示;映射错误会导致瀑布图分类或着色不准确,不影响断言。
|
|
197
197
|
|
|
198
198
|
## 边界
|
|
199
199
|
|
|
200
|
-
-
|
|
201
|
-
- **多轮会话、HITL 不归 span 管**。span 没有"等人输入"语义,会话续接也是应用协议的事——这两样照常在 `send` 里做(会话续接见[写 send](/zh/
|
|
200
|
+
- **断言数据全部来自 `send`**。工具调用需要映射进 `events`(使用内置转换器或手写映射,见[编写 Send](/zh/tutorials/write-send));用量需要包含在 `send` 返回值中。Span 中存在但 `events` 中缺失的数据不会参与断言。
|
|
201
|
+
- **多轮会话、HITL 不归 span 管**。span 没有"等人输入"语义,会话续接也是应用协议的事——这两样照常在 `send` 里做(会话续接见[写 send](/zh/tutorials/write-send),HITL 概念见 [HITL](/zh/explanation/hitl))。
|
|
202
202
|
- **收不到 span 会有提示**。整个 run 0 span 通常是端点没接上(env 没注入、服务没重启),[NiceEval](https://niceeval.com/) 会在日志里提示;瀑布图为空,断言照常判。
|
|
203
203
|
|
|
204
204
|
## 相关阅读
|
|
205
205
|
|
|
206
|
-
- [接入你的
|
|
207
|
-
- [
|
|
208
|
-
- [事件流参考](/zh/reference/events) ——
|
|
206
|
+
- [接入你的 Agent](/zh/tutorials/connect-your-agent) —— `send` 事件映射与断言的数据来源。
|
|
207
|
+
- [编写 Send](/zh/tutorials/write-send) —— 手写 Adapter 的完整教程,第六步是本页的 Adapter 侧配置。
|
|
208
|
+
- [事件流参考](/zh/reference/events) —— 断言读取的事件结构。
|