niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,600 @@
|
|
|
1
|
+
// Attempt 详情组件族的装配点(docs/feature/reports/library/attempt-detail.md)。11 个叶子
|
|
2
|
+
// 组件都是同一份 spec/data 判别联合(AttemptSectionProps<Data>):省略 input 时取当前
|
|
3
|
+
// attempt-input page 注入的 evidence;放在 scope-input page 且未显式传 input/data 时
|
|
4
|
+
// resolve 报完整用户反馈并指引移到 attempt-input page 或传入 evidence。
|
|
5
|
+
// AttemptAssessment / AttemptDetail 是组合组件,只装配叶子,不产生新的 data 或渲染面。
|
|
6
|
+
|
|
7
|
+
import type { ReactNode } from "react";
|
|
8
|
+
import { defineComponent, type ReportComponent, type ResolveContext, type TextContext, type WebContext } from "../../definition/tree.ts";
|
|
9
|
+
import { Col } from "../../definition/primitives.tsx";
|
|
10
|
+
import type { AttemptEvidence } from "../../../results/attempt-evidence.ts";
|
|
11
|
+
import { arrayProblem, dataShapeError, isObject } from "../shared.ts";
|
|
12
|
+
import type {
|
|
13
|
+
AttemptAssertionsData,
|
|
14
|
+
AttemptConversationData,
|
|
15
|
+
AttemptDiagnosticsData,
|
|
16
|
+
AttemptDiffData,
|
|
17
|
+
AttemptErrorData,
|
|
18
|
+
AttemptFixPromptData,
|
|
19
|
+
AttemptSourceData,
|
|
20
|
+
AttemptSummaryData,
|
|
21
|
+
AttemptTimelineData,
|
|
22
|
+
AttemptTraceData,
|
|
23
|
+
AttemptUsageData,
|
|
24
|
+
} from "../../model/types.ts";
|
|
25
|
+
import {
|
|
26
|
+
attemptAssertionsData,
|
|
27
|
+
attemptConversationData,
|
|
28
|
+
attemptDiagnosticsData,
|
|
29
|
+
attemptDiffData,
|
|
30
|
+
attemptErrorData,
|
|
31
|
+
attemptFixPromptData,
|
|
32
|
+
attemptSourceData,
|
|
33
|
+
attemptSummaryData,
|
|
34
|
+
attemptTimelineData,
|
|
35
|
+
attemptTraceData,
|
|
36
|
+
attemptUsageData,
|
|
37
|
+
} from "./compute.ts";
|
|
38
|
+
import {
|
|
39
|
+
attemptAssertionsText,
|
|
40
|
+
attemptConversationText,
|
|
41
|
+
attemptDiagnosticsText,
|
|
42
|
+
attemptDiffText,
|
|
43
|
+
attemptErrorText,
|
|
44
|
+
attemptFixPromptText,
|
|
45
|
+
attemptSourceText,
|
|
46
|
+
attemptSummaryText,
|
|
47
|
+
attemptTimelineText,
|
|
48
|
+
attemptTraceText,
|
|
49
|
+
attemptUsageText,
|
|
50
|
+
} from "./faces.ts";
|
|
51
|
+
import { AttemptSummary as AttemptSummaryWeb } from "./AttemptSummary.tsx";
|
|
52
|
+
import { AttemptError as AttemptErrorWeb } from "./AttemptError.tsx";
|
|
53
|
+
import { AttemptAssertions as AttemptAssertionsWeb } from "./AttemptAssertions.tsx";
|
|
54
|
+
import { AttemptSource as AttemptSourceWeb } from "./AttemptSource.tsx";
|
|
55
|
+
import { AttemptFixPrompt as AttemptFixPromptWeb } from "./AttemptFixPrompt.tsx";
|
|
56
|
+
import { AttemptTimeline as AttemptTimelineWeb } from "./AttemptTimeline.tsx";
|
|
57
|
+
import { AttemptConversation as AttemptConversationWeb } from "./AttemptConversation.tsx";
|
|
58
|
+
import { AttemptDiagnostics as AttemptDiagnosticsWeb } from "./AttemptDiagnostics.tsx";
|
|
59
|
+
import { AttemptUsage as AttemptUsageWeb } from "./AttemptUsage.tsx";
|
|
60
|
+
import { AttemptTrace as AttemptTraceWeb } from "./AttemptTrace.tsx";
|
|
61
|
+
import { AttemptDiff as AttemptDiffWeb } from "./AttemptDiff.tsx";
|
|
62
|
+
|
|
63
|
+
// ───────────────────────── spec / data 判别联合 ─────────────────────────
|
|
64
|
+
|
|
65
|
+
export type AttemptSectionProps<Data> =
|
|
66
|
+
| { input?: AttemptEvidence; data?: never; className?: string }
|
|
67
|
+
| { data: Data; input?: never; className?: string };
|
|
68
|
+
|
|
69
|
+
interface AttemptComponentDef<Data> {
|
|
70
|
+
name: string;
|
|
71
|
+
dataFnName: string;
|
|
72
|
+
shapeName: string;
|
|
73
|
+
dataFn: (evidence: AttemptEvidence) => Data | null;
|
|
74
|
+
/** 只在 data !== null 时调用。 */
|
|
75
|
+
validate: (data: unknown) => string | null;
|
|
76
|
+
web(props: { data: Data | null; className?: string }, ctx: WebContext): ReactNode;
|
|
77
|
+
text(props: { data: Data | null; className?: string }, ctx: TextContext): string;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* 11 个叶子组件共用的装配:resolve 决定 evidence 来源(显式 data > 显式 input > 当前
|
|
82
|
+
* attempt-input page 注入的 evidence),不在 scope-input page 上凭空工作;两面渲染前都
|
|
83
|
+
* 校验 data 结构,版本漂移时报完整用户反馈而不是静默展示错误字段。
|
|
84
|
+
*/
|
|
85
|
+
function makeAttemptComponent<Data>(
|
|
86
|
+
def: AttemptComponentDef<Data>,
|
|
87
|
+
): ReportComponent<AttemptSectionProps<Data>> {
|
|
88
|
+
type Props = Record<string, unknown>;
|
|
89
|
+
type Resolved = { data: Data | null; className?: string };
|
|
90
|
+
|
|
91
|
+
const assertData = (data: unknown): Data | null => {
|
|
92
|
+
if (data === null) return null;
|
|
93
|
+
const problem = def.validate(data);
|
|
94
|
+
if (problem !== null) throw dataShapeError(def.name, def.dataFnName, def.shapeName, problem);
|
|
95
|
+
return data as Data;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const resolve = (props: Props, ctx: ResolveContext): Resolved => {
|
|
99
|
+
if (props.data !== undefined) {
|
|
100
|
+
if (props.input !== undefined) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`<${def.name}> got both \`data\` and \`input\` — the two evidence sources are exclusive and niceeval will not silently pick one. ` +
|
|
103
|
+
`Keep \`data\` (precomputed with ${def.dataFnName}()) and drop \`input\`, or drop \`data\` and let the pipeline compute it from the evidence.`,
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
assertData(props.data);
|
|
107
|
+
return { data: props.data as Data, className: props.className as string | undefined };
|
|
108
|
+
}
|
|
109
|
+
const evidence =
|
|
110
|
+
(props.input as AttemptEvidence | undefined) ?? (ctx.page.input === "attempt" ? ctx.page.evidence : undefined);
|
|
111
|
+
if (evidence === undefined) {
|
|
112
|
+
throw new Error(
|
|
113
|
+
`<${def.name}> needs an attempt: the current page has no locator to derive evidence from (it is a scope-input page, or no page context is active). ` +
|
|
114
|
+
`Move it to an attempt-input page (\`input: "attempt"\`), or pass \`input\` explicitly with an AttemptEvidence.`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
return { data: def.dataFn(evidence), className: props.className as string | undefined };
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const component = defineComponent<Props, Resolved>({
|
|
121
|
+
resolve,
|
|
122
|
+
web: (props, ctx) => {
|
|
123
|
+
assertData(props.data);
|
|
124
|
+
return def.web(props, ctx);
|
|
125
|
+
},
|
|
126
|
+
text: (props, ctx) => {
|
|
127
|
+
assertData(props.data);
|
|
128
|
+
return def.text(props, ctx);
|
|
129
|
+
},
|
|
130
|
+
}) as unknown as ReportComponent<AttemptSectionProps<Data>>;
|
|
131
|
+
component.displayName = def.name;
|
|
132
|
+
return component;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ───────────────────────── 跨叶子复用的字段路径校验 ─────────────────────────
|
|
136
|
+
|
|
137
|
+
/** SourceLoc(src/shared/types.ts):`t.send` / 断言在 eval 源码里的调用点。 */
|
|
138
|
+
function sourceLocProblem(value: unknown, path: string): string | null {
|
|
139
|
+
if (!isObject(value)) return `"${path}" must be a SourceLoc { file, line }`;
|
|
140
|
+
if (typeof value.file !== "string") return `"${path}.file" must be a string`;
|
|
141
|
+
if (typeof value.line !== "number") return `"${path}.line" must be a number`;
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** AssertionResult(src/scoring/types.ts):按 outcome 判别的联合,passed/failed 要 score,unavailable 要 reason。 */
|
|
146
|
+
function assertionResultProblem(value: unknown, path: string): string | null {
|
|
147
|
+
if (!isObject(value)) return `"${path}" must be an AssertionResult object`;
|
|
148
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
149
|
+
if (value.severity !== "gate" && value.severity !== "soft") return `"${path}.severity" must be "gate" or "soft"`;
|
|
150
|
+
if (value.outcome === "passed" || value.outcome === "failed") {
|
|
151
|
+
if (typeof value.score !== "number") return `"${path}.score" must be a number`;
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
if (value.outcome === "unavailable") {
|
|
155
|
+
if (typeof value.reason !== "string") return `"${path}.reason" must be a string`;
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
return `"${path}.outcome" must be "passed" | "failed" | "unavailable"`;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** TraceSpan(src/o11y/types.ts):AttemptTimeline 的 trace 与 AttemptTrace 的 spans 共用。 */
|
|
162
|
+
function traceSpanProblem(value: unknown, path: string): string | null {
|
|
163
|
+
if (!isObject(value)) return `"${path}" must be a TraceSpan { traceId, spanId, name, startMs, endMs }`;
|
|
164
|
+
if (typeof value.traceId !== "string") return `"${path}.traceId" must be a string`;
|
|
165
|
+
if (typeof value.spanId !== "string") return `"${path}.spanId" must be a string`;
|
|
166
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
167
|
+
if (typeof value.startMs !== "number") return `"${path}.startMs" must be a number`;
|
|
168
|
+
if (typeof value.endMs !== "number") return `"${path}.endMs" must be a number`;
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ───────────────────────── AttemptSummary(恒非空) ─────────────────────────
|
|
173
|
+
|
|
174
|
+
/** AttemptIdentity(src/results/locator.ts):locator 派生自的不可变身份元组。 */
|
|
175
|
+
function attemptIdentityProblem(value: unknown, path: string): string | null {
|
|
176
|
+
if (!isObject(value)) {
|
|
177
|
+
return `"${path}" must be an AttemptIdentity { experimentId, snapshotStartedAt, evalId, attempt }`;
|
|
178
|
+
}
|
|
179
|
+
if (typeof value.experimentId !== "string") return `"${path}.experimentId" must be a string`;
|
|
180
|
+
if (typeof value.snapshotStartedAt !== "string") return `"${path}.snapshotStartedAt" must be a string`;
|
|
181
|
+
if (typeof value.evalId !== "string") return `"${path}.evalId" must be a string`;
|
|
182
|
+
if (typeof value.attempt !== "number") return `"${path}.attempt" must be a number`;
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** AttemptEvidenceCapabilities(src/results/attempt-evidence.ts):四个证据切面开关。 */
|
|
187
|
+
function capabilitiesProblem(value: unknown, path: string): string | null {
|
|
188
|
+
if (!isObject(value)) return `"${path}" must be an object { source, execution, timing, diff }`;
|
|
189
|
+
for (const key of ["source", "execution", "timing", "diff"] as const) {
|
|
190
|
+
if (typeof value[key] !== "boolean") return `"${path}.${key}" must be a boolean`;
|
|
191
|
+
}
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function validateSummaryData(data: unknown): string | null {
|
|
196
|
+
if (!isObject(data)) return "expected an object";
|
|
197
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
198
|
+
const identityProblem = attemptIdentityProblem(data.identity, "identity");
|
|
199
|
+
if (identityProblem !== null) return identityProblem;
|
|
200
|
+
if (typeof data.verdict !== "string") return 'missing "verdict" (string)';
|
|
201
|
+
if (typeof data.durationMs !== "number") return '"durationMs" must be a number';
|
|
202
|
+
if (!(data.costUSD === null || typeof data.costUSD === "number")) return '"costUSD" must be a number or null';
|
|
203
|
+
return capabilitiesProblem(data.capabilities, "capabilities");
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export const AttemptSummary = makeAttemptComponent<AttemptSummaryData>({
|
|
207
|
+
name: "AttemptSummary",
|
|
208
|
+
dataFnName: "attemptSummaryData",
|
|
209
|
+
shapeName: "AttemptSummaryData",
|
|
210
|
+
dataFn: attemptSummaryData,
|
|
211
|
+
validate: validateSummaryData,
|
|
212
|
+
web: (props, ctx) => <AttemptSummaryWeb data={props.data as AttemptSummaryData} locale={ctx.locale} className={props.className} />,
|
|
213
|
+
text: (props, ctx) => attemptSummaryText(props.data as AttemptSummaryData, ctx),
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
// ───────────────────────── AttemptError ─────────────────────────
|
|
217
|
+
|
|
218
|
+
export function validateErrorData(data: unknown): string | null {
|
|
219
|
+
if (!isObject(data)) return "expected an object";
|
|
220
|
+
if (typeof data.code !== "string") return '"code" must be a string';
|
|
221
|
+
if (typeof data.message !== "string") return '"message" must be a string';
|
|
222
|
+
if (typeof data.phase !== "string") return '"phase" must be a string';
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export const AttemptError = makeAttemptComponent<AttemptErrorData>({
|
|
227
|
+
name: "AttemptError",
|
|
228
|
+
dataFnName: "attemptErrorData",
|
|
229
|
+
shapeName: "AttemptErrorData",
|
|
230
|
+
dataFn: attemptErrorData,
|
|
231
|
+
validate: validateErrorData,
|
|
232
|
+
web: (props, ctx) => <AttemptErrorWeb data={props.data} className={props.className} />,
|
|
233
|
+
text: (props, ctx) => attemptErrorText(props.data, ctx),
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
// ───────────────────────── AttemptAssertions ─────────────────────────
|
|
237
|
+
|
|
238
|
+
export function validateAssertionsData(data: unknown): string | null {
|
|
239
|
+
if (!isObject(data)) return "expected an object";
|
|
240
|
+
const attentionProblem = arrayProblem(data.attention, "attention", assertionResultProblem);
|
|
241
|
+
if (attentionProblem !== null) return attentionProblem;
|
|
242
|
+
return arrayProblem(data.passedGroups, "passedGroups", (group, path) => {
|
|
243
|
+
if (!isObject(group) || typeof group.group !== "string") {
|
|
244
|
+
return `"${path}" must be an object with a string "group"`;
|
|
245
|
+
}
|
|
246
|
+
return arrayProblem(group.items, `${path}.items`, assertionResultProblem);
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export const AttemptAssertions = makeAttemptComponent<AttemptAssertionsData>({
|
|
251
|
+
name: "AttemptAssertions",
|
|
252
|
+
dataFnName: "attemptAssertionsData",
|
|
253
|
+
shapeName: "AttemptAssertionsData",
|
|
254
|
+
dataFn: attemptAssertionsData,
|
|
255
|
+
validate: validateAssertionsData,
|
|
256
|
+
web: (props, ctx) => <AttemptAssertionsWeb data={props.data} className={props.className} />,
|
|
257
|
+
text: (props, ctx) => attemptAssertionsText(props.data, ctx),
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
// ───────────────────────── AttemptSource ─────────────────────────
|
|
261
|
+
|
|
262
|
+
/** AnnotatedSourceLine(src/results/annotated-source.ts):一行源码 + 映射到这一行的断言 / send 标注。 */
|
|
263
|
+
function annotatedSourceLineProblem(value: unknown, path: string): string | null {
|
|
264
|
+
if (!isObject(value)) return `"${path}" must be an AttemptSourceLineData { line, text, assertions, sends, turns }`;
|
|
265
|
+
if (typeof value.line !== "number") return `"${path}.line" must be a number`;
|
|
266
|
+
if (typeof value.text !== "string") return `"${path}.text" must be a string`;
|
|
267
|
+
const assertionsProblem = arrayProblem(value.assertions, `${path}.assertions`, assertionResultProblem);
|
|
268
|
+
if (assertionsProblem !== null) return assertionsProblem;
|
|
269
|
+
if (!Array.isArray(value.sends)) return `"${path}.sends" must be an array`;
|
|
270
|
+
return arrayProblem(value.turns, `${path}.turns`, sourceTurnProblem);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function sourceTurnProblem(value: unknown, path: string): string | null {
|
|
274
|
+
if (!isObject(value)) return `"${path}" must be an AttemptSourceTurn`;
|
|
275
|
+
if (typeof value.label !== "string") return `"${path}.label" must be a string`;
|
|
276
|
+
if (value.status !== "completed" && value.status !== "failed" && value.status !== "waiting") {
|
|
277
|
+
return `"${path}.status" must be "completed", "failed", or "waiting"`;
|
|
278
|
+
}
|
|
279
|
+
if (value.durationMs !== undefined && typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
280
|
+
if (typeof value.sentText !== "string") return `"${path}.sentText" must be a string`;
|
|
281
|
+
return arrayProblem(value.replies, `${path}.replies`, conversationReplyProblem);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** AnnotatedEvalSourceSummary(src/results/annotated-source.ts):全是计数字段。 */
|
|
285
|
+
function sourceSummaryProblem(value: unknown, path: string): string | null {
|
|
286
|
+
if (!isObject(value)) return `"${path}" must be an AnnotatedEvalSourceSummary`;
|
|
287
|
+
for (const key of [
|
|
288
|
+
"totalAssertions",
|
|
289
|
+
"mappedAssertions",
|
|
290
|
+
"unmappedAssertions",
|
|
291
|
+
"passed",
|
|
292
|
+
"failed",
|
|
293
|
+
"gate",
|
|
294
|
+
"soft",
|
|
295
|
+
"totalLines",
|
|
296
|
+
"annotatedLines",
|
|
297
|
+
] as const) {
|
|
298
|
+
if (typeof value[key] !== "number") return `"${path}.${key}" must be a number`;
|
|
299
|
+
}
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export function validateSourceData(data: unknown): string | null {
|
|
304
|
+
if (!isObject(data)) return "expected an object";
|
|
305
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
306
|
+
if (typeof data.sourcePath !== "string") return 'missing "sourcePath" (string)';
|
|
307
|
+
const linesProblem = arrayProblem(data.lines, "lines", annotatedSourceLineProblem);
|
|
308
|
+
if (linesProblem !== null) return linesProblem;
|
|
309
|
+
const unmappedProblem = arrayProblem(data.unmapped, "unmapped", assertionResultProblem);
|
|
310
|
+
if (unmappedProblem !== null) return unmappedProblem;
|
|
311
|
+
const turnsProblem = arrayProblem(data.unlocatedTurns, "unlocatedTurns", sourceTurnProblem);
|
|
312
|
+
if (turnsProblem !== null) return turnsProblem;
|
|
313
|
+
return sourceSummaryProblem(data.summary, "summary");
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export const AttemptSource = makeAttemptComponent<AttemptSourceData>({
|
|
317
|
+
name: "AttemptSource",
|
|
318
|
+
dataFnName: "attemptSourceData",
|
|
319
|
+
shapeName: "AttemptSourceData",
|
|
320
|
+
dataFn: attemptSourceData,
|
|
321
|
+
validate: validateSourceData,
|
|
322
|
+
web: (props, ctx) => <AttemptSourceWeb data={props.data} className={props.className} />,
|
|
323
|
+
text: (props, ctx) => attemptSourceText(props.data, ctx),
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
327
|
+
|
|
328
|
+
export function validateFixPromptData(data: unknown): string | null {
|
|
329
|
+
if (!isObject(data)) return "expected an object";
|
|
330
|
+
if (typeof data.prompt !== "string") return 'missing "prompt" (string)';
|
|
331
|
+
return null;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
export const AttemptFixPrompt = makeAttemptComponent<AttemptFixPromptData>({
|
|
335
|
+
name: "AttemptFixPrompt",
|
|
336
|
+
dataFnName: "attemptFixPromptData",
|
|
337
|
+
shapeName: "AttemptFixPromptData",
|
|
338
|
+
dataFn: attemptFixPromptData,
|
|
339
|
+
validate: validateFixPromptData,
|
|
340
|
+
web: (props, ctx) => <AttemptFixPromptWeb data={props.data} className={props.className} />,
|
|
341
|
+
text: (props, ctx) => attemptFixPromptText(props.data, ctx),
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
// ───────────────────────── AttemptTimeline ─────────────────────────
|
|
345
|
+
|
|
346
|
+
/** PhaseTiming(src/runner/types.ts):runner 阶段计时,按执行顺序。 */
|
|
347
|
+
function phaseTimingProblem(value: unknown, path: string): string | null {
|
|
348
|
+
if (!isObject(value)) return `"${path}" must be a PhaseTiming { name, durationMs }`;
|
|
349
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
350
|
+
if (typeof value.durationMs !== "number") return `"${path}.durationMs" must be a number`;
|
|
351
|
+
return null;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
export function validateTimelineData(data: unknown): string | null {
|
|
355
|
+
if (!isObject(data)) return "expected an object";
|
|
356
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
357
|
+
const phasesProblem = arrayProblem(data.phases, "phases", phaseTimingProblem);
|
|
358
|
+
if (phasesProblem !== null) return phasesProblem;
|
|
359
|
+
if (data.trace !== null) return arrayProblem(data.trace, "trace", traceSpanProblem);
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
export const AttemptTimeline = makeAttemptComponent<AttemptTimelineData>({
|
|
364
|
+
name: "AttemptTimeline",
|
|
365
|
+
dataFnName: "attemptTimelineData",
|
|
366
|
+
shapeName: "AttemptTimelineData",
|
|
367
|
+
dataFn: attemptTimelineData,
|
|
368
|
+
validate: validateTimelineData,
|
|
369
|
+
web: (props, ctx) => <AttemptTimelineWeb data={props.data} className={props.className} />,
|
|
370
|
+
text: (props, ctx) => attemptTimelineText(props.data, ctx),
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
// ───────────────────────── AttemptConversation ─────────────────────────
|
|
374
|
+
|
|
375
|
+
const CONVERSATION_REPLY_KINDS = [
|
|
376
|
+
"assistant",
|
|
377
|
+
"user",
|
|
378
|
+
"thinking",
|
|
379
|
+
"error",
|
|
380
|
+
"tool",
|
|
381
|
+
"skill",
|
|
382
|
+
"subagent",
|
|
383
|
+
"input",
|
|
384
|
+
"compaction",
|
|
385
|
+
"raw",
|
|
386
|
+
];
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* AttemptConversationReply(src/report/model/types.ts):按 `kind` 判别的联合,每支自己的必填
|
|
390
|
+
* 字段各自校验——`raw` 是未识别事件类型的兜底分支,不吞没其余 kind 的校验。
|
|
391
|
+
*/
|
|
392
|
+
function conversationReplyProblem(value: unknown, path: string): string | null {
|
|
393
|
+
if (!isObject(value)) return `"${path}" must be an AttemptConversationReply object`;
|
|
394
|
+
switch (value.kind) {
|
|
395
|
+
case "assistant":
|
|
396
|
+
case "user":
|
|
397
|
+
case "thinking":
|
|
398
|
+
case "error":
|
|
399
|
+
if (typeof value.text !== "string") return `"${path}.text" must be a string`;
|
|
400
|
+
return null;
|
|
401
|
+
case "tool":
|
|
402
|
+
if (typeof value.callId !== "string") return `"${path}.callId" must be a string`;
|
|
403
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
404
|
+
if (!("input" in value)) return `"${path}.input" is required`;
|
|
405
|
+
return null;
|
|
406
|
+
case "skill":
|
|
407
|
+
if (typeof value.skill !== "string") return `"${path}.skill" must be a string`;
|
|
408
|
+
return null;
|
|
409
|
+
case "subagent":
|
|
410
|
+
if (typeof value.callId !== "string") return `"${path}.callId" must be a string`;
|
|
411
|
+
if (typeof value.name !== "string") return `"${path}.name" must be a string`;
|
|
412
|
+
return null;
|
|
413
|
+
case "input":
|
|
414
|
+
if (!isObject(value.request)) return `"${path}.request" must be an InputRequest object`;
|
|
415
|
+
return null;
|
|
416
|
+
case "compaction":
|
|
417
|
+
return null;
|
|
418
|
+
case "raw":
|
|
419
|
+
if (!("raw" in value)) return `"${path}.raw" is required`;
|
|
420
|
+
return null;
|
|
421
|
+
default:
|
|
422
|
+
return `"${path}.kind" must be one of ${JSON.stringify(CONVERSATION_REPLY_KINDS)}`;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function conversationRoundProblem(value: unknown, path: string): string | null {
|
|
427
|
+
if (!isObject(value)) return `"${path}" must be an AttemptConversationRound { sentText, replies }`;
|
|
428
|
+
if (typeof value.sentText !== "string") return `"${path}.sentText" must be a string`;
|
|
429
|
+
if (value.loc !== undefined) {
|
|
430
|
+
const locProblem = sourceLocProblem(value.loc, `${path}.loc`);
|
|
431
|
+
if (locProblem !== null) return locProblem;
|
|
432
|
+
}
|
|
433
|
+
return arrayProblem(value.replies, `${path}.replies`, conversationReplyProblem);
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
export function validateConversationData(data: unknown): string | null {
|
|
437
|
+
if (!isObject(data)) return "expected an object";
|
|
438
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
439
|
+
return arrayProblem(data.rounds, "rounds", conversationRoundProblem);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
export const AttemptConversation = makeAttemptComponent<AttemptConversationData>({
|
|
443
|
+
name: "AttemptConversation",
|
|
444
|
+
dataFnName: "attemptConversationData",
|
|
445
|
+
shapeName: "AttemptConversationData",
|
|
446
|
+
dataFn: attemptConversationData,
|
|
447
|
+
validate: validateConversationData,
|
|
448
|
+
web: (props, ctx) => <AttemptConversationWeb data={props.data} className={props.className} />,
|
|
449
|
+
text: (props, ctx) => attemptConversationText(props.data, ctx),
|
|
450
|
+
});
|
|
451
|
+
|
|
452
|
+
// ───────────────────────── AttemptDiagnostics ─────────────────────────
|
|
453
|
+
|
|
454
|
+
/** DiagnosticRecord(src/runner/types.ts):`level` 是消息严重度,不是 verdict 的别名。 */
|
|
455
|
+
function diagnosticRecordProblem(value: unknown, path: string): string | null {
|
|
456
|
+
if (!isObject(value)) return `"${path}" must be a DiagnosticRecord { code, level, message, phase }`;
|
|
457
|
+
if (typeof value.code !== "string") return `"${path}.code" must be a string`;
|
|
458
|
+
if (value.level !== "warning" && value.level !== "error") return `"${path}.level" must be "warning" or "error"`;
|
|
459
|
+
if (typeof value.message !== "string") return `"${path}.message" must be a string`;
|
|
460
|
+
if (typeof value.phase !== "string") return `"${path}.phase" must be a string`;
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export function validateDiagnosticsData(data: unknown): string | null {
|
|
465
|
+
if (!isObject(data)) return "expected an object";
|
|
466
|
+
return arrayProblem(data.groups, "groups", (group, path) => {
|
|
467
|
+
if (!isObject(group) || typeof group.phase !== "string") {
|
|
468
|
+
return `"${path}" must be an object with a string "phase"`;
|
|
469
|
+
}
|
|
470
|
+
return arrayProblem(group.items, `${path}.items`, diagnosticRecordProblem);
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
export const AttemptDiagnostics = makeAttemptComponent<AttemptDiagnosticsData>({
|
|
475
|
+
name: "AttemptDiagnostics",
|
|
476
|
+
dataFnName: "attemptDiagnosticsData",
|
|
477
|
+
shapeName: "AttemptDiagnosticsData",
|
|
478
|
+
dataFn: attemptDiagnosticsData,
|
|
479
|
+
validate: validateDiagnosticsData,
|
|
480
|
+
web: (props, ctx) => <AttemptDiagnosticsWeb data={props.data} className={props.className} />,
|
|
481
|
+
text: (props, ctx) => attemptDiagnosticsText(props.data, ctx),
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
// ───────────────────────── AttemptUsage ─────────────────────────
|
|
485
|
+
|
|
486
|
+
/** Usage(src/o11y/types.ts):cacheReadTokens / cacheWriteTokens / requests / costUSD 均可选。 */
|
|
487
|
+
function usageProblem(value: unknown, path: string): string | null {
|
|
488
|
+
if (!isObject(value)) return `"${path}" must be a Usage { inputTokens, outputTokens }`;
|
|
489
|
+
if (typeof value.inputTokens !== "number") return `"${path}.inputTokens" must be a number`;
|
|
490
|
+
if (typeof value.outputTokens !== "number") return `"${path}.outputTokens" must be a number`;
|
|
491
|
+
return null;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
export function validateUsageData(data: unknown): string | null {
|
|
495
|
+
if (!isObject(data)) return "expected an object";
|
|
496
|
+
const usageProb = usageProblem(data.usage, "usage");
|
|
497
|
+
if (usageProb !== null) return usageProb;
|
|
498
|
+
if (!(data.costUSD === null || typeof data.costUSD === "number")) return '"costUSD" must be a number or null';
|
|
499
|
+
return null;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
export const AttemptUsage = makeAttemptComponent<AttemptUsageData>({
|
|
503
|
+
name: "AttemptUsage",
|
|
504
|
+
dataFnName: "attemptUsageData",
|
|
505
|
+
shapeName: "AttemptUsageData",
|
|
506
|
+
dataFn: attemptUsageData,
|
|
507
|
+
validate: validateUsageData,
|
|
508
|
+
web: (props, ctx) => <AttemptUsageWeb data={props.data} className={props.className} />,
|
|
509
|
+
text: (props, ctx) => attemptUsageText(props.data, ctx),
|
|
510
|
+
});
|
|
511
|
+
|
|
512
|
+
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
513
|
+
|
|
514
|
+
export function validateTraceData(data: unknown): string | null {
|
|
515
|
+
if (!isObject(data)) return "expected an object";
|
|
516
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
517
|
+
return arrayProblem(data.spans, "spans", traceSpanProblem);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
export const AttemptTrace = makeAttemptComponent<AttemptTraceData>({
|
|
521
|
+
name: "AttemptTrace",
|
|
522
|
+
dataFnName: "attemptTraceData",
|
|
523
|
+
shapeName: "AttemptTraceData",
|
|
524
|
+
dataFn: attemptTraceData,
|
|
525
|
+
validate: validateTraceData,
|
|
526
|
+
web: (props, ctx) => <AttemptTraceWeb data={props.data} className={props.className} />,
|
|
527
|
+
text: (props, ctx) => attemptTraceText(props.data, ctx),
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
// ───────────────────────── AttemptDiff ─────────────────────────
|
|
531
|
+
|
|
532
|
+
const DIFF_FILE_NET = ["added", "modified", "deleted"];
|
|
533
|
+
|
|
534
|
+
/** AttemptDiffFileEntry(src/report/model/types.ts):`net` 恒 !== "none"(净无变化不进这份列表)。 */
|
|
535
|
+
function diffFileEntryProblem(value: unknown, path: string): string | null {
|
|
536
|
+
if (!isObject(value)) return `"${path}" must be an AttemptDiffFileEntry { path, net, lines, windows }`;
|
|
537
|
+
if (typeof value.path !== "string") return `"${path}.path" must be a string`;
|
|
538
|
+
if (typeof value.net !== "string" || !DIFF_FILE_NET.includes(value.net)) {
|
|
539
|
+
return `"${path}.net" must be one of ${JSON.stringify(DIFF_FILE_NET)}`;
|
|
540
|
+
}
|
|
541
|
+
if (!isObject(value.lines) || typeof value.lines.added !== "number" || typeof value.lines.deleted !== "number") {
|
|
542
|
+
return `"${path}.lines" must be an object { added, deleted }`;
|
|
543
|
+
}
|
|
544
|
+
if (!Array.isArray(value.windows)) return `"${path}.windows" must be an array`;
|
|
545
|
+
return null;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
export function validateDiffData(data: unknown): string | null {
|
|
549
|
+
if (!isObject(data)) return "expected an object";
|
|
550
|
+
if (typeof data.locator !== "string") return 'missing "locator" (string)';
|
|
551
|
+
return arrayProblem(data.files, "files", diffFileEntryProblem);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
export const AttemptDiff = makeAttemptComponent<AttemptDiffData>({
|
|
555
|
+
name: "AttemptDiff",
|
|
556
|
+
dataFnName: "attemptDiffData",
|
|
557
|
+
shapeName: "AttemptDiffData",
|
|
558
|
+
dataFn: attemptDiffData,
|
|
559
|
+
validate: validateDiffData,
|
|
560
|
+
web: (props, ctx) => <AttemptDiffWeb data={props.data} className={props.className} />,
|
|
561
|
+
text: (props, ctx) => attemptDiffText(props.data, ctx),
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
// ───────────────────────── 两个普通组合组件 ─────────────────────────
|
|
565
|
+
|
|
566
|
+
/** source / assertions fallback:有 source 放 AttemptSource,否则放 AttemptAssertions。 */
|
|
567
|
+
export const AttemptAssessment = defineComponent((_props: Record<string, never>, ctx) => {
|
|
568
|
+
if (ctx.page.input !== "attempt") {
|
|
569
|
+
throw new Error(
|
|
570
|
+
"AttemptAssessment requires an attempt-input page (input: \"attempt\") — it reads ctx.page.evidence to choose between AttemptSource and AttemptAssertions.",
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
return (
|
|
574
|
+
<Col>
|
|
575
|
+
<AttemptError />
|
|
576
|
+
{ctx.page.evidence.capabilities.source ? <AttemptSource /> : <AttemptAssertions />}
|
|
577
|
+
</Col>
|
|
578
|
+
);
|
|
579
|
+
});
|
|
580
|
+
AttemptAssessment.displayName = "AttemptAssessment";
|
|
581
|
+
|
|
582
|
+
/** 内建排列顺序;有 source 时回复已按 loc 展开在 AttemptSource 行内,不再重复一份 round 卡。 */
|
|
583
|
+
export const AttemptDetail = defineComponent((_props: Record<string, never>, ctx) => {
|
|
584
|
+
const conversationLivesInSource =
|
|
585
|
+
ctx.page.input === "attempt" && ctx.page.evidence.capabilities.source && ctx.page.evidence.evalSource !== null;
|
|
586
|
+
return (
|
|
587
|
+
<Col>
|
|
588
|
+
<AttemptSummary />
|
|
589
|
+
<AttemptAssessment />
|
|
590
|
+
<AttemptFixPrompt />
|
|
591
|
+
<AttemptTimeline />
|
|
592
|
+
<AttemptDiagnostics />
|
|
593
|
+
<AttemptUsage />
|
|
594
|
+
{conversationLivesInSource ? null : <AttemptConversation />}
|
|
595
|
+
<AttemptTrace />
|
|
596
|
+
<AttemptDiff />
|
|
597
|
+
</Col>
|
|
598
|
+
);
|
|
599
|
+
});
|
|
600
|
+
AttemptDetail.displayName = "AttemptDetail";
|