niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -1,310 +0,0 @@
|
|
|
1
|
-
import { useCallback, useMemo, useState, type ReactNode } from "react";
|
|
2
|
-
import { AlertCircle, CheckCircle2, ChevronRight, MessageCircle, XCircle } from "lucide-react";
|
|
3
|
-
import type { T } from "../shared.ts";
|
|
4
|
-
import type { Assertion, CodeSource, SourceTurn, TranscriptEvent } from "../types.ts";
|
|
5
|
-
import { highlightTs, indexAsserts, indexTurns, locKey } from "../lib/transcript-data.tsx";
|
|
6
|
-
import { formatScore } from "../lib/format.ts";
|
|
7
|
-
import { InputBlock, RawEventBlock, ToolBlock, Transcript } from "./Transcript.tsx";
|
|
8
|
-
|
|
9
|
-
/** soft 断言没过阈值不影响 verdict,颜色上跟 gate 失败(红)区分开,用 warn(黄);
|
|
10
|
-
* unavailable 用独立第三态(非红非绿)。 */
|
|
11
|
-
export function assertTone(a: Assertion): "good" | "warn" | "bad" | "na" {
|
|
12
|
-
if (a.outcome === "unavailable") return "na";
|
|
13
|
-
if (a.outcome === "passed") return "good";
|
|
14
|
-
return a.severity === "soft" ? "warn" : "bad";
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export function CodeView({ sources, events, assertions, t }: { sources: CodeSource[]; events: TranscriptEvent[]; assertions: Assertion[]; t: T }) {
|
|
18
|
-
const turns = useMemo(() => indexTurns(events), [events]);
|
|
19
|
-
const asserts = useMemo(() => indexAsserts(assertions), [assertions]);
|
|
20
|
-
// 断言明细单点住在源码行(契约 docs/feature/reports/view.md「Attempt 详情」):
|
|
21
|
-
// 第一条失败行默认展开,打开页面即见「为什么失败」,不用逐行找红行点开。
|
|
22
|
-
const [open, setOpen] = useState<Set<string>>(() => {
|
|
23
|
-
const first = assertions.find((a) => a.outcome === "failed" && a.loc);
|
|
24
|
-
return new Set(first?.loc ? [locKey(first.loc.file, first.loc.line)] : []);
|
|
25
|
-
});
|
|
26
|
-
const toggle = useCallback((k: string) => {
|
|
27
|
-
setOpen((prev) => {
|
|
28
|
-
const next = new Set(prev);
|
|
29
|
-
if (next.has(k)) next.delete(k); else next.add(k);
|
|
30
|
-
return next;
|
|
31
|
-
});
|
|
32
|
-
}, []);
|
|
33
|
-
|
|
34
|
-
// 哪些 loc 被源码行覆盖到了;没覆盖到的(读不到源码的文件)放底部兜底。
|
|
35
|
-
const sourceKeys = new Set<string>();
|
|
36
|
-
for (const f of sources) {
|
|
37
|
-
const n = f.content.split("\n").length;
|
|
38
|
-
for (let i = 1; i <= n; i++) sourceKeys.add(locKey(f.path, i));
|
|
39
|
-
}
|
|
40
|
-
const orphanAsserts = [...asserts.byKey.entries()]
|
|
41
|
-
.filter(([k]) => !sourceKeys.has(k))
|
|
42
|
-
.flatMap(([, v]) => v)
|
|
43
|
-
.concat(asserts.noloc);
|
|
44
|
-
|
|
45
|
-
return (
|
|
46
|
-
<div className="codeview">
|
|
47
|
-
{sources.map((file) => (
|
|
48
|
-
<CodeFile
|
|
49
|
-
key={file.path}
|
|
50
|
-
file={file}
|
|
51
|
-
turns={turns.byKey}
|
|
52
|
-
asserts={asserts.byKey}
|
|
53
|
-
open={open}
|
|
54
|
-
toggle={toggle}
|
|
55
|
-
t={t}
|
|
56
|
-
/>
|
|
57
|
-
))}
|
|
58
|
-
{orphanAsserts.length ? (
|
|
59
|
-
<div className="code-orphans">
|
|
60
|
-
<div className="code-orphans-head">{t("code.otherAssertions")}</div>
|
|
61
|
-
<AssertDetail asserts={orphanAsserts} t={t} />
|
|
62
|
-
</div>
|
|
63
|
-
) : null}
|
|
64
|
-
</div>
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
export function CodeFile({
|
|
69
|
-
file,
|
|
70
|
-
turns,
|
|
71
|
-
asserts,
|
|
72
|
-
open,
|
|
73
|
-
toggle,
|
|
74
|
-
t,
|
|
75
|
-
}: {
|
|
76
|
-
file: CodeSource;
|
|
77
|
-
turns: Map<string, SourceTurn>;
|
|
78
|
-
asserts: Map<string, Assertion[]>;
|
|
79
|
-
open: Set<string>;
|
|
80
|
-
toggle: (key: string) => void;
|
|
81
|
-
t: T;
|
|
82
|
-
}) {
|
|
83
|
-
const lines = file.content.replace(/\n$/, "").split("\n");
|
|
84
|
-
return (
|
|
85
|
-
<div className="code-file">
|
|
86
|
-
<div className="code-file-head">{file.path}</div>
|
|
87
|
-
<div className="code-lines">
|
|
88
|
-
{lines.map((text: string, i: number) => {
|
|
89
|
-
const n = i + 1;
|
|
90
|
-
const k = locKey(file.path, n);
|
|
91
|
-
return (
|
|
92
|
-
<CodeLine
|
|
93
|
-
key={n}
|
|
94
|
-
n={n}
|
|
95
|
-
text={text}
|
|
96
|
-
turn={turns.get(k)}
|
|
97
|
-
asserts={asserts.get(k)}
|
|
98
|
-
isOpen={open.has(k)}
|
|
99
|
-
onToggle={() => toggle(k)}
|
|
100
|
-
t={t}
|
|
101
|
-
/>
|
|
102
|
-
);
|
|
103
|
-
})}
|
|
104
|
-
</div>
|
|
105
|
-
</div>
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
export function CodeLine({
|
|
110
|
-
n,
|
|
111
|
-
text,
|
|
112
|
-
turn,
|
|
113
|
-
asserts,
|
|
114
|
-
isOpen,
|
|
115
|
-
onToggle,
|
|
116
|
-
t,
|
|
117
|
-
}: {
|
|
118
|
-
n: number;
|
|
119
|
-
text: string;
|
|
120
|
-
turn?: SourceTurn;
|
|
121
|
-
asserts?: Assertion[];
|
|
122
|
-
isOpen: boolean;
|
|
123
|
-
onToggle: () => void;
|
|
124
|
-
t: T;
|
|
125
|
-
}) {
|
|
126
|
-
const hasReply = !!turn;
|
|
127
|
-
const hasAsserts = !!(asserts && asserts.length);
|
|
128
|
-
// 只有 gate 断言没过才算这一行真的"fail";只剩 soft 断言没过阈值时是"warn"(不影响 verdict);
|
|
129
|
-
// 只剩 unavailable 时是第三态 "na"(非红非绿)。
|
|
130
|
-
const status = hasAsserts
|
|
131
|
-
? asserts?.every((a: Assertion) => a.outcome === "passed")
|
|
132
|
-
? "pass"
|
|
133
|
-
: asserts?.some((a: Assertion) => a.outcome === "failed" && a.severity === "gate")
|
|
134
|
-
? "fail"
|
|
135
|
-
: asserts?.some((a: Assertion) => a.outcome === "failed")
|
|
136
|
-
? "warn"
|
|
137
|
-
: "na"
|
|
138
|
-
: null;
|
|
139
|
-
const clickable = hasReply || hasAsserts;
|
|
140
|
-
const rowCls =
|
|
141
|
-
"code-line" +
|
|
142
|
-
(status ? ` line-${status}` : "") +
|
|
143
|
-
(hasReply && !status ? " line-send" : "") +
|
|
144
|
-
(clickable ? " line-clickable" : "") +
|
|
145
|
-
(isOpen ? " is-open" : "");
|
|
146
|
-
return (
|
|
147
|
-
<>
|
|
148
|
-
<div
|
|
149
|
-
className={rowCls}
|
|
150
|
-
onClick={clickable ? onToggle : undefined}
|
|
151
|
-
role={clickable ? "button" : undefined}
|
|
152
|
-
tabIndex={clickable ? 0 : undefined}
|
|
153
|
-
onKeyDown={clickable ? (e) => { if (e.key === "Enter" || e.key === " ") { e.preventDefault(); onToggle(); } } : undefined}
|
|
154
|
-
>
|
|
155
|
-
<span className="ln">{n}</span>
|
|
156
|
-
<span className="gmark">
|
|
157
|
-
{hasAsserts ? (
|
|
158
|
-
status === "pass" ? (
|
|
159
|
-
<CheckCircle2 className="gstat good" aria-hidden="true" />
|
|
160
|
-
) : status === "warn" ? (
|
|
161
|
-
<AlertCircle className="gstat warn" aria-hidden="true" />
|
|
162
|
-
) : status === "na" ? (
|
|
163
|
-
<AlertCircle className="gstat na" aria-hidden="true" />
|
|
164
|
-
) : (
|
|
165
|
-
<XCircle className="gstat bad" aria-hidden="true" />
|
|
166
|
-
)
|
|
167
|
-
) : hasReply ? (
|
|
168
|
-
<MessageCircle className="gsend" aria-hidden="true" />
|
|
169
|
-
) : null}
|
|
170
|
-
</span>
|
|
171
|
-
<code className="ctext">{highlightTs(text)}</code>
|
|
172
|
-
<span className="lbadges">
|
|
173
|
-
{hasAsserts ? asserts?.map((a: Assertion, i: number) => <AssertBadge key={i} a={a} />) : null}
|
|
174
|
-
{hasReply ? (
|
|
175
|
-
<span className="reply-hint">{isOpen ? t("code.hide") : t("code.reply")}</span>
|
|
176
|
-
) : clickable ? (
|
|
177
|
-
<ChevronRight className={`line-chev${isOpen ? " is-open" : ""}`} aria-hidden="true" />
|
|
178
|
-
) : null}
|
|
179
|
-
</span>
|
|
180
|
-
</div>
|
|
181
|
-
{isOpen && hasReply && turn ? <ReplyPanel turn={turn} t={t} /> : null}
|
|
182
|
-
{isOpen && hasAsserts && asserts ? <AssertDetail asserts={asserts} t={t} /> : null}
|
|
183
|
-
</>
|
|
184
|
-
);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/** 行尾分数徽章:judge / 带阈值的断言显示分数(过绿不过红);纯 gate 断言靠行色 + gutter 勾叉。 */
|
|
188
|
-
export function AssertBadge({ a }: { a: Assertion }) {
|
|
189
|
-
if (a.outcome === "unavailable") return <span className="abadge na">n/a</span>;
|
|
190
|
-
const showPct = a.threshold !== undefined || (a.score > 0 && a.score < 1);
|
|
191
|
-
if (!showPct) return null;
|
|
192
|
-
return (
|
|
193
|
-
<span className={`abadge ${assertTone(a)}`}>
|
|
194
|
-
{formatScore(a.score)}
|
|
195
|
-
{a.threshold !== undefined ? <span className="abadge-th">/{formatScore(a.threshold)}</span> : null}
|
|
196
|
-
</span>
|
|
197
|
-
);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
export function ReplyPanel({ turn, t }: { turn: SourceTurn; t: T }) {
|
|
201
|
-
if (!turn.replies.length) return <div className="line-detail reply-empty">{t("code.noReply")}</div>;
|
|
202
|
-
return (
|
|
203
|
-
<div className="line-detail reply-panel">
|
|
204
|
-
{turn.replies.map((r, j) => {
|
|
205
|
-
if (r.kind === "text")
|
|
206
|
-
return (
|
|
207
|
-
<div key={j} className="reply-assistant">
|
|
208
|
-
<span className="reply-role">{t("transcript.assistant")}</span>
|
|
209
|
-
<div className="reply-text">{r.text}</div>
|
|
210
|
-
</div>
|
|
211
|
-
);
|
|
212
|
-
if (r.kind === "user")
|
|
213
|
-
return (
|
|
214
|
-
<div key={j} className="reply-user">
|
|
215
|
-
<span className="reply-role">{t("transcript.user")}</span>
|
|
216
|
-
<div className="reply-text">{r.text}</div>
|
|
217
|
-
</div>
|
|
218
|
-
);
|
|
219
|
-
if (r.kind === "thinking")
|
|
220
|
-
return (
|
|
221
|
-
<details key={j} className="reply-think">
|
|
222
|
-
<summary>{t("transcript.thinking")}</summary>
|
|
223
|
-
<div className="reply-think-text">{r.text}</div>
|
|
224
|
-
</details>
|
|
225
|
-
);
|
|
226
|
-
if (r.kind === "error")
|
|
227
|
-
return <div key={j} className="reply-err">! {r.text}</div>;
|
|
228
|
-
if (r.kind === "skill")
|
|
229
|
-
return (
|
|
230
|
-
<div key={j} className="reply-skill">
|
|
231
|
-
<span className="reply-role">{t("transcript.skillLoaded")}</span> {r.skill}
|
|
232
|
-
</div>
|
|
233
|
-
);
|
|
234
|
-
if (r.kind === "raw") return <RawEventBlock key={j} raw={r.raw} t={t} />;
|
|
235
|
-
if (r.kind === "tool")
|
|
236
|
-
// 和 Transcript 同一个组件:摘要行显示工具名(入参)→ 出参预览,展开看完整出入参。
|
|
237
|
-
return <ToolBlock key={j} call={r.ev} result={r.result} t={t} />;
|
|
238
|
-
if (r.kind === "input") return <InputBlock key={j} event={{ type: "input.requested", request: r.ev.request }} t={t} />;
|
|
239
|
-
return null;
|
|
240
|
-
})}
|
|
241
|
-
</div>
|
|
242
|
-
);
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
export function AssertDetail({ asserts, t, anchor }: { asserts: Assertion[]; t: T; anchor?: (a: Assertion) => ReactNode }) {
|
|
246
|
-
return (
|
|
247
|
-
<div className="line-detail assert-detail">
|
|
248
|
-
{asserts.map((a: Assertion, i: number) => (
|
|
249
|
-
<div key={i} className="assert-row">
|
|
250
|
-
<span className={`abadge ${assertTone(a)}`}>
|
|
251
|
-
{a.outcome === "unavailable" ? t("assert.unavailable") : a.outcome === "passed" ? t("assert.pass") : t("assert.fail")}
|
|
252
|
-
</span>
|
|
253
|
-
<span className="assert-name">
|
|
254
|
-
{a.groupPath?.length ? `${a.groupPath.join(" > ")} · ` : ""}
|
|
255
|
-
{a.name}
|
|
256
|
-
{anchor?.(a)}
|
|
257
|
-
</span>
|
|
258
|
-
{a.optional ? <span className="assert-sev">{t("assert.optional")}</span> : null}
|
|
259
|
-
{a.severity === "soft" ? <span className="assert-sev">{t("assert.soft")}</span> : null}
|
|
260
|
-
{a.outcome !== "unavailable" && a.threshold !== undefined ? (
|
|
261
|
-
<span className="assert-score">
|
|
262
|
-
{formatScore(a.score)} / {formatScore(a.threshold)}
|
|
263
|
-
</span>
|
|
264
|
-
) : null}
|
|
265
|
-
{a.outcome === "unavailable" ? <div className="assert-reason">{a.reason}</div> : null}
|
|
266
|
-
{a.detail ? <div className="assert-reason">{a.detail}</div> : null}
|
|
267
|
-
{a.outcome !== "unavailable" && a.expected !== undefined ? (
|
|
268
|
-
<div className="assert-reason">expected: {a.expected}</div>
|
|
269
|
-
) : null}
|
|
270
|
-
{a.outcome !== "unavailable" && a.received !== undefined ? (
|
|
271
|
-
<div className="assert-reason">received: {a.received}</div>
|
|
272
|
-
) : null}
|
|
273
|
-
{a.outcome !== "unavailable" && a.evidence ? (
|
|
274
|
-
<details className="assert-evidence">
|
|
275
|
-
<summary>{t("assert.evidence")}</summary>
|
|
276
|
-
<pre className="assert-evidence-pre">{a.evidence}</pre>
|
|
277
|
-
</details>
|
|
278
|
-
) : null}
|
|
279
|
-
</div>
|
|
280
|
-
))}
|
|
281
|
-
</div>
|
|
282
|
-
);
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
/**
|
|
286
|
-
* 没源码可叠时(此 run 早于 source-loc,或源码不可读:远程沙箱等)。不退回老的分组视图——
|
|
287
|
-
* 用代码视图同一套视觉语言:一句说明 + checks(绿过/红不过)+ 原始会话流。重跑即可看到代码视图。
|
|
288
|
-
*/
|
|
289
|
-
export function NoSourceBody({ assertions, events, message, t }: { assertions: Assertion[]; events: TranscriptEvent[]; message?: string; t: T }) {
|
|
290
|
-
const checks = assertions || [];
|
|
291
|
-
return (
|
|
292
|
-
<div className="nosource">
|
|
293
|
-
<div className="nosource-note">
|
|
294
|
-
{message ?? t("code.noSource")}
|
|
295
|
-
</div>
|
|
296
|
-
{checks.length ? (
|
|
297
|
-
<div className="nosource-block">
|
|
298
|
-
<div className="nosource-head">{t("code.checks")}</div>
|
|
299
|
-
<AssertDetail asserts={checks} t={t} />
|
|
300
|
-
</div>
|
|
301
|
-
) : null}
|
|
302
|
-
{events?.length ? (
|
|
303
|
-
<div className="nosource-block">
|
|
304
|
-
<div className="nosource-head">{t("code.conversation")}</div>
|
|
305
|
-
<Transcript events={events} t={t} />
|
|
306
|
-
</div>
|
|
307
|
-
) : null}
|
|
308
|
-
</div>
|
|
309
|
-
);
|
|
310
|
-
}
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import React, { useState } from "react";
|
|
2
|
-
import { Check, Copy } from "lucide-react";
|
|
3
|
-
import type { T } from "../shared.ts";
|
|
4
|
-
import type { ViewResult } from "../types.ts";
|
|
5
|
-
import { reasonFor } from "../lib/verdict.ts";
|
|
6
|
-
|
|
7
|
-
/** 修复 prompt 的一条失败条目;路径均相对 view 输入根(默认 `.niceeval/`)。 */
|
|
8
|
-
export interface FixPromptEntry {
|
|
9
|
-
experiment: string;
|
|
10
|
-
evalId: string;
|
|
11
|
-
verdict: string;
|
|
12
|
-
reason: string;
|
|
13
|
-
artifactBase?: string;
|
|
14
|
-
resultPath?: string;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export function toFixPromptEntry(r: ViewResult, experimentLabel: string): FixPromptEntry {
|
|
18
|
-
return {
|
|
19
|
-
experiment: r.experimentId ?? experimentLabel,
|
|
20
|
-
evalId: r.id,
|
|
21
|
-
verdict: r.verdict,
|
|
22
|
-
reason: reasonFor(r),
|
|
23
|
-
artifactBase: r.artifactBase,
|
|
24
|
-
// result.json 与其它 artifact(events.json 等)同目录落盘,resultPath 因此复用同一个
|
|
25
|
-
// artifactBase(不再靠 attemptRef 的 {snapshot, attempt} 两段拼——locator 是不透明的,
|
|
26
|
-
// 浏览器端解不出磁盘路径)。
|
|
27
|
-
resultPath: r.artifactBase ? `${r.artifactBase}/result.json` : undefined,
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* 学 Next.js 16.3 的「Copy prompt」:把失败打包成一段可直接粘给 coding agent 的修复
|
|
33
|
-
* prompt——失败清单 + artifact 路径 + 先读随包文档 / 判断缺陷在哪一侧 / 重跑验证的步骤。
|
|
34
|
-
* prompt 面向 agent,固定英文;按钮文案走界面 i18n。
|
|
35
|
-
*/
|
|
36
|
-
export function buildFixPrompt(entries: FixPromptEntry[]): string {
|
|
37
|
-
const failures = entries
|
|
38
|
-
.map((e, i) =>
|
|
39
|
-
[
|
|
40
|
-
`${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.verdict}`,
|
|
41
|
-
e.reason ? ` reason: ${e.reason}` : null,
|
|
42
|
-
e.artifactBase ? ` artifacts: ${e.artifactBase}/` : null,
|
|
43
|
-
e.resultPath ? ` result: ${e.resultPath}` : null,
|
|
44
|
-
]
|
|
45
|
-
.filter(Boolean)
|
|
46
|
-
.join("\n"),
|
|
47
|
-
)
|
|
48
|
-
.join("\n");
|
|
49
|
-
const experiments = [...new Set(entries.map((e) => e.experiment))].join(" / ");
|
|
50
|
-
return [
|
|
51
|
-
"Fix the failing evals from this niceeval run.",
|
|
52
|
-
"",
|
|
53
|
-
"## Failures",
|
|
54
|
-
failures,
|
|
55
|
-
"",
|
|
56
|
-
"## Steps",
|
|
57
|
-
"1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
|
|
58
|
-
"2. The paths above are relative to the results directory (default `.niceeval/`). For each failure, read `events.json` in its artifacts directory — the full agent transcript including tool calls — plus `trace.json` (execution trace) and `diff.json` (workspace diff) when present, to see what actually happened.",
|
|
59
|
-
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
60
|
-
`4. Re-run: \`npx niceeval exp ${experiments || "<experiment>"} <eval-id-prefix>\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
61
|
-
"5. Run `npx niceeval show` and confirm these failures are gone.",
|
|
62
|
-
].join("\n");
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
// 批量修复 prompt 不再由壳渲染:它是内建报告首页里的 CopyFixPrompt 组件
|
|
66
|
-
// (niceeval/report 的站点组件,prompt 在 resolve 期烘进静态 HTML)。这里只留
|
|
67
|
-
// attempt 弹窗的单条版。
|
|
68
|
-
|
|
69
|
-
/** attempt 弹窗里的单条版:只打包当前 attempt 的失败,供逐条转交 agent。 */
|
|
70
|
-
export function CopyAttemptPrompt({ result, t }: { result: ViewResult; t: T }) {
|
|
71
|
-
const [copied, setCopied] = useState(false);
|
|
72
|
-
if (result.verdict !== "failed" && result.verdict !== "errored") return null;
|
|
73
|
-
|
|
74
|
-
const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
75
|
-
event.stopPropagation();
|
|
76
|
-
try {
|
|
77
|
-
await copyText(buildFixPrompt([toFixPromptEntry(result, result.agent)]));
|
|
78
|
-
setCopied(true);
|
|
79
|
-
setTimeout(() => setCopied(false), 1500);
|
|
80
|
-
} catch {
|
|
81
|
-
setCopied(false);
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
|
|
85
|
-
return (
|
|
86
|
-
<button className={`copy-all-errors${copied ? " is-copied" : ""}`} onClick={copy} title={t("action.copyPrompt")}>
|
|
87
|
-
{copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
|
|
88
|
-
<span>{copied ? t("action.copied") : t("action.copyPrompt")}</span>
|
|
89
|
-
</button>
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export async function copyText(text: string): Promise<void> {
|
|
94
|
-
if (navigator.clipboard?.writeText) {
|
|
95
|
-
await navigator.clipboard.writeText(text);
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
const ta = document.createElement("textarea");
|
|
99
|
-
ta.value = text;
|
|
100
|
-
ta.style.position = "fixed";
|
|
101
|
-
ta.style.opacity = "0";
|
|
102
|
-
document.body.appendChild(ta);
|
|
103
|
-
ta.select();
|
|
104
|
-
document.execCommand("copy");
|
|
105
|
-
ta.remove();
|
|
106
|
-
}
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
import type { ReactNode } from "react";
|
|
2
|
-
import type { T } from "../shared.ts";
|
|
3
|
-
import type { Span, ViewJson } from "../types.ts";
|
|
4
|
-
import { formatDuration } from "../lib/format.ts";
|
|
5
|
-
|
|
6
|
-
export function Trace({ spans, t }: { spans: Span[]; t: T }) {
|
|
7
|
-
if (!spans?.length) return <div className="trace-span-meta">{t("trace.noSpans")}</div>;
|
|
8
|
-
const t0 = Math.min(...spans.map((s) => s.startMs));
|
|
9
|
-
const t1 = Math.max(...spans.map((s) => s.endMs));
|
|
10
|
-
const total = Math.max(1, t1 - t0);
|
|
11
|
-
const byId = new Map(spans.map((s) => [s.spanId, s]));
|
|
12
|
-
const depthOf = (span: Span): number => {
|
|
13
|
-
let depth = 0;
|
|
14
|
-
let cur = span;
|
|
15
|
-
const seen = new Set();
|
|
16
|
-
while (cur && cur.parentSpanId && byId.has(cur.parentSpanId) && !seen.has(cur.spanId)) {
|
|
17
|
-
seen.add(cur.spanId);
|
|
18
|
-
const next = byId.get(cur.parentSpanId);
|
|
19
|
-
if (!next) break;
|
|
20
|
-
cur = next;
|
|
21
|
-
depth++;
|
|
22
|
-
if (depth > 40) break;
|
|
23
|
-
}
|
|
24
|
-
return depth;
|
|
25
|
-
};
|
|
26
|
-
const ordered = [...spans].sort((a, b) => a.startMs - b.startMs || depthOf(a) - depthOf(b));
|
|
27
|
-
return (
|
|
28
|
-
<div className="trace">
|
|
29
|
-
<div className="trace-span-meta">
|
|
30
|
-
{t("trace.total")} {formatDuration(total)} · {spans.length} {t("trace.spans")} · {t("trace.clickDetails")}
|
|
31
|
-
</div>
|
|
32
|
-
{ordered.map((span) => {
|
|
33
|
-
const left = ((span.startMs - t0) / total) * 100;
|
|
34
|
-
const width = Math.max(0.6, ((span.endMs - span.startMs) / total) * 100);
|
|
35
|
-
const kind = span.kind || "other";
|
|
36
|
-
const tone = span.status === "error" ? "bad" : "k-" + kind;
|
|
37
|
-
const detail = spanAttrs(span.attributes);
|
|
38
|
-
const row = (
|
|
39
|
-
<summary className="trace-row">
|
|
40
|
-
<div className="trace-label" style={{ paddingLeft: depthOf(span) * 12 }} title={span.name}>
|
|
41
|
-
{kind !== "other" ? <span className={`kind-chip k-${kind}`}>{kind}</span> : null}
|
|
42
|
-
{span.name}
|
|
43
|
-
</div>
|
|
44
|
-
<div className="trace-track">
|
|
45
|
-
<div className={`trace-bar ${tone}`} style={{ left: `${left}%`, width: `${width}%` }} />
|
|
46
|
-
</div>
|
|
47
|
-
<div className="trace-dur num">{formatDuration(span.endMs - span.startMs)}</div>
|
|
48
|
-
</summary>
|
|
49
|
-
);
|
|
50
|
-
return detail ? (
|
|
51
|
-
<details className="span-d" key={span.spanId}>
|
|
52
|
-
{row}
|
|
53
|
-
{detail}
|
|
54
|
-
</details>
|
|
55
|
-
) : (
|
|
56
|
-
<div className="span-d" key={span.spanId}>
|
|
57
|
-
{row}
|
|
58
|
-
</div>
|
|
59
|
-
);
|
|
60
|
-
})}
|
|
61
|
-
</div>
|
|
62
|
-
);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export function spanAttrs(attrs?: Record<string, ViewJson>): ReactNode {
|
|
66
|
-
if (!attrs) return null;
|
|
67
|
-
const hide = /^(code\.|thread\.|target$|busy_ns$|idle_ns$|rpc\.|app_server\.)/;
|
|
68
|
-
const keys = Object.keys(attrs).filter((k) => !hide.test(k));
|
|
69
|
-
if (!keys.length) return null;
|
|
70
|
-
const io = keys.filter((k) => k.startsWith("io."));
|
|
71
|
-
const rest = keys.filter((k) => !k.startsWith("io.")).sort();
|
|
72
|
-
return (
|
|
73
|
-
<div className="span-attrs">
|
|
74
|
-
{io.map((key) => {
|
|
75
|
-
const label = key.replace(/^io\./, "");
|
|
76
|
-
const value = String(attrs[key]);
|
|
77
|
-
return label === "input" || label === "output" ? (
|
|
78
|
-
<div className="attr-io" key={key}>
|
|
79
|
-
<span className="attr-k">{label}</span>
|
|
80
|
-
<pre className="attr-pre">{value}</pre>
|
|
81
|
-
</div>
|
|
82
|
-
) : (
|
|
83
|
-
<AttrRow key={key} label={label} value={value} />
|
|
84
|
-
);
|
|
85
|
-
})}
|
|
86
|
-
{rest.map((key) => (
|
|
87
|
-
<AttrRow key={key} label={key} value={typeof attrs[key] === "object" ? JSON.stringify(attrs[key]) : String(attrs[key])} />
|
|
88
|
-
))}
|
|
89
|
-
</div>
|
|
90
|
-
);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
export function AttrRow({ label, value }: { label: string; value: string }) {
|
|
94
|
-
return (
|
|
95
|
-
<div className="attr-row">
|
|
96
|
-
<span className="attr-k">{label}</span>
|
|
97
|
-
<span className="attr-v">{value}</span>
|
|
98
|
-
</div>
|
|
99
|
-
);
|
|
100
|
-
}
|
|
@@ -1,157 +0,0 @@
|
|
|
1
|
-
import type { T, ToolBlockCall } from "../shared.ts";
|
|
2
|
-
import type { ObjectRecord, ToolResultEvent, TranscriptEvent } from "../types.ts";
|
|
3
|
-
import { TOOL_VERB, resultBody, toolPrimaryArg } from "../lib/transcript-data.tsx";
|
|
4
|
-
import { prettyJson, previewText, truncate } from "../lib/format.ts";
|
|
5
|
-
|
|
6
|
-
export function Transcript({ events, t }: { events: TranscriptEvent[]; t: T }) {
|
|
7
|
-
if (!Array.isArray(events) || !events.length) return <div className="trace-span-meta">{t("transcript.noEvents")}</div>;
|
|
8
|
-
const resultByCall = new Map<string, ToolResultEvent>();
|
|
9
|
-
for (const event of events) {
|
|
10
|
-
if (event.type === "action.result" || event.type === "subagent.completed") resultByCall.set(event.callId, event);
|
|
11
|
-
}
|
|
12
|
-
const pairedResult = new Set<string>();
|
|
13
|
-
return (
|
|
14
|
-
<div className="transcript">
|
|
15
|
-
{events.map((event, index) => {
|
|
16
|
-
switch (event.type) {
|
|
17
|
-
case "message":
|
|
18
|
-
return <MessageBlock event={event} t={t} key={index} />;
|
|
19
|
-
case "thinking":
|
|
20
|
-
return <ThinkBlock event={event} t={t} key={index} />;
|
|
21
|
-
case "action.called": {
|
|
22
|
-
const result = resultByCall.get(event.callId);
|
|
23
|
-
if (result) pairedResult.add(event.callId);
|
|
24
|
-
return <ToolBlock call={event} result={result} t={t} key={index} />;
|
|
25
|
-
}
|
|
26
|
-
case "subagent.called": {
|
|
27
|
-
const result = resultByCall.get(event.callId);
|
|
28
|
-
if (result) pairedResult.add(event.callId);
|
|
29
|
-
return (
|
|
30
|
-
<ToolBlock
|
|
31
|
-
call={{ tool: "agent_task", name: event.name, input: { description: event.name, ...(event.remoteUrl ? { remoteUrl: event.remoteUrl } : {}) } }}
|
|
32
|
-
result={result}
|
|
33
|
-
t={t}
|
|
34
|
-
key={index}
|
|
35
|
-
/>
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
case "action.result":
|
|
39
|
-
case "subagent.completed":
|
|
40
|
-
return pairedResult.has(event.callId) ? null : (
|
|
41
|
-
<ToolBlock call={{ tool: "unknown", name: "result", input: null }} result={event} t={t} key={index} />
|
|
42
|
-
);
|
|
43
|
-
case "input.requested":
|
|
44
|
-
return <InputBlock event={event} t={t} key={index} />;
|
|
45
|
-
case "skill.loaded":
|
|
46
|
-
return (
|
|
47
|
-
<div className="ts-skill" key={index}>
|
|
48
|
-
<span className="ts-role">{t("transcript.skillLoaded")}</span>
|
|
49
|
-
<div className="ts-text">{event.skill}</div>
|
|
50
|
-
</div>
|
|
51
|
-
);
|
|
52
|
-
case "view.raw":
|
|
53
|
-
return <RawEventBlock raw={event.raw} t={t} key={index} />;
|
|
54
|
-
case "compaction":
|
|
55
|
-
return (
|
|
56
|
-
<div className="ts-compaction" key={index}>
|
|
57
|
-
{t("transcript.contextCompacted")}{event.reason ? " · " + event.reason : ""}
|
|
58
|
-
</div>
|
|
59
|
-
);
|
|
60
|
-
case "error":
|
|
61
|
-
return (
|
|
62
|
-
<div className="ts-error" key={index}>
|
|
63
|
-
! {event.message || "error"}
|
|
64
|
-
</div>
|
|
65
|
-
);
|
|
66
|
-
default:
|
|
67
|
-
return null;
|
|
68
|
-
}
|
|
69
|
-
})}
|
|
70
|
-
</div>
|
|
71
|
-
);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** 未识别事件的原样展示:摘要行带原始 type,展开是完整 JSON——不静默丢,方便发现新词汇后续补一等呈现。 */
|
|
75
|
-
export function RawEventBlock({ raw, t }: { raw: ObjectRecord; t: T }) {
|
|
76
|
-
const type = typeof raw.type === "string" ? raw.type : "?";
|
|
77
|
-
const body = prettyJson(raw);
|
|
78
|
-
return (
|
|
79
|
-
<details className="ts-tool-d ts-raw">
|
|
80
|
-
<summary className="ts-row">
|
|
81
|
-
<span className="ts-dot pending" />
|
|
82
|
-
<span className="ts-tool">{type}</span>
|
|
83
|
-
<span className="ts-preview">{t("transcript.rawEvent")}</span>
|
|
84
|
-
</summary>
|
|
85
|
-
<div className="ts-body">
|
|
86
|
-
<pre className="attr-pre">{truncate(body, 8000)}</pre>
|
|
87
|
-
</div>
|
|
88
|
-
</details>
|
|
89
|
-
);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
export function MessageBlock({ event, t }: { event: Extract<TranscriptEvent, { type: "message" }>; t: T }) {
|
|
93
|
-
const who = event.role === "assistant" ? "assistant" : "user";
|
|
94
|
-
return (
|
|
95
|
-
<div className={`ts-msg ts-${who}`}>
|
|
96
|
-
<span className="ts-role">{who === "assistant" ? t("transcript.assistant") : t("transcript.user")}</span>
|
|
97
|
-
<div className="ts-text">{event.text || ""}</div>
|
|
98
|
-
</div>
|
|
99
|
-
);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
export function ThinkBlock({ event, t }: { event: Extract<TranscriptEvent, { type: "thinking" }>; t: T }) {
|
|
103
|
-
return (
|
|
104
|
-
<details className="ts-think">
|
|
105
|
-
<summary>{t("transcript.thinking")}</summary>
|
|
106
|
-
<div className="ts-think-text">{event.text || ""}</div>
|
|
107
|
-
</details>
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
export function InputBlock({ event, t }: { event: Extract<TranscriptEvent, { type: "input.requested" }>; t: T }) {
|
|
112
|
-
const request = event.request || {};
|
|
113
|
-
const opts = (request.options || []).map((o: { id: string; label?: string }) => o.label || o.id).filter(Boolean).join(" / ");
|
|
114
|
-
const body = (request.prompt || t("transcript.awaitingInput")) + (opts ? "\n[ " + opts + " ]" : "");
|
|
115
|
-
return (
|
|
116
|
-
<div className="ts-msg ts-input">
|
|
117
|
-
<span className="ts-role">{t("transcript.inputRequested")}</span>
|
|
118
|
-
<div className="ts-text">{body}</div>
|
|
119
|
-
</div>
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
export function ToolBlock({ call, result, t }: { call: ToolBlockCall; result?: ToolResultEvent; t: T }) {
|
|
124
|
-
const verb = (call.tool ? TOOL_VERB[call.tool] : undefined) || call.name || call.tool || "tool";
|
|
125
|
-
const arg = toolPrimaryArg(call);
|
|
126
|
-
const label = arg ? `${verb}(${arg})` : verb;
|
|
127
|
-
const status = result ? result.status : "pending";
|
|
128
|
-
const dot = status === "failed" ? "bad" : status === "rejected" ? "warn" : status === "pending" ? "pending" : "good";
|
|
129
|
-
const inputStr = call.input == null ? "" : prettyJson(call.input);
|
|
130
|
-
const outBody = result ? resultBody(result.output) : "";
|
|
131
|
-
const preview = result ? previewText(outBody) : t("transcript.running");
|
|
132
|
-
return (
|
|
133
|
-
<details className="ts-tool-d">
|
|
134
|
-
<summary className="ts-row">
|
|
135
|
-
<span className={`ts-dot ${dot}`} />
|
|
136
|
-
<span className="ts-tool" title={label}>
|
|
137
|
-
{label}
|
|
138
|
-
</span>
|
|
139
|
-
<span className="ts-preview">{truncate(preview, 140)}</span>
|
|
140
|
-
</summary>
|
|
141
|
-
<div className="ts-body">
|
|
142
|
-
{inputStr ? (
|
|
143
|
-
<div className="ts-field">
|
|
144
|
-
<span className="ts-k">{t("transcript.input")}</span>
|
|
145
|
-
<pre className="attr-pre">{truncate(inputStr, 4000)}</pre>
|
|
146
|
-
</div>
|
|
147
|
-
) : null}
|
|
148
|
-
{result ? (
|
|
149
|
-
<div className="ts-field">
|
|
150
|
-
<span className="ts-k">{t("transcript.output")}{result.status && result.status !== "completed" ? " · " + result.status : ""}</span>
|
|
151
|
-
<pre className="attr-pre">{outBody ? truncate(outBody, 8000) : <span className="reason-empty">{t("transcript.empty")}</span>}</pre>
|
|
152
|
-
</div>
|
|
153
|
-
) : null}
|
|
154
|
-
</div>
|
|
155
|
-
</details>
|
|
156
|
-
);
|
|
157
|
-
}
|