niceeval 0.9.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +14 -6
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +10 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +74 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +97 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +297 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +239 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +547 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +4 -5
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +12 -11
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +175 -0
- package/dist/report/components/entity-lists/index.d.ts +49 -0
- package/dist/report/components/entity-lists/index.js +169 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +27 -60
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +43 -11
- package/dist/report/{tree.js → definition/tree.js} +9 -11
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +13 -7
- package/dist/report/{format.js → model/format.js} +79 -9
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +154 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +11 -1
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +151 -24
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/e2b.d.ts +0 -1
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +14 -0
- package/dist/shared/aggregate.js +31 -0
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +17 -13
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +36 -33
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +195 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +40 -16
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +102 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +6 -6
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +60 -57
- package/docs-site/zh/tutorials/write-experiment.mdx +353 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +10 -7
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +75 -59
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +27 -12
- package/src/i18n/zh-CN.ts +27 -12
- package/src/index.ts +2 -0
- package/src/o11y/prices.json +176 -99
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/assets/styles.css +1785 -0
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +158 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +224 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +758 -0
- package/src/report/components/attempt-detail/compute.ts +336 -0
- package/src/report/components/attempt-detail/faces.ts +260 -0
- package/src/report/components/attempt-detail/index.tsx +600 -0
- package/src/report/components/attempt-detail/validate.test.ts +252 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +33 -27
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +208 -0
- package/src/report/components/entity-lists/index.tsx +238 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +42 -70
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +66 -26
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +56 -20
- package/src/report/index.ts +128 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +89 -10
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +159 -22
- package/src/report/react/index.tsx +54 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +601 -53
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +53 -2
- package/src/results/select.ts +29 -5
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +160 -24
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b-reconcile.test.ts +125 -0
- package/src/sandbox/e2b.ts +47 -12
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +29 -0
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +56 -20
- package/src/show/render.ts +15 -239
- package/src/show/show.test.ts +127 -34
- package/src/view/app/App.test.tsx +1 -1
- package/src/view/app/App.tsx +119 -47
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +105 -83
- package/src/view/index.ts +1 -1
- package/src/view/server.ts +32 -2
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +0 -805
- package/src/view/view-report.test.ts +143 -22
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/react/styles.css +0 -945
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
// Attempt 详情组件族的计算函数(docs/feature/reports/library/attempt-detail.md)。每个
|
|
2
|
+
// `attempt*Data(evidence)` 都是纯同步派生:evidence 已经由 loadAttemptEvidence 一次性
|
|
3
|
+
// 装配好全部证据,这里只做适合展示与序列化的取舍,不读文件、不 fetch、不重复调用
|
|
4
|
+
// attempt.events() / trace() / diff()。
|
|
5
|
+
//
|
|
6
|
+
// 与 compute.ts(Scope → *Data)不同,这一族的输入恒为单个 AttemptEvidence,函数签名
|
|
7
|
+
// 因此不是 async——没有 IO 就没有理由返回 Promise。
|
|
8
|
+
import { attemptCostUSD } from "../../model/metrics.js";
|
|
9
|
+
import { failureSummaryOf } from "../entity-lists/compute.js";
|
|
10
|
+
// ───────────────────────── AttemptSummary(恒非空) ─────────────────────────
|
|
11
|
+
export function attemptSummaryData(evidence) {
|
|
12
|
+
const { result } = evidence;
|
|
13
|
+
return {
|
|
14
|
+
locator: evidence.locator,
|
|
15
|
+
identity: evidence.identity,
|
|
16
|
+
verdict: result.verdict,
|
|
17
|
+
startedAt: result.startedAt,
|
|
18
|
+
durationMs: result.durationMs,
|
|
19
|
+
costUSD: attemptCostUSD(result),
|
|
20
|
+
capabilities: evidence.capabilities,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
// ───────────────────────── AttemptError ─────────────────────────
|
|
24
|
+
export function attemptErrorData(evidence) {
|
|
25
|
+
const err = evidence.result.error;
|
|
26
|
+
return err ?? null;
|
|
27
|
+
}
|
|
28
|
+
// ───────────────────────── AttemptAssertions ─────────────────────────
|
|
29
|
+
export function attemptAssertionsData(evidence) {
|
|
30
|
+
const assertions = evidence.result.assertions;
|
|
31
|
+
if (!assertions || assertions.length === 0)
|
|
32
|
+
return null;
|
|
33
|
+
const attention = assertions.filter((a) => a.outcome !== "passed");
|
|
34
|
+
const passed = assertions.filter((a) => a.outcome === "passed");
|
|
35
|
+
const groups = new Map();
|
|
36
|
+
for (const a of passed) {
|
|
37
|
+
const key = a.groupPath?.join(" > ") ?? "";
|
|
38
|
+
const list = groups.get(key);
|
|
39
|
+
if (list)
|
|
40
|
+
list.push(a);
|
|
41
|
+
else
|
|
42
|
+
groups.set(key, [a]);
|
|
43
|
+
}
|
|
44
|
+
return { attention, passedGroups: [...groups.entries()].map(([group, items]) => ({ group, items })) };
|
|
45
|
+
}
|
|
46
|
+
// ───────────────────────── AttemptSource ─────────────────────────
|
|
47
|
+
export function attemptSourceData(evidence) {
|
|
48
|
+
if (!evidence.capabilities.source || evidence.evalSource === null)
|
|
49
|
+
return null;
|
|
50
|
+
const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
|
|
51
|
+
const projectedLines = lines.map((line) => ({
|
|
52
|
+
...line,
|
|
53
|
+
turns: line.sends.map((send) => ({
|
|
54
|
+
label: send.label,
|
|
55
|
+
status: send.status,
|
|
56
|
+
...(send.durationMs === undefined ? {} : { durationMs: send.durationMs }),
|
|
57
|
+
sentText: "",
|
|
58
|
+
replies: [],
|
|
59
|
+
})),
|
|
60
|
+
}));
|
|
61
|
+
const usedTurns = new Map();
|
|
62
|
+
const unlocatedTurns = [];
|
|
63
|
+
const conversation = attemptConversationData(evidence);
|
|
64
|
+
for (const [roundIndex, round] of (conversation?.rounds ?? []).entries()) {
|
|
65
|
+
const status = round.replies.some((reply) => reply.kind === "error" ||
|
|
66
|
+
((reply.kind === "tool" || reply.kind === "subagent") && reply.status === "failed"))
|
|
67
|
+
? "failed"
|
|
68
|
+
: round.replies.some((reply) => reply.kind === "input")
|
|
69
|
+
? "waiting"
|
|
70
|
+
: "completed";
|
|
71
|
+
const fallback = {
|
|
72
|
+
label: `t${roundIndex + 1}`,
|
|
73
|
+
status,
|
|
74
|
+
sentText: round.sentText,
|
|
75
|
+
replies: round.replies,
|
|
76
|
+
};
|
|
77
|
+
const loc = round.loc;
|
|
78
|
+
if (!loc || loc.file !== sourcePath || loc.line < 1 || loc.line > projectedLines.length) {
|
|
79
|
+
unlocatedTurns.push(fallback);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const line = projectedLines[loc.line - 1];
|
|
83
|
+
const turnIndex = usedTurns.get(loc.line) ?? 0;
|
|
84
|
+
usedTurns.set(loc.line, turnIndex + 1);
|
|
85
|
+
const annotated = line.turns[turnIndex];
|
|
86
|
+
if (annotated) {
|
|
87
|
+
annotated.sentText = round.sentText;
|
|
88
|
+
annotated.replies = round.replies;
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
line.turns.push(fallback);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return { locator: evidence.locator, sourcePath, lines: projectedLines, unmapped, unlocatedTurns, summary };
|
|
95
|
+
}
|
|
96
|
+
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
97
|
+
/** 单条 attempt 版的批量修复 prompt(与 CopyFixPrompt 的多条版本同一份步骤文案)。 */
|
|
98
|
+
export function attemptFixPromptData(evidence) {
|
|
99
|
+
const { result, identity } = evidence;
|
|
100
|
+
if (result.verdict !== "failed" && result.verdict !== "errored")
|
|
101
|
+
return null;
|
|
102
|
+
const { summary, more } = failureSummaryOf(result);
|
|
103
|
+
if (summary === null)
|
|
104
|
+
return null;
|
|
105
|
+
const reason = more > 0 ? `${summary} (+${more} more failures)` : summary;
|
|
106
|
+
const prompt = [
|
|
107
|
+
"Fix the failing eval from this niceeval run.",
|
|
108
|
+
"",
|
|
109
|
+
"## Failure",
|
|
110
|
+
`eval "${identity.evalId}" [experiment ${identity.experimentId}] — ${result.verdict}`,
|
|
111
|
+
` reason: ${reason}`,
|
|
112
|
+
` inspect: niceeval show ${evidence.locator}`,
|
|
113
|
+
"",
|
|
114
|
+
"## Steps",
|
|
115
|
+
"1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
|
|
116
|
+
"2. Run the inspect command above with `--source`, `--execution`, `--timing`, and `--diff` to see the assertions, transcript, timing, and workspace diff.",
|
|
117
|
+
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
118
|
+
`4. Re-run: \`npx niceeval exp ${identity.experimentId} ${identity.evalId}\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
119
|
+
"5. Run `npx niceeval show` and confirm this failure is gone.",
|
|
120
|
+
].join("\n");
|
|
121
|
+
return { prompt };
|
|
122
|
+
}
|
|
123
|
+
// ───────────────────────── AttemptTimeline ─────────────────────────
|
|
124
|
+
/** 收尾段的阶段名(见 docs/feature/results/architecture.md);两面渲染都把这些单列在主链之后,不计入主链总耗时。 */
|
|
125
|
+
export const TIMELINE_CLOSING_PHASES = new Set([
|
|
126
|
+
"eval.teardown",
|
|
127
|
+
"agent.teardown",
|
|
128
|
+
"sandbox.teardown",
|
|
129
|
+
"sandbox.suspend",
|
|
130
|
+
"sandbox.stop",
|
|
131
|
+
]);
|
|
132
|
+
export function attemptTimelineData(evidence) {
|
|
133
|
+
const phases = evidence.result.phases;
|
|
134
|
+
if (!phases || phases.length === 0)
|
|
135
|
+
return null;
|
|
136
|
+
return { locator: evidence.locator, phases, trace: evidence.trace };
|
|
137
|
+
}
|
|
138
|
+
// ───────────────────────── AttemptConversation ─────────────────────────
|
|
139
|
+
/**
|
|
140
|
+
* 标准事件流按 `loc` 分轮(docs/feature/reports/library/attempt-detail.md「Attempt 详情组件」):
|
|
141
|
+
* 带 loc 的 user 消息开一轮;无 loc 的 user 消息不开新轮——与当前轮 sent 同文本的回显直接
|
|
142
|
+
* 吃掉,其它(stop-hook 反馈、skill 注入等轮内注入)作为回复条目留在当前轮。流首出现无 loc
|
|
143
|
+
* 的 user 消息(没有当前轮可归入)时退化开一条 loc 缺省的兜底轮,不丢弃。未识别的事件类型
|
|
144
|
+
* 包成 `raw` 条目原样呈现,不吞没其余事件——StreamEvent 是随 artifact 版本演进的开放词表,
|
|
145
|
+
* 这份纯函数不能假设自己认识每一种将来会出现的 type。
|
|
146
|
+
*/
|
|
147
|
+
export function attemptConversationData(evidence) {
|
|
148
|
+
const events = evidence.events;
|
|
149
|
+
if (!events || events.length === 0)
|
|
150
|
+
return null;
|
|
151
|
+
const rounds = [];
|
|
152
|
+
const toolByCallId = new Map();
|
|
153
|
+
const subagentByCallId = new Map();
|
|
154
|
+
let current = null;
|
|
155
|
+
for (const ev of events) {
|
|
156
|
+
if (ev.type === "message" && ev.role === "user") {
|
|
157
|
+
if (!ev.loc && current) {
|
|
158
|
+
if (current.replies.length === 0 && (ev.text || "") === current.sentText)
|
|
159
|
+
continue;
|
|
160
|
+
current.replies.push({ kind: "user", text: ev.text || "" });
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
current = { loc: ev.loc, sentText: ev.text || "", replies: [] };
|
|
164
|
+
rounds.push(current);
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
if (!current) {
|
|
168
|
+
// 流首没有开轮的用户消息就先来了其它事件(旧 artifact 的边界情况):开一条无 loc 兜底轮。
|
|
169
|
+
current = { sentText: "", replies: [] };
|
|
170
|
+
rounds.push(current);
|
|
171
|
+
}
|
|
172
|
+
current.replies.push(...conversationReplyOf(ev, toolByCallId, subagentByCallId));
|
|
173
|
+
}
|
|
174
|
+
return { locator: evidence.locator, rounds };
|
|
175
|
+
}
|
|
176
|
+
/** 单条事件 → 0 或 1 条回复条目;action.result/subagent.completed 只更新已有条目,不新增。 */
|
|
177
|
+
function conversationReplyOf(ev, toolByCallId, subagentByCallId) {
|
|
178
|
+
switch (ev.type) {
|
|
179
|
+
case "message":
|
|
180
|
+
// role === "user" 已在主循环处理(开轮 / 回显吃掉 / 轮内注入),这里只剩 assistant。
|
|
181
|
+
return [{ kind: "assistant", text: ev.text }];
|
|
182
|
+
case "thinking":
|
|
183
|
+
return [{ kind: "thinking", text: ev.text }];
|
|
184
|
+
case "error":
|
|
185
|
+
return [{ kind: "error", text: ev.message }];
|
|
186
|
+
case "skill.loaded":
|
|
187
|
+
return [{ kind: "skill", skill: ev.skill }];
|
|
188
|
+
case "input.requested":
|
|
189
|
+
return [{ kind: "input", request: ev.request }];
|
|
190
|
+
case "compaction":
|
|
191
|
+
return [{ kind: "compaction", reason: ev.reason }];
|
|
192
|
+
case "action.called": {
|
|
193
|
+
const reply = {
|
|
194
|
+
kind: "tool",
|
|
195
|
+
callId: ev.callId,
|
|
196
|
+
name: ev.name,
|
|
197
|
+
tool: ev.tool,
|
|
198
|
+
input: ev.input,
|
|
199
|
+
};
|
|
200
|
+
toolByCallId.set(ev.callId, reply);
|
|
201
|
+
return [reply];
|
|
202
|
+
}
|
|
203
|
+
case "action.result": {
|
|
204
|
+
const tool = toolByCallId.get(ev.callId);
|
|
205
|
+
if (tool) {
|
|
206
|
+
tool.output = ev.output;
|
|
207
|
+
tool.status = ev.status;
|
|
208
|
+
}
|
|
209
|
+
return [];
|
|
210
|
+
}
|
|
211
|
+
case "subagent.called": {
|
|
212
|
+
const reply = {
|
|
213
|
+
kind: "subagent",
|
|
214
|
+
callId: ev.callId,
|
|
215
|
+
name: ev.name,
|
|
216
|
+
remoteUrl: ev.remoteUrl,
|
|
217
|
+
};
|
|
218
|
+
subagentByCallId.set(ev.callId, reply);
|
|
219
|
+
return [reply];
|
|
220
|
+
}
|
|
221
|
+
case "subagent.completed": {
|
|
222
|
+
const subagent = subagentByCallId.get(ev.callId);
|
|
223
|
+
if (subagent) {
|
|
224
|
+
subagent.output = ev.output;
|
|
225
|
+
subagent.status = ev.status;
|
|
226
|
+
}
|
|
227
|
+
return [];
|
|
228
|
+
}
|
|
229
|
+
default: {
|
|
230
|
+
// 穷尽性检查在这里刻意不做:StreamEvent 是随 artifact 版本演进的开放词表,未识别的
|
|
231
|
+
// 事件(将来的新 type,或第三方 harness 的自定义变体)包成 raw 原样呈现,不静默丢弃、
|
|
232
|
+
// 也不因为一个不认识的条目让整个装配失败。
|
|
233
|
+
return [{ kind: "raw", raw: ev }];
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
// ───────────────────────── AttemptDiagnostics ─────────────────────────
|
|
238
|
+
export function attemptDiagnosticsData(evidence) {
|
|
239
|
+
const diagnostics = evidence.result.diagnostics;
|
|
240
|
+
if (!diagnostics || diagnostics.length === 0)
|
|
241
|
+
return null;
|
|
242
|
+
const groups = new Map();
|
|
243
|
+
for (const d of diagnostics) {
|
|
244
|
+
const list = groups.get(d.phase);
|
|
245
|
+
if (list)
|
|
246
|
+
list.push(d);
|
|
247
|
+
else
|
|
248
|
+
groups.set(d.phase, [d]);
|
|
249
|
+
}
|
|
250
|
+
return { groups: [...groups.entries()].map(([phase, items]) => ({ phase, items })) };
|
|
251
|
+
}
|
|
252
|
+
// ───────────────────────── AttemptUsage ─────────────────────────
|
|
253
|
+
export function attemptUsageData(evidence) {
|
|
254
|
+
const usage = evidence.result.usage;
|
|
255
|
+
if (!usage)
|
|
256
|
+
return null;
|
|
257
|
+
return { usage, costUSD: attemptCostUSD(evidence.result) };
|
|
258
|
+
}
|
|
259
|
+
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
260
|
+
export function attemptTraceData(evidence) {
|
|
261
|
+
const spans = evidence.trace;
|
|
262
|
+
if (!spans || spans.length === 0)
|
|
263
|
+
return null;
|
|
264
|
+
return { locator: evidence.locator, spans };
|
|
265
|
+
}
|
|
266
|
+
// ───────────────────────── AttemptDiff ─────────────────────────
|
|
267
|
+
/** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似;与 `niceeval show --diff` 同一算法。 */
|
|
268
|
+
function lineDelta(before, after) {
|
|
269
|
+
const a = before === undefined ? [] : before.split("\n");
|
|
270
|
+
const b = after === undefined ? [] : after.split("\n");
|
|
271
|
+
let prefix = 0;
|
|
272
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix])
|
|
273
|
+
prefix++;
|
|
274
|
+
let suffix = 0;
|
|
275
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix])
|
|
276
|
+
suffix++;
|
|
277
|
+
return { added: b.length - prefix - suffix, deleted: a.length - prefix - suffix };
|
|
278
|
+
}
|
|
279
|
+
export function attemptDiffData(evidence) {
|
|
280
|
+
if (!evidence.capabilities.diff || evidence.diff === null)
|
|
281
|
+
return null;
|
|
282
|
+
const diff = evidence.diff;
|
|
283
|
+
const files = [];
|
|
284
|
+
for (const [path, summary] of Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b))) {
|
|
285
|
+
if (summary.net === "none")
|
|
286
|
+
continue;
|
|
287
|
+
const windows = diff.windows.filter((w) => w.changes[path] !== undefined).map((w) => w.window);
|
|
288
|
+
if (summary.binary) {
|
|
289
|
+
files.push({ path, net: summary.net, lines: { added: 0, deleted: 0 }, binary: true, windows });
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
const before = summary.net === "added" ? undefined : diff.windows.find((w) => w.changes[path]?.before !== undefined)?.changes[path]?.before;
|
|
293
|
+
const after = summary.net === "deleted" ? undefined : diff.get(path);
|
|
294
|
+
files.push({ path, net: summary.net, lines: lineDelta(before, after), windows });
|
|
295
|
+
}
|
|
296
|
+
return files.length > 0 ? { locator: evidence.locator, files } : null;
|
|
297
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { AttemptAssertionsData, AttemptConversationData, AttemptDiagnosticsData, AttemptDiffData, AttemptErrorData, AttemptFixPromptData, AttemptSourceData, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, AttemptUsageData } from "../../model/types.ts";
|
|
2
|
+
import type { TextContext } from "../../definition/tree.ts";
|
|
3
|
+
export declare function attemptSummaryText(data: AttemptSummaryData, ctx: TextContext): string;
|
|
4
|
+
export declare function attemptErrorText(data: AttemptErrorData | null, _ctx: TextContext): string;
|
|
5
|
+
export declare function attemptAssertionsText(data: AttemptAssertionsData | null, _ctx: TextContext): string;
|
|
6
|
+
export declare function attemptSourceText(data: AttemptSourceData | null, ctx: TextContext): string;
|
|
7
|
+
/** text 面零输出:终端已有 attemptSummaryText 里的 locator,直接跑 `niceeval show @<locator>` 即可;这里不重复整段 prompt。 */
|
|
8
|
+
export declare function attemptFixPromptText(_data: AttemptFixPromptData | null, _ctx: TextContext): string;
|
|
9
|
+
export declare function attemptTimelineText(data: AttemptTimelineData | null, ctx: TextContext): string;
|
|
10
|
+
export declare function attemptConversationText(data: AttemptConversationData | null, ctx: TextContext): string;
|
|
11
|
+
export declare function attemptDiagnosticsText(data: AttemptDiagnosticsData | null, _ctx: TextContext): string;
|
|
12
|
+
export declare function attemptUsageText(data: AttemptUsageData | null, _ctx: TextContext): string;
|
|
13
|
+
export declare function attemptTraceText(data: AttemptTraceData | null, ctx: TextContext): string;
|
|
14
|
+
export declare function attemptDiffText(data: AttemptDiffData | null, ctx: TextContext): string;
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
// Attempt 详情组件族的 text 面(docs/feature/reports/library/attempt-detail.md「在 show 与
|
|
2
|
+
// view 怎样渲染」):与 web 面共享同一次 resolve 产出的 data 事实(verdict、计数、能力位、
|
|
3
|
+
// 引用),允许把大块内容折成摘要 + 专用证据命令,但不得改变判定、计数或引用。
|
|
4
|
+
// 零 react、零 IO、纯同步——text 宿主不需要 react-dom 的那一半。
|
|
5
|
+
import { localeText } from "../../model/locale.js";
|
|
6
|
+
import { formatDurationMs, formatMetricValue, formatReportDateTime, formatUSD, verdictMark } from "../../model/format.js";
|
|
7
|
+
import { TIMELINE_CLOSING_PHASES } from "./compute.js";
|
|
8
|
+
import { summaryText } from "../../../scoring/display.js";
|
|
9
|
+
/**
|
|
10
|
+
* 证据切面下钻命令:`ctx.attemptCommand` 由宿主按当前 report 是否有 attempt-input page
|
|
11
|
+
* 注入(见 report.ts DEFAULT_ATTEMPT_COMMAND);没有时不生成假命令,调用方直接省略这一段
|
|
12
|
+
* (与 traceWaterfallText 同一套退化规则,architecture.md「Attempt 详情是一张参数化 page」)。
|
|
13
|
+
*/
|
|
14
|
+
function evidenceCommand(ctx, locator, flag) {
|
|
15
|
+
return ctx.attemptCommand ? `${ctx.attemptCommand(locator)} ${flag}` : undefined;
|
|
16
|
+
}
|
|
17
|
+
// ───────────────────────── AttemptSummary ─────────────────────────
|
|
18
|
+
export function attemptSummaryText(data, ctx) {
|
|
19
|
+
const locale = ctx.locale;
|
|
20
|
+
const parts = [
|
|
21
|
+
data.locator,
|
|
22
|
+
data.identity.evalId,
|
|
23
|
+
data.identity.experimentId,
|
|
24
|
+
localeText(locale, "attemptSummary.attempt", { n: data.identity.attempt + 1 }),
|
|
25
|
+
`${verdictMark(data.verdict)} ${localeText(locale, `verdict.${data.verdict}`)}`,
|
|
26
|
+
];
|
|
27
|
+
if (data.startedAt !== undefined)
|
|
28
|
+
parts.push(formatReportDateTime(data.startedAt, locale));
|
|
29
|
+
parts.push(formatDurationMs(data.durationMs));
|
|
30
|
+
if (data.costUSD !== null)
|
|
31
|
+
parts.push(formatUSD(data.costUSD));
|
|
32
|
+
return parts.join(" · ");
|
|
33
|
+
}
|
|
34
|
+
// ───────────────────────── AttemptError ─────────────────────────
|
|
35
|
+
export function attemptErrorText(data, _ctx) {
|
|
36
|
+
if (data === null)
|
|
37
|
+
return "";
|
|
38
|
+
// message/cause 折单行加上限(同 assertionLine 的 summaryText 规则);stack 是唯一没有替代
|
|
39
|
+
// 查看入口的自由文本,原样保留多行,不折。
|
|
40
|
+
const lines = [`error: ${data.code}`, ` phase: ${data.phase}`, ` message: ${summaryText(data.message)}`];
|
|
41
|
+
if (data.cause) {
|
|
42
|
+
const causeMessage = summaryText(data.cause.message);
|
|
43
|
+
lines.push(` cause: ${data.cause.name ? `${data.cause.name} · ${causeMessage}` : causeMessage}`);
|
|
44
|
+
}
|
|
45
|
+
const stack = data.stack?.replace(/\n+$/, "");
|
|
46
|
+
return stack ? `${lines.join("\n")}\n\n${stack}` : lines.join("\n");
|
|
47
|
+
}
|
|
48
|
+
// ───────────────────────── AttemptAssertions ─────────────────────────
|
|
49
|
+
/** `loc` 是断言在 eval 源码里的调用点,独立于整份源码是否被捕获(`AttemptSource` 的能力位);
|
|
50
|
+
* 失败断言只要带 loc 就给可复制定位的源码锚,不必等 AttemptSource 可用才显示。 */
|
|
51
|
+
function locAnchor(loc) {
|
|
52
|
+
return `${loc.file}:${loc.line}${loc.column ? `:${loc.column}` : ""}`;
|
|
53
|
+
}
|
|
54
|
+
function assertionLine(a) {
|
|
55
|
+
const group = a.groupPath && a.groupPath.length > 0 ? `${a.groupPath.join(" > ")} · ` : "";
|
|
56
|
+
if (a.outcome === "unavailable")
|
|
57
|
+
return `◌ unavailable · ${group}${a.name} — ${a.reason}`;
|
|
58
|
+
const mark = a.outcome === "passed" ? "✓" : "✗";
|
|
59
|
+
const detail = a.detail && a.detail !== a.name ? `: ${a.detail}` : "";
|
|
60
|
+
// expected/received 折单行 + 加字符上限(与 scoring 摘要面同一条 summaryText 规则):
|
|
61
|
+
// received 常常就是被检查的整份文本(如 includes() 对全文匹配),不收口会把整份源码/工具
|
|
62
|
+
// 输出灌进这一行;完整值仍在 web 面(可展开的完整结构化细节)与 --source 里原样可查。
|
|
63
|
+
const evidence = [
|
|
64
|
+
a.expected !== undefined ? `expected: ${summaryText(a.expected)}` : undefined,
|
|
65
|
+
a.received !== undefined ? `received: ${summaryText(a.received)}` : undefined,
|
|
66
|
+
a.outcome === "failed" && a.loc ? `source: ${locAnchor(a.loc)}` : undefined,
|
|
67
|
+
].filter((part) => part !== undefined);
|
|
68
|
+
const evidenceSuffix = evidence.length > 0 ? ` · ${evidence.join(" · ")}` : "";
|
|
69
|
+
return `${mark} ${a.severity} · ${group}${a.name}${detail}${evidenceSuffix}`;
|
|
70
|
+
}
|
|
71
|
+
export function attemptAssertionsText(data, _ctx) {
|
|
72
|
+
if (data === null)
|
|
73
|
+
return "";
|
|
74
|
+
const lines = data.attention.map(assertionLine);
|
|
75
|
+
for (const { group, items } of data.passedGroups) {
|
|
76
|
+
lines.push(`✓ passed · ${group || "(ungrouped)"} · ${items.length}`);
|
|
77
|
+
}
|
|
78
|
+
return lines.join("\n");
|
|
79
|
+
}
|
|
80
|
+
// ───────────────────────── AttemptSource ─────────────────────────
|
|
81
|
+
export function attemptSourceText(data, ctx) {
|
|
82
|
+
if (data === null)
|
|
83
|
+
return "";
|
|
84
|
+
const command = evidenceCommand(ctx, data.locator, "--source");
|
|
85
|
+
const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
|
|
86
|
+
if (command)
|
|
87
|
+
headerParts.push(command);
|
|
88
|
+
const hasConversation = data.unlocatedTurns.length > 0 || data.lines.some((line) => line.turns.length > 0);
|
|
89
|
+
const executionCommand = hasConversation ? evidenceCommand(ctx, data.locator, "--execution") : null;
|
|
90
|
+
if (executionCommand)
|
|
91
|
+
headerParts.push(executionCommand);
|
|
92
|
+
// 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
|
|
93
|
+
// 挑出非 passed 的条目,不重复算锚点。
|
|
94
|
+
const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
|
|
95
|
+
const lines = failed.map((a) => ` ${assertionLine(a)}`);
|
|
96
|
+
return [headerParts.join(" · "), ...lines].join("\n");
|
|
97
|
+
}
|
|
98
|
+
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
99
|
+
/** text 面零输出:终端已有 attemptSummaryText 里的 locator,直接跑 `niceeval show @<locator>` 即可;这里不重复整段 prompt。 */
|
|
100
|
+
export function attemptFixPromptText(_data, _ctx) {
|
|
101
|
+
return "";
|
|
102
|
+
}
|
|
103
|
+
// ───────────────────────── AttemptTimeline ─────────────────────────
|
|
104
|
+
function fmtMs(ms) {
|
|
105
|
+
if (ms >= 60_000)
|
|
106
|
+
return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
|
|
107
|
+
if (ms >= 1000)
|
|
108
|
+
return `${(ms / 1000).toFixed(1)}s`;
|
|
109
|
+
return `${ms}ms`;
|
|
110
|
+
}
|
|
111
|
+
function countTimingNodes(nodes) {
|
|
112
|
+
return nodes.reduce((sum, n) => sum + 1 + countTimingNodes(n.children ?? []), 0);
|
|
113
|
+
}
|
|
114
|
+
export function attemptTimelineText(data, ctx) {
|
|
115
|
+
if (data === null)
|
|
116
|
+
return "";
|
|
117
|
+
const main = data.phases.filter((p) => !TIMELINE_CLOSING_PHASES.has(p.name));
|
|
118
|
+
const closing = data.phases.filter((p) => TIMELINE_CLOSING_PHASES.has(p.name));
|
|
119
|
+
const total = main.reduce((sum, p) => sum + p.durationMs, 0);
|
|
120
|
+
const anyFailed = data.phases.some((p) => p.failed);
|
|
121
|
+
const command = evidenceCommand(ctx, data.locator, "--timing");
|
|
122
|
+
const head = [`timing: ${fmtMs(total)}${anyFailed ? " ✗" : ""}`];
|
|
123
|
+
if (command)
|
|
124
|
+
head.push(command);
|
|
125
|
+
const lines = [head.join(" · ")];
|
|
126
|
+
for (const p of main) {
|
|
127
|
+
const kids = p.children ?? [];
|
|
128
|
+
const suffix = kids.length > 0 ? ` (${countTimingNodes(kids)} children collapsed)` : "";
|
|
129
|
+
lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}${suffix}`);
|
|
130
|
+
}
|
|
131
|
+
if (closing.length > 0) {
|
|
132
|
+
lines.push(" teardown:");
|
|
133
|
+
for (const p of closing)
|
|
134
|
+
lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}`);
|
|
135
|
+
}
|
|
136
|
+
return lines.join("\n");
|
|
137
|
+
}
|
|
138
|
+
// ───────────────────────── AttemptConversation ─────────────────────────
|
|
139
|
+
// 每条回复的自由文本(消息正文、发起请求的 prompt、未识别事件的原始 payload)都可能是任意大小的
|
|
140
|
+
// 块——system prompt、完整工具输出等——同一条 summaryText 规则折单行加上限;完整内容仍在
|
|
141
|
+
// --execution 里原样可查。
|
|
142
|
+
function replySummary(reply) {
|
|
143
|
+
switch (reply.kind) {
|
|
144
|
+
case "assistant":
|
|
145
|
+
return `assistant: ${summaryText(reply.text)}`;
|
|
146
|
+
case "user":
|
|
147
|
+
return `user: ${summaryText(reply.text)}`;
|
|
148
|
+
case "thinking":
|
|
149
|
+
return `thinking: ${summaryText(reply.text)}`;
|
|
150
|
+
case "error":
|
|
151
|
+
return `error: ${summaryText(reply.text)}`;
|
|
152
|
+
case "tool":
|
|
153
|
+
return `tool ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
|
|
154
|
+
case "skill":
|
|
155
|
+
return `skill loaded: ${reply.skill}`;
|
|
156
|
+
case "subagent":
|
|
157
|
+
return `subagent ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
|
|
158
|
+
case "input":
|
|
159
|
+
return `input requested${reply.request.prompt ? `: ${summaryText(reply.request.prompt)}` : ""}`;
|
|
160
|
+
case "compaction":
|
|
161
|
+
return `compaction${reply.reason ? `: ${summaryText(reply.reason)}` : ""}`;
|
|
162
|
+
case "raw":
|
|
163
|
+
return `unrecognized event: ${summaryText(JSON.stringify(reply.raw))}`;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
export function attemptConversationText(data, ctx) {
|
|
167
|
+
if (data === null)
|
|
168
|
+
return "";
|
|
169
|
+
const command = evidenceCommand(ctx, data.locator, "--execution");
|
|
170
|
+
const head = [`conversation: ${data.rounds.length} round${data.rounds.length === 1 ? "" : "s"}`];
|
|
171
|
+
if (command)
|
|
172
|
+
head.push(command);
|
|
173
|
+
const lines = [head.join(" · ")];
|
|
174
|
+
data.rounds.forEach((round, i) => {
|
|
175
|
+
lines.push(` round ${i + 1}${round.sentText ? `: ${summaryText(round.sentText)}` : ""}`);
|
|
176
|
+
for (const reply of round.replies)
|
|
177
|
+
lines.push(` ${replySummary(reply)}`);
|
|
178
|
+
});
|
|
179
|
+
return lines.join("\n");
|
|
180
|
+
}
|
|
181
|
+
// ───────────────────────── AttemptDiagnostics ─────────────────────────
|
|
182
|
+
export function attemptDiagnosticsText(data, _ctx) {
|
|
183
|
+
if (data === null)
|
|
184
|
+
return "";
|
|
185
|
+
const lines = [];
|
|
186
|
+
for (const { phase, items } of data.groups) {
|
|
187
|
+
lines.push(`${phase}:`);
|
|
188
|
+
for (const d of items) {
|
|
189
|
+
const count = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
|
|
190
|
+
lines.push(` ${d.level} · ${d.code} — ${summaryText(d.message)}${count}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return lines.join("\n");
|
|
194
|
+
}
|
|
195
|
+
// ───────────────────────── AttemptUsage ─────────────────────────
|
|
196
|
+
export function attemptUsageText(data, _ctx) {
|
|
197
|
+
if (data === null)
|
|
198
|
+
return "";
|
|
199
|
+
const { usage } = data;
|
|
200
|
+
const total = usage.inputTokens + usage.outputTokens;
|
|
201
|
+
const parts = [`usage: ${formatMetricValue(total)} tokens (${formatMetricValue(usage.inputTokens)} in / ${formatMetricValue(usage.outputTokens)} out)`];
|
|
202
|
+
if (usage.cacheReadTokens !== undefined)
|
|
203
|
+
parts.push(`cache read ${formatMetricValue(usage.cacheReadTokens)}`);
|
|
204
|
+
if (usage.cacheWriteTokens !== undefined)
|
|
205
|
+
parts.push(`cache write ${formatMetricValue(usage.cacheWriteTokens)}`);
|
|
206
|
+
if (usage.requests !== undefined)
|
|
207
|
+
parts.push(`${usage.requests} requests`);
|
|
208
|
+
if (data.costUSD !== null)
|
|
209
|
+
parts.push(formatUSD(data.costUSD));
|
|
210
|
+
return parts.join(" · ");
|
|
211
|
+
}
|
|
212
|
+
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
213
|
+
export function attemptTraceText(data, ctx) {
|
|
214
|
+
if (data === null)
|
|
215
|
+
return "";
|
|
216
|
+
const command = evidenceCommand(ctx, data.locator, "--timing");
|
|
217
|
+
const head = [`trace: ${data.spans.length} span${data.spans.length === 1 ? "" : "s"}`];
|
|
218
|
+
if (command)
|
|
219
|
+
head.push(command);
|
|
220
|
+
return head.join(" · ");
|
|
221
|
+
}
|
|
222
|
+
// ───────────────────────── AttemptDiff ─────────────────────────
|
|
223
|
+
function netLetter(net) {
|
|
224
|
+
return net === "added" ? "A" : net === "deleted" ? "D" : "M";
|
|
225
|
+
}
|
|
226
|
+
export function attemptDiffText(data, ctx) {
|
|
227
|
+
if (data === null)
|
|
228
|
+
return "";
|
|
229
|
+
const command = evidenceCommand(ctx, data.locator, "--diff");
|
|
230
|
+
const head = [`changes: ${data.files.length} file${data.files.length === 1 ? "" : "s"} changed by agent`];
|
|
231
|
+
if (command)
|
|
232
|
+
head.push(command);
|
|
233
|
+
const lines = [head.join(" · ")];
|
|
234
|
+
for (const f of data.files) {
|
|
235
|
+
const delta = f.binary ? "binary" : `+${f.lines.added}/-${f.lines.deleted}`;
|
|
236
|
+
lines.push(` ${netLetter(f.net)} ${f.path} (${delta})`);
|
|
237
|
+
}
|
|
238
|
+
return lines.join("\n");
|
|
239
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { type ReportComponent } from "../../definition/tree.ts";
|
|
2
|
+
import type { AttemptEvidence } from "../../../results/attempt-evidence.ts";
|
|
3
|
+
import type { AttemptAssertionsData, AttemptConversationData, AttemptDiagnosticsData, AttemptDiffData, AttemptFixPromptData, AttemptSourceData, AttemptSummaryData, AttemptTimelineData, AttemptTraceData, AttemptUsageData } from "../../model/types.ts";
|
|
4
|
+
export type AttemptSectionProps<Data> = {
|
|
5
|
+
input?: AttemptEvidence;
|
|
6
|
+
data?: never;
|
|
7
|
+
className?: string;
|
|
8
|
+
} | {
|
|
9
|
+
data: Data;
|
|
10
|
+
input?: never;
|
|
11
|
+
className?: string;
|
|
12
|
+
};
|
|
13
|
+
export declare function validateSummaryData(data: unknown): string | null;
|
|
14
|
+
export declare const AttemptSummary: ReportComponent<AttemptSectionProps<AttemptSummaryData>>;
|
|
15
|
+
export declare function validateErrorData(data: unknown): string | null;
|
|
16
|
+
export declare const AttemptError: ReportComponent<AttemptSectionProps<import("../../../types.ts").AttemptError>>;
|
|
17
|
+
export declare function validateAssertionsData(data: unknown): string | null;
|
|
18
|
+
export declare const AttemptAssertions: ReportComponent<AttemptSectionProps<AttemptAssertionsData>>;
|
|
19
|
+
export declare function validateSourceData(data: unknown): string | null;
|
|
20
|
+
export declare const AttemptSource: ReportComponent<AttemptSectionProps<AttemptSourceData>>;
|
|
21
|
+
export declare function validateFixPromptData(data: unknown): string | null;
|
|
22
|
+
export declare const AttemptFixPrompt: ReportComponent<AttemptSectionProps<AttemptFixPromptData>>;
|
|
23
|
+
export declare function validateTimelineData(data: unknown): string | null;
|
|
24
|
+
export declare const AttemptTimeline: ReportComponent<AttemptSectionProps<AttemptTimelineData>>;
|
|
25
|
+
export declare function validateConversationData(data: unknown): string | null;
|
|
26
|
+
export declare const AttemptConversation: ReportComponent<AttemptSectionProps<AttemptConversationData>>;
|
|
27
|
+
export declare function validateDiagnosticsData(data: unknown): string | null;
|
|
28
|
+
export declare const AttemptDiagnostics: ReportComponent<AttemptSectionProps<AttemptDiagnosticsData>>;
|
|
29
|
+
export declare function validateUsageData(data: unknown): string | null;
|
|
30
|
+
export declare const AttemptUsage: ReportComponent<AttemptSectionProps<AttemptUsageData>>;
|
|
31
|
+
export declare function validateTraceData(data: unknown): string | null;
|
|
32
|
+
export declare const AttemptTrace: ReportComponent<AttemptSectionProps<AttemptTraceData>>;
|
|
33
|
+
export declare function validateDiffData(data: unknown): string | null;
|
|
34
|
+
export declare const AttemptDiff: ReportComponent<AttemptSectionProps<AttemptDiffData>>;
|
|
35
|
+
/** source / assertions fallback:有 source 放 AttemptSource,否则放 AttemptAssertions。 */
|
|
36
|
+
export declare const AttemptAssessment: ReportComponent<Record<string, never>>;
|
|
37
|
+
/** 内建排列顺序;有 source 时回复已按 loc 展开在 AttemptSource 行内,不再重复一份 round 卡。 */
|
|
38
|
+
export declare const AttemptDetail: ReportComponent<Record<string, never>>;
|