niceeval 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +31 -33
- package/dist/agents/types.d.ts +20 -17
- package/dist/context/types.d.ts +21 -21
- package/dist/i18n/en.d.ts +10 -2
- package/dist/i18n/zh-CN.d.ts +15 -7
- package/dist/o11y/execution-tree.d.ts +103 -0
- package/dist/o11y/otlp/select.d.ts +22 -0
- package/dist/report/{react → assets}/colors.d.ts +8 -0
- package/dist/report/{react → assets}/colors.js +23 -0
- package/dist/report/built-in/index.d.ts +2 -2
- package/dist/report/built-in/index.js +2 -2
- package/dist/report/built-in/standard.d.ts +8 -1
- package/dist/report/built-in/standard.js +17 -7
- package/dist/report/components/attempt-detail/AttemptAssertions.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptAssertions.js +17 -0
- package/dist/report/components/attempt-detail/AttemptConversation.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptConversation.js +34 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiagnostics.js +7 -0
- package/dist/report/components/attempt-detail/AttemptDiff.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptDiff.js +11 -0
- package/dist/report/components/attempt-detail/AttemptError.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptError.js +16 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptFixPrompt.js +7 -0
- package/dist/report/components/attempt-detail/AttemptSource.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptSource.js +27 -0
- package/dist/report/components/attempt-detail/AttemptSummary.d.ts +8 -0
- package/dist/report/components/attempt-detail/AttemptSummary.js +20 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTimeline.js +28 -0
- package/dist/report/components/attempt-detail/AttemptTrace.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptTrace.js +29 -0
- package/dist/report/components/attempt-detail/AttemptUsage.d.ts +6 -0
- package/dist/report/components/attempt-detail/AttemptUsage.js +20 -0
- package/dist/report/components/attempt-detail/compute.d.ts +24 -0
- package/dist/report/components/attempt-detail/compute.js +254 -0
- package/dist/report/components/attempt-detail/faces.d.ts +14 -0
- package/dist/report/components/attempt-detail/faces.js +235 -0
- package/dist/report/components/attempt-detail/index.d.ts +38 -0
- package/dist/report/components/attempt-detail/index.js +527 -0
- package/dist/report/{react → components}/cell.d.ts +5 -3
- package/dist/report/{react → components}/cell.js +3 -3
- package/dist/report/{react → components/entity-lists}/AttemptList.d.ts +8 -4
- package/dist/report/{react → components/entity-lists}/AttemptList.js +16 -10
- package/dist/report/{react → components/entity-lists}/EvalList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/EvalList.js +0 -0
- package/dist/report/{react → components/entity-lists}/ExperimentList.d.ts +3 -3
- package/dist/report/{react → components/entity-lists}/ExperimentList.js +10 -10
- package/dist/report/components/entity-lists/compute.d.ts +23 -0
- package/dist/report/components/entity-lists/compute.js +171 -0
- package/dist/report/components/entity-lists/faces.d.ts +5 -0
- package/dist/report/components/entity-lists/faces.js +174 -0
- package/dist/report/components/entity-lists/index.d.ts +51 -0
- package/dist/report/components/entity-lists/index.js +165 -0
- package/dist/report/{react → components}/fixtures.d.ts +2 -2
- package/dist/report/{react → components}/fixtures.js +1 -1
- package/dist/report/{react → components/metric-views}/DeltaTable.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/DeltaTable.js +4 -4
- package/dist/report/{react → components/metric-views}/MetricBars.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricBars.js +3 -3
- package/dist/report/{react → components/metric-views}/MetricLine.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/MetricLine.js +8 -5
- package/dist/report/{react → components/metric-views}/MetricMatrix.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricMatrix.js +4 -4
- package/dist/report/components/metric-views/MetricScatter.d.ts +11 -0
- package/dist/report/{react → components/metric-views}/MetricScatter.js +26 -23
- package/dist/report/{react → components/metric-views}/MetricTable.d.ts +3 -3
- package/dist/report/{react → components/metric-views}/MetricTable.js +4 -4
- package/dist/report/{react → components/metric-views}/Scoreboard.d.ts +2 -2
- package/dist/report/{react → components/metric-views}/Scoreboard.js +3 -3
- package/dist/report/components/metric-views/compute.d.ts +89 -0
- package/dist/report/{compute.js → components/metric-views/compute.js} +32 -415
- package/dist/report/components/metric-views/faces.d.ts +13 -0
- package/dist/report/components/metric-views/faces.js +381 -0
- package/dist/report/components/metric-views/index.d.ts +50 -0
- package/dist/report/components/metric-views/index.js +273 -0
- package/dist/report/{text → components/metric-views}/plot.js +1 -1
- package/dist/report/components/shared-compute.d.ts +22 -0
- package/dist/report/components/shared-compute.js +47 -0
- package/dist/report/components/shared-faces.d.ts +9 -0
- package/dist/report/components/shared-faces.js +26 -0
- package/dist/report/components/shared.d.ts +68 -0
- package/dist/report/components/shared.js +125 -0
- package/dist/report/{react → components/site-components}/CopyFixPrompt.d.ts +2 -2
- package/dist/report/{react → components/site-components}/CopyFixPrompt.js +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.d.ts +2 -2
- package/dist/report/{react → components/site-components}/HeroCard.js +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.d.ts +2 -2
- package/dist/report/{react → components/site-components}/ScopeWarnings.js +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.d.ts +3 -3
- package/dist/report/{react → components/site-components}/TraceWaterfall.js +5 -5
- package/dist/report/components/site-components/compute.d.ts +28 -0
- package/dist/report/components/site-components/compute.js +132 -0
- package/dist/report/components/site-components/faces.d.ts +21 -0
- package/dist/report/components/site-components/faces.js +75 -0
- package/dist/report/components/site-components/index.d.ts +74 -0
- package/dist/report/components/site-components/index.js +220 -0
- package/dist/report/{scope-warnings.d.ts → components/site-components/scope-warnings.d.ts} +2 -2
- package/dist/report/{scope-warnings.js → components/site-components/scope-warnings.js} +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.d.ts +2 -2
- package/dist/report/{react → components/summaries}/ScopeSummary.js +17 -8
- package/dist/report/components/summaries/compute.d.ts +7 -0
- package/dist/report/components/summaries/compute.js +51 -0
- package/dist/report/components/summaries/faces.d.ts +7 -0
- package/dist/report/components/summaries/faces.js +38 -0
- package/dist/report/components/summaries/index.d.ts +27 -0
- package/dist/report/components/summaries/index.js +78 -0
- package/dist/report/definition/grid-layout.d.ts +50 -0
- package/dist/report/definition/grid-layout.js +89 -0
- package/dist/report/{primitives.d.ts → definition/primitives.d.ts} +34 -4
- package/dist/report/{primitives.js → definition/primitives.js} +101 -10
- package/dist/report/{report.d.ts → definition/report.d.ts} +36 -65
- package/dist/report/{report.js → definition/report.js} +58 -90
- package/dist/report/{text/table.d.ts → definition/table-text.d.ts} +2 -2
- package/dist/report/{text/table.js → definition/table-text.js} +2 -2
- package/dist/report/{tree.d.ts → definition/tree.d.ts} +44 -12
- package/dist/report/{tree.js → definition/tree.js} +10 -12
- package/dist/report/index.d.ts +33 -18
- package/dist/report/index.js +23 -14
- package/dist/report/{aggregate.d.ts → model/aggregate.d.ts} +12 -4
- package/dist/report/{aggregate.js → model/aggregate.js} +28 -8
- package/dist/report/{flag.d.ts → model/flag.d.ts} +14 -1
- package/dist/report/{flag.js → model/flag.js} +35 -0
- package/dist/report/{format.d.ts → model/format.d.ts} +11 -5
- package/dist/report/{format.js → model/format.js} +48 -4
- package/dist/report/{locale.d.ts → model/locale.d.ts} +17 -17
- package/dist/report/{locale.js → model/locale.js} +31 -33
- package/dist/report/{metrics.d.ts → model/metrics.d.ts} +2 -2
- package/dist/report/{metrics.js → model/metrics.js} +2 -2
- package/dist/report/{types.d.ts → model/types.d.ts} +140 -20
- package/dist/report/react/index.d.ts +32 -21
- package/dist/report/react/index.js +33 -21
- package/dist/report/{load.d.ts → runtime/load.d.ts} +1 -1
- package/dist/report/{load.js → runtime/load.js} +1 -1
- package/dist/report/runtime/text.d.ts +67 -0
- package/dist/report/runtime/text.js +114 -0
- package/dist/report/runtime/web.d.ts +35 -0
- package/dist/report/runtime/web.js +70 -0
- package/dist/results/annotated-source.d.ts +87 -0
- package/dist/results/attempt-evidence.d.ts +74 -0
- package/dist/results/attempt-source.d.ts +15 -0
- package/dist/results/select.d.ts +12 -2
- package/dist/results/select.js +27 -5
- package/dist/runner/eval-selection.d.ts +26 -0
- package/dist/runner/feedback/sink.d.ts +26 -1
- package/dist/runner/types.d.ts +149 -23
- package/dist/sandbox/docker.d.ts +6 -0
- package/dist/sandbox/types.d.ts +23 -21
- package/dist/scoring/display.d.ts +7 -2
- package/dist/scoring/display.js +42 -4
- package/dist/scoring/types.d.ts +3 -3
- package/dist/shared/aggregate.d.ts +15 -1
- package/dist/shared/aggregate.js +32 -1
- package/dist/shared/types.d.ts +6 -1
- package/docs-site/images/logo-dark.svg +7 -0
- package/docs-site/images/logo.svg +6 -5
- package/docs-site/zh/README.md +8 -7
- package/docs-site/zh/examples/ai-agent-application.mdx +5 -5
- package/docs-site/zh/examples/coding-agent-extensions.mdx +4 -4
- package/docs-site/zh/examples/index.mdx +3 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +1 -1
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/langgraph.mdx +3 -3
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +2 -2
- package/docs-site/zh/explanation/adapter.mdx +19 -19
- package/docs-site/zh/explanation/assert.mdx +9 -9
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/evals.mdx +8 -8
- package/docs-site/zh/explanation/experiment.mdx +8 -8
- package/docs-site/zh/explanation/hitl.mdx +13 -13
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/explanation/overview.mdx +7 -7
- package/docs-site/zh/explanation/runner.mdx +11 -11
- package/docs-site/zh/explanation/tier.mdx +6 -6
- package/docs-site/zh/index.mdx +14 -14
- package/docs-site/zh/introduction.mdx +13 -16
- package/docs-site/zh/reference/builtin-agents.mdx +61 -28
- package/docs-site/zh/reference/capabilities.mdx +7 -27
- package/docs-site/zh/reference/cli.mdx +35 -32
- package/docs-site/zh/reference/define-agent.mdx +32 -31
- package/docs-site/zh/reference/define-config.mdx +6 -6
- package/docs-site/zh/reference/define-eval.mdx +33 -20
- package/docs-site/zh/reference/events.mdx +4 -4
- package/docs-site/zh/reference/expect.mdx +3 -3
- package/docs-site/zh/reference/official-adapters.mdx +17 -17
- package/docs-site/zh/reference/report-components.mdx +193 -121
- package/docs-site/zh/reference/results-data.mdx +13 -13
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +11 -11
- package/docs-site/zh/troubleshooting/debugging.mdx +43 -19
- package/docs-site/zh/{how-to → tutorials}/agent-feedback-loop.mdx +38 -38
- package/docs-site/zh/tutorials/agent-onboarding.mdx +100 -0
- package/docs-site/zh/tutorials/authoring.mdx +211 -0
- package/docs-site/zh/{how-to → tutorials}/ci-integration.mdx +8 -8
- package/docs-site/zh/{how-to → tutorials}/connect-otel.mdx +22 -22
- package/docs-site/zh/{how-to → tutorials}/connect-your-agent.mdx +58 -63
- package/docs-site/zh/tutorials/custom-reports.mdx +453 -0
- package/docs-site/zh/{how-to → tutorials}/dataset-fanout.mdx +4 -4
- package/docs-site/zh/tutorials/experiments.mdx +103 -0
- package/docs-site/zh/{how-to → tutorials}/fixtures.mdx +9 -9
- package/docs-site/zh/{how-to → tutorials}/publish-report.mdx +11 -5
- package/docs-site/zh/tutorials/quickstart.mdx +18 -18
- package/docs-site/zh/{how-to → tutorials}/reporters.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-agent.mdx +6 -6
- package/docs-site/zh/{how-to → tutorials}/sandbox-providers.mdx +41 -19
- package/docs-site/zh/{how-to → tutorials}/scoring-guide.mdx +4 -4
- package/docs-site/zh/{how-to → tutorials}/viewing-results.mdx +59 -50
- package/docs-site/zh/tutorials/write-experiment.mdx +355 -0
- package/docs-site/zh/{how-to → tutorials}/write-send.mdx +40 -40
- package/package.json +8 -4
- package/src/agents/bub.ts +21 -8
- package/src/agents/claude-code.ts +23 -10
- package/src/agents/codex.test.ts +22 -7
- package/src/agents/codex.ts +22 -9
- package/src/agents/index.ts +2 -2
- package/src/agents/openai-compat.ts +1 -1
- package/src/agents/post-setup.ts +45 -22
- package/src/agents/streaming.ts +1 -1
- package/src/agents/types.ts +20 -17
- package/src/agents/ui-message-stream.test.ts +10 -0
- package/src/agents/ui-message-stream.ts +12 -1
- package/src/cli.ts +77 -64
- package/src/context/types.ts +21 -21
- package/src/define.ts +13 -0
- package/src/i18n/en.ts +30 -15
- package/src/i18n/zh-CN.ts +30 -15
- package/src/index.ts +2 -0
- package/src/report/{react → assets}/colors.ts +22 -0
- package/src/report/{react → assets}/enhance.js +1 -33
- package/src/report/{react → assets}/styles.css +136 -112
- package/src/report/built-in/index.tsx +2 -2
- package/src/report/built-in/standard.tsx +18 -6
- package/src/report/components/attempt-detail/AttemptAssertions.tsx +67 -0
- package/src/report/components/attempt-detail/AttemptConversation.tsx +110 -0
- package/src/report/components/attempt-detail/AttemptDiagnostics.tsx +38 -0
- package/src/report/components/attempt-detail/AttemptDiff.tsx +35 -0
- package/src/report/components/attempt-detail/AttemptError.tsx +31 -0
- package/src/report/components/attempt-detail/AttemptFixPrompt.tsx +26 -0
- package/src/report/components/attempt-detail/AttemptSource.tsx +80 -0
- package/src/report/components/attempt-detail/AttemptSummary.tsx +74 -0
- package/src/report/components/attempt-detail/AttemptTimeline.tsx +108 -0
- package/src/report/components/attempt-detail/AttemptTrace.tsx +63 -0
- package/src/report/components/attempt-detail/AttemptUsage.tsx +29 -0
- package/src/report/components/attempt-detail/attempt-components.test.tsx +680 -0
- package/src/report/components/attempt-detail/compute.ts +287 -0
- package/src/report/components/attempt-detail/faces.ts +257 -0
- package/src/report/components/attempt-detail/index.tsx +583 -0
- package/src/report/components/attempt-detail/validate.test.ts +235 -0
- package/src/report/{react → components}/cell.tsx +6 -3
- package/src/report/{report.test.ts → components/compute.test.ts} +316 -46
- package/src/report/{react → components/entity-lists}/AttemptList.tsx +24 -16
- package/src/report/{react → components/entity-lists}/EvalList.tsx +0 -0
- package/src/report/{react → components/entity-lists}/ExperimentList.tsx +23 -20
- package/src/report/components/entity-lists/compute.ts +196 -0
- package/src/report/components/entity-lists/faces.ts +207 -0
- package/src/report/components/entity-lists/index.tsx +241 -0
- package/src/report/components/entity-lists/validate.test.ts +114 -0
- package/src/report/{react → components}/fixtures.ts +2 -2
- package/src/report/{react → components/metric-views}/DeltaTable.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricBars.tsx +5 -5
- package/src/report/{react → components/metric-views}/MetricLine.tsx +9 -6
- package/src/report/{react → components/metric-views}/MetricMatrix.tsx +6 -6
- package/src/report/{react → components/metric-views}/MetricScatter.tsx +41 -33
- package/src/report/{react → components/metric-views}/MetricTable.tsx +6 -6
- package/src/report/{react → components/metric-views}/Scoreboard.tsx +4 -4
- package/src/report/{compute.ts → components/metric-views/compute.ts} +51 -477
- package/src/report/components/metric-views/faces.ts +423 -0
- package/src/report/components/metric-views/index.tsx +352 -0
- package/src/report/{text → components/metric-views}/plot.ts +1 -1
- package/src/report/components/metric-views/validate.test.ts +211 -0
- package/src/report/{react → components}/render.test.tsx +47 -17
- package/src/report/components/shared-compute.ts +59 -0
- package/src/report/components/shared-faces.ts +30 -0
- package/src/report/components/shared.ts +192 -0
- package/src/report/{react → components/site-components}/CopyFixPrompt.tsx +3 -3
- package/src/report/{react → components/site-components}/HeroCard.tsx +3 -3
- package/src/report/{react → components/site-components}/ScopeWarnings.tsx +4 -4
- package/src/report/{react → components/site-components}/TraceWaterfall.tsx +13 -10
- package/src/report/components/site-components/compute.ts +144 -0
- package/src/report/components/site-components/faces.ts +80 -0
- package/src/report/components/site-components/index.tsx +265 -0
- package/src/report/{scope-warnings.ts → components/site-components/scope-warnings.ts} +3 -3
- package/src/report/{site-components.test.tsx → components/site-components/site-components.test.tsx} +23 -21
- package/src/report/components/site-components/validate.test.ts +89 -0
- package/src/report/{react → components/summaries}/ScopeSummary.tsx +26 -11
- package/src/report/components/summaries/compute.ts +56 -0
- package/src/report/components/summaries/faces.ts +48 -0
- package/src/report/components/summaries/index.tsx +127 -0
- package/src/report/components/summaries/validate.test.ts +49 -0
- package/src/report/definition/grid-layout.test.ts +124 -0
- package/src/report/definition/grid-layout.ts +146 -0
- package/src/report/{primitives.tsx → definition/primitives.tsx} +168 -18
- package/src/report/{report.ts → definition/report.ts} +98 -165
- package/src/report/{shell-head.test.ts → definition/shell-head.test.ts} +2 -2
- package/src/report/{text/table.ts → definition/table-text.ts} +4 -4
- package/src/report/{tree.ts → definition/tree.ts} +58 -22
- package/src/report/index.ts +126 -66
- package/src/report/{aggregate.ts → model/aggregate.ts} +31 -9
- package/src/report/{flag.ts → model/flag.ts} +39 -1
- package/src/report/{format.ts → model/format.ts} +59 -5
- package/src/report/{locale.ts → model/locale.ts} +34 -36
- package/src/report/{metrics.ts → model/metrics.ts} +3 -3
- package/src/report/{types.ts → model/types.ts} +143 -22
- package/src/report/react/index.tsx +52 -26
- package/src/report/{dual-render.test.tsx → runtime/dual-render.test.tsx} +564 -49
- package/src/report/runtime/host.test.ts +44 -0
- package/src/report/runtime/host.ts +138 -0
- package/src/report/{load.ts → runtime/load.ts} +1 -1
- package/src/report/runtime/text.ts +192 -0
- package/src/report/runtime/web.ts +106 -0
- package/src/results/attempt-evidence.ts +5 -1
- package/src/results/host-equivalence.test.ts +54 -3
- package/src/results/select.ts +30 -6
- package/src/runner/attempt.test.ts +48 -1
- package/src/runner/attempt.ts +62 -42
- package/src/runner/cleanup-timeout.test.ts +16 -0
- package/src/runner/cleanup-timeout.ts +23 -0
- package/src/runner/discover.ts +1 -2
- package/src/runner/eval-selection.test.ts +187 -0
- package/src/runner/eval-selection.ts +109 -0
- package/src/runner/experiment-cleanup-registry.test.ts +89 -0
- package/src/runner/experiment-cleanup-registry.ts +39 -0
- package/src/runner/experiment-labels.test.ts +71 -0
- package/src/runner/feedback/agent.test.ts +33 -1
- package/src/runner/feedback/agent.ts +12 -16
- package/src/runner/feedback/ci.test.ts +35 -1
- package/src/runner/feedback/ci.ts +16 -18
- package/src/runner/feedback/coordinator.ts +28 -0
- package/src/runner/feedback/human.test.ts +108 -1
- package/src/runner/feedback/human.ts +71 -21
- package/src/runner/feedback/reducer.test.ts +64 -0
- package/src/runner/feedback/reducer.ts +26 -0
- package/src/runner/feedback/sink.ts +40 -1
- package/src/runner/fingerprint.ts +2 -1
- package/src/runner/report.test.ts +14 -0
- package/src/runner/report.ts +3 -1
- package/src/runner/run.test.ts +390 -12
- package/src/runner/run.ts +259 -26
- package/src/runner/sandbox-selection.test.ts +13 -3
- package/src/runner/sandbox-selection.ts +4 -2
- package/src/runner/types.ts +158 -23
- package/src/sandbox/docker.ts +7 -0
- package/src/sandbox/e2b.ts +12 -13
- package/src/sandbox/types.ts +23 -21
- package/src/scoring/display.test.ts +28 -2
- package/src/scoring/display.ts +38 -5
- package/src/scoring/types.ts +3 -3
- package/src/shared/aggregate.test.ts +52 -0
- package/src/shared/aggregate.ts +30 -1
- package/src/shared/types.ts +6 -1
- package/src/show/command.test.ts +34 -0
- package/src/show/command.ts +16 -0
- package/src/show/compose.ts +1 -1
- package/src/show/index.ts +58 -22
- package/src/show/render.ts +16 -240
- package/src/show/show.test.ts +128 -35
- package/src/view/app/App.test.tsx +8 -7
- package/src/view/app/App.tsx +135 -51
- package/src/view/app/i18n.ts +7 -147
- package/src/view/app/lib/attempt-dialog.test.ts +65 -0
- package/src/view/app/lib/attempt-dialog.ts +69 -0
- package/src/view/app/main.tsx +0 -1
- package/src/view/app/types.ts +3 -86
- package/src/view/artifact-serving.test.ts +22 -38
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +15 -22
- package/src/view/data.test.ts +48 -36
- package/src/view/data.ts +122 -89
- package/src/view/index.ts +3 -13
- package/src/view/server.ts +36 -6
- package/src/view/shared/types.ts +6 -31
- package/src/view/site-parity.test.ts +24 -1
- package/src/view/site.ts +144 -17
- package/src/view/styles.css +18 -806
- package/src/view/view-report.test.ts +182 -34
- package/dist/report/components.d.ts +0 -179
- package/dist/report/components.js +0 -544
- package/dist/report/compute.d.ts +0 -139
- package/dist/report/react/ExperimentComparison.d.ts +0 -10
- package/dist/report/react/ExperimentComparison.js +0 -12
- package/dist/report/react/MetricScatter.d.ts +0 -9
- package/dist/report/react/format.d.ts +0 -3
- package/dist/report/react/format.js +0 -7
- package/dist/report/text/faces.d.ts +0 -45
- package/dist/report/text/faces.js +0 -671
- package/dist/report/web.d.ts +0 -32
- package/dist/report/web.js +0 -48
- package/docs-site/zh/how-to/authoring.mdx +0 -162
- package/docs-site/zh/how-to/custom-reports.mdx +0 -414
- package/docs-site/zh/how-to/experiments.mdx +0 -86
- package/docs-site/zh/how-to/write-experiment.mdx +0 -164
- package/src/report/components.tsx +0 -925
- package/src/report/react/ExperimentComparison.tsx +0 -73
- package/src/report/react/format.ts +0 -9
- package/src/report/text/faces.ts +0 -767
- package/src/report/web.ts +0 -77
- package/src/show/report-host.test.ts +0 -205
- package/src/show/report-host.ts +0 -389
- package/src/view/app/components/AttemptModal.tsx +0 -496
- package/src/view/app/components/CodeView.test.tsx +0 -142
- package/src/view/app/components/CodeView.tsx +0 -310
- package/src/view/app/components/CopyControls.tsx +0 -106
- package/src/view/app/components/Trace.tsx +0 -100
- package/src/view/app/components/Transcript.tsx +0 -157
- package/src/view/app/components/ui/badge.tsx +0 -21
- package/src/view/app/lib/artifact-url.ts +0 -17
- package/src/view/app/lib/attempt-route.test.ts +0 -80
- package/src/view/app/lib/attempt-route.ts +0 -52
- package/src/view/app/lib/format.ts +0 -70
- package/src/view/app/lib/guards.test.ts +0 -108
- package/src/view/app/lib/guards.ts +0 -71
- package/src/view/app/lib/rows.ts +0 -22
- package/src/view/app/lib/transcript-data.tsx +0 -151
- package/src/view/app/lib/verdict.ts +0 -23
- package/src/view/app/shared.ts +0 -8
- /package/dist/report/{react → components/metric-views}/chart-math.d.ts +0 -0
- /package/dist/report/{react → components/metric-views}/chart-math.js +0 -0
- /package/dist/report/{text → components/metric-views}/plot.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.d.ts +0 -0
- /package/dist/report/{react → components/site-components}/PoweredBy.js +0 -0
- /package/dist/report/{text/layout.d.ts → model/text-layout.d.ts} +0 -0
- /package/dist/report/{text/layout.js → model/text-layout.js} +0 -0
- /package/dist/report/{types.js → model/types.js} +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.test.ts +0 -0
- /package/src/report/{react → components/metric-views}/chart-math.ts +0 -0
- /package/src/report/{react → components/site-components}/PoweredBy.tsx +0 -0
- /package/src/report/{text/layout.ts → model/text-layout.ts} +0 -0
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
// Attempt 详情组件族的计算函数(docs/feature/reports/library/attempt-detail.md)。每个
|
|
2
|
+
// `attempt*Data(evidence)` 都是纯同步派生:evidence 已经由 loadAttemptEvidence 一次性
|
|
3
|
+
// 装配好全部证据,这里只做适合展示与序列化的取舍,不读文件、不 fetch、不重复调用
|
|
4
|
+
// attempt.events() / trace() / diff()。
|
|
5
|
+
//
|
|
6
|
+
// 与 compute.ts(Scope → *Data)不同,这一族的输入恒为单个 AttemptEvidence,函数签名
|
|
7
|
+
// 因此不是 async——没有 IO 就没有理由返回 Promise。
|
|
8
|
+
|
|
9
|
+
import type { AttemptEvidence } from "../../../results/attempt-evidence.ts";
|
|
10
|
+
import type {
|
|
11
|
+
AttemptAssertionsData,
|
|
12
|
+
AttemptConversationData,
|
|
13
|
+
AttemptConversationReply,
|
|
14
|
+
AttemptConversationRound,
|
|
15
|
+
AttemptDiagnosticsData,
|
|
16
|
+
AttemptDiffData,
|
|
17
|
+
AttemptDiffFileEntry,
|
|
18
|
+
AttemptErrorData,
|
|
19
|
+
AttemptFixPromptData,
|
|
20
|
+
AttemptSourceData,
|
|
21
|
+
AttemptSummaryData,
|
|
22
|
+
AttemptTimelineData,
|
|
23
|
+
AttemptTraceData,
|
|
24
|
+
AttemptUsageData,
|
|
25
|
+
} from "../../model/types.ts";
|
|
26
|
+
import type { AssertionResult, DiagnosticRecord, JsonValue, StreamEvent } from "../../../types.ts";
|
|
27
|
+
import { attemptCostUSD } from "../../model/metrics.ts";
|
|
28
|
+
import { failureSummaryOf } from "../entity-lists/compute.ts";
|
|
29
|
+
|
|
30
|
+
// ───────────────────────── AttemptSummary(恒非空) ─────────────────────────
|
|
31
|
+
|
|
32
|
+
export function attemptSummaryData(evidence: AttemptEvidence): AttemptSummaryData {
|
|
33
|
+
const { result } = evidence;
|
|
34
|
+
return {
|
|
35
|
+
locator: evidence.locator,
|
|
36
|
+
identity: evidence.identity,
|
|
37
|
+
verdict: result.verdict,
|
|
38
|
+
startedAt: result.startedAt,
|
|
39
|
+
durationMs: result.durationMs,
|
|
40
|
+
costUSD: attemptCostUSD(result),
|
|
41
|
+
capabilities: evidence.capabilities,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ───────────────────────── AttemptError ─────────────────────────
|
|
46
|
+
|
|
47
|
+
export function attemptErrorData(evidence: AttemptEvidence): AttemptErrorData | null {
|
|
48
|
+
const err = evidence.result.error;
|
|
49
|
+
return err ?? null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ───────────────────────── AttemptAssertions ─────────────────────────
|
|
53
|
+
|
|
54
|
+
export function attemptAssertionsData(evidence: AttemptEvidence): AttemptAssertionsData | null {
|
|
55
|
+
const assertions = evidence.result.assertions;
|
|
56
|
+
if (!assertions || assertions.length === 0) return null;
|
|
57
|
+
const attention = assertions.filter((a) => a.outcome !== "passed");
|
|
58
|
+
const passed = assertions.filter((a) => a.outcome === "passed");
|
|
59
|
+
const groups = new Map<string, AssertionResult[]>();
|
|
60
|
+
for (const a of passed) {
|
|
61
|
+
const key = a.groupPath?.join(" > ") ?? "";
|
|
62
|
+
const list = groups.get(key);
|
|
63
|
+
if (list) list.push(a);
|
|
64
|
+
else groups.set(key, [a]);
|
|
65
|
+
}
|
|
66
|
+
return { attention, passedGroups: [...groups.entries()].map(([group, items]) => ({ group, items })) };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ───────────────────────── AttemptSource ─────────────────────────
|
|
70
|
+
|
|
71
|
+
export function attemptSourceData(evidence: AttemptEvidence): AttemptSourceData | null {
|
|
72
|
+
if (!evidence.capabilities.source || evidence.evalSource === null) return null;
|
|
73
|
+
const { sourcePath, lines, unmapped, summary } = evidence.evalSource;
|
|
74
|
+
return { locator: evidence.locator, sourcePath, lines, unmapped, summary };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
78
|
+
|
|
79
|
+
/** 单条 attempt 版的批量修复 prompt(与 CopyFixPrompt 的多条版本同一份步骤文案)。 */
|
|
80
|
+
export function attemptFixPromptData(evidence: AttemptEvidence): AttemptFixPromptData | null {
|
|
81
|
+
const { result, identity } = evidence;
|
|
82
|
+
if (result.verdict !== "failed" && result.verdict !== "errored") return null;
|
|
83
|
+
const { summary, more } = failureSummaryOf(result);
|
|
84
|
+
if (summary === null) return null;
|
|
85
|
+
const reason = more > 0 ? `${summary} (+${more} more failures)` : summary;
|
|
86
|
+
const prompt = [
|
|
87
|
+
"Fix the failing eval from this niceeval run.",
|
|
88
|
+
"",
|
|
89
|
+
"## Failure",
|
|
90
|
+
`eval "${identity.evalId}" [experiment ${identity.experimentId}] — ${result.verdict}`,
|
|
91
|
+
` reason: ${reason}`,
|
|
92
|
+
` inspect: niceeval show ${evidence.locator}`,
|
|
93
|
+
"",
|
|
94
|
+
"## Steps",
|
|
95
|
+
"1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
|
|
96
|
+
"2. Run the inspect command above with `--source`, `--execution`, `--timing`, and `--diff` to see the assertions, transcript, timing, and workspace diff.",
|
|
97
|
+
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
98
|
+
`4. Re-run: \`npx niceeval exp ${identity.experimentId} ${identity.evalId}\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
99
|
+
"5. Run `npx niceeval show` and confirm this failure is gone.",
|
|
100
|
+
].join("\n");
|
|
101
|
+
return { prompt };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ───────────────────────── AttemptTimeline ─────────────────────────
|
|
105
|
+
|
|
106
|
+
/** 收尾段的阶段名(见 docs/feature/results/architecture.md);两面渲染都把这些单列在主链之后,不计入主链总耗时。 */
|
|
107
|
+
export const TIMELINE_CLOSING_PHASES: ReadonlySet<string> = new Set([
|
|
108
|
+
"eval.teardown",
|
|
109
|
+
"agent.teardown",
|
|
110
|
+
"sandbox.teardown",
|
|
111
|
+
"sandbox.suspend",
|
|
112
|
+
"sandbox.stop",
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
export function attemptTimelineData(evidence: AttemptEvidence): AttemptTimelineData | null {
|
|
116
|
+
const phases = evidence.result.phases;
|
|
117
|
+
if (!phases || phases.length === 0) return null;
|
|
118
|
+
return { locator: evidence.locator, phases, trace: evidence.trace };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ───────────────────────── AttemptConversation ─────────────────────────
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* 标准事件流按 `loc` 分轮(docs/feature/reports/library/attempt-detail.md「Attempt 详情组件」):
|
|
125
|
+
* 带 loc 的 user 消息开一轮;无 loc 的 user 消息不开新轮——与当前轮 sent 同文本的回显直接
|
|
126
|
+
* 吃掉,其它(stop-hook 反馈、skill 注入等轮内注入)作为回复条目留在当前轮。流首出现无 loc
|
|
127
|
+
* 的 user 消息(没有当前轮可归入)时退化开一条 loc 缺省的兜底轮,不丢弃。未识别的事件类型
|
|
128
|
+
* 包成 `raw` 条目原样呈现,不吞没其余事件——StreamEvent 是随 artifact 版本演进的开放词表,
|
|
129
|
+
* 这份纯函数不能假设自己认识每一种将来会出现的 type。
|
|
130
|
+
*/
|
|
131
|
+
export function attemptConversationData(evidence: AttemptEvidence): AttemptConversationData | null {
|
|
132
|
+
const events = evidence.events;
|
|
133
|
+
if (!events || events.length === 0) return null;
|
|
134
|
+
|
|
135
|
+
const rounds: AttemptConversationRound[] = [];
|
|
136
|
+
const toolByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>();
|
|
137
|
+
const subagentByCallId = new Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>();
|
|
138
|
+
let current: AttemptConversationRound | null = null;
|
|
139
|
+
|
|
140
|
+
for (const ev of events) {
|
|
141
|
+
if (ev.type === "message" && ev.role === "user") {
|
|
142
|
+
if (!ev.loc && current) {
|
|
143
|
+
if (current.replies.length === 0 && (ev.text || "") === current.sentText) continue;
|
|
144
|
+
current.replies.push({ kind: "user", text: ev.text || "" });
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
current = { loc: ev.loc, sentText: ev.text || "", replies: [] };
|
|
148
|
+
rounds.push(current);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
if (!current) {
|
|
152
|
+
// 流首没有开轮的用户消息就先来了其它事件(旧 artifact 的边界情况):开一条无 loc 兜底轮。
|
|
153
|
+
current = { sentText: "", replies: [] };
|
|
154
|
+
rounds.push(current);
|
|
155
|
+
}
|
|
156
|
+
current.replies.push(...conversationReplyOf(ev, toolByCallId, subagentByCallId));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return { locator: evidence.locator, rounds };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** 单条事件 → 0 或 1 条回复条目;action.result/subagent.completed 只更新已有条目,不新增。 */
|
|
163
|
+
function conversationReplyOf(
|
|
164
|
+
ev: StreamEvent,
|
|
165
|
+
toolByCallId: Map<string, Extract<AttemptConversationReply, { kind: "tool" }>>,
|
|
166
|
+
subagentByCallId: Map<string, Extract<AttemptConversationReply, { kind: "subagent" }>>,
|
|
167
|
+
): AttemptConversationReply[] {
|
|
168
|
+
switch (ev.type) {
|
|
169
|
+
case "message":
|
|
170
|
+
// role === "user" 已在主循环处理(开轮 / 回显吃掉 / 轮内注入),这里只剩 assistant。
|
|
171
|
+
return [{ kind: "assistant", text: ev.text }];
|
|
172
|
+
case "thinking":
|
|
173
|
+
return [{ kind: "thinking", text: ev.text }];
|
|
174
|
+
case "error":
|
|
175
|
+
return [{ kind: "error", text: ev.message }];
|
|
176
|
+
case "skill.loaded":
|
|
177
|
+
return [{ kind: "skill", skill: ev.skill }];
|
|
178
|
+
case "input.requested":
|
|
179
|
+
return [{ kind: "input", request: ev.request }];
|
|
180
|
+
case "compaction":
|
|
181
|
+
return [{ kind: "compaction", reason: ev.reason }];
|
|
182
|
+
case "action.called": {
|
|
183
|
+
const reply: Extract<AttemptConversationReply, { kind: "tool" }> = {
|
|
184
|
+
kind: "tool",
|
|
185
|
+
callId: ev.callId,
|
|
186
|
+
name: ev.name,
|
|
187
|
+
tool: ev.tool,
|
|
188
|
+
input: ev.input,
|
|
189
|
+
};
|
|
190
|
+
toolByCallId.set(ev.callId, reply);
|
|
191
|
+
return [reply];
|
|
192
|
+
}
|
|
193
|
+
case "action.result": {
|
|
194
|
+
const tool = toolByCallId.get(ev.callId);
|
|
195
|
+
if (tool) {
|
|
196
|
+
tool.output = ev.output;
|
|
197
|
+
tool.status = ev.status;
|
|
198
|
+
}
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
case "subagent.called": {
|
|
202
|
+
const reply: Extract<AttemptConversationReply, { kind: "subagent" }> = {
|
|
203
|
+
kind: "subagent",
|
|
204
|
+
callId: ev.callId,
|
|
205
|
+
name: ev.name,
|
|
206
|
+
remoteUrl: ev.remoteUrl,
|
|
207
|
+
};
|
|
208
|
+
subagentByCallId.set(ev.callId, reply);
|
|
209
|
+
return [reply];
|
|
210
|
+
}
|
|
211
|
+
case "subagent.completed": {
|
|
212
|
+
const subagent = subagentByCallId.get(ev.callId);
|
|
213
|
+
if (subagent) {
|
|
214
|
+
subagent.output = ev.output;
|
|
215
|
+
subagent.status = ev.status;
|
|
216
|
+
}
|
|
217
|
+
return [];
|
|
218
|
+
}
|
|
219
|
+
default: {
|
|
220
|
+
// 穷尽性检查在这里刻意不做:StreamEvent 是随 artifact 版本演进的开放词表,未识别的
|
|
221
|
+
// 事件(将来的新 type,或第三方 harness 的自定义变体)包成 raw 原样呈现,不静默丢弃、
|
|
222
|
+
// 也不因为一个不认识的条目让整个装配失败。
|
|
223
|
+
return [{ kind: "raw", raw: ev as unknown as JsonValue }];
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// ───────────────────────── AttemptDiagnostics ─────────────────────────
|
|
229
|
+
|
|
230
|
+
export function attemptDiagnosticsData(evidence: AttemptEvidence): AttemptDiagnosticsData | null {
|
|
231
|
+
const diagnostics = evidence.result.diagnostics;
|
|
232
|
+
if (!diagnostics || diagnostics.length === 0) return null;
|
|
233
|
+
const groups = new Map<string, DiagnosticRecord[]>();
|
|
234
|
+
for (const d of diagnostics) {
|
|
235
|
+
const list = groups.get(d.phase);
|
|
236
|
+
if (list) list.push(d);
|
|
237
|
+
else groups.set(d.phase, [d]);
|
|
238
|
+
}
|
|
239
|
+
return { groups: [...groups.entries()].map(([phase, items]) => ({ phase, items })) };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// ───────────────────────── AttemptUsage ─────────────────────────
|
|
243
|
+
|
|
244
|
+
export function attemptUsageData(evidence: AttemptEvidence): AttemptUsageData | null {
|
|
245
|
+
const usage = evidence.result.usage;
|
|
246
|
+
if (!usage) return null;
|
|
247
|
+
return { usage, costUSD: attemptCostUSD(evidence.result) };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
251
|
+
|
|
252
|
+
export function attemptTraceData(evidence: AttemptEvidence): AttemptTraceData | null {
|
|
253
|
+
const spans = evidence.trace;
|
|
254
|
+
if (!spans || spans.length === 0) return null;
|
|
255
|
+
return { locator: evidence.locator, spans };
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// ───────────────────────── AttemptDiff ─────────────────────────
|
|
259
|
+
|
|
260
|
+
/** 有界行 diff(公共前后缀修剪):对单区域编辑精确,复杂编辑给出上界近似;与 `niceeval show --diff` 同一算法。 */
|
|
261
|
+
function lineDelta(before: string | undefined, after: string | undefined): { added: number; deleted: number } {
|
|
262
|
+
const a = before === undefined ? [] : before.split("\n");
|
|
263
|
+
const b = after === undefined ? [] : after.split("\n");
|
|
264
|
+
let prefix = 0;
|
|
265
|
+
while (prefix < a.length && prefix < b.length && a[prefix] === b[prefix]) prefix++;
|
|
266
|
+
let suffix = 0;
|
|
267
|
+
while (suffix < a.length - prefix && suffix < b.length - prefix && a[a.length - 1 - suffix] === b[b.length - 1 - suffix]) suffix++;
|
|
268
|
+
return { added: b.length - prefix - suffix, deleted: a.length - prefix - suffix };
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function attemptDiffData(evidence: AttemptEvidence): AttemptDiffData | null {
|
|
272
|
+
if (!evidence.capabilities.diff || evidence.diff === null) return null;
|
|
273
|
+
const diff = evidence.diff;
|
|
274
|
+
const files: AttemptDiffFileEntry[] = [];
|
|
275
|
+
for (const [path, summary] of Object.entries(diff.files).sort(([a], [b]) => a.localeCompare(b))) {
|
|
276
|
+
if (summary.net === "none") continue;
|
|
277
|
+
const windows = diff.windows.filter((w) => w.changes[path] !== undefined).map((w) => w.window);
|
|
278
|
+
if (summary.binary) {
|
|
279
|
+
files.push({ path, net: summary.net, lines: { added: 0, deleted: 0 }, binary: true, windows });
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
const before = summary.net === "added" ? undefined : diff.windows.find((w) => w.changes[path]?.before !== undefined)?.changes[path]?.before;
|
|
283
|
+
const after = summary.net === "deleted" ? undefined : diff.get(path);
|
|
284
|
+
files.push({ path, net: summary.net, lines: lineDelta(before, after), windows });
|
|
285
|
+
}
|
|
286
|
+
return files.length > 0 ? { locator: evidence.locator, files } : null;
|
|
287
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
// Attempt 详情组件族的 text 面(docs/feature/reports/library/attempt-detail.md「在 show 与
|
|
2
|
+
// view 怎样渲染」):与 web 面共享同一次 resolve 产出的 data 事实(verdict、计数、能力位、
|
|
3
|
+
// 引用),允许把大块内容折成摘要 + 专用证据命令,但不得改变判定、计数或引用。
|
|
4
|
+
// 零 react、零 IO、纯同步——text 宿主不需要 react-dom 的那一半。
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
AttemptAssertionsData,
|
|
8
|
+
AttemptConversationData,
|
|
9
|
+
AttemptConversationReply,
|
|
10
|
+
AttemptDiagnosticsData,
|
|
11
|
+
AttemptDiffData,
|
|
12
|
+
AttemptErrorData,
|
|
13
|
+
AttemptFixPromptData,
|
|
14
|
+
AttemptSourceData,
|
|
15
|
+
AttemptSummaryData,
|
|
16
|
+
AttemptTimelineData,
|
|
17
|
+
AttemptTraceData,
|
|
18
|
+
AttemptUsageData,
|
|
19
|
+
} from "../../model/types.ts";
|
|
20
|
+
import type { AssertionResult, TimingNode } from "../../../types.ts";
|
|
21
|
+
import type { AttemptLocator } from "../../../results/locator.ts";
|
|
22
|
+
import type { TextContext } from "../../definition/tree.ts";
|
|
23
|
+
import { localeText } from "../../model/locale.ts";
|
|
24
|
+
import { formatDurationMs, formatMetricValue, formatReportDateTime, formatUSD, verdictMark } from "../../model/format.ts";
|
|
25
|
+
import { TIMELINE_CLOSING_PHASES } from "./compute.ts";
|
|
26
|
+
import { summaryText } from "../../../scoring/display.ts";
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* 证据切面下钻命令:`ctx.attemptCommand` 由宿主按当前 report 是否有 attempt-input page
|
|
30
|
+
* 注入(见 report.ts DEFAULT_ATTEMPT_COMMAND);没有时不生成假命令,调用方直接省略这一段
|
|
31
|
+
* (与 traceWaterfallText 同一套退化规则,architecture.md「Attempt 详情是一张参数化 page」)。
|
|
32
|
+
*/
|
|
33
|
+
function evidenceCommand(ctx: TextContext, locator: AttemptLocator, flag: string): string | undefined {
|
|
34
|
+
return ctx.attemptCommand ? `${ctx.attemptCommand(locator)} ${flag}` : undefined;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ───────────────────────── AttemptSummary ─────────────────────────
|
|
38
|
+
|
|
39
|
+
export function attemptSummaryText(data: AttemptSummaryData, ctx: TextContext): string {
|
|
40
|
+
const locale = ctx.locale;
|
|
41
|
+
const parts = [
|
|
42
|
+
data.locator,
|
|
43
|
+
data.identity.evalId,
|
|
44
|
+
data.identity.experimentId,
|
|
45
|
+
localeText(locale, "attemptSummary.attempt", { n: data.identity.attempt + 1 }),
|
|
46
|
+
`${verdictMark(data.verdict)} ${localeText(locale, `verdict.${data.verdict}`)}`,
|
|
47
|
+
];
|
|
48
|
+
if (data.startedAt !== undefined) parts.push(formatReportDateTime(data.startedAt, locale));
|
|
49
|
+
parts.push(formatDurationMs(data.durationMs));
|
|
50
|
+
if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
|
|
51
|
+
return parts.join(" · ");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ───────────────────────── AttemptError ─────────────────────────
|
|
55
|
+
|
|
56
|
+
export function attemptErrorText(data: AttemptErrorData | null, _ctx: TextContext): string {
|
|
57
|
+
if (data === null) return "";
|
|
58
|
+
// message/cause 折单行加上限(同 assertionLine 的 summaryText 规则);stack 是唯一没有替代
|
|
59
|
+
// 查看入口的自由文本,原样保留多行,不折。
|
|
60
|
+
const lines = [`error: ${data.code}`, ` phase: ${data.phase}`, ` message: ${summaryText(data.message)}`];
|
|
61
|
+
if (data.cause) {
|
|
62
|
+
const causeMessage = summaryText(data.cause.message);
|
|
63
|
+
lines.push(` cause: ${data.cause.name ? `${data.cause.name} · ${causeMessage}` : causeMessage}`);
|
|
64
|
+
}
|
|
65
|
+
const stack = data.stack?.replace(/\n+$/, "");
|
|
66
|
+
return stack ? `${lines.join("\n")}\n\n${stack}` : lines.join("\n");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ───────────────────────── AttemptAssertions ─────────────────────────
|
|
70
|
+
|
|
71
|
+
/** `loc` 是断言在 eval 源码里的调用点,独立于整份源码是否被捕获(`AttemptSource` 的能力位);
|
|
72
|
+
* 失败断言只要带 loc 就给可复制定位的源码锚,不必等 AttemptSource 可用才显示。 */
|
|
73
|
+
function locAnchor(loc: { file: string; line: number; column?: number }): string {
|
|
74
|
+
return `${loc.file}:${loc.line}${loc.column ? `:${loc.column}` : ""}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function assertionLine(a: AssertionResult): string {
|
|
78
|
+
const group = a.groupPath && a.groupPath.length > 0 ? `${a.groupPath.join(" > ")} · ` : "";
|
|
79
|
+
if (a.outcome === "unavailable") return `◌ unavailable · ${group}${a.name} — ${a.reason}`;
|
|
80
|
+
const mark = a.outcome === "passed" ? "✓" : "✗";
|
|
81
|
+
const detail = a.detail && a.detail !== a.name ? `: ${a.detail}` : "";
|
|
82
|
+
// expected/received 折单行 + 加字符上限(与 scoring 摘要面同一条 summaryText 规则):
|
|
83
|
+
// received 常常就是被检查的整份文本(如 includes() 对全文匹配),不收口会把整份源码/工具
|
|
84
|
+
// 输出灌进这一行;完整值仍在 web 面(可展开的完整结构化细节)与 --source 里原样可查。
|
|
85
|
+
const evidence = [
|
|
86
|
+
a.expected !== undefined ? `expected: ${summaryText(a.expected)}` : undefined,
|
|
87
|
+
a.received !== undefined ? `received: ${summaryText(a.received)}` : undefined,
|
|
88
|
+
a.outcome === "failed" && a.loc ? `source: ${locAnchor(a.loc)}` : undefined,
|
|
89
|
+
].filter((part): part is string => part !== undefined);
|
|
90
|
+
const evidenceSuffix = evidence.length > 0 ? ` · ${evidence.join(" · ")}` : "";
|
|
91
|
+
return `${mark} ${a.severity} · ${group}${a.name}${detail}${evidenceSuffix}`;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function attemptAssertionsText(data: AttemptAssertionsData | null, _ctx: TextContext): string {
|
|
95
|
+
if (data === null) return "";
|
|
96
|
+
const lines = data.attention.map(assertionLine);
|
|
97
|
+
for (const { group, items } of data.passedGroups) {
|
|
98
|
+
lines.push(`✓ passed · ${group || "(ungrouped)"} · ${items.length}`);
|
|
99
|
+
}
|
|
100
|
+
return lines.join("\n");
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ───────────────────────── AttemptSource ─────────────────────────
|
|
104
|
+
|
|
105
|
+
export function attemptSourceText(data: AttemptSourceData | null, ctx: TextContext): string {
|
|
106
|
+
if (data === null) return "";
|
|
107
|
+
const command = evidenceCommand(ctx, data.locator, "--source");
|
|
108
|
+
const headerParts = [`${data.sourcePath} · ${data.summary.annotatedLines}/${data.summary.totalLines} lines annotated`];
|
|
109
|
+
if (command) headerParts.push(command);
|
|
110
|
+
// 源码锚由 assertionLine 自己按 a.loc 拼(与 AttemptAssertions 共用同一份逻辑);这里只负责
|
|
111
|
+
// 挑出非 passed 的条目,不重复算锚点。
|
|
112
|
+
const failed = data.lines.flatMap((line) => line.assertions.filter((a) => a.outcome !== "passed"));
|
|
113
|
+
const lines = failed.map((a) => ` ${assertionLine(a)}`);
|
|
114
|
+
return [headerParts.join(" · "), ...lines].join("\n");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ───────────────────────── AttemptFixPrompt ─────────────────────────
|
|
118
|
+
|
|
119
|
+
/** text 面零输出:终端已有 attemptSummaryText 里的 locator,直接跑 `niceeval show @<locator>` 即可;这里不重复整段 prompt。 */
|
|
120
|
+
export function attemptFixPromptText(_data: AttemptFixPromptData | null, _ctx: TextContext): string {
|
|
121
|
+
return "";
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ───────────────────────── AttemptTimeline ─────────────────────────
|
|
125
|
+
|
|
126
|
+
function fmtMs(ms: number): string {
|
|
127
|
+
if (ms >= 60_000) return `${Math.floor(ms / 60_000)}m ${Math.round((ms % 60_000) / 1000)}s`;
|
|
128
|
+
if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
|
|
129
|
+
return `${ms}ms`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function countTimingNodes(nodes: readonly TimingNode[]): number {
|
|
133
|
+
return nodes.reduce((sum, n) => sum + 1 + countTimingNodes(n.children ?? []), 0);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export function attemptTimelineText(data: AttemptTimelineData | null, ctx: TextContext): string {
|
|
137
|
+
if (data === null) return "";
|
|
138
|
+
const main = data.phases.filter((p) => !TIMELINE_CLOSING_PHASES.has(p.name));
|
|
139
|
+
const closing = data.phases.filter((p) => TIMELINE_CLOSING_PHASES.has(p.name));
|
|
140
|
+
const total = main.reduce((sum, p) => sum + p.durationMs, 0);
|
|
141
|
+
const anyFailed = data.phases.some((p) => p.failed);
|
|
142
|
+
const command = evidenceCommand(ctx, data.locator, "--timing");
|
|
143
|
+
const head = [`timing: ${fmtMs(total)}${anyFailed ? " ✗" : ""}`];
|
|
144
|
+
if (command) head.push(command);
|
|
145
|
+
const lines = [head.join(" · ")];
|
|
146
|
+
for (const p of main) {
|
|
147
|
+
const kids = p.children ?? [];
|
|
148
|
+
const suffix = kids.length > 0 ? ` (${countTimingNodes(kids)} children collapsed)` : "";
|
|
149
|
+
lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}${suffix}`);
|
|
150
|
+
}
|
|
151
|
+
if (closing.length > 0) {
|
|
152
|
+
lines.push(" teardown:");
|
|
153
|
+
for (const p of closing) lines.push(` ${p.failed ? "✗" : "·"} ${p.name} ${fmtMs(p.durationMs)}`);
|
|
154
|
+
}
|
|
155
|
+
return lines.join("\n");
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// ───────────────────────── AttemptConversation ─────────────────────────
|
|
159
|
+
|
|
160
|
+
// 每条回复的自由文本(消息正文、发起请求的 prompt、未识别事件的原始 payload)都可能是任意大小的
|
|
161
|
+
// 块——system prompt、完整工具输出等——同一条 summaryText 规则折单行加上限;完整内容仍在
|
|
162
|
+
// --execution 里原样可查。
|
|
163
|
+
function replySummary(reply: AttemptConversationReply): string {
|
|
164
|
+
switch (reply.kind) {
|
|
165
|
+
case "assistant":
|
|
166
|
+
return `assistant: ${summaryText(reply.text)}`;
|
|
167
|
+
case "user":
|
|
168
|
+
return `user: ${summaryText(reply.text)}`;
|
|
169
|
+
case "thinking":
|
|
170
|
+
return `thinking: ${summaryText(reply.text)}`;
|
|
171
|
+
case "error":
|
|
172
|
+
return `error: ${summaryText(reply.text)}`;
|
|
173
|
+
case "tool":
|
|
174
|
+
return `tool ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
|
|
175
|
+
case "skill":
|
|
176
|
+
return `skill loaded: ${reply.skill}`;
|
|
177
|
+
case "subagent":
|
|
178
|
+
return `subagent ${reply.name}${reply.status ? ` (${reply.status})` : ""}`;
|
|
179
|
+
case "input":
|
|
180
|
+
return `input requested${reply.request.prompt ? `: ${summaryText(reply.request.prompt)}` : ""}`;
|
|
181
|
+
case "compaction":
|
|
182
|
+
return `compaction${reply.reason ? `: ${summaryText(reply.reason)}` : ""}`;
|
|
183
|
+
case "raw":
|
|
184
|
+
return `unrecognized event: ${summaryText(JSON.stringify(reply.raw))}`;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function attemptConversationText(data: AttemptConversationData | null, ctx: TextContext): string {
|
|
189
|
+
if (data === null) return "";
|
|
190
|
+
const command = evidenceCommand(ctx, data.locator, "--execution");
|
|
191
|
+
const head = [`conversation: ${data.rounds.length} round${data.rounds.length === 1 ? "" : "s"}`];
|
|
192
|
+
if (command) head.push(command);
|
|
193
|
+
const lines = [head.join(" · ")];
|
|
194
|
+
data.rounds.forEach((round, i) => {
|
|
195
|
+
lines.push(` round ${i + 1}${round.sentText ? `: ${summaryText(round.sentText)}` : ""}`);
|
|
196
|
+
for (const reply of round.replies) lines.push(` ${replySummary(reply)}`);
|
|
197
|
+
});
|
|
198
|
+
return lines.join("\n");
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// ───────────────────────── AttemptDiagnostics ─────────────────────────
|
|
202
|
+
|
|
203
|
+
export function attemptDiagnosticsText(data: AttemptDiagnosticsData | null, _ctx: TextContext): string {
|
|
204
|
+
if (data === null) return "";
|
|
205
|
+
const lines: string[] = [];
|
|
206
|
+
for (const { phase, items } of data.groups) {
|
|
207
|
+
lines.push(`${phase}:`);
|
|
208
|
+
for (const d of items) {
|
|
209
|
+
const count = d.count && d.count > 1 ? ` (${d.count} occurrences)` : "";
|
|
210
|
+
lines.push(` ${d.level} · ${d.code} — ${summaryText(d.message)}${count}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return lines.join("\n");
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// ───────────────────────── AttemptUsage ─────────────────────────
|
|
217
|
+
|
|
218
|
+
export function attemptUsageText(data: AttemptUsageData | null, _ctx: TextContext): string {
|
|
219
|
+
if (data === null) return "";
|
|
220
|
+
const { usage } = data;
|
|
221
|
+
const total = usage.inputTokens + usage.outputTokens;
|
|
222
|
+
const parts = [`usage: ${formatMetricValue(total)} tokens (${formatMetricValue(usage.inputTokens)} in / ${formatMetricValue(usage.outputTokens)} out)`];
|
|
223
|
+
if (usage.cacheReadTokens !== undefined) parts.push(`cache read ${formatMetricValue(usage.cacheReadTokens)}`);
|
|
224
|
+
if (usage.cacheWriteTokens !== undefined) parts.push(`cache write ${formatMetricValue(usage.cacheWriteTokens)}`);
|
|
225
|
+
if (usage.requests !== undefined) parts.push(`${usage.requests} requests`);
|
|
226
|
+
if (data.costUSD !== null) parts.push(formatUSD(data.costUSD));
|
|
227
|
+
return parts.join(" · ");
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ───────────────────────── AttemptTrace ─────────────────────────
|
|
231
|
+
|
|
232
|
+
export function attemptTraceText(data: AttemptTraceData | null, ctx: TextContext): string {
|
|
233
|
+
if (data === null) return "";
|
|
234
|
+
const command = evidenceCommand(ctx, data.locator, "--timing");
|
|
235
|
+
const head = [`trace: ${data.spans.length} span${data.spans.length === 1 ? "" : "s"}`];
|
|
236
|
+
if (command) head.push(command);
|
|
237
|
+
return head.join(" · ");
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ───────────────────────── AttemptDiff ─────────────────────────
|
|
241
|
+
|
|
242
|
+
function netLetter(net: "added" | "modified" | "deleted"): string {
|
|
243
|
+
return net === "added" ? "A" : net === "deleted" ? "D" : "M";
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export function attemptDiffText(data: AttemptDiffData | null, ctx: TextContext): string {
|
|
247
|
+
if (data === null) return "";
|
|
248
|
+
const command = evidenceCommand(ctx, data.locator, "--diff");
|
|
249
|
+
const head = [`changes: ${data.files.length} file${data.files.length === 1 ? "" : "s"} changed by agent`];
|
|
250
|
+
if (command) head.push(command);
|
|
251
|
+
const lines = [head.join(" · ")];
|
|
252
|
+
for (const f of data.files) {
|
|
253
|
+
const delta = f.binary ? "binary" : `+${f.lines.added}/-${f.lines.deleted}`;
|
|
254
|
+
lines.push(` ${netLetter(f.net)} ${f.path} (${delta})`);
|
|
255
|
+
}
|
|
256
|
+
return lines.join("\n");
|
|
257
|
+
}
|